├── Get-StakRank.ps1
├── License.md
└── README.md


/Get-StakRank.ps1:
--------------------------------------------------------------------------------
  1 | <#
  2 | .SYNOPSIS
  3 | Stacks csv/tsv input by frequency of occurence. Header and delimiter may be passed as arguments.
  4 | Output is written to tsv files.
  5 | .DESCRIPTION
  6 | Get-StakRank.ps1 parses multiple separated values input files, the user may specify the delimiter and 
  7 | header just as with import-csv, if not specified csv is assumed with the first row assumed to be the 
  8 | header row. The user specifies the fields by which to stack the data, defaulting in ascending order, 
  9 | creating a table where less frequently occuring items bubble up, if mutliple fields are provided as 
 10 | an argument, those fields will be ranked in combination.
 11 | 
 12 | If you don't know the fields and you're frequently working with various separated values files, 
 13 | https://github.com/davehull/Get-Fields.ps1, may be useful, alternatively, providing incorrect fields
 14 | throws an error that lists the fields found in the first input file that matches the supplied
 15 | file name pattern.
 16 | 
 17 | More details, including a use-case scenario that explains more:
 18 | https://github.com/davehull/Get-StakRank#get-stakrank
 19 | 
 20 | .PARAMETER FileNamePattern
 21 | Specifies the pattern common to the files to be ranked.
 22 | .PARAMETER Delimiter
 23 | Specifies the single character delimiter.
 24 | .PARAMETER Header
 25 | Specifies header values for the delimited file.
 26 | .PARAMETER Desc
 27 | Specifies output should be in descending order. Ascending is default.
 28 | .PARAMETER Key
 29 | Data should be sorted by the key.
 30 | .PARAMETER Value
 31 | Data should be sorted by the value, this is the default. The values sorted by are the
 32 | elements supplied by the user via the Fields argument.
 33 | .PARAMETER Roles
 34 | Output should be ranked by roles -- assumes input file names contain some role identifier.
 35 | .PARAMETER Fields
 36 | Specifies the field or fields to rank.
 37 | .EXAMPLE
 38 | Get-StakRank -FileNamePattern .\CADataCenter\*autoruns.csv -Roles .\CADataCenter\ServerRoles.txt -Fields MD5, "Image Path"
 39 | #>
 40 | 
 41 | [CmdletBinding()]
 42 | Param(
 43 |     [Parameter(Mandatory=$True,Position=0)]
 44 |         [string]$FileNamePattern,
 45 |     [Parameter(Mandatory=$False)]
 46 |         [char]$Delimiter=",",
 47 |     [Parameter(Mandatory=$False)]
 48 |         [string]$Header="",
 49 |     [Parameter(Mandatory=$False)]
 50 |         [switch]$Desc=$False,
 51 |     [Parameter(Mandatory=$False)]
 52 |         [switch]$Key=$False,
 53 |     [Parameter(Mandatory=$False)]
 54 |         [string]$RoleFile="",
 55 |     [Parameter(Mandatory=$True)]
 56 |         [array]$Fields
 57 | )
 58 | 
 59 | function Check-Fields {
 60 | <#
 61 | .SYNOPSIS
 62 | Verifies the user supplied fields are found in the input file.
 63 | If user supplied fields are not found in input file header, an 
 64 | error is written and the script exits.
 65 | #>
 66 | Param(
 67 |     [Parameter(Mandatory=$True,Position=0)]
 68 |         [Array]$FileFields,
 69 |     [Parameter(Mandatory=$True,Position=1)]
 70 |         [Array]$UserFields,
 71 |     [Parameter(Mandatory=$True,Position=2)]
 72 |         [Char]$Delimiter
 73 | )
 74 |     Write-Verbose "Entering $($MyInvocation.MyCommand)"
 75 |     Write-Debug "Parameters: `$FileFields is ${FileFields}; `$UserFields is ${UserFields}; `$Delimiter is `"$($Delimiter -replace "`t", "``t")`""
 76 |     $MissingFields = @()
 77 |     $FileFields = $FileFields -replace "`"", ""
 78 |     Write-Debug "`$FileFields is $($FileFields -join $Delimiter)"
 79 |     foreach($Field in $UserFields) {
 80 |         Write-Debug "`$Field is $Field"
 81 |         if ($FileFields -notcontains $Field) {
 82 |             Write-Debug "User supplied $Field was not found. Verify your delimiter is `"${Delimiter}`"."
 83 |             $MissingFields += $Field
 84 |         }
 85 |     }
 86 |     if ($MissingFields.Length -gt 1) {
 87 |         Write-Error "[!] Error: User supplied fields, $($MissingFields -join ", "), were not found in `n`t$($FileFields -join $Delimiter)"
 88 |         Write-Error "[!] Error: You may want to verify that you've specified the correct delimiter for the input files."
 89 |         Write-Verbose "Exiting $($MyInvocation.MyCommand)"
 90 |         exit
 91 |     } elseif ($MissingFields.Length -eq 1) {
 92 |         Write-Error "[!] Error: User supplied field, $MissingFields, was not found in `n`t$($FileFields -join $Delimiter)"
 93 |         Write-Error "[!] Error: You may want to verify that you've specified the correct delimiter for the input files."
 94 |         Write-Verbose "Exiting $($MyInvocation.MyCommand)"
 95 |         exit
 96 |     }
 97 |     Write-Verbose "Exiting $($MyInvocation.MyCommand)"
 98 | }
 99 | 
100 | function Get-Files {
101 | <#
102 | .SYNOPSIS
103 | Returns the list of input files matching the user supplied file name pattern.
104 | Traverses subdirectories.
105 | #>
106 | Param(
107 |     [Parameter(Mandatory=$True,Position=0)]
108 |         [String]$FileNamePattern
109 | )
110 |     Write-Verbose "Entering $($MyInvocation.MyCommand)"
111 |     Write-Verbose "Looking for files matching user supplied pattern, $FileNamePattern"
112 |     Write-Verbose "This process traverses subdirectories so it may take some time."
113 |     $Files = @(ls -r $FileNamePattern | % { $_.FullName })
114 |     if ($Files) {
115 |         Write-Verbose "File(s) matching pattern, ${FileNamePattern}:`n$($Files -join "`n")"
116 |         $Files
117 |     } else {
118 |         Write-Error "No input files were found matching the user supplied pattern, ${FileNamePattern}."
119 |         Write-Verbose "Exiting $($MyInvocation.MyCommand)"
120 |         exit
121 |     }
122 |     Write-Verbose "Exiting $($MyInvocation.MyCommand)"
123 | }
124 | 
125 | function Get-FileHeader {
126 | <#
127 | .SYNOPSIS
128 | Get the header row from the first file in the list of files supplied by the user.
129 | #>
130 | Param(
131 |     [Parameter(Mandatory=$True,Position=0)]
132 |         [string]$File,
133 |     [Parameter(Mandatory=$True,Position=1)]
134 |         [char]$Delimiter
135 | )
136 |     Write-Verbose "Entering $($MyInvocation.MyCommand)"
137 |     $Fields = @()
138 |     Write-Verbose "Attempting to extract input file headers from ${File}."
139 |     $HeaderRow = gc $File -TotalCount 1
140 |     $Fields = @($HeaderRow -split $Delimiter)
141 |     $Fields = $Fields -replace "`"", ""
142 |     Write-Verbose "Extracted the following fields: $($Fields -join $Delimiter)"
143 |     $Fields
144 |     Write-Verbose "Exiting $($MyInvocation.MyCommand)"
145 | }
146 | 
147 | function Get-Roles {
148 | <#
149 | .SYNOPSIS
150 | Reads roles from a text file, one role per line. Think of roles as smart name elements 
151 | common to specific groups of things (i.e. computernames from HR dept. containing "HR")
152 | #>
153 | Param(
154 |     [Parameter(Mandatory=$True,Position=0)]
155 |         [string]$RoleFile
156 | )
157 |     Write-Verbose "Entering $($MyInvocation.MyCommand)"
158 |     if (Test-Path $RoleFile) {
159 |         $Roles = gc $RoleFile
160 |         Write-Verbose "Found the following roles in ${RoleFile}: ${Roles}"
161 |     } else {
162 |         Write-Error "User specified role file, ${RoleFile}, was not found."
163 |         Write-Verbose "Exiting $($MyInvocation.MyCommand)"
164 |         exit
165 |     }
166 |     $Roles
167 |     Write-Verbose "Exiting $($MyInvocation.MyCommand)"
168 | }
169 | 
170 | function Get-FieldList {
171 | <#
172 | .SYNOPSIS
173 | Returns the user supplied fields for use in a scriptblock to be used in the ranking.
174 | #>
175 | Param(
176 |     [Parameter(Mandatory=$True,Position=0)]
177 |         [array]$Fields
178 | )
179 |     Write-Verbose "Entering $($MyInvocation.MyCommand)"
180 |     $FieldList = $Fields -join "`" + `"``t`" + `$_.`""
181 |     $FieldList += "`""
182 |     $FieldList = "`$_.`"" + $FieldList
183 |     Write-Verbose "`$FieldList is ${FieldList}."
184 |     $FieldList
185 |     Write-Verbose "Exiting $($MyInvocation.MyCommand)"
186 | }
187 | 
188 | function Get-Rank {
189 | <#
190 | .SYNOPSIS
191 | The heart of the script where the actual ranking of the data happens. I'm looking for way to refactor and improve
192 | readability and performance.
193 | #>
194 | Param(
195 |     [Parameter(Mandatory=$True,Position=0)]
196 |         [array]$Files,
197 |     [Parameter(Mandatory=$True,Position=1)]
198 |         [char]$Delimiter,
199 |     [Parameter(Mandatory=$True,Position=2)]
200 |         [Array]$Header,
201 |     [Parameter(Mandatory=$False)]
202 |         [array]$Roles,
203 |     [Parameter(Mandatory=$False)]
204 |         [boolean]$Desc,
205 |     [Parameter(Mandatory=$False)]
206 |         [boolean]$Key,
207 |     [Parameter(Mandatory=$True)]
208 |         [Array]$Fields
209 | )        
210 |     Write-Verbose "Entering $($MyInvocation.MyCommand)"
211 |     $FieldList = Get-FieldList $Fields
212 |     Write-Debug "`$FieldList is $FieldList"
213 | 
214 |     $DictScriptblock = {
215 |         if ($Dict.ContainsKey($Element)) {
216 |             Write-Verbose "Incrementing ${Element}."
217 |             $Dict.Set_Item($Element, $Dict.Get_Item($Element) + 1)
218 |         } else {
219 |             Write-Verbose "Adding ${Element}."
220 |             $Dict.add($Element, 1)
221 |         }
222 |     }
223 | 
224 |     $OutScriptblock = {
225 |         if ($Role) {
226 |             $Outheader = "Count`tRole`t"
227 |             Write-Debug "`$Outheader is $($Outheader -replace "`t", "``t")"
228 |         } else {
229 |             $Outheader = "Count`t"
230 |         }
231 |         $Outheader += $Fields -join "`t"
232 |         $Outheader += "`r`n"
233 |         Write-Debug "`$Outheader is $($Outheader -replace "`t", "``t")"
234 |         $Output = ""
235 |         if ($Key) {
236 |             Write-Verbose "Writing out by key."
237 |             if ($Desc) {
238 |                 $Output += $Dict.GetEnumerator() | Sort-Object -Desc key,value | % {[string]$_.Value + "`t" + $_.Key + "`r`n"}
239 |             } else {
240 |                 $Output += $Dict.GetEnumerator() | Sort-Object key,value | % {[string]$_.Value + "`t" + $_.Key + "`r`n"}
241 |             }
242 |         } else {
243 |             Write-Verbose "Writing out by value."
244 |             if ($Desc) {
245 |                 $Output += $Dict.GetEnumerator() | Sort-Object -Desc value,key | % {[string]$_.Value + "`t" + $_.Key + "`r`n"}
246 |             } else {
247 |                 $Output += $Dict.GetEnumerator() | Sort-Object value,key | % {[string]$_.value + "`t" + $_.key + "`r`n"}
248 |             }
249 |         }
250 |         $Output = $Outheader += $Output
251 |         Write-Debug "`$Output is $($Output -replace "`t", "``t")"
252 |         $FieldsFileName = $Fields -join "-"
253 |         if ($Role) {
254 |             $Output | Set-Content -Encoding Ascii ${Role}-${FieldsFileName}.tsv
255 |         } else {
256 |             $Output | Set-Content -Encoding Ascii -Path $(${FieldsFileName} + ".tsv")
257 |         }
258 |     }
259 | 
260 |     if ($Roles) {
261 |         Write-Verbose "We have roles..."
262 |         $PrefixFieldList = "`$Element = `$Role + `"``t`" + "
263 |         $PrefixFieldList += $FieldList
264 |         $FieldList = $PrefixFieldList
265 |         $FieldList += $DictScriptblock
266 |         $Scriptblock = [scriptblock]::Create($FieldList)
267 |         foreach ($Role in $Roles) {
268 |             Write-Verbose "Processing role ${Role}."
269 |             $InputData = @()
270 |             $Dict = @{}
271 |             $FilesInRole = $Files | ? { $_ -match $Role}
272 |             if ($FilesInRole) {
273 |                 foreach ($File in $FilesInRole) {
274 |                     Write-Verbose "Reading ${File}."
275 |                     $InputData += Import-Csv -Path $File -Delimiter $Delimiter -Header $Header
276 |                 }
277 |             } else {
278 |                 Write-Verbose "No files found matching role, ${Role}. Continuing."
279 |                 Continue
280 |             }
281 |             Write-Verbose "Building dictionary of stack ranked elements for ${Role}."
282 |             $InputData | % $Scriptblock
283 |             & $OutScriptblock
284 |         }
285 |     } else {
286 |         Write-Verbose "We have no roles..."
287 |         # $PrefixFieldList = "`$Element = `"``t`" + "
288 |         $PrefixFieldList = "`$Element = `"`" + "
289 |         $PrefixFieldList += $FieldList
290 |         $FieldList = $PrefixFieldList
291 |         $FieldList += $DictScriptblock
292 |         Write-Debug "`$FieldList is $FieldList"
293 |         $Scriptblock = [scriptblock]::Create($FieldList)
294 |         Write-Verbose "Processing all up."
295 |         $InputData = @()
296 |         $Dict = @{}
297 |         foreach($File in $Files) {
298 |             Write-Verbose "Reading ${File}."
299 |             $InputData += Import-Csv -Path $File -Delimiter $Delimiter -Header $Header
300 |         }
301 |         Write-Verbose "Building dictionary of stack ranked elements all up."
302 |         $InputData | % $Scriptblock
303 |         & $OutScriptblock
304 |     }
305 |     Write-Verbose "Exiting $($MyInvocation.MyCommand)"
306 | }
307 | 
308 | $Files, $Roles, $InputFileHeader = @()
309 | $FieldList = ""
310 | Write-Verbose "Starting up $($MyInvocation.MyCommand)"
311 | if ($RoleFile) {
312 |     $Roles = @(Get-Roles $RoleFile)
313 | }
314 | $Files = @(Get-Files $FileNamePattern)
315 | if ($Header) {
316 |     $InputFileHeader = @($Header -split $Delimiter)
317 | } else {
318 |     $InputFileHeader = @(Get-FileHeader $Files[0] $Delimiter)
319 | }
320 | Check-Fields $InputFileHeader $Fields $Delimiter
321 | Write-Debug "User supplied fields, ${Fields}, found in input file."
322 | Get-Rank -Files $Files -Delimiter $Delimiter -Header $InputFileHeader -Roles $Roles -Desc $Desc -Key $Key -Fields $Fields
323 | Write-Verbose "Exiting $($MyInvocation.MyCommand)"
324 | 


--------------------------------------------------------------------------------
/License.md:
--------------------------------------------------------------------------------
  1 | Apache License
  2 |                            Version 2.0, January 2004
  3 |                         http://www.apache.org/licenses/
  4 | 
  5 |    TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
  6 | 
  7 |    1. Definitions.
  8 | 
  9 |       "License" shall mean the terms and conditions for use, reproduction,
 10 |       and distribution as defined by Sections 1 through 9 of this document.
 11 | 
 12 |       "Licensor" shall mean the copyright owner or entity authorized by
 13 |       the copyright owner that is granting the License.
 14 | 
 15 |       "Legal Entity" shall mean the union of the acting entity and all
 16 |       other entities that control, are controlled by, or are under common
 17 |       control with that entity. For the purposes of this definition,
 18 |       "control" means (i) the power, direct or indirect, to cause the
 19 |       direction or management of such entity, whether by contract or
 20 |       otherwise, or (ii) ownership of fifty percent (50%) or more of the
 21 |       outstanding shares, or (iii) beneficial ownership of such entity.
 22 | 
 23 |       "You" (or "Your") shall mean an individual or Legal Entity
 24 |       exercising permissions granted by this License.
 25 | 
 26 |       "Source" form shall mean the preferred form for making modifications,
 27 |       including but not limited to software source code, documentation
 28 |       source, and configuration files.
 29 | 
 30 |       "Object" form shall mean any form resulting from mechanical
 31 |       transformation or translation of a Source form, including but
 32 |       not limited to compiled object code, generated documentation,
 33 |       and conversions to other media types.
 34 | 
 35 |       "Work" shall mean the work of authorship, whether in Source or
 36 |       Object form, made available under the License, as indicated by a
 37 |       copyright notice that is included in or attached to the work
 38 |       (an example is provided in the Appendix below).
 39 | 
 40 |       "Derivative Works" shall mean any work, whether in Source or Object
 41 |       form, that is based on (or derived from) the Work and for which the
 42 |       editorial revisions, annotations, elaborations, or other modifications
 43 |       represent, as a whole, an original work of authorship. For the purposes
 44 |       of this License, Derivative Works shall not include works that remain
 45 |       separable from, or merely link (or bind by name) to the interfaces of,
 46 |       the Work and Derivative Works thereof.
 47 | 
 48 |       "Contribution" shall mean any work of authorship, including
 49 |       the original version of the Work and any modifications or additions
 50 |       to that Work or Derivative Works thereof, that is intentionally
 51 |       submitted to Licensor for inclusion in the Work by the copyright owner
 52 |       or by an individual or Legal Entity authorized to submit on behalf of
 53 |       the copyright owner. For the purposes of this definition, "submitted"
 54 |       means any form of electronic, verbal, or written communication sent
 55 |       to the Licensor or its representatives, including but not limited to
 56 |       communication on electronic mailing lists, source code control systems,
 57 |       and issue tracking systems that are managed by, or on behalf of, the
 58 |       Licensor for the purpose of discussing and improving the Work, but
 59 |       excluding communication that is conspicuously marked or otherwise
 60 |       designated in writing by the copyright owner as "Not a Contribution."
 61 | 
 62 |       "Contributor" shall mean Licensor and any individual or Legal Entity
 63 |       on behalf of whom a Contribution has been received by Licensor and
 64 |       subsequently incorporated within the Work.
 65 | 
 66 |    2. Grant of Copyright License. Subject to the terms and conditions of
 67 |       this License, each Contributor hereby grants to You a perpetual,
 68 |       worldwide, non-exclusive, no-charge, royalty-free, irrevocable
 69 |       copyright license to reproduce, prepare Derivative Works of,
 70 |       publicly display, publicly perform, sublicense, and distribute the
 71 |       Work and such Derivative Works in Source or Object form.
 72 | 
 73 |    3. Grant of Patent License. Subject to the terms and conditions of
 74 |       this License, each Contributor hereby grants to You a perpetual,
 75 |       worldwide, non-exclusive, no-charge, royalty-free, irrevocable
 76 |       (except as stated in this section) patent license to make, have made,
 77 |       use, offer to sell, sell, import, and otherwise transfer the Work,
 78 |       where such license applies only to those patent claims licensable
 79 |       by such Contributor that are necessarily infringed by their
 80 |       Contribution(s) alone or by combination of their Contribution(s)
 81 |       with the Work to which such Contribution(s) was submitted. If You
 82 |       institute patent litigation against any entity (including a
 83 |       cross-claim or counterclaim in a lawsuit) alleging that the Work
 84 |       or a Contribution incorporated within the Work constitutes direct
 85 |       or contributory patent infringement, then any patent licenses
 86 |       granted to You under this License for that Work shall terminate
 87 |       as of the date such litigation is filed.
 88 | 
 89 |    4. Redistribution. You may reproduce and distribute copies of the
 90 |       Work or Derivative Works thereof in any medium, with or without
 91 |       modifications, and in Source or Object form, provided that You
 92 |       meet the following conditions:
 93 | 
 94 |       (a) You must give any other recipients of the Work or
 95 |           Derivative Works a copy of this License; and
 96 | 
 97 |       (b) You must cause any modified files to carry prominent notices
 98 |           stating that You changed the files; and
 99 | 
100 |       (c) You must retain, in the Source form of any Derivative Works
101 |           that You distribute, all copyright, patent, trademark, and
102 |           attribution notices from the Source form of the Work,
103 |           excluding those notices that do not pertain to any part of
104 |           the Derivative Works; and
105 | 
106 |       (d) If the Work includes a "NOTICE" text file as part of its
107 |           distribution, then any Derivative Works that You distribute must
108 |           include a readable copy of the attribution notices contained
109 |           within such NOTICE file, excluding those notices that do not
110 |           pertain to any part of the Derivative Works, in at least one
111 |           of the following places: within a NOTICE text file distributed
112 |           as part of the Derivative Works; within the Source form or
113 |           documentation, if provided along with the Derivative Works; or,
114 |           within a display generated by the Derivative Works, if and
115 |           wherever such third-party notices normally appear. The contents
116 |           of the NOTICE file are for informational purposes only and
117 |           do not modify the License. You may add Your own attribution
118 |           notices within Derivative Works that You distribute, alongside
119 |           or as an addendum to the NOTICE text from the Work, provided
120 |           that such additional attribution notices cannot be construed
121 |           as modifying the License.
122 | 
123 |       You may add Your own copyright statement to Your modifications and
124 |       may provide additional or different license terms and conditions
125 |       for use, reproduction, or distribution of Your modifications, or
126 |       for any such Derivative Works as a whole, provided Your use,
127 |       reproduction, and distribution of the Work otherwise complies with
128 |       the conditions stated in this License.
129 | 
130 |    5. Submission of Contributions. Unless You explicitly state otherwise,
131 |       any Contribution intentionally submitted for inclusion in the Work
132 |       by You to the Licensor shall be under the terms and conditions of
133 |       this License, without any additional terms or conditions.
134 |       Notwithstanding the above, nothing herein shall supersede or modify
135 |       the terms of any separate license agreement you may have executed
136 |       with Licensor regarding such Contributions.
137 | 
138 |    6. Trademarks. This License does not grant permission to use the trade
139 |       names, trademarks, service marks, or product names of the Licensor,
140 |       except as required for reasonable and customary use in describing the
141 |       origin of the Work and reproducing the content of the NOTICE file.
142 | 
143 |    7. Disclaimer of Warranty. Unless required by applicable law or
144 |       agreed to in writing, Licensor provides the Work (and each
145 |       Contributor provides its Contributions) on an "AS IS" BASIS,
146 |       WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
147 |       implied, including, without limitation, any warranties or conditions
148 |       of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
149 |       PARTICULAR PURPOSE. You are solely responsible for determining the
150 |       appropriateness of using or redistributing the Work and assume any
151 |       risks associated with Your exercise of permissions under this License.
152 | 
153 |    8. Limitation of Liability. In no event and under no legal theory,
154 |       whether in tort (including negligence), contract, or otherwise,
155 |       unless required by applicable law (such as deliberate and grossly
156 |       negligent acts) or agreed to in writing, shall any Contributor be
157 |       liable to You for damages, including any direct, indirect, special,
158 |       incidental, or consequential damages of any character arising as a
159 |       result of this License or out of the use or inability to use the
160 |       Work (including but not limited to damages for loss of goodwill,
161 |       work stoppage, computer failure or malfunction, or any and all
162 |       other commercial damages or losses), even if such Contributor
163 |       has been advised of the possibility of such damages.
164 | 
165 |    9. Accepting Warranty or Additional Liability. While redistributing
166 |       the Work or Derivative Works thereof, You may choose to offer,
167 |       and charge a fee for, acceptance of support, warranty, indemnity,
168 |       or other liability obligations and/or rights consistent with this
169 |       License. However, in accepting such obligations, You may act only
170 |       on Your own behalf and on Your sole responsibility, not on behalf
171 |       of any other Contributor, and only if You agree to indemnify,
172 |       defend, and hold each Contributor harmless for any liability
173 |       incurred by, or claims asserted against, such Contributor by reason
174 |       of your accepting any such warranty or additional liability.
175 | 
176 |    END OF TERMS AND CONDITIONS
177 | 
178 |    APPENDIX: How to apply the Apache License to your work.
179 | 
180 |       To apply the Apache License to your work, attach the following
181 |       boilerplate notice, with the fields enclosed by brackets "{}"
182 |       replaced with your own identifying information. (Don't include
183 |       the brackets!)  The text should be enclosed in the appropriate
184 |       comment syntax for the file format. We also recommend that a
185 |       file or class name and description of purpose be included on the
186 |       same "printed page" as the copyright notice for easier
187 |       identification within third-party archives.
188 | 
189 |    Copyright {yyyy} {name of copyright owner}
190 | 
191 |    Licensed under the Apache License, Version 2.0 (the "License");
192 |    you may not use this file except in compliance with the License.
193 |    You may obtain a copy of the License at
194 | 
195 |        http://www.apache.org/licenses/LICENSE-2.0
196 | 
197 |    Unless required by applicable law or agreed to in writing, software
198 |    distributed under the License is distributed on an "AS IS" BASIS,
199 |    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
200 |    See the License for the specific language governing permissions and
201 |    limitations under the License.
202 | 
203 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
  1 | ##Get-Stakrank
  2 | =============
  3 | Get-Stakrank came out of the need to parse hundreds, thousands, tens of thousands... of csv files containing various kinds of data collected from distributed computer systems. Consider csv output from tools like Sysinternals autorunsc.exe, sigcheck.exe, or the native Windows tasklist.exe with loaded modules, collected from many hosts during the process of hunting for indications of compromise in large distributed networks. Many attackers want to maintain persistence in environments and there are a variety of means for accomplishing this from credential harvesting, to planting code on the host. If the attacker plants code on a small number of hosts, small being a relative term given the overall environment size, stack ranking the frequency of occurrence for a given autorun, process or loaded module can be a useful lead generation tool for incident response teams.
  4 | 
  5 | For more on the idea of stack ranking as a means of hunting evil, check out Mandiant's post, "An In-Depth Look Into Data Stacking," here: https://www.mandiant.com/blog/indepth-data-stacking/.
  6 | 
  7 | One of the capability goals for the script was to be able to process any separated values file. This is an early release and has been through limited testing. If you find bugs, I'm sure there will be some, please use the Issues function in github and let me know about it. I'll do what I can to fix it, or if you're ambitious, fork the repo and fix it yourself.
  8 | 
  9 | ####Example scenario:
 10 | Say your organization is comprised of Sales &amp; Marketing, R&amp;D, Operations, HR, Finance and IT. You want to engage in systematic, goal oriented breach hunts across your organization and decide to focus on one aspect of each business unit -- Windows Autostart Extension Points or ASEPs more commonly known as autoruns.
 11 | 
 12 | Your IT group uses smart naming conventions for computers in your organization that tie each resource to its role, for example, computers belonging to Sales &amp; Marketing have "MKTG" in the hostname; computers belonging to R&amp;D have "RND" in the hostname; Operations systems have "OPS" in the hostname; "HRES" for HR; "FIN" for Finance; and "INFO" for IT.
 13 | 
 14 | A sample listing of hostnames for each environment might look like the following:<br />
 15 | ```
 16 | 534AZMKTG -- a Sales & Marketing host
 17 | 277AARND  -- a R&D host
 18 | 182AROPS  -- an Operations host
 19 | 011AHRES  -- a HR host
 20 | 1040EZFIN -- a Finance host
 21 | 1337INFO  -- a host from IT
 22 | ```
 23 | Knowing your organization's smart naming convention, allows you to sort data and make comparisons within each role. Say for example it's common for your R&amp;D folks to have certain software development tools installed on their systems, but outside of that role, it's relatively uncommon. You will likely find intra-role commonalities within each division. Your Finance team will likely have software packages installed that are unique to their role. Of course, if your organization doesn't use such naming conventions, you can still collect the data and analyze it all up, without regard to role, but being able to sort it out by role has the advantage of helping you spot a malicious process on an IT machine that an attacker has named to resemble a process common to Finance machines.
 24 | 
 25 | You create a script that uses Sysinternals Autorunsc.exe to collect all ASEPs from every profile on every host, complete with MD5, SHA1 and SHA256 hashes of each ASEP file and write the output to csv files that include the name of each host the data came from, very roughly, something like the following executed on every host:<br />
 26 | ```Powershell
 27 | & \\hunter\tools\autorunsc.exe -a -v -f -c '*' > \\hunter\data\$env:computername.autoruns.csv
 28 | ```
 29 | You orchestrate this collection in whatever way you can, SCCM, Powershell Remoting, PSExec, GPO push, etc. Maybe your organization is large and highly geographically distributed and collection takes a week or two. The result is a pile of data in the \\\hunter\data share, maybe hundreds, thousands, tens or hundreds of thousands of csv files listing every Autorun for every system in your organization.
 30 | 
 31 | You can now use Get-Stakrank to perform frequency analysis of this data and help you find follow up items that may warrant further investigation. Since our scenario involves an organization that uses smart system naming conventions, you can use that to your advantage and sort the data by system role. You do this by putting the role identifiers in a text file, one per line and saving that file to disk, maybe call it roles.txt.
 32 | 
 33 | You then call Get-Stakrank.ps1 from within the directory (or a parent directory) where your collected Autoruns data is, as follows:<br />
 34 | ```Powershell
 35 | .\Get-Stakrank -FileNamePattern *autoruns.csv -RoleFile .\roles.txt -Fields MD5, "Image Path"
 36 | ```
 37 | Where did the -Fields arguments come from? Those are fields in the Autoruns output that you're using to stack rank the data, you can choose whatever fields you want, so long as they are present in the input file's header or match the user supplied header, which can be passed as an argument. If you run the command as shown above, depending on the size of your data collection, it may run for a few seconds, or for hours, the script, as run above, provides no feedback as to its progress, for that you may want to run it with the -Verbose flag. When run with the -Verbose flag, you may see something like the following:<br />
 38 | <br />
 39 | ```
 40 | VERBOSE: Starting up Get-StakRank.ps1
 41 | VERBOSE: Entering Get-Roles
 42 | VERBOSE: Found the following roles in .\roles.txt: MKTG RND OPS HRES FIN INFO
 43 | VERBOSE: Exiting Get-Roles
 44 | VERBOSE: Entering Get-Files
 45 | VERBOSE: Looking for files matching user supplied pattern, .\data\*autoruns.csv
 46 | VERBOSE: This process traverses subdirectories so it may take some time.
 47 | VERBOSE: File(s) matching pattern, .\data\*autoruns.csv:
 48 | E:\hunt\data\ATL001FIN_autoruns.csv
 49 | E:\hunt\data\ATL002FIN_autoruns.csv
 50 | E:\hunt\data\ATL003FIN_autoruns.csv
 51 | E:\hunt\data\ATL004FIN_autoruns.csv
 52 | E:\hunt\data\ATL001HRES_autoruns.csv
 53 | E:\hunt\data\ATL002HRES_autoruns.csv
 54 | E:\hunt\data\ATL003HRES_autoruns.csv
 55 | E:\hunt\data\ATL004HRES_autoruns.csv
 56 | E:\hunt\data\ATL005HRES_autoruns.csv
 57 | E:\hunt\data\ATL006HRES_autoruns.csv
 58 | E:\hunt\data\ATL007HRES_autoruns.csv
 59 | E:\hunt\data\ATL001INFO_autoruns.csv
 60 | E:\hunt\data\ATL002INFO_autoruns.csv
 61 | ...
 62 | E:\hunt\data\ATLFFFINFO_autoruns.csv
 63 | E:\hunt\data\ATL001MKTG_autoruns.csv
 64 | E:\hunt\data\ATL002MKTG_autoruns.csv
 65 | E:\hunt\data\ATL003MKTG_autoruns.csv
 66 | E:\hunt\data\ATL004MKTG_autoruns.csv
 67 | ...
 68 | E:\hunt\data\ATLFFFMKTG_autoruns.csv
 69 | ...
 70 | VERBOSE: Exiting Get-Files
 71 | VERBOSE: Entering Get-FileHeader
 72 | VERBOSE: Attempting to extract input file headers from E:\hunt\data\ATL0001FIN_autoruns.csv.
 73 | VERBOSE: Extracted the following fields: Entry Location,Entry,Enabled,Description,Publisher,Image Path,Launch String,MD5,SHA-1,SHA-256
 74 | VERBOSE: Exiting Get-FileHeader
 75 | VERBOSE: Entering Check-Fields
 76 | VERBOSE: Exiting Check-Fields
 77 | VERBOSE: Entering Get-Rank
 78 | VERBOSE: Entering Get-FieldList
 79 | VERBOSE: $FieldList is $_."MD5" + "`t" + $_."Image Path".
 80 | VERBOSE: Exiting Get-FieldList
 81 | VERBOSE: We have roles...
 82 | VERBOSE: Processing role FIN.
 83 | VERBOSE: E:\hunt\data\ATL001FIN_autoruns.csv
 84 | VERBOSE: E:\hunt\data\ATL002FIN_autoruns.csv
 85 | VERBOSE: E:\hunt\data\ATL003FIN_autoruns.csv
 86 | VERBOSE: E:\hunt\data\ATL004FIN_autoruns.csv
 87 | VERBOSE: Building dictionary of stack ranked elements for FIN.
 88 | VERBOSE: Adding FIN MD5 Image Path.
 89 | VERBOSE: Adding FIN 3b536a8bec3b4f23ffdfd78b11a2ab93 c:\windows\system32\autochk.exe.
 90 | VERBOSE: Adding FIN 3290d6946b5e30e70414990574883ddb c:\windows\system32\alg.exe.
 91 | VERBOSE: Adding FIN f23fef6d569fce88671949894a8becf1 c:\windows\system32\audiosrv.dll.
 92 | VERBOSE: Incrementing FIN f23fef6d569fce88671949894a8becf1 c:\windows\system32\audiosrv.dll.
 93 | VERBOSE: Adding FIN f22f7f2395560a83e7d1da705e3ba759 c:\windows\system32\bfe.dll.
 94 | VERBOSE: Adding FIN 1ea7969e3271cbc59e1730697dc74682 c:\windows\system32\qmgr.dll.
 95 | VERBOSE: Adding FIN a8edb86fc2a4d6d1285e4c70384ac35a c:\windows\system32\dllhost.exe.
 96 | VERBOSE: Adding FIN c118a82cd78818c29ab228366ebf81c3 c:\windows\system32\lsass.exe.
 97 | VERBOSE: Adding FIN b4447f606bb19fd8ad0bafb59b90f5d9 c:\windows\system32\fntcache.dll.
 98 | VERBOSE: Incrementing FIN c118a82cd78818c29ab228366ebf81c3 c:\windows\system32\lsass.exe.
 99 | ...
100 | VERBOSE: Writing out by value.
101 | VERBOSE: Exiting Get-Rank
102 | VERBOSE: Exiting Get-StakRank.ps1
103 | ```
104 | When the script completes, you're left with one tsv file (per role in the role scenario) summarizing the frequency of each Autorun for each of the -Fields you provided. Of course, if you run the script without the -RolesFile option because you don't have a naming convention you can rely on, you'll end up with a single tsv file summarizing the frequency of each Autorun across all the systems for which you have collected data. Incidentally, for those still reading, a better way to run this analysis specifically for Autoruns data would be:<br />
105 | ```Powershell
106 | .\Get-Stakrank -FileNamePattern *autoruns.csv -RoleFile .\roles.txt -key -Fields "Image Path", MD5 -Verbose
107 | ```
108 | By swapping the "Image Path" and MD5 fields and sorting by the key, which in this case will be the "Image Path" and MD5 tuple, rather than sorting by value, which is the frequency count, you end up with a result that shows the frequency of each entry by role with "Image Path" values clustered together, something like this:<br />
109 | ```
110 | Count    Role    Image Path    MD5
111 | ----------------------------------
112 | 10    FIN
113 | 7    FIN    c:\program files\hp\cissesrv\cissesrv.exe    b7de9eab067dc76047b7e46707914807
114 | 1    FIN    c:\program files\hp\cissesrv\cissesrv.exe    bf68a382c43a5721eef03ff45faece4a
115 | 1    FIN    c:\program files\hpwbem\storage\service\hpwmistor.exe    5534ed475c61188fffa4168f28a0d893
116 | 1    FIN    c:\program files\hpwbem\storage\service\hpwmistor.exe    85fea3a46d528ed62e6d3ba4bd1c3fcd
117 | 9    FIN    c:\program files\microsoft security client\antimalware\mpcmdrun.exe    180e295d3c0b0e30cab63b8a50b38122
118 | 1    FIN    c:\program files\microsoft security client\antimalware\mpcmdrun.exe    705c190bf4a86b35c97a7622a539edd1
119 | 1    FIN    c:\program files\microsoft security client\antimalware\msmpeng.exe    157e9e498206a3366baa7e4697bdd947
120 | 9    FIN    c:\program files\microsoft security client\antimalware\msmpeng.exe    64e69a217d861776ca848b453fb96d71
121 | 1    FIN    c:\program files\microsoft security client\antimalware\nissrv.exe    566ddd5d82520da01d75f81428ac4c38
122 | 9    FIN    c:\program files\microsoft security client\antimalware\nissrv.exe    c67e39d2968400b38f54a10822e6eacf
123 | 9    FIN    c:\program files\microsoft security client\msseces.exe    46ee88d1ee4562186987b525aefe58b6
124 | ```
125 | Viewing the data this way allows you to quickly review for outliers that may be hiding as binaries with the same name as legit versions, or it could be that different hosts are running different versions of the same software. Again, the idea is lead generation for further investigation, this is a method for finding anomalies and not every anomaly is an indicator of something malicious.
126 | 
127 | Get-Stakrank is not limited to frequency analysis of Autoruns output. It should be applicable for any collection of separated values files.
128 | 
129 | ####Update:
130 | Some have asked, "Cool story bro, but how do I go from the output of this script, back to the source machine(s) a given line of data may have come from?"
131 | 
132 | Fair question. I do this using Powershell as follows:
133 | ```Powershell
134 | Select-String -pattern "bf68a382c43a5721eef03ff45faece4a" *autoruns.csv
135 | ```
136 | from within the directory where all my Autoruns data was stashed.
137 | 


--------------------------------------------------------------------------------