├── template-csvs ├── simplecsv2.csv ├── simplecsv1.csv ├── csv1.csv ├── csv3.csv └── csv2.csv ├── merge-csv-example-github.png ├── README.md ├── LICENSE ├── MergeCsv.psd1 ├── MergeCsv.Tests.ps1 └── MergeCsv.psm1 /template-csvs/simplecsv2.csv: -------------------------------------------------------------------------------- 1 | Username,Department 2 | John,HR 3 | Jane,IT 4 | Janet,Maintenance 5 | -------------------------------------------------------------------------------- /merge-csv-example-github.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/EliteLoser/MergeCsv/HEAD/merge-csv-example-github.png -------------------------------------------------------------------------------- /template-csvs/simplecsv1.csv: -------------------------------------------------------------------------------- 1 | Username,Email 2 | John,john@example.com 3 | Jane,jane@example.com 4 | Janet,janet@maintexample.com 5 | -------------------------------------------------------------------------------- /template-csvs/csv1.csv: -------------------------------------------------------------------------------- 1 | ComputerName,Uh,OSFamily 2 | ServerA,UhA,Windows 3 | ServerA,UhA,Linux 4 | ServerB,UhB,Windows 5 | ServerB,UhB,Linux 6 | -------------------------------------------------------------------------------- /template-csvs/csv3.csv: -------------------------------------------------------------------------------- 1 | ComputerName,Uh,OrderFile3 2 | ServerA,UhA,1 3 | ServerA,UhA,2 4 | ServerA,UhA,3 5 | ServerA,UhA,4 6 | ServerB,UhB,5 7 | ServerB,UhB,6 8 | 9 | -------------------------------------------------------------------------------- /template-csvs/csv2.csv: -------------------------------------------------------------------------------- 1 | ComputerName,Uh,PSVer,Env 2 | ServerA,UhA,5.1,Production 3 | ServerA,UhA,6.0,Test 4 | ServerA,UhA,3.0,Production 5 | ServerB,UhB,5.1,Test 6 | ServerB,UhB,6.0,Legacy -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # MergeCsv 2 | 3 | Merge-Csv is a function in the MergeCsv module for PowerShell which is now also here on GitHub. 4 | 5 | It works well with PowerShell version 3 and up - and might sort of work with PowerShell version 2. 6 | PowerShell version 3 or higher is recommended. 7 | 8 | Online blog documentation: https://www.powershelladmin.com/wiki/Merge_CSV_files_or_PSObjects_in_PowerShell 9 | 10 | It's also published to the PowerShell Gallery: https://www.powershellgallery.com/packages/MergeCsv/ 11 | 12 | If you have PSv5 or newer (default in Win 10 / Server 2016), you can install with: 13 | ``` 14 | Install-Module -Name MergeCsv 15 | ``` 16 | 17 | or for your user only (elevation not required): 18 | ``` 19 | Install-Module -Name MergeCsv -Scope CurrentUser 20 | ``` 21 | 22 | I will add more information here later, but for now check out the online docs which are comprehensive. 23 | 24 | Here's a screenshot demonstrating some of the more sophisticated features. 25 | 26 | ![alt tag](/merge-csv-example-github.png) 27 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2013 Joakim Svendsen 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /MergeCsv.psd1: -------------------------------------------------------------------------------- 1 | # 2 | # Module manifest for module 'PSGet_MergeCsv' 3 | # 4 | # Generated by: Joakim Borger Svendsen 5 | # 6 | # Generated on: 12/12/2017 7 | # 8 | # v 1.7.0.3. Doc fixes. Module metadata added (tags, file list). 9 | 10 | @{ 11 | 12 | # Script module or binary module file associated with this manifest. 13 | RootModule = 'MergeCsv.psm1' 14 | 15 | # Version number of this module. 16 | ModuleVersion = '1.7.0.3' 17 | 18 | # Supported PSEditions 19 | # CompatiblePSEditions = @() 20 | 21 | # ID used to uniquely identify this module 22 | GUID = 'b3611e3d-f337-494b-9f16-50b424f8471c' 23 | 24 | # Author of this module 25 | Author = 'Joakim Borger Svendsen' 26 | 27 | # Company or vendor of this module 28 | CompanyName = 'Svendsen Tech' 29 | 30 | # Copyright statement for this module 31 | Copyright = '(c) 2014-2017 Joakim Borger Svendsen. Svendsen Tech. All rights reserved.' 32 | 33 | # Description of the functionality provided by this module 34 | Description = 'Use Svendsen Tech''s Merge-Csv function to merge CSV files and/or custom PowerShell objects based one or more shared ID properties. Online documentation here: http://www.powershelladmin.com/wiki/Merge_CSV_files_or_PSObjects_in_PowerShell' 35 | 36 | # Minimum version of the Windows PowerShell engine required by this module 37 | PowerShellVersion = '3.0' 38 | 39 | # Name of the Windows PowerShell host required by this module 40 | # PowerShellHostName = '' 41 | 42 | # Minimum version of the Windows PowerShell host required by this module 43 | # PowerShellHostVersion = '' 44 | 45 | # Minimum version of Microsoft .NET Framework required by this module. This prerequisite is valid for the PowerShell Desktop edition only. 46 | # DotNetFrameworkVersion = '' 47 | 48 | # Minimum version of the common language runtime (CLR) required by this module. This prerequisite is valid for the PowerShell Desktop edition only. 49 | # CLRVersion = '' 50 | 51 | # Processor architecture (None, X86, Amd64) required by this module 52 | # ProcessorArchitecture = '' 53 | 54 | # Modules that must be imported into the global environment prior to importing this module 55 | # RequiredModules = @() 56 | 57 | # Assemblies that must be loaded prior to importing this module 58 | # RequiredAssemblies = @() 59 | 60 | # Script files (.ps1) that are run in the caller's environment prior to importing this module. 61 | # ScriptsToProcess = @() 62 | 63 | # Type files (.ps1xml) to be loaded when importing this module 64 | # TypesToProcess = @() 65 | 66 | # Format files (.ps1xml) to be loaded when importing this module 67 | # FormatsToProcess = @() 68 | 69 | # Modules to import as nested modules of the module specified in RootModule/ModuleToProcess 70 | # NestedModules = @() 71 | 72 | # Functions to export from this module, for best performance, do not use wildcards and do not delete the entry, use an empty array if there are no functions to export. 73 | FunctionsToExport = 'Merge-Csv' 74 | 75 | # Cmdlets to export from this module, for best performance, do not use wildcards and do not delete the entry, use an empty array if there are no cmdlets to export. 76 | CmdletsToExport = @() 77 | 78 | # Variables to export from this module 79 | # VariablesToExport = @() 80 | 81 | # Aliases to export from this module, for best performance, do not use wildcards and do not delete the entry, use an empty array if there are no aliases to export. 82 | AliasesToExport = @() 83 | 84 | # DSC resources to export from this module 85 | # DscResourcesToExport = @() 86 | 87 | # List of all modules packaged with this module 88 | # ModuleList = @() 89 | 90 | # List of all files packaged with this module 91 | FileList = @("MergeCsv.psm1", "MergeCsv.psd1") 92 | 93 | # Private data to pass to the module specified in RootModule/ModuleToProcess. This may also contain a PSData hashtable with additional module metadata used by PowerShell. 94 | PrivateData = @{ 95 | 96 | PSData = @{ 97 | 98 | # Tags applied to this module. These help with module discovery in online galleries. 99 | Tags = 'CSV','Object','PSObject','Merge','Join' 100 | 101 | # A URL to the license for this module. 102 | # LicenseUri = '' 103 | 104 | # A URL to the main website for this project. 105 | # ProjectUri = '' 106 | 107 | # A URL to an icon representing this module. 108 | # IconUri = '' 109 | 110 | # ReleaseNotes of this module 111 | # ReleaseNotes = '' 112 | 113 | # External dependent modules of this module 114 | # ExternalModuleDependencies = '' 115 | 116 | } # End of PSData hashtable 117 | 118 | } # End of PrivateData hashtable 119 | 120 | # HelpInfo URI of this module 121 | # HelpInfoURI = '' 122 | 123 | # Default prefix for commands exported from this module. Override the default prefix using Import-Module -Prefix. 124 | # DefaultCommandPrefix = '' 125 | 126 | } 127 | 128 | -------------------------------------------------------------------------------- /MergeCsv.Tests.ps1: -------------------------------------------------------------------------------- 1 | #requires -version 3 2 | [CmdletBinding()] 3 | Param() 4 | # Pester >4.x tests for Svendsen Tech's Merge-Csv function/module. 5 | # Created: 2017-11-18. 6 | # Author: Joakim Borger Svendsen 7 | 8 | Import-Module -Name Pester -ErrorAction Stop 9 | # Putting this in the wild... 10 | Remove-Module -Name MergeCsv -ErrorAction SilentlyContinue 11 | Import-Module -Name MergeCsv -ErrorAction Stop 12 | #$MyScriptRoot = Split-Path -Path $MyInvocation.MyCommand.Path -Parent 13 | 14 | # Doing this instead, at least for myself, to avoid having to copy files to 15 | # the profile/computer PowerShell modules directory each time... 16 | #Copy-Item -Path ..\MergeCsv.psm1 -Destination $PSScriptRoot\MergeCsv.ps1 17 | ###Move-Item -Path .\MergeCsv.psm1 -Destination "$PSScriptRoot\MergeCsv.ps1" -Force 18 | #. "$PSScriptRoot\MergeCsv.ps1" 19 | 20 | Describe "Merge-Csv" { 21 | 22 | function InternalTestPathCSV { 23 | [CmdletBinding()] 24 | Param([String] $FilePath) 25 | if (-not (Test-Path -Path "$PSScriptRoot\template-csvs\$FilePath" -PathType Leaf)) { 26 | if (-not (Test-Path -Path "$PSScriptRoot\$FilePath" -PathType Leaf)) { 27 | throw "'$FilePath' isn't in the same directory as the test script or in a subfolder called 'template-csvs'." 28 | } 29 | else { 30 | "$PSScriptRoot\$FilePath" 31 | } 32 | } 33 | else { 34 | "$PSScriptRoot\template-csvs\$FilePath" 35 | } 36 | } 37 | 38 | It "Merges two simple objects with three IDs correctly" { 39 | 40 | $EmailObjects = @([PSCustomObject] @{ 41 | Username = "John" 42 | Email = "john@example.com" 43 | }, [PSCustomObject] @{ 44 | Username = "Jane" 45 | Email = "jane@example.com" 46 | }, [PSCustomObject] @{ 47 | Username = "Janet" 48 | Email = "janet@maintexample.com" 49 | }) 50 | $DepartmentObjects = @([PSCustomObject] @{ 51 | Username = "John" 52 | Department = "HR" 53 | }, [PSCustomObject] @{ 54 | Username = "Jane" 55 | Department = "IT" 56 | }, [PSCustomObject] @{ 57 | Username = "Janet" 58 | Department = "Maintenance" 59 | }) 60 | ((Merge-Csv -InputObject $EmailObjects, $DepartmentObjects -Identity Username | 61 | Sort-Object Username | 62 | ConvertTo-Json -Depth 100 -Compress) 3> $null) -eq ` 63 | '[{"Username":"Jane","Email":"jane@example.com","Department":"IT"},{"Username":"Janet","Email":"janet@maintexample.com","Department":"Maintenance"},{"Username":"John","Email":"john@example.com","Department":"HR"}]' | 64 | Should -Be $True 65 | 66 | } 67 | 68 | It "Merges two simple CSV files with three IDs correctly" { 69 | 70 | $FirstPath, $SecondPath = "simplecsv1.csv", "simplecsv2.csv" | 71 | ForEach-Object { 72 | InternalTestPathCSV -FilePath $_ 73 | } 74 | Write-Verbose -Message "First path: $FirstPath. Second path: $SecondPath." #-Verbose 75 | (Merge-Csv -Path $FirstPath, $SecondPath -Identity Username | 76 | Sort-Object Username | 77 | ConvertTo-Json -Depth 100 -Compress) -eq ` 78 | '[{"Username":"Jane","Email":"jane@example.com","Department":"IT"},{"Username":"Janet","Email":"janet@maintexample.com","Department":"Maintenance"},{"Username":"John","Email":"john@example.com","Department":"HR"}]' | 79 | Should -Be $True 80 | 81 | } 82 | 83 | It "Merges three somewhat complex CSV files with two IDs properly" { 84 | 85 | $FirstPath, $SecondPath, $ThirdPath = "csv1.csv", "csv2.csv", "csv3.csv" | 86 | ForEach-Object { 87 | InternalTestPathCSV -FilePath $_ 88 | } 89 | Write-Verbose ("`n" + ($FirstPath, $SecondPath, $ThirdPath -join "`n")) #-Verbose 90 | ((Merge-Csv -Path $FirstPath, $SecondPath, $ThirdPath -Identity ComputerName, Uh -WarningVariable Warnings | 91 | Sort-Object -Property ComputerName, Uh | 92 | ConvertTo-Json -Depth 100 -Compress) 3> $null) -eq ` 93 | '[{"ComputerName":"ServerA","Uh":"UhA","OSFamily":"Windows","Env":"Production","PSVer":"5.1","OrderFile3":"1"},{"ComputerName":"ServerB","Uh":"UhB","OSFamily":"Windows","Env":"Test","PSVer":"5.1","OrderFile3":"5"}]' | 94 | Should -Be $True 95 | $Warnings.Count | Should -Be 9 96 | 97 | } 98 | 99 | It "Merges three somewhat complex CSV files with two IDs properly, with -AllowDuplicates" { 100 | 101 | $FirstPath, $SecondPath, $ThirdPath = "csv1.csv", "csv2.csv", "csv3.csv" | 102 | ForEach-Object { 103 | InternalTestPathCSV -FilePath $_ 104 | } 105 | ((Merge-Csv -Path $FirstPath, $SecondPath, $ThirdPath -Identity ComputerName, Uh -AllowDuplicates -WarningVariable Warnings | 106 | Sort-Object -Property ComputerName, Uh, OrderFile3 | 107 | ConvertTo-Json -Depth 100 -Compress) 3> $null) -eq ` 108 | '[{"ComputerName":"ServerA","Uh":"UhA","OSFamily":"Windows","Env":"Production","PSVer":"5.1","OrderFile3":"1"},{"ComputerName":"ServerA","Uh":"UhA","OSFamily":"Linux","Env":"Test","PSVer":"6.0","OrderFile3":"2"},{"ComputerName":"ServerA","Uh":"UhA","OSFamily":null,"Env":"Production","PSVer":"3.0","OrderFile3":"3"},{"ComputerName":"ServerA","Uh":"UhA","OSFamily":null,"Env":null,"PSVer":null,"OrderFile3":"4"},{"ComputerName":"ServerB","Uh":"UhB","OSFamily":"Windows","Env":"Test","PSVer":"5.1","OrderFile3":"5"},{"ComputerName":"ServerB","Uh":"UhB","OSFamily":"Linux","Env":"Legacy","PSVer":"6.0","OrderFile3":"6"}]' | 109 | Should -Be $True 110 | $Warnings.Count | Should -Be 0 111 | 112 | } 113 | 114 | It "Warns about duplicates" { 115 | 116 | $Object1 = @([PSCustomObject] @{ 117 | Username = "Repeated" 118 | foo = "bar" 119 | }, [PSCustomObject] @{ 120 | Username = "Repeated" 121 | foo = "barf" 122 | }) 123 | $Object2 = [PSCustomObject] @{ 124 | Username = "Repeated" 125 | bar = "foo" 126 | } 127 | 128 | (Merge-Csv -InputObject $Object1, $Object2 -Identity Username -WarningVariable Warnings) 3> $null | Out-Null 129 | $Warnings.Message | 130 | Should -Match "Duplicate identifying \(shared column\(s\) ID\) entry found in CSV data/file 1: Repeated" 131 | # Check in reverse order. 132 | (Merge-Csv -InputObject $Object2, $Object1 -Identity Username -WarningVariable Warnings) 3> $null | Out-Null 133 | $Warnings.Message | 134 | Should -Match "Duplicate identifying \(shared column\(s\) ID\) entry found in CSV data/file 2: Repeated" 135 | 136 | } 137 | 138 | It "Warns about duplicates with two ID fields" { 139 | 140 | $Object1 = @([PSCustomObject] @{ 141 | Username = "Repeated" 142 | ID2 = "a" 143 | foo = "bar" 144 | }, [PSCustomObject] @{ 145 | Username = "Repeated" 146 | ID2 = "a" 147 | foo = "barf" 148 | }) 149 | $Object2 = [PSCustomObject] @{ 150 | Username = "Repeated" 151 | ID2 = "a" 152 | bar = "foo" 153 | } 154 | 155 | (Merge-Csv -InputObject $Object1, $Object2 -Identity Username, ID2 -WarningVariable Warnings) 3> $null | Out-Null 156 | $Warnings.Message | 157 | Should -Match "Duplicate identifying \(shared column\(s\) ID\) entry found in CSV data/file 1: Repeated, a" 158 | 159 | # Check in reverse order. 160 | (Merge-Csv -InputObject $Object2, $Object1 -Identity Username, ID2 -WarningVariable Warnings) 3> $null | Out-Null 161 | $Warnings.Message | 162 | Should -Match "Duplicate identifying \(shared column\(s\) ID\) entry found in CSV data/file 2: Repeated, a" 163 | 164 | } 165 | 166 | It "Warns about duplicates with two ID fields and >2 objects" { 167 | 168 | $Object1 = @([PSCustomObject] @{ 169 | Username = "Repeated" 170 | ID2 = "a" 171 | foo = "bar" 172 | }, [PSCustomObject] @{ 173 | Username = "Repeated" 174 | ID2 = "a" 175 | foo = "barf" 176 | }) 177 | $Object2 = [PSCustomObject] @{ 178 | Username = "Repeated" 179 | ID2 = "a" 180 | bar = "foo" 181 | } 182 | $Object3 = [PSCustomObject] @{ 183 | Username = "Repeated" 184 | ID2 = "a" 185 | baz = "boo" 186 | } 187 | 188 | # Check that position 3 is reported correctly. 189 | (Merge-Csv -InputObject $Object3, $Object2, $Object1 -Identity Username, ID2 -WarningVariable Warnings) 3> $null | Out-Null 190 | $Warnings.Message | 191 | Should -Match "Duplicate identifying \(shared column\(s\) ID\) entry found in CSV data/file 3: Repeated, a" 192 | 193 | # Check as second. 194 | (Merge-Csv -InputObject $Object2, $Object1, $Object3 -Identity Username, ID2 -WarningVariable Warnings) 3> $null | Out-Null 195 | $Warnings.Message | 196 | Should -Match "Duplicate identifying \(shared column\(s\) ID\) entry found in CSV data/file 2: Repeated, a" 197 | 198 | # Check as first. 199 | (Merge-Csv -InputObject $Object1, $Object2, $Object3 -Identity Username, ID2 -WarningVariable Warnings) 3> $null | Out-Null 200 | $Warnings.Message | 201 | Should -Match "Duplicate identifying \(shared column\(s\) ID\) entry found in CSV data/file 1: Repeated, a" 202 | 203 | } 204 | 205 | It "Warns about a missing ID property in one or more of three objects" { 206 | 207 | $Object1 = @([PSCustomObject] @{ 208 | Username = "Repeated" 209 | foo = "bar" 210 | }, [PSCustomObject] @{ 211 | Username = "MissingInTheOther" 212 | foo = "bar2" 213 | }) 214 | $Object2 = [PSCustomObject] @{ 215 | Username = "Repeated" 216 | bar = "foo" 217 | } 218 | $Object3 = [PSCustomObject] @{ 219 | Username = "Repeated" 220 | baz = "boo" 221 | } 222 | 223 | # Check position 1. 224 | (Merge-Csv -InputObject $Object1, $Object2, $Object3 -Identity Username -WarningVariable Warnings) 3> $null | Out-Null 225 | $Warnings.Message | 226 | Should -Match "Identifying column entry '$($Object1[1].Username 227 | )' was not found in all CSV data objects/files. Found in object/file no.: 1" 228 | 229 | # Check position 2. 230 | (Merge-Csv -InputObject $Object2, $Object1, $Object3 -Identity Username -WarningVariable Warnings) 3> $null | Out-Null 231 | $Warnings.Message | 232 | Should -Match "Identifying column entry '$($Object1[1].Username 233 | )' was not found in all CSV data objects/files. Found in object/file no.: 2" 234 | 235 | # Check position 3. 236 | (Merge-Csv -InputObject $Object3, $Object2, $Object1 -Identity Username -WarningVariable Warnings) 3> $null | Out-Null 237 | $Warnings.Message | 238 | Should -Match "Identifying column entry '$($Object1[1].Username 239 | )' was not found in all CSV data objects/files. Found in object/file no.: 3" 240 | 241 | # Check two scenarios where 242 | $Object2 = @([PSCustomObject] @{ 243 | Username = "Repeated" 244 | bar = "foo" 245 | }, [PSCustomObject] @{ 246 | Username = "MissingInTheOther" 247 | bar = "foo2" 248 | }) 249 | 250 | # Check when it's missing in position 1. 251 | (Merge-Csv -InputObject $Object3, $Object2, $Object1 -Identity Username -WarningVariable Warnings) 3> $null | Out-Null 252 | $Warnings.Message | 253 | Should -Match "Identifying column entry '$($Object2[1].Username 254 | )' was not found in all CSV data objects/files. Found in object/file no.: 2, 3" 255 | 256 | # Check when it's missing in position 2. 257 | (Merge-Csv -InputObject $Object2, $Object3, $Object1 -Identity Username -WarningVariable Warnings) 3> $null | Out-Null 258 | $Warnings.Message | 259 | Should -Match "Identifying column entry '$($Object1[1].Username 260 | )' was not found in all CSV data objects/files. Found in object/file no.: 1, 3" 261 | 262 | # Check when it's missing in position 3. 263 | (Merge-Csv -InputObject $Object1, $Object2, $Object3 -Identity Username -WarningVariable Warnings) 3> $null | Out-Null 264 | $Warnings.Message | 265 | Should -Match "Identifying column entry '$($Object1[1].Username 266 | )' was not found in all CSV data objects/files. Found in object/file no.: 1, 2" 267 | 268 | } 269 | 270 | It "Warns about a missing, combined ID property (two properties) in one or more of three objects" { 271 | 272 | $Object1 = @([PSCustomObject] @{ 273 | Username = "Repeated" 274 | ID2 = "a" 275 | foo = "bar" 276 | }, [PSCustomObject] @{ 277 | Username = "MissingInTheOther" 278 | ID2 = "m" 279 | foo = "bar2" 280 | }) 281 | $Object2 = [PSCustomObject] @{ 282 | Username = "Repeated" 283 | ID2 = "a" 284 | bar = "foo" 285 | } 286 | $Object3 = [PSCustomObject] @{ 287 | Username = "Repeated" 288 | ID2 = "a" 289 | baz = "boo" 290 | } 291 | 292 | # Check position 1. 293 | (Merge-Csv -InputObject $Object1, $Object2, $Object3 -Identity Username, ID2 -WarningVariable Warnings) 3> $null | Out-Null 294 | $Warnings.Message | 295 | Should -Match "Identifying column entry '$(($Object1[1].Username, 296 | $Object1[1].ID2) -join ', ')' was not found in all CSV data objects/files. Found in object/file no.: 1" 297 | 298 | # Check position 2. 299 | (Merge-Csv -InputObject $Object2, $Object1, $Object3 -Identity Username, ID2 -WarningVariable Warnings) 3> $null | Out-Null 300 | $Warnings.Message | 301 | Should -Match "Identifying column entry '$(($Object1[1].Username, 302 | $Object1[1].ID2) -join ', ')' was not found in all CSV data objects/files. Found in object/file no.: 2" 303 | 304 | # Check position 3. 305 | (Merge-Csv -InputObject $Object3, $Object2, $Object1 -Identity Username, ID2 -WarningVariable Warnings) 3> $null | Out-Null 306 | $Warnings.Message | 307 | Should -Match "Identifying column entry '$(($Object1[1].Username, 308 | $Object1[1].ID2) -join ', ')' was not found in all CSV data objects/files. Found in object/file no.: 3" 309 | 310 | $Object2 = @([PSCustomObject] @{ 311 | Username = "Repeated" 312 | ID2 = "a" 313 | food = "bar33" 314 | }, [PSCustomObject] @{ 315 | Username = "MissingInTheOther" 316 | ID2 = "m" 317 | food = "bar3" 318 | }) 319 | 320 | # Check when it's missing in position 1. 321 | (Merge-Csv -InputObject $Object3, $Object2, $Object1 -Identity Username, ID2 -WarningVariable Warnings) 3> $null | Out-Null 322 | $Warnings.Message | 323 | Should -Match "Identifying column entry '$(($Object1[1].Username, 324 | $Object1[1].ID2) -join ', ')' was not found in all CSV data objects/files. Found in object/file no.: 2, 3" 325 | 326 | # Check when it's missing in position 2. 327 | (Merge-Csv -InputObject $Object2, $Object3, $Object1 -Identity Username, ID2 -WarningVariable Warnings) 3> $null | Out-Null 328 | $Warnings.Message | 329 | Should -Match "Identifying column entry '$(($Object1[1].Username, 330 | $Object1[1].ID2) -join ', ')' was not found in all CSV data objects/files. Found in object/file no.: 1, 3" 331 | 332 | # Check when it's missing in position 3. 333 | (Merge-Csv -InputObject $Object1, $Object2, $Object3 -Identity Username, ID2 -WarningVariable Warnings) 3> $null | Out-Null 334 | $Warnings.Message | 335 | Should -Match "Identifying column entry '$(($Object1[1].Username, 336 | $Object1[1].ID2) -join ', ')' was not found in all CSV data objects/files. Found in object/file no.: 1, 2" 337 | 338 | } 339 | 340 | } 341 | -------------------------------------------------------------------------------- /MergeCsv.psm1: -------------------------------------------------------------------------------- 1 | #requires -version 3 2 | 3 | 4 | <# 5 | .SYNOPSIS 6 | Merges an arbitrary amount of CSV files or PowerShell objects based on an ID column or 7 | several combined ID columns. Works on custom PowerShell objects with the InputObject parameter. 8 | 9 | .DESCRIPTION 10 | Slapping parentheses around Import-Csv like, say, "-InputObject (ipcsv csvfile.csv), $objectHere" 11 | is good for a mix of objects and CSV files. 12 | 13 | PowerShell version 3 or higher is needed. 14 | 15 | Copyright Joakim Borger Svendsen (C) 2014-present 16 | All rights reserved. 17 | Svendsen Tech 18 | 19 | MIT license. 20 | 21 | Online documentation: 22 | http://www.powershelladmin.com/wiki/Merge_CSV_files_or_PSObjects_in_PowerShell 23 | 24 | GitHub: 25 | https://github.com/EliteLoser/MergeCsv/ 26 | 27 | The PowerShell Gallery: 28 | https://www.powershellgallery.com/packages/MergeCsv/ 29 | 30 | .PARAMETER Identity 31 | Shared ID property/header (multiple supported). 32 | 33 | .PARAMETER Path 34 | CSV files to process. 35 | 36 | .PARAMETER InputObject 37 | Custom PowerShell objects to process. 38 | 39 | .PARAMETER Delimiter 40 | Optional delimiter that's used if you pass file paths (default is a comma). 41 | 42 | .PARAMETER Separator 43 | Optional multi-ID column string separator (default "#Merge-Csv-Separator#"). 44 | 45 | .PARAMETER AllowDuplicates 46 | Allow and aggregate duplicate entries (IDs) in the order they occur. 47 | 48 | .PARAMETER IncludeAliasProperty 49 | Include alias properties in addition to note properties. 50 | 51 | .EXAMPLE 52 | ipcsv users.csv | ft -AutoSize 53 | 54 | Username Department 55 | -------- ---------- 56 | John IT 57 | Jane HR 58 | 59 | PS C:\> ipcsv user-mail.csv | ft -AutoSize 60 | 61 | Username Email 62 | -------- ----- 63 | John john@example.com 64 | Jane jane@example.com 65 | 66 | PS C:\> Merge-Csv -Path users.csv, user-mail.csv -Id Username | Export-Csv -enc UTF8 merged.csv 67 | 68 | PS C:\> ipcsv .\merged.csv | ft -AutoSize 69 | 70 | Username Department Email 71 | -------- ---------- ----- 72 | John IT john@example.com 73 | Jane HR jane@example.com 74 | 75 | .EXAMPLE 76 | Merge-Csv -In (ipcsv .\csv1.csv), (ipcsv csv2.csv), (ipcsv csv3.csv) -Id Username | Sort-Object username | ft -AutoSize 77 | 78 | Merging three files. 79 | 80 | WARNING: Duplicate identifying (shared column(s) ID) entry found in CSV data/file 0: user42 81 | WARNING: Identifying column entry 'firstOnly' was not found in all CSV data objects/files. Found in object/file no.: 1 82 | WARNING: Identifying column entry '2only' was not found in all CSV data objects/files. Found in object/file no.: 2 83 | WARNING: Identifying column entry 'user2and3only' was not found in all CSV data objects/files. Found in object/file no.: 2, 3 84 | 85 | Username File1A File1B TestID File2A File2B TestX File3 86 | -------- ------ ------ ------ ------ ------ ----- ----- 87 | 2only a b c 88 | firstOnly firstOnlyA1 firstOnlyB1 foo 89 | user1 1A1 1B1 same 1A3 2A3 same same 90 | user2 2A1 2B1 diff2 2A3 2B3 diff2_2 testC2 91 | user2and3only 2and3A2 2and3B2 test2and3X testID 92 | user3 3A1 3B1 same 3A3 3B3 same same 93 | user42 42A1 42B1 same42 testA42 testB42 testX42 testC42 94 | 95 | .EXAMPLE 96 | Merge-Csv -Path csvmerge1.csv, csvmerge2.csv, csvmerge3.csv -Id Username, TestID | Sort-Object username | ft -a 97 | 98 | Two shared/ID column, three files. 99 | 100 | WARNING: Duplicate identifying (shared column(s) ID) entry found in CSV data/file 1: user42, same42 101 | WARNING: Identifying column entry 'user2, diff2' was not found in all CSV data objects/files. Found in object/file no.: 1 102 | WARNING: Identifying column entry 'user2and3only, testID' was not found in all CSV data objects/files. Found in object/file no.: 3 103 | WARNING: Identifying column entry 'user2, testC2' was not found in all CSV data objects/files. Found in object/file no.: 3 104 | WARNING: Identifying column entry '2only, c' was not found in all CSV data objects/files. Found in object/file no.: 2 105 | WARNING: Identifying column entry 'user2and3only, test2and3X' was not found in all CSV data objects/files. Found in object/file no.: 2 106 | WARNING: Identifying column entry 'user2, diff2_2' was not found in all CSV data objects/files. Found in object/file no.: 2 107 | WARNING: Identifying column entry 'firstOnly, foo' was not found in all CSV data objects/files. Found in object/file no.: 1 108 | 109 | Username TestID File1A File1B File2A File2B 110 | -------- ------ ------ ------ ------ ------ 111 | 2only c a b 112 | firstOnly foo firstOnlyA1 firstOnlyB1 113 | user1 same 1A1 1B1 1A3 2A3 114 | user2 diff2 2A1 2B1 115 | user2 diff2_2 2A3 2B3 116 | user2 testC2 117 | user2and3only testID 118 | user2and3only test2and3X 2and3A2 2and3B2 119 | user3 same 3A1 3B1 3A3 3B3 120 | user42 same42 42A1 42B1 testA42 testB42 121 | 122 | .EXAMPLE 123 | Merge-Csv -Path csv1.csv, csv2.csv, csv3.csv -Id ID -AllowDuplicates | ft -AutoSize 124 | 125 | ID 1Title1 1Title2 1Title3 2Title1 2Title2 3Title1 126 | -- ------- ------- ------- ------- ------- ------- 127 | FooBar x y z blorp dongs first3 128 | FooBar xxx yyy second3 129 | FooBar third3 130 | Svendsen a b c e f SvenData3 131 | Svendsen aa bb cc ee ff 132 | Svendsen aaa eee fff 133 | #> 134 | function Merge-Csv { 135 | [CmdletBinding( 136 | DefaultParameterSetName='Files' 137 | )] 138 | param( 139 | # Shared ID column(s)/header(s). 140 | [Parameter(Mandatory=$true)] 141 | [ValidateNotNullOrEmpty()] 142 | [String[]] $Identity, 143 | 144 | # CSV files to process. 145 | [Parameter(ParameterSetName='Files',Mandatory=$true)] 146 | [ValidateScript({Test-Path $_ -PathType Leaf})] 147 | [String[]] $Path, 148 | 149 | # Custom PowerShell objects to process. 150 | [Parameter(ParameterSetName='Objects',Mandatory=$true)] 151 | [PSObject[]] $InputObject, 152 | 153 | # Optional delimiter that's used if you pass file paths (default is a comma). 154 | [Parameter(ParameterSetName='Files')] 155 | [String] $Delimiter = ',', 156 | 157 | # Optional multi-ID column string separator (default "#Merge-Csv-Separator#"). 158 | [String] $Separator = '#Merge-Csv-Separator#', 159 | 160 | # Allow duplicate entries (IDs). 161 | [Switch] $AllowDuplicates, 162 | 163 | # Include alias properties. 164 | [Switch] $IncludeAliasProperty) 165 | # v1.4 as a module - 2016-10-28 - adding module format prerequisites, cleaning up redundant code 166 | # v1.4 - 2016-09-16 - Added support for handling duplicate IDs. 167 | # v1.5. Forgot to make a note here, see wiki. 168 | # v1.6 - Allowing duplicates, see wiki. 169 | # v1.7 - 2017-09-13 - Adding -IncludeAliasProperty parameter. 170 | # Non-default to not break old stuff people might have. 171 | # Id parameter changed to full form: Identity. 172 | # v1.7.0.1 - 2017-09-14 - Empty strings instead of bullshit objects. wtf was I thinking. 173 | # Now I have to decide if I manipulate the IDs, possibly "Header1: Value1. Header2: Value2."? 174 | # with the title as "Header1. Header2." But periods aren't unique, so I don't know what else 175 | # to do right now than keep the silly, presumed unique separator string. 176 | # v1.7.0.2 - 2017-09-14 - Found a good way to handle multiple IDs with regards to the 177 | # presentation aspect! So clever it almost hurts - that's how it feels now anyway. 178 | [String[]] $PropertyTypes = @() 179 | if ($IncludeAliasProperty) { 180 | $PropertyTypes = @("NoteProperty", "AliasProperty") 181 | } 182 | else { 183 | $PropertyTypes = @("NoteProperty") 184 | } 185 | [PSObject[]] $CsvObjects = @() 186 | if ($PSCmdlet.ParameterSetName -eq 'Files') { 187 | $CsvObjects = foreach ($File in $Path) { 188 | ,@(Import-Csv -Delimiter $Delimiter -Path $File) 189 | } 190 | } 191 | else { 192 | $CsvObjects = $InputObject 193 | } 194 | $Headers = @() 195 | foreach ($Csv in $CsvObjects) { 196 | $Headers += , @($Csv | Get-Member -MemberType $PropertyTypes | Select-Object -ExpandProperty Name) 197 | } 198 | $Counter = 0 199 | foreach ($h in $Headers) { 200 | $Counter++ 201 | foreach ($Column in $Identity) { 202 | if ($h -notcontains $Column) { 203 | Write-Error "Headers in object/file $Counter don't include $Column. Exiting." 204 | return 205 | } 206 | } 207 | } 208 | $HeadersFlatNoShared = @($Headers | ForEach-Object { $_ } | Where-Object { $Identity -notcontains $_ }) 209 | if ($HeadersFlatNoShared.Count -ne @($HeadersFlatNoShared | Sort-Object -Unique).Count) { 210 | Write-Error "Some headers are shared. Are you just looking for '@(ipcsv csv1) + @(ipcsv csv2) | Export-Csv ...'?`nTo remove duplicate (between the files to merge) headers from a CSV file, Import-Csv it, pass it to Select-Object, and omit the duplicate header(s)/column(s).`nExiting." 211 | return 212 | } 213 | $SharedColumnHashes = @() 214 | $SharedColumnCount = $Identity.Count 215 | $Counter = 0 216 | foreach ($Csv in $CsvObjects) { 217 | $SharedColumnHashes += @{} 218 | $Csv | ForEach-Object { 219 | $CurrentID = $(for ($i = 0; $i -lt $SharedColumnCount; $i++) { 220 | $_ | Select-Object -ExpandProperty $Identity[$i] -EA SilentlyContinue 221 | }) -join $Separator 222 | if (-not $SharedColumnHashes[$Counter].ContainsKey($CurrentID)) { 223 | $SharedColumnHashes[$Counter].Add($CurrentID, @($_ | Select-Object -Property $Headers[$Counter])) 224 | } 225 | else { 226 | if ($AllowDuplicates) { 227 | $SharedColumnHashes[$Counter].$CurrentID += $_ | Select-Object -Property $Headers[$Counter] 228 | } 229 | else { 230 | Write-Warning ("Duplicate identifying (shared column(s) ID) entry found in CSV data/file $($Counter+1): " + ($CurrentID -replace [regex]::Escape($Separator), ', ')) 231 | } 232 | } 233 | } 234 | $Counter++ 235 | } 236 | $Result = @{} 237 | $NotFound = @{} 238 | foreach ($Counter in 0..($SharedColumnHashes.Count-1)) { 239 | foreach ($InnerCounter in (0..($SharedColumnHashes.Count-1) | Where-Object { $_ -ne $Counter })) { 240 | foreach ($Key in $SharedColumnHashes[$Counter].Keys) { 241 | Write-Verbose "Key: $Key, Counter: $Counter, InnerCounter: $InnerCounter" 242 | $Obj = New-Object -TypeName PSObject 243 | if ($SharedColumnHashes[$InnerCounter].ContainsKey($Key)) { 244 | foreach ($Header in $Headers[$InnerCounter] | Where-Object { $Identity -notcontains $_ }) { 245 | Add-Member -InputObject $Obj -MemberType NoteProperty -Name $Header -Value ($SharedColumnHashes[$InnerCounter].$Key | Select-Object $Header) 246 | } 247 | } 248 | else { 249 | foreach ($Header in $Headers[$Counter]) { 250 | if ($Identity -notcontains $Header) { 251 | Add-Member -InputObject $Obj -MemberType NoteProperty -Name $Header -Value ($SharedColumnHashes[$Counter].$Key | Select-Object $Header) 252 | } 253 | } 254 | if (-not $NotFound.ContainsKey($Key)) { 255 | $NotFound.Add($Key, @($Counter)) 256 | } 257 | else { 258 | $NotFound[$Key] += $Counter 259 | } 260 | } 261 | if (-not $Result.ContainsKey($Key)) { 262 | $Result.$Key = $Obj 263 | } 264 | else { 265 | foreach ($Property in @($Obj | Get-Member -MemberType $PropertyTypes | Select-Object -ExpandProperty Name)) { 266 | if (-not ($Result.$Key | Get-Member -MemberType $PropertyTypes -Name $Property)) { 267 | Add-Member -InputObject $Result.$Key -MemberType NoteProperty -Name $Property -Value $Obj.$Property #-EA SilentlyContinue 268 | } 269 | } 270 | } 271 | 272 | } 273 | } 274 | } 275 | if ($NotFound) { 276 | foreach ($Key in $NotFound.Keys) { 277 | Write-Warning "Identifying column entry '$($Key -replace [regex]::Escape($Separator), ', ')' was not found in all CSV data objects/files. Found in object/file no.: $( 278 | if ($NotFound.$Key) { ($NotFound.$Key | ForEach-Object { ([int]$_)+1 } | Sort-Object -Unique) -join ', '} 279 | elseif ($CsvObjects.Count -eq 2) { '1' } 280 | else { 'none' } 281 | )" 282 | } 283 | } 284 | #$Global:Result = $Result 285 | $Counter = 0 286 | [hashtable[]] $SharedHeadersNoDuplicate = $Identity | ForEach-Object { 287 | @{n="$($Identity[$Counter])";e=[scriptblock]::Create("(`$_.Name -split ([regex]::Escape('$Separator')))[$Counter]")} 288 | $Counter++ 289 | } 290 | [hashtable[]] $HeaderPropertiesNoDuplicate = $HeadersFlatNoShared | ForEach-Object { 291 | @{n=$_.ToString(); e=[scriptblock]::Create("`$_.Value.'$_' | Select -ExpandProperty '$_'")} 292 | } 293 | [int] $HeadersFlatNoSharedCount = $HeadersFlatNoShared.Count 294 | # Return results. 295 | if (-not $AllowDuplicates) { 296 | $Result.GetEnumerator() | Select-Object -Property ($SharedHeadersNoDuplicate + $HeaderPropertiesNoDuplicate) 297 | } 298 | else { 299 | $Result.GetEnumerator() | ForEach-Object { 300 | # Latching on support for duplicate objects. Insanely inefficient. 301 | # Variable for the count of duplicates we find. Initialize to 1 for each array of PSobjects for each ID. 302 | $MaxDuplicateCount = 1 303 | foreach ($Title in $_.Value | Get-Member -MemberType $PropertyTypes | Select-Object -ExpandProperty Name) { 304 | $Count = @($_.Value.$Title).Count 305 | # find max count for this instance (if at all higher than 1) 306 | # duplicates are processed in the order they occur 307 | if ($MaxDuplicateCount -lt $Count) { 308 | $MaxDuplicateCount = $Count 309 | } 310 | } 311 | Write-Verbose "Max duplicate count: $MaxDuplicateCount" 312 | foreach ($i in 0..($MaxDuplicateCount-1)) { 313 | # Add ID(s) once to each object. 314 | $Obj = $null 315 | $Obj = New-Object -TypeName PSObject 316 | $IDSplitCounter = 0 317 | foreach ($TempID in $Identity) { 318 | Add-Member -InputObject $Obj -MemberType NoteProperty -Name $TempID -Value @($_.Name -split [Regex]::Escape($Separator))[$IDSplitCounter] 319 | ++$IDSplitCounter 320 | } 321 | foreach ($NumHeader in 0..($HeadersFlatNoSharedCount-1)) { 322 | try { 323 | $Value = ($_.Value.($HeadersFlatNoShared[$NumHeader]))[$i] | Select-Object -ExpandProperty $HeadersFlatNoShared[$NumHeader] 324 | } 325 | catch { 326 | Write-Verbose "Caught out of bounds in array." 327 | $Value = '' #| Select-Object -Property $HeadersFlatNoShared[$NumHeader] 328 | } 329 | Add-Member -InputObject $Obj -MemberType NoteProperty -Name $HeadersFlatNoShared[$NumHeader] -Value $Value 330 | } 331 | $Obj | Select-Object -Property ($Identity + $HeadersFlatNoShared) 332 | } 333 | } 334 | } 335 | } 336 | #Export-ModuleMember -Function Merge-Csv 337 | --------------------------------------------------------------------------------