├── .github └── workflows │ └── CI.yml ├── .project ├── .smalltalk.ston ├── Magritte.md ├── README.md ├── license.txt └── repository ├── .properties ├── BaselineOfNeoCSV ├── BaselineOfNeoCSV.class.st └── package.st ├── ConfigurationOfNeoCSV ├── ConfigurationOfNeoCSV.class.st └── package.st ├── Neo-CSV-Core ├── NeoCSVData.class.st ├── NeoCSVReader.class.st ├── NeoCSVWriter.class.st ├── NeoNumberParser.class.st └── package.st ├── Neo-CSV-GT ├── NeoCSVData.extension.st └── package.st ├── Neo-CSV-Magritte ├── MACSVField.class.st ├── MACSVImporter.class.st ├── MACSVImporterTests.class.st ├── MACSVMappedPragmaBuilder.class.st ├── MACSVTestPerson.class.st ├── MACSVTwoStageImporter.class.st ├── MACSVWriter.class.st ├── MACSVWriterTests.class.st ├── MAElementDescription.extension.st ├── ManifestNeoCSVMagritte.class.st ├── NeoCSVReader.extension.st ├── Object.extension.st └── package.st └── Neo-CSV-Tests ├── NeoCSVBenchmark.class.st ├── NeoCSVReaderTests.class.st ├── NeoCSVTestObject.class.st ├── NeoCSVTestObject2.class.st ├── NeoCSVWriterTests.class.st ├── NeoNumberParserTests.class.st └── package.st /.github/workflows/CI.yml: -------------------------------------------------------------------------------- 1 | name: CI 2 | 3 | on: [ push, pull_request, workflow_dispatch ] 4 | 5 | jobs: 6 | build: 7 | runs-on: ubuntu-latest 8 | strategy: 9 | matrix: 10 | smalltalk: [ Pharo64-12, Pharo64-11, Pharo64-10, Pharo64-9.0, Pharo64-8.0, Pharo64-7.0 ] 11 | name: ${{ matrix.smalltalk }} 12 | steps: 13 | - uses: actions/checkout@v2 14 | - uses: hpi-swa/setup-smalltalkCI@v1 15 | with: 16 | smalltalk-version: ${{ matrix.smalltalk }} 17 | - run: smalltalkci -s ${{ matrix.smalltalk }} 18 | shell: bash 19 | timeout-minutes: 1 20 | -------------------------------------------------------------------------------- /.project: -------------------------------------------------------------------------------- 1 | { 2 | 'srcDirectory' : 'repository' 3 | } -------------------------------------------------------------------------------- /.smalltalk.ston: -------------------------------------------------------------------------------- 1 | SmalltalkCISpec { 2 | #loading : [ 3 | SCIMetacelloLoadSpec { 4 | #baseline : 'NeoCSV', 5 | #directory : 'repository', 6 | #platforms : [ #pharo ] 7 | } 8 | ] 9 | } 10 | -------------------------------------------------------------------------------- /Magritte.md: -------------------------------------------------------------------------------- 1 | # Automate with Magritte 2 | 3 | ### 1. Meta-Describe the CSV Field You Care About 4 | - Enable field mapping with `anElementDesciption propertyAt: #csvFieldName put: aString`. CSV fields which do not have a header corresponding to a Magritte #csvFieldName will be ignored. 5 | - Customize CSV-to-object conversion via `anElementDesciption propertyAt: #csvReader put: aValuable`. 6 | 7 | Example: 8 | ```smalltalk 9 | MyDomainClass>>#descriptionPhone 10 | 11 | ^ MAStringDescription new 12 | accessor: #phone; 13 | propertyAt: #csvFieldName put: 'Phone'; 14 | propertyAt: #csvReader put: [ :s | s select: #isDigit ]; 15 | yourself 16 | ``` 17 | 18 | ### 2. Read a CSV File 19 | - The main entry point is `MyDomainClass class>>#fromCSV: file`. 20 | - For exotic headers (i.e. not a single line with field names), override `MyDomainClass class>>#readCSVHeaderWith:`. You may for example want to skip irrevelant/blank lines). 21 | 22 | Example: 23 | ```smalltalk 24 | file := self myFolder / 'myfile.csv'. 25 | collectionOfMyDomainObjects := self fromCSV: file 26 | ``` 27 | where: 28 | ```smalltalk 29 | MyDomainClass class>>#readCSVHeaderWith: reader 30 | 31 | ^ reader next; next; readHeader 32 | ``` 33 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # NeoCSV 2 | 3 | NeoCSV is an elegant and efficient standalone Smalltalk framework to read and write CSV converting to or from Smalltalk objects. 4 | 5 | [![CI](https://github.com/svenvc/NeoCSV/actions/workflows/CI.yml/badge.svg)](https://github.com/svenvc/NeoCSV/actions/workflows/CI.yml) 6 | 7 | MIT Licensed. 8 | 9 | A chapter in the [Enterprise Pharo](https://books.pharo.org/enterprise-pharo/) book is a good introduction to [NeoCSV](https://ci.inria.fr/pharo-contribution/job/EnterprisePharoBook/lastSuccessfulBuild/artifact/book-result/NeoCSV/NeoCSV.html). 10 | 11 | Go ahead and read the [NeoCSV paper](https://github.com/svenvc/docs/blob/master/neo/neo-csv-paper.md). 12 | 13 | Basically, NeoCSV deals with a format that 14 | - is text based (ASCII, Latin1, Unicode) 15 | - consists of records, 1 per line (any line ending convention) 16 | - where records consist of fields separated by a delimiter (comma, tab, semicolon) 17 | - where every record has the same number of fields 18 | - where fields can be quoted should they contain separators or line endings 19 | 20 | https://en.wikipedia.org/wiki/Comma-separated_values 21 | 22 | ## Installation 23 | 24 | You can load NeoCSV using Metacello 25 | 26 | ```Smalltalk 27 | Metacello new 28 | repository: 'github://svenvc/NeoCSV/repository'; 29 | baseline: 'NeoCSV'; 30 | load. 31 | ``` 32 | 33 | You can use the following dependency from your own Metacello configuration or baseline 34 | 35 | ```Smalltalk 36 | spec baseline: 'NeoCSV' with: [ spec repository: 'github://svenvc/NeoCSV/repository' ]. 37 | ``` 38 | 39 | -------------------------------------------------------------------------------- /license.txt: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (C) 2012 Sven Van Caekenberghe 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 6 | 7 | The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. 8 | 9 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 10 | 11 | -------------------------------------------------------------------------------- /repository/.properties: -------------------------------------------------------------------------------- 1 | { 2 | #format : #tonel 3 | }e 4 | } -------------------------------------------------------------------------------- /repository/BaselineOfNeoCSV/BaselineOfNeoCSV.class.st: -------------------------------------------------------------------------------- 1 | " 2 | I am BaselineOfNeoCSV, a way to load the code of the NeoCSV project. 3 | I am a BaselineOf. 4 | " 5 | Class { 6 | #name : #BaselineOfNeoCSV, 7 | #superclass : #BaselineOf, 8 | #category : #BaselineOfNeoCSV 9 | } 10 | 11 | { #category : #baselines } 12 | BaselineOfNeoCSV >> baseline: spec [ 13 | 14 | spec 15 | for: #common 16 | do: [ spec 17 | package: 'Neo-CSV-Core'; 18 | package: 'Neo-CSV-Tests' with: [ spec requires: #('Neo-CSV-Core') ]; 19 | package: 'Neo-CSV-Magritte' with: [ spec requires: #('Neo-CSV-Core') ]; 20 | group: 'default' with: #('core' 'tests'); 21 | group: 'core' with: #('Neo-CSV-Core'); 22 | group: 'magritte' with: #('Neo-CSV-Magritte'); 23 | group: 'tests' with: #('Neo-CSV-Tests') ] 24 | ] 25 | -------------------------------------------------------------------------------- /repository/BaselineOfNeoCSV/package.st: -------------------------------------------------------------------------------- 1 | Package { #name : #BaselineOfNeoCSV } 2 | -------------------------------------------------------------------------------- /repository/ConfigurationOfNeoCSV/ConfigurationOfNeoCSV.class.st: -------------------------------------------------------------------------------- 1 | " 2 | I am ConfigurationOfNeoCSV, a Metacello configuration for NeoCSV. 3 | 4 | NeoCSV is a flexible and efficient reader and writer for CSV and other delimiter-separated-value formats like TSV. 5 | 6 | License MIT. 7 | " 8 | Class { 9 | #name : #ConfigurationOfNeoCSV, 10 | #superclass : #Object, 11 | #instVars : [ 12 | 'project' 13 | ], 14 | #classVars : [ 15 | 'LastVersionLoad' 16 | ], 17 | #category : 'ConfigurationOfNeoCSV' 18 | } 19 | 20 | { #category : #'development support' } 21 | ConfigurationOfNeoCSV class >> DevelopmentSupport [ 22 | 23 | "See the methods in the 'development support' category on the class-side of MetacelloBaseConfiguration. Decide what development support methods you would like to use and copy them the the class-side of your configuration." 24 | 25 | 26 | ] 27 | 28 | { #category : #private } 29 | ConfigurationOfNeoCSV class >> baseConfigurationClassIfAbsent: aBlock [ 30 | 31 | ^Smalltalk 32 | at: #MetacelloBaseConfiguration 33 | ifAbsent: [ 34 | self ensureMetacelloBaseConfiguration. 35 | Smalltalk at: #MetacelloBaseConfiguration ifAbsent: aBlock ]. 36 | 37 | ] 38 | 39 | { #category : #accessing } 40 | ConfigurationOfNeoCSV class >> catalogContactInfo [ 41 | ^ 'Written and maintained by Sven Van Caekenberghe (http://stfx.eu) and the community. Discussions on the Pharo mailing lists.' 42 | ] 43 | 44 | { #category : #accessing } 45 | ConfigurationOfNeoCSV class >> catalogDescription [ 46 | ^ 'CSV (Comma Separated Values) is a popular data-interchange format. NeoCSV is an elegant and efficient standalone framework to read and write CSV converting to or from Smalltalk objects.' 47 | ] 48 | 49 | { #category : #accessing } 50 | ConfigurationOfNeoCSV class >> catalogKeywords [ 51 | ^ #(format input output #'comma-separated-values' csv #'tab-separated-values' tsv #'tabular-data' records fields #'rfc-4180' text ascii) 52 | ] 53 | 54 | { #category : #private } 55 | ConfigurationOfNeoCSV class >> ensureMetacello [ 56 | 57 | (self baseConfigurationClassIfAbsent: []) ensureMetacello 58 | ] 59 | 60 | { #category : #private } 61 | ConfigurationOfNeoCSV class >> ensureMetacelloBaseConfiguration [ 62 | 63 | Smalltalk 64 | at: #MetacelloBaseConfiguration 65 | ifAbsent: [ 66 | | repository version | 67 | repository := MCHttpRepository location: 'http://seaside.gemstone.com/ss/metacello' user: '' password: ''. 68 | repository 69 | versionReaderForFileNamed: 'Metacello-Base-DaleHenrichs.2.mcz' 70 | do: [ :reader | 71 | version := reader version. 72 | version load. 73 | version workingCopy repositoryGroup addRepository: repository ] ] 74 | ] 75 | 76 | { #category : #'metacello tool support' } 77 | ConfigurationOfNeoCSV class >> isMetacelloConfig [ 78 | "Answer true and the Metacello tools will operate on you" 79 | 80 | ^true 81 | ] 82 | 83 | { #category : #loading } 84 | ConfigurationOfNeoCSV class >> load [ 85 | "Load the #stable version defined for this platform. The #stable version is the version that is recommended to be used on this platform." 86 | 87 | "self load" 88 | 89 | 90 | ^(self project version: #stable) load 91 | ] 92 | 93 | { #category : #loading } 94 | ConfigurationOfNeoCSV class >> loadBleedingEdge [ 95 | "Load the latest versions of the mcz files defined for this project. It is not likely that the #bleedingEdge has been tested." 96 | 97 | "self loadBleedingEdge" 98 | 99 | 100 | ^(self project version: #bleedingEdge) load 101 | ] 102 | 103 | { #category : #loading } 104 | ConfigurationOfNeoCSV class >> loadDevelopment [ 105 | "Load the #development version defined for this platform. The #development version will change over time and is not expected to be stable." 106 | 107 | "self loadDevelopment" 108 | 109 | 110 | ^(self project version: #development) load 111 | ] 112 | 113 | { #category : #accessing } 114 | ConfigurationOfNeoCSV class >> project [ 115 | 116 | ^self new project 117 | ] 118 | 119 | { #category : #'development support' } 120 | ConfigurationOfNeoCSV class >> validate [ 121 | "Check the configuration for Errors, Critical Warnings, and Warnings (see class comment for MetacelloMCVersionValidator for more information). 122 | Errors identify specification issues that will result in unexpected behaviour when you load the configuration. 123 | Critical Warnings identify specification issues that may result in unexpected behavior when you load the configuration. 124 | Warnings identify specification issues that are technically correct, but are worth take a look at." 125 | 126 | "self validate" 127 | 128 | 129 | self ensureMetacello. 130 | ^ ((Smalltalk at: #MetacelloToolBox) validateConfiguration: self debug: #() recurse: false) explore 131 | ] 132 | 133 | { #category : #baselines } 134 | ConfigurationOfNeoCSV >> baseline1: spec [ 135 | 136 | 137 | spec for: #common do: [ 138 | spec 139 | blessing: #baseline; 140 | repository: 'http://mc.stfx.eu/Neo'; 141 | package: 'Neo-CSV-Core'; 142 | package: 'Neo-CSV-Tests' with: [ spec requires: 'Neo-CSV-Core' ]; 143 | group: 'default' with: #('Neo-CSV-Core' 'Neo-CSV-Tests'); 144 | group: 'Core' with: #('Neo-CSV-Core'); 145 | group: 'Tests' with: #('Neo-CSV-Tests') ] 146 | ] 147 | 148 | { #category : #accessing } 149 | ConfigurationOfNeoCSV >> project [ 150 | 151 | ^ project ifNil: [ | constructor | 152 | "Bootstrap Metacello if it is not already loaded" 153 | (self class baseConfigurationClassIfAbsent: []) ensureMetacello. 154 | "Construct Metacello project" 155 | constructor := (Smalltalk at: #MetacelloVersionConstructor) on: self. 156 | project := constructor project. 157 | project loadType: #linear. "change to #atomic if desired" 158 | project ] 159 | ] 160 | 161 | { #category : #'symbolic versions' } 162 | ConfigurationOfNeoCSV >> stable: spec [ 163 | 164 | 165 | spec for: #common version: '15' 166 | ] 167 | 168 | { #category : #versions } 169 | ConfigurationOfNeoCSV >> version10: spec [ 170 | 171 | 172 | spec for: #common do: [ 173 | spec 174 | blessing: #release; 175 | package: 'Neo-CSV-Core' with: 'Neo-CSV-Core-SvenVanCaekenberghe.16'; 176 | package: 'Neo-CSV-Tests' with: 'Neo-CSV-Tests-SvenVanCaekenberghe.15' ] 177 | ] 178 | 179 | { #category : #versions } 180 | ConfigurationOfNeoCSV >> version11: spec [ 181 | 182 | 183 | spec for: #common do: [ 184 | spec 185 | blessing: #release; 186 | package: 'Neo-CSV-Core' with: 'Neo-CSV-Core-SvenVanCaekenberghe.20'; 187 | package: 'Neo-CSV-Tests' with: 'Neo-CSV-Tests-SvenVanCaekenberghe.17' ] 188 | ] 189 | 190 | { #category : #versions } 191 | ConfigurationOfNeoCSV >> version12: spec [ 192 | 193 | 194 | spec for: #common do: [ 195 | spec 196 | blessing: #release; 197 | package: 'Neo-CSV-Core' with: 'Neo-CSV-Core-SvenVanCaekenberghe.21'; 198 | package: 'Neo-CSV-Tests' with: 'Neo-CSV-Tests-SvenVanCaekenberghe.18' ] 199 | ] 200 | 201 | { #category : #versions } 202 | ConfigurationOfNeoCSV >> version13: spec [ 203 | 204 | 205 | spec for: #common do: [ 206 | spec 207 | blessing: #release; 208 | package: 'Neo-CSV-Core' with: 'Neo-CSV-Core-SvenVanCaekenberghe.22'; 209 | package: 'Neo-CSV-Tests' with: 'Neo-CSV-Tests-SvenVanCaekenberghe.19' ] 210 | ] 211 | 212 | { #category : #versions } 213 | ConfigurationOfNeoCSV >> version14: spec [ 214 | 215 | 216 | spec for: #common do: [ 217 | spec 218 | blessing: #release; 219 | package: 'Neo-CSV-Core' with: 'Neo-CSV-Core-SvenVanCaekenberghe.24'; 220 | package: 'Neo-CSV-Tests' with: 'Neo-CSV-Tests-SvenVanCaekenberghe.20' ] 221 | ] 222 | 223 | { #category : #versions } 224 | ConfigurationOfNeoCSV >> version15: spec [ 225 | 226 | 227 | spec for: #common do: [ 228 | spec 229 | blessing: #release; 230 | package: 'Neo-CSV-Core' with: 'Neo-CSV-Core-SvenVanCaekenberghe.26'; 231 | package: 'Neo-CSV-Tests' with: 'Neo-CSV-Tests-SvenVanCaekenberghe.23' ] 232 | ] 233 | 234 | { #category : #versions } 235 | ConfigurationOfNeoCSV >> version1: spec [ 236 | 237 | 238 | spec for: #common do: [ 239 | spec 240 | blessing: #release; 241 | package: 'Neo-CSV-Core' with: 'Neo-CSV-Core-SvenVanCaekenberghe.6'; 242 | package: 'Neo-CSV-Tests' with: 'Neo-CSV-Tests-SvenVanCaekenberghe.6' ] 243 | ] 244 | 245 | { #category : #versions } 246 | ConfigurationOfNeoCSV >> version2: spec [ 247 | 248 | 249 | spec for: #common do: [ 250 | spec 251 | blessing: #release; 252 | package: 'Neo-CSV-Core' with: 'Neo-CSV-Core-SvenVanCaekenberghe.8'; 253 | package: 'Neo-CSV-Tests' with: 'Neo-CSV-Tests-SvenVanCaekenberghe.6' ] 254 | ] 255 | 256 | { #category : #versions } 257 | ConfigurationOfNeoCSV >> version3: spec [ 258 | 259 | 260 | spec for: #common do: [ 261 | spec 262 | blessing: #release; 263 | package: 'Neo-CSV-Core' with: 'Neo-CSV-Core-SvenVanCaekenberghe.9'; 264 | package: 'Neo-CSV-Tests' with: 'Neo-CSV-Tests-SvenVanCaekenberghe.7' ] 265 | ] 266 | 267 | { #category : #versions } 268 | ConfigurationOfNeoCSV >> version4: spec [ 269 | 270 | 271 | spec for: #common do: [ 272 | spec 273 | blessing: #release; 274 | package: 'Neo-CSV-Core' with: 'Neo-CSV-Core-SvenVanCaekenberghe.9'; 275 | package: 'Neo-CSV-Tests' with: 'Neo-CSV-Tests-SvenVanCaekenberghe.8' ] 276 | ] 277 | 278 | { #category : #versions } 279 | ConfigurationOfNeoCSV >> version5: spec [ 280 | 281 | 282 | spec for: #common do: [ 283 | spec 284 | blessing: #release; 285 | package: 'Neo-CSV-Core' with: 'Neo-CSV-Core-SvenVanCaekenberghe.10'; 286 | package: 'Neo-CSV-Tests' with: 'Neo-CSV-Tests-SvenVanCaekenberghe.9' ] 287 | ] 288 | 289 | { #category : #versions } 290 | ConfigurationOfNeoCSV >> version6: spec [ 291 | 292 | 293 | spec for: #common do: [ 294 | spec 295 | blessing: #release; 296 | package: 'Neo-CSV-Core' with: 'Neo-CSV-Core-SvenVanCaekenberghe.12'; 297 | package: 'Neo-CSV-Tests' with: 'Neo-CSV-Tests-SvenVanCaekenberghe.11' ] 298 | ] 299 | 300 | { #category : #versions } 301 | ConfigurationOfNeoCSV >> version7: spec [ 302 | 303 | 304 | spec for: #common do: [ 305 | spec 306 | blessing: #release; 307 | package: 'Neo-CSV-Core' with: 'Neo-CSV-Core-SvenVanCaekenberghe.13'; 308 | package: 'Neo-CSV-Tests' with: 'Neo-CSV-Tests-SvenVanCaekenberghe.12' ] 309 | ] 310 | 311 | { #category : #versions } 312 | ConfigurationOfNeoCSV >> version8: spec [ 313 | 314 | 315 | spec for: #common do: [ 316 | spec 317 | blessing: #release; 318 | package: 'Neo-CSV-Core' with: 'Neo-CSV-Core-SvenVanCaekenberghe.14'; 319 | package: 'Neo-CSV-Tests' with: 'Neo-CSV-Tests-SvenVanCaekenberghe.13' ] 320 | ] 321 | 322 | { #category : #versions } 323 | ConfigurationOfNeoCSV >> version9: spec [ 324 | 325 | 326 | spec for: #common do: [ 327 | spec 328 | blessing: #release; 329 | package: 'Neo-CSV-Core' with: 'Neo-CSV-Core-SvenVanCaekenberghe.15'; 330 | package: 'Neo-CSV-Tests' with: 'Neo-CSV-Tests-SvenVanCaekenberghe.14' ] 331 | ] 332 | -------------------------------------------------------------------------------- /repository/ConfigurationOfNeoCSV/package.st: -------------------------------------------------------------------------------- 1 | Package { #name : #ConfigurationOfNeoCSV } 2 | -------------------------------------------------------------------------------- /repository/Neo-CSV-Core/NeoCSVData.class.st: -------------------------------------------------------------------------------- 1 | " 2 | I am NeoCSVData, representing the data of a CSV file. 3 | 4 | I hold an optional header and a collection of records with the same fields. 5 | 6 | I am a convenience object. 7 | 8 | You can create an instance of me by reading me from a stream (NeoCSVData class>>readFrom:). 9 | You can write me to a stream (NeoCSVData>>writeTo:). 10 | 11 | Note that I assume my records are indexable collections, not dictionaries. 12 | 13 | No field conversions are done, all fields remain strings. 14 | 15 | Using NeoCSVReader and NeoCSVWriter directly gives you much more options. 16 | 17 | I can also be used by construction from a collection (NeoCSVData class>>with:) with the header being set manually. 18 | " 19 | Class { 20 | #name : #NeoCSVData, 21 | #superclass : #Object, 22 | #instVars : [ 23 | 'header', 24 | 'data', 25 | 'types' 26 | ], 27 | #category : #'Neo-CSV-Core' 28 | } 29 | 30 | { #category : #'instance creation' } 31 | NeoCSVData class >> readAsDictionariesFrom: characterReadStream [ 32 | ^ self new readAsDictionariesFrom: characterReadStream; yourself 33 | ] 34 | 35 | { #category : #'instance creation' } 36 | NeoCSVData class >> readFrom: characterReadStream [ 37 | ^ self new readFrom: characterReadStream; yourself 38 | ] 39 | 40 | { #category : #accessing } 41 | NeoCSVData class >> typesMap [ 42 | ^ { 43 | #string -> [ :string | string ]. 44 | #number -> [ :string | NeoNumberParser parse: string ]. 45 | #timestamp -> [ :string | DateAndTime fromString: string ]. 46 | #boolean -> [ :string | #(true t yes y '1') includes: string asLowercase ] 47 | } asDictionary 48 | ] 49 | 50 | { #category : #'instance creation' } 51 | NeoCSVData class >> with: collectionOfRecords [ 52 | ^ self new 53 | data: collectionOfRecords; 54 | yourself 55 | ] 56 | 57 | { #category : #accessing } 58 | NeoCSVData >> data [ 59 | ^ data 60 | ] 61 | 62 | { #category : #accessing } 63 | NeoCSVData >> data: collectionOfRecords [ 64 | data := collectionOfRecords 65 | ] 66 | 67 | { #category : #testing } 68 | NeoCSVData >> hasData [ 69 | ^ data notNil and: [ data size > 0 ] 70 | ] 71 | 72 | { #category : #testing } 73 | NeoCSVData >> hasHeader [ 74 | ^ header notNil and: [ header size > 0 ] 75 | ] 76 | 77 | { #category : #testing } 78 | NeoCSVData >> hasTypes [ 79 | ^ types notNil and: [ types size > 0 ] 80 | ] 81 | 82 | { #category : #accessing } 83 | NeoCSVData >> header [ 84 | ^ header 85 | ] 86 | 87 | { #category : #accessing } 88 | NeoCSVData >> header: collectionOfFieldNames [ 89 | header := collectionOfFieldNames 90 | ] 91 | 92 | { #category : #accessing } 93 | NeoCSVData >> headerForPresentation [ 94 | ^ self header 95 | ifNil: [ self data ifNotNil: [ (1 to: self data first size) collect: #asString ] ] 96 | ] 97 | 98 | { #category : #accessing } 99 | NeoCSVData >> numberOfColumns [ 100 | ^ header 101 | ifNil: [ 102 | self data ifNil: [ 0 ] ifNotNil: [ self data first size ] ] 103 | ifNotNil: [ header size ] 104 | ] 105 | 106 | { #category : #accessing } 107 | NeoCSVData >> numberOfRows [ 108 | ^ data ifNil: [ 0 ] ifNotNil: [ data size ] 109 | ] 110 | 111 | { #category : #printing } 112 | NeoCSVData >> printOn: stream [ 113 | super printOn: stream. 114 | stream 115 | nextPut: $(; 116 | print: self numberOfRows; 117 | space; 118 | nextPutAll: 'row'. 119 | self numberOfRows = 1 ifFalse: [ stream nextPut: $s ]. 120 | stream 121 | space; 122 | nextPut: $x; 123 | space; 124 | print: self numberOfColumns; 125 | space; 126 | nextPutAll: 'column'. 127 | self numberOfColumns = 1 ifFalse: [ stream nextPut: $s ]. 128 | stream 129 | nextPut: $) 130 | ] 131 | 132 | { #category : #reading } 133 | NeoCSVData >> readAsDictionariesFrom: characterReadStream [ 134 | | reader | 135 | reader := NeoCSVReader on: characterReadStream. 136 | reader recordClass: Dictionary. 137 | self hasHeader 138 | ifTrue: [ reader fieldCount: header size ] 139 | ifFalse: [ self header: reader readHeader ]. 140 | self hasTypes 141 | ifTrue: [ 142 | self types with: self header do: [ :type :column | 143 | reader addFieldAt: column converter: (self class typesMap at: type) ] ]. 144 | self data: reader upToEnd 145 | ] 146 | 147 | { #category : #reading } 148 | NeoCSVData >> readFrom: characterReadStream [ 149 | | reader | 150 | reader := NeoCSVReader on: characterReadStream. 151 | self hasHeader 152 | ifTrue: [ reader fieldCount: header size ] 153 | ifFalse: [ self header: reader readHeader ]. 154 | self hasTypes 155 | ifTrue: [ 156 | self types do: [ :type | 157 | reader addFieldConverter: (self class typesMap at: type) ] ]. 158 | self data: reader upToEnd 159 | ] 160 | 161 | { #category : #accessing } 162 | NeoCSVData >> typeForColumnAt: index [ 163 | self hasTypes ifFalse: [ ^ #string ]. 164 | ^ self types at: index 165 | ] 166 | 167 | { #category : #accessing } 168 | NeoCSVData >> typeForColumnNamed: columnName [ 169 | self hasHeader ifFalse: [ ^ #string ]. 170 | ^ self typeForColumnAt: (self header indexOf: columnName) 171 | ] 172 | 173 | { #category : #accessing } 174 | NeoCSVData >> types [ 175 | ^ types 176 | ] 177 | 178 | { #category : #accessing } 179 | NeoCSVData >> types: collectionOfTypeSymbols [ 180 | self assert: (collectionOfTypeSymbols asSet \ self class typesMap keys) isEmpty. 181 | types := collectionOfTypeSymbols 182 | ] 183 | 184 | { #category : #writing } 185 | NeoCSVData >> writeOn: characterWriteStream [ 186 | | writer | 187 | self data isEmptyOrNil ifTrue: [ ^ self ]. 188 | writer := NeoCSVWriter on: characterWriteStream. 189 | self data first isDictionary 190 | ifTrue: [ 191 | writer namedColumnsConfiguration: self header ] 192 | ifFalse: [ 193 | self header ifNotNil: [ writer writeHeader: self header ] ]. 194 | writer nextPutAll: self data 195 | ] 196 | -------------------------------------------------------------------------------- /repository/Neo-CSV-Core/NeoCSVReader.class.st: -------------------------------------------------------------------------------- 1 | " 2 | I am NeoCSVReader. 3 | 4 | I read a format that 5 | - is text based (ASCII, Latin1, Unicode) 6 | - consists of records, 1 per line (any line ending convention) 7 | - where records consist of fields separated by a delimiter (comma, tab, semicolon) 8 | - where every record has the same number of fields 9 | - where fields can be quoted should they contain separators or line endings 10 | 11 | Without further configuration, records will become Arrays of Strings. 12 | 13 | By specifiying a recordClass and fields with optional converters most objects can be read and instanciated correctly. 14 | 15 | MIT License. 16 | 17 | " 18 | Class { 19 | #name : #NeoCSVReader, 20 | #superclass : #Object, 21 | #instVars : [ 22 | 'readStream', 23 | 'charBuffer', 24 | 'separator', 25 | 'stringStream', 26 | 'fieldCount', 27 | 'recordClass', 28 | 'recordClassIsIndexable', 29 | 'fieldAccessors', 30 | 'emptyFieldValue', 31 | 'emptyFieldInput', 32 | 'strict' 33 | ], 34 | #category : #'Neo-CSV-Core' 35 | } 36 | 37 | { #category : #'instance creation' } 38 | NeoCSVReader class >> on: readStream [ 39 | "Initialize on readStream, which should be a character stream that 40 | implements #next, #atEnd and (optionally) #close." 41 | 42 | ^ self new 43 | on: readStream; 44 | yourself 45 | ] 46 | 47 | { #category : #'initialize-release' } 48 | NeoCSVReader >> addField [ 49 | "Add the next indexable field with a pass through converter" 50 | 51 | self 52 | recordClassIsIndexable: true; 53 | addFieldAccessor: [ :string | string ] 54 | ] 55 | 56 | { #category : #'initialize-release' } 57 | NeoCSVReader >> addField: accessor [ 58 | "Add a field based on a mutator accessor accepting a field 59 | String as argument to be sent to an instance of recordClass. 60 | Accessor can be a Symbol or a Block" 61 | 62 | self 63 | recordClassIsIndexable: false; 64 | addFieldAccessor: [ :object :string | 65 | self applyAccessor: accessor on: object with: string ] 66 | ] 67 | 68 | { #category : #'initialize-release' } 69 | NeoCSVReader >> addField: accessor converter: converter [ 70 | "Add a field based on a mutator accessor accepting the result of 71 | applying the converter block on the field String read as argument 72 | to be sent to an instance of recordClass. 73 | Accessor can be a Symbol or a Block" 74 | 75 | self 76 | recordClassIsIndexable: false; 77 | addFieldAccessor: [ :object :string | 78 | self applyAccessor: accessor on: object with: (converter value: string) ] 79 | ] 80 | 81 | { #category : #private } 82 | NeoCSVReader >> addFieldAccessor: block [ 83 | fieldAccessors 84 | ifNil: [ 85 | fieldAccessors := Array with: block ] 86 | ifNotNil: [ 87 | fieldAccessors := fieldAccessors copyWith: block ] 88 | ] 89 | 90 | { #category : #'initialize-release' } 91 | NeoCSVReader >> addFieldAt: key [ 92 | "Add a field that will be stored under key in recordClass as String" 93 | 94 | self 95 | recordClassIsIndexable: false; 96 | addFieldAccessor: [ :object :string | 97 | object at: key put: string ] 98 | ] 99 | 100 | { #category : #'initialize-release' } 101 | NeoCSVReader >> addFieldAt: key converter: converter [ 102 | "Add a field that will be stored under key in recordClass as the result of 103 | applying the converter block on the field String read as argument" 104 | 105 | self 106 | recordClassIsIndexable: false; 107 | addFieldAccessor: [ :object :string | 108 | object at: key put: (converter value: string) ] 109 | ] 110 | 111 | { #category : #'initialize-release' } 112 | NeoCSVReader >> addFieldConverter: converter [ 113 | "Add the next indexable field with converter block, 114 | accepting a String and returning a specific object" 115 | 116 | self 117 | recordClassIsIndexable: true; 118 | addFieldAccessor: converter 119 | ] 120 | 121 | { #category : #convenience } 122 | NeoCSVReader >> addFields: accessors [ 123 | "Add fields based on a collection of accessors, not doing any conversions." 124 | 125 | accessors do: [ :each | 126 | self addField: each ] 127 | ] 128 | 129 | { #category : #convenience } 130 | NeoCSVReader >> addFieldsAt: keys [ 131 | "Add fields based on a collection of keys for #at:put: not doing any conversions" 132 | 133 | keys do: [ :each | 134 | self addFieldAt: each ] 135 | ] 136 | 137 | { #category : #convenience } 138 | NeoCSVReader >> addFloatField [ 139 | "Add a field for indexable records parsed as Float" 140 | 141 | self addFieldConverter: [ :string | NeoNumberParser parse: string ] 142 | ] 143 | 144 | { #category : #convenience } 145 | NeoCSVReader >> addFloatField: accessor [ 146 | "Add a field with accessor parsed as Float" 147 | 148 | self 149 | addField: accessor 150 | converter: [ :string | NeoNumberParser parse: string ] 151 | ] 152 | 153 | { #category : #convenience } 154 | NeoCSVReader >> addFloatFieldAt: key [ 155 | "Add a field for key for #at:put: parsed as Float" 156 | 157 | self 158 | addFieldAt: key 159 | converter: [ :string | NeoNumberParser parse: string ] 160 | ] 161 | 162 | { #category : #convenience } 163 | NeoCSVReader >> addFloatFieldRadixPointComma [ 164 | "Add a field for indexable records parsed as Float using a comma as radix point" 165 | 166 | self addFieldConverter: [ :string | (NeoNumberParser on: string) radixPoint: $, ; next ] 167 | ] 168 | 169 | { #category : #convenience } 170 | NeoCSVReader >> addFloatFieldRadixPointComma: accessor [ 171 | "Add a field for indexable records parsed as Float using a comma as radix point" 172 | 173 | self 174 | addField: accessor 175 | converter: [ :string | (NeoNumberParser on: string) radixPoint: $, ; next ] 176 | ] 177 | 178 | { #category : #convenience } 179 | NeoCSVReader >> addFloatFieldRadixPointCommaAt: key [ 180 | "Add a field for key for #at:put: parsed as Float using a comma as radix point" 181 | 182 | self 183 | addFieldAt: key 184 | converter: [ :string | (NeoNumberParser on: string) radixPoint: $, ; next ] 185 | ] 186 | 187 | { #category : #'initialize-release' } 188 | NeoCSVReader >> addIgnoredField [ 189 | "Add a field that should be ignored, should not become part of the record" 190 | 191 | self addFieldAccessor: nil 192 | ] 193 | 194 | { #category : #convenience } 195 | NeoCSVReader >> addIgnoredFields: count [ 196 | "Add a count of consecutive ignored fields to receiver." 197 | 198 | count timesRepeat: [ self addIgnoredField ] 199 | ] 200 | 201 | { #category : #convenience } 202 | NeoCSVReader >> addIntegerField [ 203 | "Add a field for indexable records parsed as Integer" 204 | 205 | self addFieldConverter: [ :string | NeoNumberParser parse: string ] 206 | ] 207 | 208 | { #category : #convenience } 209 | NeoCSVReader >> addIntegerField: accessor [ 210 | "Add a field with accessor parsed as Integer" 211 | 212 | self 213 | addField: accessor 214 | converter: [ :string | NeoNumberParser parse: string ] 215 | ] 216 | 217 | { #category : #convenience } 218 | NeoCSVReader >> addIntegerFieldAt: key [ 219 | "Add a field for key for #at:put: parsed as Integer" 220 | 221 | self 222 | addFieldAt: key 223 | converter: [ :string | NeoNumberParser parse: string ] 224 | ] 225 | 226 | { #category : #convenience } 227 | NeoCSVReader >> addSymbolField [ 228 | "Add a field for indexable records read as Symbol" 229 | 230 | self addFieldConverter: [ :string | string asSymbol ] 231 | ] 232 | 233 | { #category : #convenience } 234 | NeoCSVReader >> addSymbolField: accessor [ 235 | "Add a field with accessor read as Symbol" 236 | 237 | self 238 | addField: accessor 239 | converter: [ :string | string asSymbol ] 240 | ] 241 | 242 | { #category : #convenience } 243 | NeoCSVReader >> addSymbolFieldAt: key [ 244 | "Add a field for key for #at:put: read as Symbol" 245 | 246 | self 247 | addFieldAt: key 248 | converter: [ :string | string asSymbol ] 249 | ] 250 | 251 | { #category : #private } 252 | NeoCSVReader >> applyAccessor: accessor on: object with: value [ 253 | "Use accessor to assign value on a property of object. 254 | Accessor can be a block or mutator symbol." 255 | 256 | "If Symbol implemented #value:value: this could be implemented more elegantly." 257 | 258 | accessor isBlock 259 | ifTrue: [ accessor value: object value: value ] 260 | ifFalse: [ object perform: accessor with: value ] 261 | ] 262 | 263 | { #category : #testing } 264 | NeoCSVReader >> atEnd [ 265 | ^ charBuffer == nil and: [ readStream atEnd ] 266 | ] 267 | 268 | { #category : #'initialize-release' } 269 | NeoCSVReader >> beStrict [ 270 | "Configure me to signal errors when the input does not match my field configration" 271 | 272 | strict := true 273 | ] 274 | 275 | { #category : #'initialize-release' } 276 | NeoCSVReader >> close [ 277 | readStream ifNotNil: [ 278 | readStream close. 279 | readStream := charBuffer := nil ] 280 | ] 281 | 282 | { #category : #enumerating } 283 | NeoCSVReader >> collect: block [ 284 | "Execute block for each record until I am at end, returning the results in an Array" 285 | 286 | ^ Array streamContents: [ :out | 287 | [ self atEnd ] 288 | whileFalse: [ 289 | out nextPut: (block value: self next) ] ] 290 | ] 291 | 292 | { #category : #enumerating } 293 | NeoCSVReader >> do: block [ 294 | "Execute block for each record until I am at end." 295 | 296 | [ self atEnd ] 297 | whileFalse: [ 298 | block value: self next ] 299 | ] 300 | 301 | { #category : #'initialize-release' } 302 | NeoCSVReader >> emptyFieldInput: block [ 303 | "Set an optional block to test for field input values that should be considered empty. 304 | The default is nil, equivalent to [ :field | field isEmpty ]." 305 | 306 | emptyFieldInput := block 307 | ] 308 | 309 | { #category : #'initialize-release' } 310 | NeoCSVReader >> emptyFieldValue: object [ 311 | "Set the value to be used when reading empty or missing fields. 312 | The default is nil. Empty or missing fields are never set 313 | when the record class is non-indexeabe, nor are they passed to converters. 314 | The special #passNil can be set to force that in the case of 315 | an empty or missing field nil *is* passed to a converter block 316 | so that per field empty values or specific behavior are possible." 317 | 318 | emptyFieldValue := object 319 | ] 320 | 321 | { #category : #'initialize-release' } 322 | NeoCSVReader >> fieldCount: count [ 323 | "Set the field count up front. 324 | This will be used when reading records as Arrays. 325 | This instance variable will be set and used automatically based on the first record seen. 326 | If set, the fieldAccessors collection defines (overrides) the fieldCount." 327 | 328 | fieldCount := count 329 | ] 330 | 331 | { #category : #'private - reading' } 332 | NeoCSVReader >> handleEndOfRecord [ 333 | strict 334 | ifTrue: [ 335 | self readAtEndOrEndOfLine 336 | ifFalse: [ self error: 'Excess fields for CSV input' ] ] 337 | ifFalse: [ self skipLine ] 338 | ] 339 | 340 | { #category : #'private - reading' } 341 | NeoCSVReader >> handleSeparator [ 342 | self readSeparator 343 | ifFalse: [ 344 | strict 345 | ifTrue: [ self error: 'Insufficient fields for CSV input' ] ] 346 | ] 347 | 348 | { #category : #'initialize-release' } 349 | NeoCSVReader >> initialize [ 350 | super initialize. 351 | recordClass := Array. 352 | recordClassIsIndexable := true. 353 | separator := $,. 354 | strict := false 355 | ] 356 | 357 | { #category : #convenience } 358 | NeoCSVReader >> namedColumnsConfiguration [ 359 | "Assuming there is a header row that has not yet been read, 360 | configure the receiver to read each row as a Dictionary where 361 | each field is stored under a column name. 362 | Note that each field is read as a string." 363 | 364 | | header | 365 | self recordClass: Dictionary. 366 | header := self readHeader. 367 | self addFieldsAt: (header collect: [ :each | each asSymbol ]). 368 | ^ header 369 | ] 370 | 371 | { #category : #accessing } 372 | NeoCSVReader >> next [ 373 | "Read the next record. 374 | I will return an instance of recordClass." 375 | 376 | ^ recordClassIsIndexable 377 | ifTrue: [ self readNextRecordAsArray ] 378 | ifFalse: [ self readNextRecordAsObject ] 379 | ] 380 | 381 | { #category : #private } 382 | NeoCSVReader >> nextChar [ 383 | ^ charBuffer 384 | ifNil: [ 385 | readStream next ] 386 | ifNotNil: [ | char | 387 | char := charBuffer. 388 | charBuffer := nil. 389 | ^ char ] 390 | ] 391 | 392 | { #category : #'initialize-release' } 393 | NeoCSVReader >> on: aReadStream [ 394 | "Initialize on aReadStream, which should be a character stream that 395 | implements #next, #atEnd and (optionally) #close." 396 | 397 | readStream := aReadStream 398 | ] 399 | 400 | { #category : #private } 401 | NeoCSVReader >> peekChar [ 402 | ^ charBuffer 403 | ifNil: [ 404 | charBuffer := readStream next ] 405 | ] 406 | 407 | { #category : #private } 408 | NeoCSVReader >> peekEndOfLine [ 409 | | char | 410 | char := self peekChar codePoint. 411 | ^ (char == 10 "Character lf" ) or: [ char == 13 "Character cr" ] 412 | ] 413 | 414 | { #category : #private } 415 | NeoCSVReader >> peekFor: character [ 416 | self peekChar == character 417 | ifTrue: [ 418 | self nextChar. 419 | ^ true ]. 420 | ^ false 421 | ] 422 | 423 | { #category : #private } 424 | NeoCSVReader >> peekQuote [ 425 | ^ self peekChar == $" 426 | ] 427 | 428 | { #category : #private } 429 | NeoCSVReader >> peekSeparator [ 430 | ^ self peekChar == separator 431 | ] 432 | 433 | { #category : #private } 434 | NeoCSVReader >> readAtEndOrEndOfLine [ 435 | ^ self atEnd or: [ self readEndOfLine ] 436 | 437 | ] 438 | 439 | { #category : #private } 440 | NeoCSVReader >> readEndOfLine [ 441 | | char | 442 | char := self peekChar codePoint. 443 | char == 10 "Character lf" 444 | ifTrue: [ 445 | self nextChar. 446 | ^ true ]. 447 | char == 13 "Character cr" 448 | ifTrue: [ 449 | self nextChar. 450 | (self atEnd not and: [ self peekChar codePoint == 10 "Character lf" ]) 451 | ifTrue: [ 452 | self nextChar ]. 453 | ^ true ]. 454 | ^ false 455 | 456 | ] 457 | 458 | { #category : #private } 459 | NeoCSVReader >> readEndOfQuotedField [ 460 | "A double quote inside a quoted field is an embedded quote (escaped)" 461 | 462 | ^ self readQuote and: [ self peekQuote not ] 463 | ] 464 | 465 | { #category : #'private - reading' } 466 | NeoCSVReader >> readField [ 467 | | field | 468 | field := self peekQuote 469 | ifTrue: [ self readQuotedField ] 470 | ifFalse: [ self readUnquotedField ]. 471 | (field ~= emptyFieldValue and: [ emptyFieldInput notNil ]) ifTrue: [ 472 | (emptyFieldInput value: field) 473 | ifTrue: [ ^ emptyFieldValue ] ]. 474 | ^ field 475 | ] 476 | 477 | { #category : #'private - reading' } 478 | NeoCSVReader >> readFirstRecord [ 479 | "This is only used for array based records when there are no field accessors or 480 | when there is no field count, to obtain a field count based on the first record" 481 | 482 | ^ self recordClassStreamContents: [ :stream | 483 | [ self readAtEndOrEndOfLine ] 484 | whileFalse: [ 485 | stream nextPut: self readField. 486 | (self readSeparator and: [ self atEnd or: [ self peekEndOfLine ] ]) 487 | ifTrue: [ stream nextPut: emptyFieldValue ] ] ] 488 | ] 489 | 490 | { #category : #accessing } 491 | NeoCSVReader >> readHeader [ 492 | "Read a record, presumably a header and return the header field names. 493 | This should normally be called only at the beginning and only once. 494 | This sets the fieldCount (but fieldAccessors overrides fieldCount)." 495 | 496 | | names | 497 | names := Array streamContents: [ :out | 498 | [ self readAtEndOrEndOfLine ] 499 | whileFalse: [ 500 | out nextPut: self readField. 501 | (self readSeparator and: [ self atEnd or: [ self peekEndOfLine ] ]) 502 | ifTrue: [ out nextPut: emptyFieldValue ] ] ]. 503 | self fieldCount: names size. 504 | ^ names 505 | ] 506 | 507 | { #category : #'private - reading' } 508 | NeoCSVReader >> readNextRecord [ 509 | | record | 510 | record := self recordClassNew: fieldCount. 511 | fieldAccessors 512 | ifNil: [ self readNextRecordWithoutFieldAccessors: record ] 513 | ifNotNil: [ self readNextRecordWithFieldAccessors: record ]. 514 | self handleEndOfRecord. 515 | ^ record 516 | ] 517 | 518 | { #category : #'private - reading' } 519 | NeoCSVReader >> readNextRecordAsArray [ 520 | fieldAccessors ifNotNil: [ 521 | self fieldCount: (fieldAccessors count: [ :each | each notNil ]) ]. 522 | ^ fieldCount 523 | ifNil: [ | record | 524 | record := self readFirstRecord. 525 | self fieldCount: record size. 526 | record ] 527 | ifNotNil: [ 528 | self readNextRecord ] 529 | ] 530 | 531 | { #category : #'private - reading' } 532 | NeoCSVReader >> readNextRecordAsObject [ 533 | | object | 534 | object := self recordClassNew. 535 | fieldAccessors 536 | do: [ :each | | rawValue | 537 | rawValue := self readField. 538 | "nil field accessors are used to ignore fields" 539 | each 540 | ifNotNil: [ 541 | rawValue = emptyFieldValue 542 | ifTrue: [ 543 | emptyFieldValue = #passNil 544 | ifTrue: [ each value: object value: nil ] ] 545 | ifFalse: [ each value: object value: rawValue ] ] ] 546 | separatedBy: [ self handleSeparator ]. 547 | self handleEndOfRecord. 548 | ^ object 549 | ] 550 | 551 | { #category : #'private - reading' } 552 | NeoCSVReader >> readNextRecordWithFieldAccessors: record [ 553 | | fieldIndex | 554 | fieldIndex := 1. 555 | fieldAccessors 556 | do: [ :each | | rawValue | 557 | rawValue := self readField. 558 | "nil field accessors are used to ignore fields" 559 | each 560 | ifNotNil: [ 561 | rawValue = emptyFieldValue 562 | ifTrue: [ 563 | emptyFieldValue = #passNil 564 | ifTrue: [ record at: fieldIndex put: (each value: nil) ] 565 | ifFalse: [ record at: fieldIndex put: emptyFieldValue ] ] 566 | ifFalse: [ record at: fieldIndex put: (each value: rawValue) ]. 567 | fieldIndex := fieldIndex + 1 ] ] 568 | separatedBy: [ self handleSeparator ] 569 | ] 570 | 571 | { #category : #'private - reading' } 572 | NeoCSVReader >> readNextRecordWithoutFieldAccessors: record [ 573 | 1 to: fieldCount do: [ :each | 574 | record at: each put: self readField. 575 | each = fieldCount ifFalse: [ self handleSeparator ] ] 576 | ] 577 | 578 | { #category : #private } 579 | NeoCSVReader >> readQuote [ 580 | ^ self peekFor: $" 581 | ] 582 | 583 | { #category : #'private - reading' } 584 | NeoCSVReader >> readQuotedField [ 585 | | field | 586 | self readQuote. 587 | field := self stringStreamContents: [ :stream | 588 | [ self atEnd or: [ self readEndOfQuotedField ] ] 589 | whileFalse: [ 590 | stream nextPut: self nextChar ] ]. 591 | ^ field isEmpty 592 | ifTrue: [ emptyFieldValue ] 593 | ifFalse: [ field ] 594 | ] 595 | 596 | { #category : #private } 597 | NeoCSVReader >> readSeparator [ 598 | ^ self peekFor: separator 599 | ] 600 | 601 | { #category : #'private - reading' } 602 | NeoCSVReader >> readUnquotedField [ 603 | (self atEnd or: [ self peekSeparator or: [ self peekEndOfLine ] ]) 604 | ifTrue: [ ^ emptyFieldValue ]. 605 | ^ self stringStreamContents: [ :stream | 606 | [ self atEnd or: [ self peekSeparator or: [ self peekEndOfLine ] ] ] 607 | whileFalse: [ 608 | stream nextPut: self nextChar ] ] 609 | ] 610 | 611 | { #category : #'initialize-release' } 612 | NeoCSVReader >> recordClass: anObject [ 613 | "Set the object class to instanciate while reading records. 614 | Unless the objets are integer indexable, you have to specify fields as well." 615 | 616 | recordClass := anObject 617 | ] 618 | 619 | { #category : #'initialize-release' } 620 | NeoCSVReader >> recordClassIsIndexable: boolean [ 621 | "Set whether recordClass should be treated as an indexable sequenceable collection 622 | class that implements #new: and #streamContents and whose instances implement #at:put: 623 | If false, fields accessors have to be provided. The default is true. 624 | Will be set automatically when field accessors or converters are set." 625 | 626 | recordClassIsIndexable := boolean 627 | ] 628 | 629 | { #category : #private } 630 | NeoCSVReader >> recordClassNew [ 631 | ^ recordClass new 632 | ] 633 | 634 | { #category : #private } 635 | NeoCSVReader >> recordClassNew: size [ 636 | ^ recordClass new: size 637 | ] 638 | 639 | { #category : #private } 640 | NeoCSVReader >> recordClassStreamContents: block [ 641 | ^ recordClass streamContents: block 642 | ] 643 | 644 | { #category : #enumerating } 645 | NeoCSVReader >> reject: filter [ 646 | "Read and collect records that do not satisfy filter into an Array until there are none left. 647 | Return the array." 648 | 649 | ^ Array streamContents: [ :stream | 650 | self 651 | reject: filter 652 | thenDo: [ :each | stream nextPut: each ] ] 653 | ] 654 | 655 | { #category : #enumerating } 656 | NeoCSVReader >> reject: filter thenDo: block [ 657 | "Execute block for each record that does not satisfy filter until I am at end." 658 | 659 | self do: [ :record | 660 | (filter value: record) 661 | ifFalse: [ block value: record ] ] 662 | ] 663 | 664 | { #category : #'initialize-release' } 665 | NeoCSVReader >> resetStream [ 666 | charBuffer := nil. 667 | readStream reset 668 | ] 669 | 670 | { #category : #enumerating } 671 | NeoCSVReader >> select: filter [ 672 | "Read and collect records that satisfy filter into an Array until there are none left. 673 | Return the array." 674 | 675 | ^ Array streamContents: [ :stream | 676 | self 677 | select: filter 678 | thenDo: [ :each | stream nextPut: each ] ] 679 | ] 680 | 681 | { #category : #enumerating } 682 | NeoCSVReader >> select: filter thenDo: block [ 683 | "Execute block for each record that satisfies filter until I am at end." 684 | 685 | self do: [ :record | 686 | (filter value: record) 687 | ifTrue: [ block value: record ] ] 688 | ] 689 | 690 | { #category : #'initialize-release' } 691 | NeoCSVReader >> separator: character [ 692 | "Set the field separator character to use, defaults to comma" 693 | 694 | self assert: character isCharacter. 695 | separator := character 696 | ] 697 | 698 | { #category : #accessing } 699 | NeoCSVReader >> skip [ 700 | self skipRecord 701 | ] 702 | 703 | { #category : #convenience } 704 | NeoCSVReader >> skip: count [ 705 | "Skip count records by reading until end of line." 706 | 707 | count timesRepeat: [ self skip ] 708 | ] 709 | 710 | { #category : #'private - reading' } 711 | NeoCSVReader >> skipField [ 712 | self peekQuote 713 | ifTrue: [ 714 | self readQuote. 715 | [ self atEnd or: [ self readEndOfQuotedField ] ] 716 | whileFalse: [ self nextChar ] ] 717 | ifFalse: [ 718 | [ self atEnd or: [ self peekSeparator or: [ self peekEndOfLine ] ] ] 719 | whileFalse: [ self nextChar ] ] 720 | ] 721 | 722 | { #category : #convenience } 723 | NeoCSVReader >> skipHeader [ 724 | "Read a record, presumably a header, with the intention of skipping it. 725 | This should normally be called only at the beginning and only once. 726 | This sets the fieldCount (but fieldAccessors overrides fieldCount)." 727 | 728 | self readHeader 729 | ] 730 | 731 | { #category : #accessing } 732 | NeoCSVReader >> skipLine [ 733 | "Skip one (the current) record by reading until end of line. 734 | This is fast and has no side effects but does not honor quoted newlines." 735 | 736 | [ self readAtEndOrEndOfLine ] whileFalse: [ self nextChar ] 737 | ] 738 | 739 | { #category : #accessing } 740 | NeoCSVReader >> skipRecord [ 741 | "Skip one (the current) record by reading fields until end of line. 742 | This is fast, has no side effects and honors quoted newlines." 743 | 744 | [ self readAtEndOrEndOfLine ] 745 | whileFalse: [ 746 | self skipField. 747 | self readSeparator ] 748 | ] 749 | 750 | { #category : #private } 751 | NeoCSVReader >> stringStreamContents: block [ 752 | "Like String streamContents: block 753 | but reusing the underlying buffer for improved efficiency" 754 | 755 | stringStream 756 | ifNil: [ 757 | stringStream := (String new: 32) writeStream ]. 758 | stringStream reset. 759 | block value: stringStream. 760 | ^ stringStream contents 761 | ] 762 | 763 | { #category : #accessing } 764 | NeoCSVReader >> upToEnd [ 765 | "Read and collect records into an Array until there are none left. 766 | Return the array." 767 | 768 | ^ Array streamContents: [ :stream | 769 | self do: [ :each | stream nextPut: each ] ] 770 | ] 771 | -------------------------------------------------------------------------------- /repository/Neo-CSV-Core/NeoCSVWriter.class.st: -------------------------------------------------------------------------------- 1 | " 2 | I am NeoCSVWriter. 3 | 4 | I write a format that 5 | - is text based (ASCII, Latin1, Unicode) 6 | - consists of records, 1 per line (any line ending convention) 7 | - where records consist of fields separated by a delimiter (comma, tab, semicolon) 8 | - where every record has the same number of fields 9 | - where fields can be quoted should they contain separators or line endings 10 | 11 | Without further configuration, I write record objects whose fields can be enumerated using #do: such as SequenceableCollections 12 | 13 | By specifiying fields any object can be written converting and/or quoting each field as needed. 14 | 15 | MIT License. 16 | " 17 | Class { 18 | #name : #NeoCSVWriter, 19 | #superclass : #Object, 20 | #instVars : [ 21 | 'writeStream', 22 | 'separator', 23 | 'fieldWriter', 24 | 'lineEnd', 25 | 'fieldAccessors', 26 | 'emptyFieldValue' 27 | ], 28 | #category : 'Neo-CSV-Core' 29 | } 30 | 31 | { #category : #'instance creation' } 32 | NeoCSVWriter class >> on: writeStream [ 33 | "Initialize on writeStream, which should be a character stream that 34 | implements #nextPut:, #nextPutAll:, #space and (optionally) #close." 35 | 36 | ^ self new 37 | on: writeStream; 38 | yourself 39 | ] 40 | 41 | { #category : #'initialize-release' } 42 | NeoCSVWriter >> addConstantField: string [ 43 | "Add a constant field to be written using fieldWriter" 44 | 45 | self addFieldAccessor: [ :object | 46 | self writeField: string ] 47 | ] 48 | 49 | { #category : #'initialize-release' } 50 | NeoCSVWriter >> addEmptyField [ 51 | "Add an empty field to be written using fieldWriter" 52 | 53 | self addFieldAccessor: [ :object | 54 | self writeField: '' ] 55 | ] 56 | 57 | { #category : #'initialize-release' } 58 | NeoCSVWriter >> addField: accessor [ 59 | "Add a field based on an accessor to be written using fieldWriter. 60 | Accessor can be a Symbol or a Block" 61 | 62 | self addFieldAccessor: [ :object | 63 | self writeField: (accessor value: object) ] 64 | ] 65 | 66 | { #category : #private } 67 | NeoCSVWriter >> addFieldAccessor: block [ 68 | fieldAccessors 69 | ifNil: [ 70 | fieldAccessors := Array with: block ] 71 | ifNotNil: [ 72 | fieldAccessors := fieldAccessors copyWith: block ] 73 | ] 74 | 75 | { #category : #'initialize-release' } 76 | NeoCSVWriter >> addFieldAt: key [ 77 | "Add a field based on a key to be written using fieldWriter" 78 | 79 | self addFieldAccessor: [ :object | 80 | self writeField: (object at: key ifAbsent: [ '' ]) ] 81 | ] 82 | 83 | { #category : #convenience } 84 | NeoCSVWriter >> addFields: accessors [ 85 | accessors do: [ :each | 86 | self addField: each ] 87 | ] 88 | 89 | { #category : #convenience } 90 | NeoCSVWriter >> addFieldsAt: keys [ 91 | keys do: [ :each | 92 | self addFieldAt: each ] 93 | ] 94 | 95 | { #category : #'initialize-release' } 96 | NeoCSVWriter >> addObjectField: accessor [ 97 | "Add a field based on an accessor to be written as an #object field. 98 | Accessor can be a Symbol or a Block" 99 | 100 | self addFieldAccessor: [ :object | 101 | self writeObjectField: (accessor value: object) ] 102 | ] 103 | 104 | { #category : #'initialize-release' } 105 | NeoCSVWriter >> addObjectFieldAt: key [ 106 | "Add a field based on a key to be written as an #object field" 107 | 108 | self addFieldAccessor: [ :object | 109 | self writeObjectField: (object at: key) ] 110 | ] 111 | 112 | { #category : #convenience } 113 | NeoCSVWriter >> addObjectFields: accessors [ 114 | accessors do: [ :each | 115 | self addObjectField: each ] 116 | ] 117 | 118 | { #category : #convenience } 119 | NeoCSVWriter >> addObjectFieldsAt: keys [ 120 | keys do: [ :each | 121 | self addObjectFieldAt: each ] 122 | ] 123 | 124 | { #category : #'initialize-release' } 125 | NeoCSVWriter >> addOptionalQuotedField: accessor [ 126 | "Add a field based on an accessor to be written as a #optionalQuoted field. 127 | Accessor can be a Symbol or a Block" 128 | 129 | self addFieldAccessor: [ :object | 130 | self writeOptionalQuotedField: (accessor value: object) ] 131 | ] 132 | 133 | { #category : #'initialize-release' } 134 | NeoCSVWriter >> addOptionalQuotedFieldAt: key [ 135 | "Add a field based on a key to be written as a #optionalQuoted field" 136 | 137 | self addFieldAccessor: [ :object | 138 | self writeOptionalQuotedField: (object at: key) ] 139 | ] 140 | 141 | { #category : #convenience } 142 | NeoCSVWriter >> addOptionalQuotedFields: accessors [ 143 | accessors do: [ :each | 144 | self addOptionalQuotedField: each ] 145 | ] 146 | 147 | { #category : #convenience } 148 | NeoCSVWriter >> addOptionalQuotedFieldsAt: keys [ 149 | keys do: [ :each | 150 | self addOptionalQuotedFieldAt: each ] 151 | ] 152 | 153 | { #category : #'initialize-release' } 154 | NeoCSVWriter >> addQuotedField: accessor [ 155 | "Add a field based on an accessor to be written as a #quoted field. 156 | Accessor can be a Symbol or a Block" 157 | 158 | self addFieldAccessor: [ :object | 159 | self writeQuotedField: (accessor value: object) ] 160 | ] 161 | 162 | { #category : #'initialize-release' } 163 | NeoCSVWriter >> addQuotedFieldAt: key [ 164 | "Add a field based on a key to be written as a #quoted field" 165 | 166 | self addFieldAccessor: [ :object | 167 | self writeQuotedField: (object at: key) ] 168 | ] 169 | 170 | { #category : #convenience } 171 | NeoCSVWriter >> addQuotedFields: accessors [ 172 | accessors do: [ :each | 173 | self addQuotedField: each ] 174 | ] 175 | 176 | { #category : #convenience } 177 | NeoCSVWriter >> addQuotedFieldsAt: keys [ 178 | keys do: [ :each | 179 | self addQuotedFieldAt: each ] 180 | ] 181 | 182 | { #category : #'initialize-release' } 183 | NeoCSVWriter >> addRawField: accessor [ 184 | "Add a field based on an accessor to be written as a #raw field. 185 | Accessor can be a Symbol or a Block" 186 | 187 | self addFieldAccessor: [ :object | 188 | self writeRawField: (accessor value: object) ] 189 | ] 190 | 191 | { #category : #'initialize-release' } 192 | NeoCSVWriter >> addRawFieldAt: key [ 193 | "Add a field based on a key to be written as a #raw field" 194 | 195 | self addFieldAccessor: [ :object | 196 | self writeRawField: (object at: key) ] 197 | ] 198 | 199 | { #category : #convenience } 200 | NeoCSVWriter >> addRawFields: accessors [ 201 | accessors do: [ :each | 202 | self addRawField: each ] 203 | ] 204 | 205 | { #category : #convenience } 206 | NeoCSVWriter >> addRawFieldsAt: keys [ 207 | keys do: [ :each | 208 | self addRawFieldAt: each ] 209 | ] 210 | 211 | { #category : #'initialize-release' } 212 | NeoCSVWriter >> close [ 213 | writeStream ifNotNil: [ 214 | writeStream close. 215 | writeStream := nil ] 216 | ] 217 | 218 | { #category : #'initialize-release' } 219 | NeoCSVWriter >> emptyFieldValue: object [ 220 | "Set the empty field value to object. 221 | When reading fields from records to be written out, 222 | if the field value equals the emptyFieldValue, 223 | it will be considered an empty field and written as such." 224 | 225 | emptyFieldValue := object 226 | ] 227 | 228 | { #category : #accessing } 229 | NeoCSVWriter >> fieldWriter: symbol [ 230 | "Set the field write to be used, either #quoted (the default), #raw or #object. 231 | This determines how field values will be written in the general case. 232 | #quoted will wrap fields #asString in double quotes and escape embedded double quotes 233 | #raw will write fields #asString as such (no separator, double quote or end of line chars allowed) 234 | #optionalQuoted will write fields using #raw if possible (no separators, ...), and #quoted otherwise 235 | #object will #print: fields (no separator, double quote or end of line chars allowed)" 236 | 237 | self assert: (#(quoted raw object optionalQuoted) includes: symbol). 238 | fieldWriter := ('write', symbol capitalized, 'Field:') asSymbol 239 | 240 | ] 241 | 242 | { #category : #accessing } 243 | NeoCSVWriter >> flush [ 244 | writeStream flush 245 | ] 246 | 247 | { #category : #'initialize-release' } 248 | NeoCSVWriter >> initialize [ 249 | super initialize. 250 | lineEnd := OSPlatform current lineEnding. 251 | separator := $, . 252 | fieldWriter := #writeQuotedField: 253 | 254 | ] 255 | 256 | { #category : #'initialize-release' } 257 | NeoCSVWriter >> lineEndConvention: symbol [ 258 | "Set the end of line convention to be used. 259 | Either #cr, #lf or #crlf (the default)." 260 | 261 | self assert: (#(cr lf crlf) includes: symbol). 262 | lineEnd := String perform: symbol 263 | ] 264 | 265 | { #category : #convenience } 266 | NeoCSVWriter >> namedColumnsConfiguration: columns [ 267 | "Configure the receiver to output the named columns as keyed properties. 268 | The objects to be written should respond to #at: like a Dictionary. 269 | Writes a header first. Uses the configured field writer." 270 | 271 | self writeHeader: columns. 272 | self addFieldsAt: columns 273 | ] 274 | 275 | { #category : #accessing } 276 | NeoCSVWriter >> nextPut: anObject [ 277 | "Write anObject as single record. 278 | Depending on configuration fieldAccessors or a #do: enumeration will be used." 279 | 280 | fieldAccessors 281 | ifNil: [ 282 | self writeFieldsUsingDo: anObject ] 283 | ifNotNil: [ 284 | self writeFieldsUsingAccessors: anObject ]. 285 | self writeEndOfLine 286 | ] 287 | 288 | { #category : #accessing } 289 | NeoCSVWriter >> nextPutAll: collection [ 290 | "Write a collection of objects as records" 291 | 292 | collection do: [ :each | 293 | self nextPut: each ] 294 | ] 295 | 296 | { #category : #'initialize-release' } 297 | NeoCSVWriter >> on: aWriteStream [ 298 | "Initialize on aWriteStream, which should be a character stream that 299 | implements #nextPut:, #nextPutAll:, #space and (optionally) #close." 300 | 301 | writeStream := aWriteStream 302 | 303 | ] 304 | 305 | { #category : #'initialize-release' } 306 | NeoCSVWriter >> separator: character [ 307 | "Set character to be used as separator" 308 | 309 | self assert: character isCharacter. 310 | separator := character 311 | ] 312 | 313 | { #category : #private } 314 | NeoCSVWriter >> writeEndOfLine [ 315 | writeStream nextPutAll: lineEnd 316 | ] 317 | 318 | { #category : #private } 319 | NeoCSVWriter >> writeField: object [ 320 | self perform: fieldWriter with: object 321 | ] 322 | 323 | { #category : #private } 324 | NeoCSVWriter >> writeFieldsUsingAccessors: anObject [ 325 | | first | 326 | first := true. 327 | fieldAccessors do: [ :each | | fieldValue | 328 | first ifTrue: [ first := false ] ifFalse: [ self writeSeparator ]. 329 | fieldValue := each value: anObject ] 330 | ] 331 | 332 | { #category : #private } 333 | NeoCSVWriter >> writeFieldsUsingDo: anObject [ 334 | | first | 335 | first := true. 336 | anObject do: [ :each | 337 | first ifTrue: [ first := false ] ifFalse: [ self writeSeparator ]. 338 | self writeField: each ] 339 | ] 340 | 341 | { #category : #accessing } 342 | NeoCSVWriter >> writeHeader: fieldNames [ 343 | "Write the header, a collection of field names. 344 | This should normally be called only at the beginning and only once." 345 | 346 | fieldNames 347 | do: [ :each | self writeField: each ] 348 | separatedBy: [ self writeSeparator ]. 349 | self writeEndOfLine 350 | ] 351 | 352 | { #category : #private } 353 | NeoCSVWriter >> writeObjectField: object [ 354 | object = emptyFieldValue 355 | ifFalse: [ object printOn: writeStream ] 356 | ] 357 | 358 | { #category : #writing } 359 | NeoCSVWriter >> writeOptionalQuotedField: object [ 360 | | string | 361 | object = emptyFieldValue 362 | ifTrue: [ ^ self ]. 363 | string := object asString. 364 | ({lineEnd asString. 365 | separator asString. 366 | '"'} anySatisfy: [ :each | string includesSubstring: each ]) 367 | ifTrue: [ self writeQuotedField: object ] 368 | ifFalse: [ self writeRawField: object ] 369 | ] 370 | 371 | { #category : #private } 372 | NeoCSVWriter >> writeQuotedField: object [ 373 | object = emptyFieldValue 374 | ifTrue: [ writeStream nextPut: $" ; nextPut: $" ] 375 | ifFalse: [ | string | 376 | string := object asString. 377 | writeStream nextPut: $". 378 | string do: [ :each | 379 | each == $" 380 | ifTrue: [ writeStream nextPut: $"; nextPut: $" ] 381 | ifFalse: [ writeStream nextPut: each ] ]. 382 | writeStream nextPut: $" ] 383 | ] 384 | 385 | { #category : #private } 386 | NeoCSVWriter >> writeRawField: object [ 387 | object = emptyFieldValue 388 | ifFalse: [ writeStream nextPutAll: object asString ] 389 | ] 390 | 391 | { #category : #private } 392 | NeoCSVWriter >> writeSeparator [ 393 | writeStream nextPut: separator 394 | ] 395 | -------------------------------------------------------------------------------- /repository/Neo-CSV-Core/NeoNumberParser.class.st: -------------------------------------------------------------------------------- 1 | " 2 | I am NeoNumberParser, an alternative number parser that needs only a minimal read stream protocol. 3 | 4 | I accept the following syntax: 5 | 6 | number 7 | int 8 | int frac 9 | int exp 10 | int frac exp 11 | int 12 | digits 13 | - digits 14 | frac 15 | . digits 16 | exp 17 | e digits 18 | digits 19 | digit 20 | digit digits 21 | e 22 | e 23 | e+ 24 | e- 25 | E 26 | E+ 27 | E- 28 | 29 | where digit depends on the base (2 to 36), 0 .. 9, A-Z, a-z. 30 | " 31 | Class { 32 | #name : #NeoNumberParser, 33 | #superclass : #Object, 34 | #instVars : [ 35 | 'stream', 36 | 'base', 37 | 'radixPoint', 38 | 'digitGroupSeparator' 39 | ], 40 | #category : #'Neo-CSV-Core' 41 | } 42 | 43 | { #category : #'instance creation' } 44 | NeoNumberParser class >> on: stringOrStream [ 45 | | stream | 46 | stream := stringOrStream isString 47 | ifTrue: [ stringOrStream readStream ] 48 | ifFalse: [ stringOrStream ]. 49 | ^ self new 50 | on: stream; 51 | yourself 52 | ] 53 | 54 | { #category : #queries } 55 | NeoNumberParser class >> parse: stringOrStream [ 56 | ^ (self on: stringOrStream) next 57 | ] 58 | 59 | { #category : #queries } 60 | NeoNumberParser class >> parse: stringOrStream base: base [ 61 | ^ (self on: stringOrStream) 62 | base: base; 63 | next 64 | ] 65 | 66 | { #category : #queries } 67 | NeoNumberParser class >> parse: stringOrStream base: base ifFail: block [ 68 | ^ [ self parse: stringOrStream base: base ] 69 | on: Error 70 | do: block 71 | ] 72 | 73 | { #category : #queries } 74 | NeoNumberParser class >> parse: stringOrStream ifFail: block [ 75 | ^ [ self parse: stringOrStream ] 76 | on: Error 77 | do: block 78 | ] 79 | 80 | { #category : #testing } 81 | NeoNumberParser >> atEnd [ 82 | ^ stream atEnd 83 | ] 84 | 85 | { #category : #'initialize-release' } 86 | NeoNumberParser >> base: integer [ 87 | "Set the base of the numbers that I parse to integer. 88 | The default is 10" 89 | 90 | self assert: (integer between: 2 and: 36) description: 'Number base must be between 2 and 36'. 91 | base := integer 92 | ] 93 | 94 | { #category : #parsing } 95 | NeoNumberParser >> consumeWhitespace [ 96 | "Strip whitespaces from the input stream." 97 | 98 | [ stream atEnd not and: [ stream peek isSeparator ] ] 99 | whileTrue: [ stream next ] 100 | 101 | ] 102 | 103 | { #category : #'initialize-release' } 104 | NeoNumberParser >> digitGroupSeparator: separatorCharacter [ 105 | "Set the digit group separator to separatorCharacter. 106 | The are skipped while parsing digit characters. 107 | The default is nil (nothing being skipped)" 108 | 109 | digitGroupSeparator := separatorCharacter 110 | ] 111 | 112 | { #category : #parsing } 113 | NeoNumberParser >> digitsDo: oneArgumentBlock [ 114 | "Evaluate oneArgumentBlock with integer digit values from the input stream, 115 | while the stream is not at end and the digit value is within [0, base). 116 | Skip digit group separator characters." 117 | 118 | | character digitValue | 119 | [ (character := stream peek) notNil ] 120 | whileTrue: [ 121 | character = digitGroupSeparator 122 | ifFalse: [ 123 | ((digitValue := character digitValue) >= 0 and: [ digitValue < base ]) 124 | ifTrue: [ oneArgumentBlock value: digitValue ] 125 | ifFalse: [ ^ self ] ]. 126 | stream next ] 127 | ] 128 | 129 | { #category : #parsing } 130 | NeoNumberParser >> failIfNotAtEnd [ 131 | self atEnd 132 | ifFalse: [ self error: 'extraneous input detected' ] 133 | ] 134 | 135 | { #category : #'initialize-release' } 136 | NeoNumberParser >> initialize [ 137 | super initialize. 138 | self base: 10. 139 | self radixPoint: $. 140 | ] 141 | 142 | { #category : #accessing } 143 | NeoNumberParser >> next [ 144 | ^ self parseNumber 145 | ] 146 | 147 | { #category : #'initialize-release' } 148 | NeoNumberParser >> on: readStream [ 149 | stream := readStream 150 | ] 151 | 152 | { #category : #parsing } 153 | NeoNumberParser >> parseNumber [ 154 | | negated number isFloat | 155 | negated := stream peekFor: $-. 156 | number := self parseNumberInteger. 157 | isFloat := (stream peekFor: radixPoint) 158 | ifTrue: [ number := number + self parseNumberFraction. true ] 159 | ifFalse: [ false ]. 160 | ((stream peekFor: $e) or: [ stream peekFor: $E ]) 161 | ifTrue: [ number := number * self parseNumberExponent ]. 162 | isFloat ifTrue: [ number := number asFloat ]. 163 | negated 164 | ifTrue: [ number := number negated ]. 165 | ^ number 166 | ] 167 | 168 | { #category : #parsing } 169 | NeoNumberParser >> parseNumberExponent [ 170 | | number negated | 171 | number := 0. 172 | (negated := stream peekFor: $-) 173 | ifFalse: [ stream peekFor: $+ ]. 174 | self digitsDo: [ :x | 175 | number := base * number + x ]. 176 | negated 177 | ifTrue: [ number := number negated ]. 178 | ^ base raisedTo: number 179 | ] 180 | 181 | { #category : #parsing } 182 | NeoNumberParser >> parseNumberFraction [ 183 | | number power | 184 | number := 0. 185 | power := 1. 186 | self digitsDo: [ :x | 187 | number := base * number + x. 188 | power := power * base ]. 189 | ^ number / power 190 | ] 191 | 192 | { #category : #parsing } 193 | NeoNumberParser >> parseNumberInteger [ 194 | | number found | 195 | number := 0. 196 | found := false. 197 | self digitsDo: [ :x | 198 | found := true. 199 | number := base * number + x ]. 200 | found 201 | ifFalse: [ self error: 'Integer digit expected' ]. 202 | ^ number 203 | ] 204 | 205 | { #category : #'initialize-release' } 206 | NeoNumberParser >> radixPoint: radixCharacter [ 207 | "Set the radix of the numbers that I parse to radixCharacter. 208 | The default is $." 209 | 210 | radixPoint := radixCharacter 211 | ] 212 | -------------------------------------------------------------------------------- /repository/Neo-CSV-Core/package.st: -------------------------------------------------------------------------------- 1 | Package { #name : #'Neo-CSV-Core' } 2 | -------------------------------------------------------------------------------- /repository/Neo-CSV-GT/NeoCSVData.extension.st: -------------------------------------------------------------------------------- 1 | Extension { #name : #NeoCSVData } 2 | 3 | { #category : #'*Neo-CSV-GT' } 4 | NeoCSVData >> gtViewDataFOr: composite [ 5 | 6 | | columnedList | 7 | self data ifNil: [ ^ composite empty ]. 8 | self headerForPresentation ifNil: [ ^ composite empty ]. 9 | columnedList := composite columnedList 10 | title: 'Data'; 11 | items: [ self data ]; 12 | priority: 35. 13 | columnedList column: 'nr' text: [ :_ :nr | nr asRopedText foreground: Color gray ] weight: 1. 14 | self headerForPresentation doWithIndex: [ :column :index | 15 | columnedList column: column do: [ :aColumn | 16 | aColumn 17 | item: [ :eachRow | 18 | eachRow isDictionary 19 | ifTrue: [ eachRow at: column ifAbsent: nil ] 20 | ifFalse: [ eachRow at: index ] ]; 21 | weight: 1 ] ]. 22 | ^ columnedList 23 | ] 24 | 25 | { #category : #'*Neo-CSV-GT' } 26 | NeoCSVData >> gtViewHeaderFor: composite [ 27 | 28 | self header ifNil: [ ^ composite empty ]. 29 | ^ composite columnedList 30 | title: 'Header'; 31 | priority: 30; 32 | items: [ self header ]; 33 | column: 'Position' text: [ :_ :position | position ]; 34 | column: 'Field' text: #yourself weight: 2; 35 | column: 'Type' text: [ :_ :position | self typeForColumnAt: position ] 36 | ] 37 | -------------------------------------------------------------------------------- /repository/Neo-CSV-GT/package.st: -------------------------------------------------------------------------------- 1 | Package { #name : #'Neo-CSV-GT' } 2 | -------------------------------------------------------------------------------- /repository/Neo-CSV-Magritte/MACSVField.class.st: -------------------------------------------------------------------------------- 1 | Class { 2 | #name : #MACSVField, 3 | #superclass : #Object, 4 | #instVars : [ 5 | 'descriptionSource', 6 | 'name', 7 | 'encoder', 8 | 'decoder' 9 | ], 10 | #category : #'Neo-CSV-Magritte' 11 | } 12 | 13 | { #category : #accessing } 14 | MACSVField >> configureDescriptionFor: aWriter [ 15 | 16 | | description | 17 | self descriptionSource isSymbol ifTrue: [ 18 | description := aWriter subjectDescription detect: [ :fieldDesc | 19 | fieldDesc definingContext methodSelector = self descriptionSource ] ]. 20 | 21 | self descriptionSource isBlock ifTrue: [ 22 | description := self descriptionSource value. 23 | aWriter subjectDescription add: description ]. 24 | 25 | description 26 | propertyAt: aWriter fieldNamePropertyKey 27 | put: self name. 28 | 29 | self encoder ifNotNil: [ :anEncoder | 30 | description 31 | propertyAt: aWriter fieldWriterPropertyKey 32 | put: anEncoder ]. 33 | 34 | self decoder ifNotNil: [ :aDecoder | 35 | description 36 | propertyAt: aWriter fieldWriterPropertyKey 37 | put: aDecoder ]. 38 | 39 | ^ description 40 | ] 41 | 42 | { #category : #accessing } 43 | MACSVField >> decoder [ 44 | ^ decoder 45 | ] 46 | 47 | { #category : #accessing } 48 | MACSVField >> decoder: anObject [ 49 | decoder := anObject 50 | ] 51 | 52 | { #category : #accessing } 53 | MACSVField >> descriptionSource [ 54 | ^ descriptionSource 55 | ] 56 | 57 | { #category : #accessing } 58 | MACSVField >> descriptionSource: anObject [ 59 | "anObject - either a block returning a description, or the defining selector of a description already defined by the domain objects being written" 60 | 61 | descriptionSource := anObject 62 | ] 63 | 64 | { #category : #accessing } 65 | MACSVField >> encoder [ 66 | ^ encoder 67 | ] 68 | 69 | { #category : #accessing } 70 | MACSVField >> encoder: anObject [ 71 | encoder := anObject 72 | ] 73 | 74 | { #category : #accessing } 75 | MACSVField >> name [ 76 | ^ name 77 | ] 78 | 79 | { #category : #accessing } 80 | MACSVField >> name: anObject [ 81 | name := anObject 82 | ] 83 | -------------------------------------------------------------------------------- /repository/Neo-CSV-Magritte/MACSVImporter.class.st: -------------------------------------------------------------------------------- 1 | Class { 2 | #name : #MACSVImporter, 3 | #superclass : #Object, 4 | #instVars : [ 5 | 'reader', 6 | 'readerClass', 7 | 'recordClass', 8 | 'source', 9 | 'columnNames', 10 | 'recordMagritteDescription' 11 | ], 12 | #category : #'Neo-CSV-Magritte-Visitors' 13 | } 14 | 15 | { #category : #accessing } 16 | MACSVImporter >> columnNames [ 17 | "See setter comment" 18 | 19 | ^ columnNames 20 | ] 21 | 22 | { #category : #accessing } 23 | MACSVImporter >> columnNames: aCollection [ 24 | "If not supplied, will assume the first row contains them" 25 | 26 | columnNames := aCollection 27 | ] 28 | 29 | { #category : #private } 30 | MACSVImporter >> configureReaderFor: aStream [ 31 | self reader on: aStream. 32 | self columnNames ifNil: [ self columnNames: self readHeader ]. 33 | self reader recordClass: self recordClass. 34 | ] 35 | 36 | { #category : #accessing } 37 | MACSVImporter >> execute [ 38 | ^ self source isStream 39 | ifTrue: [ self importStream: self source ] 40 | ifFalse: [ self source readStreamDo: [ :str | self importStream: str ] ] 41 | ] 42 | 43 | { #category : #accessing } 44 | MACSVImporter >> fieldNamePropertyKey [ 45 | "The property where the element description stores the field name; override to customize" 46 | 47 | ^ #csvFieldName 48 | ] 49 | 50 | { #category : #accessing } 51 | MACSVImporter >> fieldReaderPropertyKey [ 52 | "The property where the element description stores the field reader. Override to customize. See `MAElementDescription>>#csvReader:` method comment for more info" 53 | 54 | ^ #csvReader 55 | ] 56 | 57 | { #category : #private } 58 | MACSVImporter >> importStream: aStream [ 59 | | fields | 60 | self configureReaderFor: aStream. 61 | 62 | fields := self recordMagritteDescription children. 63 | self columnNames 64 | do: [ :h | 65 | fields 66 | detect: [ :f | 67 | f 68 | propertyAt: self fieldNamePropertyKey 69 | ifPresent: [ :fieldName | fieldName = h asString trimmed ] 70 | ifAbsent: [ false ] ] 71 | ifFound: [ :e | 72 | self flag: 'need a way to customize the reader here'. 73 | self reader addFieldDescribedByMagritte: e ] 74 | ifNone: [ self reader addIgnoredField ] ]. 75 | ^ self reader upToEnd "or do more processing e.g. `select: [ :record | record lastName isNotNil ]`" 76 | ] 77 | 78 | { #category : #private } 79 | MACSVImporter >> readHeader [ 80 | "For exotic headers (i.e. not a single line with field names), override this. For example, you may for example want to skip irrevelant/blank lines." 81 | ^ self reader readHeader 82 | ] 83 | 84 | { #category : #accessing } 85 | MACSVImporter >> reader [ 86 | ^ reader ifNil: [ reader := self readerClass new ] 87 | ] 88 | 89 | { #category : #accessing } 90 | MACSVImporter >> readerClass [ 91 | ^ readerClass ifNil: [ NeoCSVReader ] 92 | ] 93 | 94 | { #category : #accessing } 95 | MACSVImporter >> readerClass: aClass [ 96 | readerClass := aClass 97 | ] 98 | 99 | { #category : #accessing } 100 | MACSVImporter >> recordClass [ 101 | ^ recordClass 102 | ] 103 | 104 | { #category : #accessing } 105 | MACSVImporter >> recordClass: aClass [ 106 | recordClass := aClass 107 | ] 108 | 109 | { #category : #'as yet unclassified' } 110 | MACSVImporter >> recordMagritteDescription [ 111 | 112 | ^ recordMagritteDescription ifNil: [ recordMagritteDescription := self recordClass new magritteDescription ] 113 | ] 114 | 115 | { #category : #'as yet unclassified' } 116 | MACSVImporter >> recordMagritteDescription: anMAContainer [ 117 | 118 | recordMagritteDescription := anMAContainer 119 | ] 120 | 121 | { #category : #accessing } 122 | MACSVImporter >> source [ 123 | 124 | ^ source 125 | ] 126 | 127 | { #category : #accessing } 128 | MACSVImporter >> source: aFileOrStream [ 129 | 130 | source := aFileOrStream 131 | ] 132 | -------------------------------------------------------------------------------- /repository/Neo-CSV-Magritte/MACSVImporterTests.class.st: -------------------------------------------------------------------------------- 1 | Class { 2 | #name : #MACSVImporterTests, 3 | #superclass : #TestCase, 4 | #instVars : [ 5 | 'input', 6 | 'importer', 7 | 'objects' 8 | ], 9 | #category : #'Neo-CSV-Magritte-Tests' 10 | } 11 | 12 | { #category : #'as yet unclassified' } 13 | MACSVImporterTests >> readInput: aString [ 14 | objects := importer 15 | source: aString readStream; 16 | execute 17 | ] 18 | 19 | { #category : #running } 20 | MACSVImporterTests >> setUp [ 21 | importer := MACSVTestPerson maCSVImporter: MACSVImporter. 22 | ] 23 | 24 | { #category : #tests } 25 | MACSVImporterTests >> testCustomConversion [ 26 | self readInput: 'Phone Number,Ignored Field 27 | 1-203-555-0100,something'. 28 | self assert: objects first phoneNumber equals: 12035550100. 29 | ] 30 | 31 | { #category : #tests } 32 | MACSVImporterTests >> testDateDefaultConversion [ 33 | self readInput: 'DOB,Ignored Field 34 | 2/28/2020,something'. 35 | self assert: objects first birthdate equals: '2/28/2020' asDate. 36 | ] 37 | 38 | { #category : #tests } 39 | MACSVImporterTests >> testEmptyField [ 40 | self readInput: 'Phone Number,Ignored Field 41 | ,something'. 42 | self assert: objects first phoneNumber equals: nil. 43 | ] 44 | 45 | { #category : #tests } 46 | MACSVImporterTests >> testStringDefaultConversion [ 47 | self readInput: 'Name,Ignored Field 48 | Alan Kay,something'. 49 | self assert: objects first name equals: 'Alan Kay'. 50 | ] 51 | 52 | { #category : #tests } 53 | MACSVImporterTests >> testTrailingLeadingSpaceIgnored [ 54 | self readInput: 'Name,Ignored Field 55 | Alan Kay ,something'. 56 | self assert: objects first name equals: 'Alan Kay'. 57 | ] 58 | -------------------------------------------------------------------------------- /repository/Neo-CSV-Magritte/MACSVMappedPragmaBuilder.class.st: -------------------------------------------------------------------------------- 1 | " 2 | I provide a way to extend Magritte element descriptions for CSV reading without modifying the containing domain class. I help avoid bloating domain classes with extensions for each supported CSV source. For example, for contacts, Google has its field names, Outlook has others… By using me, you will not have to extend existing element descriptions via Magritte's pragma (the one that takes the description selector as an argument). 3 | 4 | To configure me, you should: 5 | - set the `#fieldNamePropertyKey` that your reader expects to map descriptions to CSV fields 6 | - as needed, specify the `#fieldReaderPropertyKey` that your reader uses to convert the CSV field string to a domain object; otherwise the default reader will be used 7 | 8 | The two primary ways to map fields to descriptions are: 9 | - `aBuilder map: fieldName fieldTo: descriptionSelector` 10 | - `aBuilder map: fieldName fieldTo: descriptionSelector using: reader` 11 | " 12 | Class { 13 | #name : #MACSVMappedPragmaBuilder, 14 | #superclass : #MAPragmaBuilder, 15 | #instVars : [ 16 | 'map', 17 | 'fieldNamePropertyKey', 18 | 'fieldReaderPropertyKey' 19 | ], 20 | #category : #'Neo-CSV-Magritte-Support' 21 | } 22 | 23 | { #category : #private } 24 | MACSVMappedPragmaBuilder >> description: description extendedBy: descriptionExtensions for: descriptionSelector of: anObject [ 25 | | result | 26 | result := super 27 | description: description 28 | extendedBy: descriptionExtensions 29 | for: descriptionSelector 30 | of: anObject. 31 | 32 | self ensureFieldPropertiesForDescription: result from: descriptionSelector. 33 | 34 | ^ result 35 | ] 36 | 37 | { #category : #private } 38 | MACSVMappedPragmaBuilder >> ensureFieldPropertiesForDescription: description from: descriptionSelector [ 39 | 40 | self map 41 | at: descriptionSelector 42 | ifPresent: [ :anArray | 43 | description 44 | propertyAt: self fieldNamePropertyKey 45 | put: anArray first. 46 | 47 | anArray second ifNil: [ ^ self ]. 48 | description 49 | propertyAt: self fieldReaderPropertyKey 50 | put: [ :trimmed | 51 | anArray second 52 | cull: trimmed 53 | cull: description ] ]. 54 | ] 55 | 56 | { #category : #accessing } 57 | MACSVMappedPragmaBuilder >> fieldNamePropertyKey [ 58 | ^ fieldNamePropertyKey 59 | ] 60 | 61 | { #category : #accessing } 62 | MACSVMappedPragmaBuilder >> fieldNamePropertyKey: anObject [ 63 | fieldNamePropertyKey := anObject 64 | ] 65 | 66 | { #category : #accessing } 67 | MACSVMappedPragmaBuilder >> fieldReaderPropertyKey [ 68 | ^ fieldReaderPropertyKey 69 | ] 70 | 71 | { #category : #accessing } 72 | MACSVMappedPragmaBuilder >> fieldReaderPropertyKey: anObject [ 73 | fieldReaderPropertyKey := anObject 74 | ] 75 | 76 | { #category : #accessing } 77 | MACSVMappedPragmaBuilder >> map [ 78 | ^ map ifNil: [ map := Dictionary new ] 79 | ] 80 | 81 | { #category : #accessing } 82 | MACSVMappedPragmaBuilder >> map: anObject [ 83 | map := anObject 84 | ] 85 | 86 | { #category : #accessing } 87 | MACSVMappedPragmaBuilder >> map: fieldName fieldTo: descriptionSelector [ 88 | 89 | self map: fieldName fieldTo: descriptionSelector using: nil 90 | ] 91 | 92 | { #category : #accessing } 93 | MACSVMappedPragmaBuilder >> map: fieldName fieldTo: descriptionSelector using: reader [ 94 | " 95 | descriptionSelector - returning the description to be used 96 | reader - typically a block, where arguments are: 97 | 1) the field string input and 98 | 2) the description 99 | 100 | A typical pattern is to customize the description and then use the default reader with something like this: 101 | `desc defaultCsvReader cull: input trimmed`" 102 | 103 | self map at: descriptionSelector put: { fieldName. reader } 104 | ] 105 | -------------------------------------------------------------------------------- /repository/Neo-CSV-Magritte/MACSVTestPerson.class.st: -------------------------------------------------------------------------------- 1 | Class { 2 | #name : #MACSVTestPerson, 3 | #superclass : #Object, 4 | #instVars : [ 5 | 'phoneNumber', 6 | 'birthdate', 7 | 'name', 8 | 'birthplace', 9 | 'wikipediaUrl' 10 | ], 11 | #category : #'Neo-CSV-Magritte-Tests' 12 | } 13 | 14 | { #category : #accessing } 15 | MACSVTestPerson class >> exampleAlanKay [ 16 | 17 | ^ self new 18 | name: 'Alan Kay'; 19 | birthdate: '5/17/1940' asDate; 20 | birthplace: 'Springfield, Massachusetts, U.S.'; 21 | wikipediaUrl: 'https://en.wikipedia.org/wiki/Alan_Kay' asUrl 22 | yourself 23 | ] 24 | 25 | { #category : #'magritte-accessing' } 26 | MACSVTestPerson >> addressDescription [ 27 | 28 | ^ MAStringDescription new 29 | accessor: #address; 30 | priority: 40; 31 | yourself 32 | ] 33 | 34 | { #category : #accessing } 35 | MACSVTestPerson >> birthdate [ 36 | ^ self maLazyInstVarUsing: self birthdateDescription 37 | ] 38 | 39 | { #category : #accessing } 40 | MACSVTestPerson >> birthdate: aDate [ 41 | birthdate := aDate 42 | ] 43 | 44 | { #category : #'magritte-accessing' } 45 | MACSVTestPerson >> birthdateDescription [ 46 | 47 | ^ MADateDescription new 48 | accessor: #birthdate; 49 | csvFieldName: 'DOB'; 50 | priority: 30; 51 | yourself 52 | ] 53 | 54 | { #category : #accessing } 55 | MACSVTestPerson >> birthplace [ 56 | ^ birthplace 57 | ] 58 | 59 | { #category : #accessing } 60 | MACSVTestPerson >> birthplace: anObject [ 61 | birthplace := anObject 62 | ] 63 | 64 | { #category : #'magritte-accessing' } 65 | MACSVTestPerson >> birthplaceDescription [ 66 | 67 | ^ MAStringDescription new 68 | accessor: #birthplace; 69 | priority: 40; 70 | yourself 71 | ] 72 | 73 | { #category : #accessing } 74 | MACSVTestPerson >> name [ 75 | ^ self maLazyInstVarUsing: self nameDescription 76 | ] 77 | 78 | { #category : #accessing } 79 | MACSVTestPerson >> name: aString [ 80 | name := aString 81 | ] 82 | 83 | { #category : #'magritte-accessing' } 84 | MACSVTestPerson >> nameDescription [ 85 | 86 | ^ MAStringDescription new 87 | accessor: #name; 88 | csvFieldName: 'Name'; 89 | priority: 10; 90 | yourself 91 | ] 92 | 93 | { #category : #accessing } 94 | MACSVTestPerson >> phoneNumber [ 95 | ^ self maLazyInstVarUsing: self phoneNumberDescription 96 | ] 97 | 98 | { #category : #accessing } 99 | MACSVTestPerson >> phoneNumber: aNumber [ 100 | phoneNumber := aNumber 101 | ] 102 | 103 | { #category : #'magritte-accessing' } 104 | MACSVTestPerson >> phoneNumberDescription [ 105 | 106 | ^ MANumberDescription new 107 | accessor: #phoneNumber; 108 | csvFieldName: 'Phone Number'; 109 | priority: 20; 110 | csvReader: [ :s | (s select: #isDigit) asNumber ]; 111 | yourself 112 | ] 113 | 114 | { #category : #accessing } 115 | MACSVTestPerson >> wikipediaUrl [ 116 | ^ wikipediaUrl 117 | ] 118 | 119 | { #category : #accessing } 120 | MACSVTestPerson >> wikipediaUrl: anObject [ 121 | wikipediaUrl := anObject 122 | ] 123 | -------------------------------------------------------------------------------- /repository/Neo-CSV-Magritte/MACSVTwoStageImporter.class.st: -------------------------------------------------------------------------------- 1 | " 2 | Sometimes, you just can't fully create your domain objects from CSV in one pass. For example: 3 | - Multiple rows combine into one object (`inject:into:` can be helpful here) 4 | - Multiple cells combine into one object field 5 | 6 | To handle such a case, subclass me and, at minimum, override `#domainObjectFromDictionary:`. See its comment for more info. Implementations of that method often utilize my `#initializeDomainObject:fromRecord:` helper method. 7 | 8 | ##Combining Multiple Cells into One Field 9 | Typically, you would override `domainObjectFromDictionary:`, do a super send, and then 10 | Ideally, you would have a domain value object whose state is represented by these fields. If so, a common pattern is to and pass this object to `` like so: 11 | ```smalltalk 12 | result := super domainObjectFromDictionary: aDictionary. 13 | 14 | aValueObject := self 15 | initializeDomainObject: MyValueObject new 16 | fromRecord: aDictionary. 17 | 18 | ^ result 19 | fooBar: aValueObject; 20 | yourself 21 | ``` 22 | 23 | If the object can't be created anew - say it lives in a registry - you can always just pull 24 | " 25 | Class { 26 | #name : #MACSVTwoStageImporter, 27 | #superclass : #MACSVImporter, 28 | #instVars : [ 29 | 'selectBlock', 30 | 'preprocessor' 31 | ], 32 | #category : #'Neo-CSV-Magritte-Visitors' 33 | } 34 | 35 | { #category : #accessing } 36 | MACSVTwoStageImporter >> configureReaderFor: aStream [ 37 | 38 | super configureReaderFor: aStream. 39 | self reader recordClass: Dictionary. 40 | 41 | self reader 42 | emptyFieldValue: #passNil; 43 | addFieldsAt: (self columnNames collect: [ :each | each asSymbol ]) "Adapted from NeoCSVReader>>#namedColumnsConfiguration" 44 | ] 45 | 46 | { #category : #accessing } 47 | MACSVTwoStageImporter >> convertToDomainObjects: aCollectionOfDictionaries [ 48 | "aCollectionOfDictionaries - the result of a naive processing of CSV 49 | Return - a collection of domain objects" 50 | 51 | ^ aCollectionOfDictionaries 52 | inject: OrderedCollection new 53 | into: [ :col :rowDict | 54 | | result | 55 | self preprocessor value: rowDict. 56 | (self selectBlock value: rowDict) 57 | ifTrue: [ 58 | result := self domainObjectFromDictionary: rowDict. 59 | self prepareRecord: result. 60 | col add: result ]. 61 | col ] 62 | ] 63 | 64 | { #category : #accessing } 65 | MACSVTwoStageImporter >> domainObjectFromDictionary: aDictionary [ 66 | 67 | ^ self 68 | initializeDomainObject: self recordClass new 69 | fromRecord: aDictionary 70 | ] 71 | 72 | { #category : #accessing } 73 | MACSVTwoStageImporter >> importStream: aStream [ 74 | | rows preparedStream | 75 | preparedStream := self prepareStream: aStream. 76 | self configureReaderFor: preparedStream. 77 | rows := self reader upToEnd. 78 | ^ self convertToDomainObjects: rows 79 | ] 80 | 81 | { #category : #accessing } 82 | MACSVTwoStageImporter >> initialize [ 83 | 84 | super initialize. 85 | self recordClass: Dictionary. 86 | ] 87 | 88 | { #category : #accessing } 89 | MACSVTwoStageImporter >> initializeDomainObject: anObject fromRecord: aDictionary [ 90 | "We needed an instance-side version because some objects may need configuration during instance creation" 91 | 92 | ^ self 93 | initializeDomainObject: anObject 94 | fromRecord: aDictionary 95 | mapping: [ :builder | ] 96 | ] 97 | 98 | { #category : #accessing } 99 | MACSVTwoStageImporter >> initializeDomainObject: anObject fromRecord: aDictionary mapping: aBlock [ 100 | 101 | ^ self 102 | initializeDomainObject: anObject 103 | fromRecord: aDictionary 104 | mapping: aBlock 105 | descriptionDo: #yourself 106 | ] 107 | 108 | { #category : #accessing } 109 | MACSVTwoStageImporter >> initializeDomainObject: anObject fromRecord: aDictionary mapping: mapBlock descriptionDo: descriptionBlock [ 110 | "Send me only if you want to customize the domain object's Magritte description. Otherwise send one of the similar messages omitting the #descriptionDo: argument because my usage may seem a bit verbose due to creation of MADescriptions by hand, but I provide advantages - like gracefully handling nils. My arguments are as follows: 111 | anObject - domain object to be initialized 112 | aDictionary - keys are CSV column names 113 | mapBlock - use to modify existing descriptions; argument will be an MACSVMappedPragmaBuilder to configure 114 | descriptionBlock - argument is the container description for anObject, for further configuration e.g. adding an element description" 115 | 116 | | contDesc builder | 117 | builder := MACSVMappedPragmaBuilder new 118 | fieldNamePropertyKey: self fieldNamePropertyKey; 119 | fieldReaderPropertyKey: self fieldReaderPropertyKey; 120 | yourself. 121 | mapBlock value: builder. 122 | 123 | contDesc := builder for: anObject. 124 | descriptionBlock value: contDesc. 125 | contDesc do: [ :desc | 126 | desc 127 | propertyAt: self fieldNamePropertyKey 128 | ifPresent: [ :fieldName | 129 | | stringValue value fieldReader | 130 | stringValue := aDictionary at: fieldName. 131 | self flag: 'This next part looks very memento-like'. 132 | stringValue ifNotNil: [ 133 | fieldReader := desc 134 | propertyAt: self fieldReaderPropertyKey 135 | ifAbsent: [ desc csvReader ]. 136 | value := fieldReader cull: stringValue cull: desc. 137 | desc write: value to: anObject ] ] ]. 138 | ^ anObject 139 | ] 140 | 141 | { #category : #accessing } 142 | MACSVTwoStageImporter >> prepareRecord: aDomainObject [ 143 | "Override this to fix input problems which are easier to fix utilizing real accessors and domain objects instead of in the preprocessor where you have dictionary keys and strings" 144 | ] 145 | 146 | { #category : #accessing } 147 | MACSVTwoStageImporter >> prepareStream: aStream [ 148 | "Override me e.g. if the input is not proper CSV. One might remove a standard number of header or footer lines, handle blank columns, etc" 149 | 150 | ^ aStream 151 | ] 152 | 153 | { #category : #accessing } 154 | MACSVTwoStageImporter >> preprocessor [ 155 | ^ preprocessor ifNil: [ #yourself ] 156 | ] 157 | 158 | { #category : #accessing } 159 | MACSVTwoStageImporter >> preprocessor: aValuable [ 160 | "Consider #prepareObject: first because it is generally easier to fix the record after initialization because we have real accessors and domain objects e.g. aDate instead of aDateString" 161 | 162 | preprocessor := aValuable 163 | ] 164 | 165 | { #category : #accessing } 166 | MACSVTwoStageImporter >> selectBlock [ 167 | ^ selectBlock ifNil: [ #isNotNil ] 168 | ] 169 | 170 | { #category : #accessing } 171 | MACSVTwoStageImporter >> selectBlock: anObject [ 172 | selectBlock := anObject 173 | ] 174 | -------------------------------------------------------------------------------- /repository/Neo-CSV-Magritte/MACSVWriter.class.st: -------------------------------------------------------------------------------- 1 | Class { 2 | #name : #MACSVWriter, 3 | #superclass : #Object, 4 | #instVars : [ 5 | 'writer', 6 | 'writerClass', 7 | 'target', 8 | 'subjects', 9 | 'map', 10 | 'subjectDescription', 11 | 'ignoresUnkownFields', 12 | 'includesHeader' 13 | ], 14 | #category : #'Neo-CSV-Magritte-Visitors' 15 | } 16 | 17 | { #category : #accessing } 18 | MACSVWriter >> execute [ 19 | ^ self target isStream 20 | ifTrue: [ self writeToStream: self target ] 21 | ifFalse: [ self target ensureCreateFile writeStreamDo: [ :str | self writeToStream: str ] ] 22 | ] 23 | 24 | { #category : #private } 25 | MACSVWriter >> fieldDescriptions [ 26 | ^ self subjectDescription 27 | select: [ :desc | desc hasProperty: self fieldNamePropertyKey ] 28 | ] 29 | 30 | { #category : #accessing } 31 | MACSVWriter >> fieldNamePropertyKey [ 32 | "The property where the element description stores the field name; override to customize" 33 | 34 | ^ #csvFieldName 35 | ] 36 | 37 | { #category : #accessing } 38 | MACSVWriter >> fieldWriterPropertyKey [ 39 | "The property where the element description stores the field reader. Override to customize. See `MAElementDescription>>#csvReader:` method comment for more info" 40 | 41 | ^ #csvWriter 42 | ] 43 | 44 | { #category : #private } 45 | MACSVWriter >> header [ 46 | ^ self fieldDescriptions children 47 | collect: [ :field | field propertyAt: self fieldNamePropertyKey ifAbsent: [ field name ] ] 48 | ] 49 | 50 | { #category : #accessing } 51 | MACSVWriter >> ignoresUnknownFields [ 52 | ^ ignoresUnkownFields ifNil: [ false ] 53 | ] 54 | 55 | { #category : #accessing } 56 | MACSVWriter >> ignoresUnknownFields: anObject [ 57 | ignoresUnkownFields := anObject 58 | ] 59 | 60 | { #category : #accessing } 61 | MACSVWriter >> includesHeader [ 62 | ^ includesHeader ifNil: [ true ]. 63 | ] 64 | 65 | { #category : #accessing } 66 | MACSVWriter >> includesHeader: anObject [ 67 | includesHeader := anObject 68 | ] 69 | 70 | { #category : #accessing } 71 | MACSVWriter >> map [ 72 | 73 | ^ map ifNil: [ map := OrderedCollection new ] 74 | ] 75 | 76 | { #category : #accessing } 77 | MACSVWriter >> map: aString fieldDo: aBlock [ 78 | 79 | | field | 80 | field := MACSVField new 81 | name: aString; 82 | yourself. 83 | 84 | aBlock value: field. 85 | 86 | self map add: field. 87 | ] 88 | 89 | { #category : #accessing } 90 | MACSVWriter >> map: aString fieldSource: aBlock [ 91 | 92 | self 93 | map: aString 94 | fieldDo: [ :field | field descriptionSource: aBlock ] 95 | ] 96 | 97 | { #category : #accessing } 98 | MACSVWriter >> subjectDescription [ 99 | ^ subjectDescription ifNil: [ subjectDescription := self subjects atRandom magritteDescription ] 100 | ] 101 | 102 | { #category : #accessing } 103 | MACSVWriter >> subjectDescription: anMAContainer [ 104 | subjectDescription := anMAContainer 105 | ] 106 | 107 | { #category : #accessing } 108 | MACSVWriter >> subjects [ 109 | ^ subjects 110 | ] 111 | 112 | { #category : #accessing } 113 | MACSVWriter >> subjects: anObject [ 114 | subjects := anObject 115 | ] 116 | 117 | { #category : #accessing } 118 | MACSVWriter >> target [ 119 | 120 | ^ target 121 | ] 122 | 123 | { #category : #accessing } 124 | MACSVWriter >> target: aFileOrStream [ 125 | 126 | target := aFileOrStream 127 | ] 128 | 129 | { #category : #private } 130 | MACSVWriter >> writeToStream: aStream [ 131 | self subjects isEmptyOrNil ifTrue: [ ^ self ]. 132 | 133 | self map do: [ :field | field configureDescriptionFor: self ]. 134 | 135 | self fieldDescriptions 136 | do: [ :field | 137 | | converter wrappedConverter | 138 | converter := field 139 | propertyAt: self fieldWriterPropertyKey 140 | ifAbsent: #yourself. 141 | wrappedConverter := [ :anObject | 142 | (self ignoresUnknownFields not or: [ field accessor canRead: anObject ]) 143 | ifTrue: [ 144 | | aValue | 145 | aValue := field read: anObject. 146 | 147 | "`#read: returns either a non-nil value or #undefinedValue, so the following test is sufficient e.g. not checking for nil" 148 | aValue = field undefinedValue ifTrue: [ 149 | aValue := field default ]. 150 | 151 | converter cull: aValue cull: anObject ] 152 | ifFalse: [ nil ] ]. 153 | self writer addField: wrappedConverter ]. 154 | 155 | writer := self writer on: aStream. 156 | 157 | self includesHeader ifTrue: [ writer writeHeader: self header ]. 158 | 159 | writer nextPutAll: self subjects 160 | ] 161 | 162 | { #category : #accessing } 163 | MACSVWriter >> writer [ 164 | ^ writer ifNil: [ writer := self writerClass new ] 165 | ] 166 | 167 | { #category : #accessing } 168 | MACSVWriter >> writerClass [ 169 | ^ writerClass ifNil: [ NeoCSVWriter ] 170 | ] 171 | 172 | { #category : #accessing } 173 | MACSVWriter >> writerClass: aClass [ 174 | writerClass := aClass 175 | ] 176 | -------------------------------------------------------------------------------- /repository/Neo-CSV-Magritte/MACSVWriterTests.class.st: -------------------------------------------------------------------------------- 1 | Class { 2 | #name : #MACSVWriterTests, 3 | #superclass : #TestCase, 4 | #instVars : [ 5 | 'input', 6 | 'objects', 7 | 'person', 8 | 'target', 9 | 'writer' 10 | ], 11 | #category : #'Neo-CSV-Magritte-Tests' 12 | } 13 | 14 | { #category : #running } 15 | MACSVWriterTests >> setUp [ 16 | person := MACSVTestPerson exampleAlanKay. 17 | target := FileSystem memory / 'people.csv'. 18 | writer := MACSVWriter new 19 | target: target; 20 | subjects: { person }; 21 | yourself 22 | ] 23 | 24 | { #category : #tests } 25 | MACSVWriterTests >> testAddedDescription [ 26 | 27 | writer 28 | map: 'Wikipedia' fieldDo: [ :field | 29 | field descriptionSource: [ 30 | MAUrlDescription new 31 | accessor: #wikipediaUrl; 32 | priority: 25; 33 | yourself ] ]; 34 | execute. 35 | 36 | self assert: target contents equals: '"Name","Phone Number","Wikipedia","DOB" 37 | "Alan Kay","","https://en.wikipedia.org/wiki/Alan_Kay","17 May 1940" 38 | ' withUnixLineEndings. 39 | ] 40 | 41 | { #category : #tests } 42 | MACSVWriterTests >> testAlteredDescription [ 43 | 44 | writer 45 | map: 'Birthplace String' fieldDo: [ :field | 46 | field descriptionSource: #birthplaceDescription ]; 47 | execute. 48 | 49 | self assert: target contents equals: '"Name","Phone Number","DOB","Birthplace String" 50 | "Alan Kay","","17 May 1940","Springfield, Massachusetts, U.S." 51 | ' withUnixLineEndings. 52 | ] 53 | 54 | { #category : #tests } 55 | MACSVWriterTests >> testDefaultMapping [ 56 | writer execute. 57 | self assert: target contents equals: '"Name","Phone Number","DOB" 58 | "Alan Kay","","17 May 1940" 59 | ' withUnixLineEndings. 60 | ] 61 | 62 | { #category : #tests } 63 | MACSVWriterTests >> testIgnoreUnknownFields [ 64 | 65 | | field substitution unknownField | 66 | field := MACSVTestPerson magritteTemplate birthdateDescription. 67 | unknownField := MAStringDescription new 68 | accessor: #unknownSelector; 69 | csvFieldName: 'Unknown'; 70 | priority: 10; 71 | yourself. 72 | substitution := MAContainer withAll: { field. unknownField }. 73 | 74 | writer 75 | subjectDescription: substitution; 76 | ignoresUnknownFields: true; 77 | execute. 78 | 79 | self assert: target contents equals: '"DOB","Unknown" 80 | "17 May 1940","" 81 | ' withUnixLineEndings. 82 | ] 83 | 84 | { #category : #tests } 85 | MACSVWriterTests >> testSubjectDescriptionSubstitution [ 86 | 87 | writer 88 | subjectDescription: MACSVTestPerson magritteTemplate birthdateDescription asContainer; 89 | execute. 90 | 91 | self assert: target contents equals: '"DOB" 92 | "17 May 1940" 93 | ' withUnixLineEndings. 94 | ] 95 | 96 | { #category : #tests } 97 | MACSVWriterTests >> testUnknownFields [ 98 | 99 | | unknownField | 100 | unknownField := MAStringDescription new 101 | accessor: #unknownSelector; 102 | csvFieldName: 'Unknown'; 103 | priority: 10; 104 | yourself. 105 | 106 | self 107 | should: [ 108 | writer 109 | subjectDescription: unknownField asContainer; 110 | execute ] 111 | raise: MessageNotUnderstood. 112 | ] 113 | -------------------------------------------------------------------------------- /repository/Neo-CSV-Magritte/MAElementDescription.extension.st: -------------------------------------------------------------------------------- 1 | Extension { #name : #MAElementDescription } 2 | 3 | { #category : #'*Neo-CSV-Magritte' } 4 | MAElementDescription >> csvFieldName [ 5 | ^ self propertyAt: #csvFieldName ifAbsent: [ nil ] 6 | ] 7 | 8 | { #category : #'*Neo-CSV-Magritte' } 9 | MAElementDescription >> csvFieldName: aString [ 10 | ^ self propertyAt: #csvFieldName put: aString 11 | ] 12 | 13 | { #category : #'*Neo-CSV-Magritte' } 14 | MAElementDescription >> csvReader [ 15 | 16 | ^ self propertyAt: #csvReader ifAbsent: [ self defaultCsvReader ] 17 | ] 18 | 19 | { #category : #'*Neo-CSV-Magritte' } 20 | MAElementDescription >> csvReader: aBlock [ 21 | " 22 | aBlock 23 | - 1st argument - the input string 24 | - 2nd (optional) argument - this description 25 | - return value - a value appropriate to the field e.g. aDate for MADateDescription." 26 | ^ self propertyAt: #csvReader put: aBlock 27 | ] 28 | 29 | { #category : #'*Neo-CSV-Magritte' } 30 | MAElementDescription >> defaultCsvReader [ 31 | 32 | ^ [ :trimmed | self fromString: trimmed ]. 33 | ] 34 | -------------------------------------------------------------------------------- /repository/Neo-CSV-Magritte/ManifestNeoCSVMagritte.class.st: -------------------------------------------------------------------------------- 1 | " 2 | # Automate Neo-CSV with Magritte 3 | 4 | ### 1. Meta-Describe the CSV Field You Care About 5 | - Enable field mapping with `anElementDesciption csvFieldName: aString`. CSV fields which do not have a header corresponding to a Magritte #csvFieldName will be ignored. 6 | - Customize CSV-to-object conversion via `anElementDesciption csvReader: aValuable`. 7 | 8 | Example: 9 | ```smalltalk 10 | MyPerson>>#descriptionPhone 11 | 12 | ^ MANumberDescription new 13 | accessor: #phone; 14 | csvFieldName: 'Phone'; 15 | csvReader: [ :s | (s select: #isDigit) asNumber ]; 16 | yourself 17 | ``` 18 | 19 | ### 2. Read a CSV File 20 | - The main entry point is `MyDomainClass class>>#maCSVImporter: aVisitorClass`. 21 | - The header must be a single line with field names. See the {{gtMethod:MACSVImporter>>#readHeader}} hook, provided to e.g. manually skip leading irrevelant/blank lines. 22 | 23 | Example: 24 | ```smalltalk 25 | file := self myFolder / 'my_contacts.csv'. 26 | importer := MyPerson maCSVImporter: MACSVImporter. 27 | collectionOfMyDomainObjects := importer 28 | source: file; 29 | execute. 30 | ``` 31 | " 32 | Class { 33 | #name : #ManifestNeoCSVMagritte, 34 | #superclass : #PackageManifest, 35 | #category : 'Neo-CSV-Magritte-Manifest' 36 | } 37 | -------------------------------------------------------------------------------- /repository/Neo-CSV-Magritte/NeoCSVReader.extension.st: -------------------------------------------------------------------------------- 1 | Extension { #name : #NeoCSVReader } 2 | 3 | { #category : #'*Neo-CSV-Magritte' } 4 | NeoCSVReader >> addFieldDescribedByMagritte: aDescription [ 5 | 6 | self 7 | addField: [ :obj :value | aDescription accessor write: value to: obj ] 8 | converter: [ :s | 9 | s trimmed 10 | ifNotEmpty: aDescription csvReader 11 | ifEmpty: [ aDescription undefinedValue ] ] 12 | ] 13 | -------------------------------------------------------------------------------- /repository/Neo-CSV-Magritte/Object.extension.st: -------------------------------------------------------------------------------- 1 | Extension { #name : #Object } 2 | 3 | { #category : #'*Neo-CSV-Magritte' } 4 | Object class >> maCSVImporter: importerClass [ 5 | ^ importerClass new 6 | recordClass: self; 7 | yourself. 8 | ] 9 | 10 | { #category : #'*Neo-CSV-Magritte' } 11 | Object class >> maGenerateFieldsFromCSVHeaders: aString [ 12 | "Given a class-side `#headers` message returning a tab-separated string (e.g. pasted from MS Excel), generate a field (i.e. constructor and accessors) for each token" 13 | 14 | | headers usedHeaders | 15 | headers := Character tab split: aString. 16 | usedHeaders := headers reject: #isEmpty. 17 | usedHeaders do: [ :h | self maAddField: h asCamelCase uncapitalized asSymbol ] 18 | ] 19 | -------------------------------------------------------------------------------- /repository/Neo-CSV-Magritte/package.st: -------------------------------------------------------------------------------- 1 | Package { #name : #'Neo-CSV-Magritte' } 2 | -------------------------------------------------------------------------------- /repository/Neo-CSV-Tests/NeoCSVBenchmark.class.st: -------------------------------------------------------------------------------- 1 | " 2 | I am NeoCSVBenchmark. 3 | 4 | | benchmark | 5 | benchmark := NeoCSVBenchmark new. 6 | benchmark cleanup. 7 | [ benchmark write1 ] timeToRun. 8 | 9 | | benchmark | 10 | benchmark := NeoCSVBenchmark new. 11 | [ benchmark read0 ] timeToRun. 12 | 13 | | benchmark | 14 | benchmark := NeoCSVBenchmark new. 15 | [ benchmark read1 ] timeToRun. 16 | 17 | | benchmark | 18 | benchmark := NeoCSVBenchmark new. 19 | benchmark cleanup. 20 | [ benchmark write2 ] timeToRun. 21 | 22 | | benchmark | 23 | benchmark := NeoCSVBenchmark new. 24 | benchmark cleanup. 25 | [ benchmark write3 ] timeToRun. 26 | 27 | | benchmark | 28 | benchmark := NeoCSVBenchmark new. 29 | benchmark cleanup. 30 | [ benchmark write4 ] timeToRun. 31 | 32 | | benchmark | 33 | benchmark := NeoCSVBenchmark new. 34 | benchmark cleanup. 35 | [ benchmark write5 ] timeToRun. 36 | 37 | | benchmark | 38 | benchmark := NeoCSVBenchmark new. 39 | [ benchmark read2 ] timeToRun. 40 | 41 | | benchmark | 42 | benchmark := NeoCSVBenchmark new. 43 | [ benchmark read3 ] timeToRun. 44 | 45 | 46 | " 47 | Class { 48 | #name : #NeoCSVBenchmark, 49 | #superclass : #Object, 50 | #instVars : [ 51 | 'data' 52 | ], 53 | #category : 'Neo-CSV-Tests' 54 | } 55 | 56 | { #category : #public } 57 | NeoCSVBenchmark >> cleanup [ 58 | self filename ensureDelete 59 | 60 | ] 61 | 62 | { #category : #accessing } 63 | NeoCSVBenchmark >> filename [ 64 | ^ 'NeoCSVBenchmark.csv' asFileReference 65 | ] 66 | 67 | { #category : #'initialize-release' } 68 | NeoCSVBenchmark >> initialize [ 69 | data := Array new: 100000 streamContents: [ :stream | 70 | 1 to: 100000 do: [ :each | 71 | stream nextPut: (Array with: each with: each negated with: (100000 - each)) ] ] 72 | ] 73 | 74 | { #category : #public } 75 | NeoCSVBenchmark >> read0 [ 76 | self filename readStreamDo: [ :stream | 77 | (NeoCSVReader on: stream) 78 | upToEnd ] 79 | ] 80 | 81 | { #category : #public } 82 | NeoCSVBenchmark >> read1 [ 83 | self filename readStreamDo: [ :stream | 84 | (NeoCSVReader on: (ZnBufferedReadStream on: stream)) 85 | upToEnd ] 86 | ] 87 | 88 | { #category : #public } 89 | NeoCSVBenchmark >> read2 [ 90 | self filename readStreamDo: [ :stream | 91 | (NeoCSVReader on: stream) 92 | recordClass: NeoCSVTestObject; 93 | addIntegerField: #x: ; 94 | addIntegerField: #y: ; 95 | addIntegerField: #z: ; 96 | upToEnd ] 97 | ] 98 | 99 | { #category : #public } 100 | NeoCSVBenchmark >> read3 [ 101 | self filename readStreamDo: [ :stream | 102 | (NeoCSVReader on: (ZnBufferedReadStream on: stream)) 103 | recordClass: NeoCSVTestObject; 104 | addIntegerField: #x: ; 105 | addIntegerField: #y: ; 106 | addIntegerField: #z: ; 107 | upToEnd ] 108 | ] 109 | 110 | { #category : #public } 111 | NeoCSVBenchmark >> write0 [ 112 | self filename writeStreamDo: [ :stream | 113 | (NeoCSVWriter on: stream) 114 | nextPutAll: data ] 115 | ] 116 | 117 | { #category : #public } 118 | NeoCSVBenchmark >> write1 [ 119 | self filename writeStreamDo: [ :stream | 120 | (NeoCSVWriter on: (ZnBufferedWriteStream on: stream)) 121 | nextPutAll: data; 122 | flush ] 123 | ] 124 | 125 | { #category : #public } 126 | NeoCSVBenchmark >> write2 [ 127 | self filename writeStreamDo: [ :stream | 128 | (NeoCSVWriter on: (ZnBufferedWriteStream on: stream)) 129 | addRawFields: #(first second third); 130 | nextPutAll: data; 131 | flush ] 132 | ] 133 | 134 | { #category : #public } 135 | NeoCSVBenchmark >> write3 [ 136 | self filename writeStreamDo: [ :stream | 137 | (NeoCSVWriter on: (ZnBufferedWriteStream on: stream)) 138 | addObjectFields: #(first second third); 139 | nextPutAll: data; 140 | flush ] 141 | ] 142 | 143 | { #category : #public } 144 | NeoCSVBenchmark >> write4 [ 145 | self filename writeStreamDo: [ :stream | 146 | (NeoCSVWriter on: (ZnBufferedWriteStream on: stream)) 147 | fieldWriter: #raw; 148 | nextPutAll: data; 149 | flush ] 150 | ] 151 | 152 | { #category : #public } 153 | NeoCSVBenchmark >> write5 [ 154 | self filename writeStreamDo: [ :stream | 155 | (NeoCSVWriter on: (ZnBufferedWriteStream on: stream)) 156 | fieldWriter: #object; 157 | nextPutAll: data; 158 | flush ] 159 | ] 160 | -------------------------------------------------------------------------------- /repository/Neo-CSV-Tests/NeoCSVReaderTests.class.st: -------------------------------------------------------------------------------- 1 | " 2 | I am NeoCSVReaderTests, a suite of unit tests for NeoCSVReader. 3 | 4 | " 5 | Class { 6 | #name : #NeoCSVReaderTests, 7 | #superclass : #TestCase, 8 | #category : #'Neo-CSV-Tests' 9 | } 10 | 11 | { #category : #testing } 12 | NeoCSVReaderTests >> testAvailableAndDefinedFieldsMismatch [ 13 | | input | 14 | input := 'foo,1\bar,2\foobar,3' withCRs. 15 | "if we configure the reader for 1 field, the second one in the input should be ignored" 16 | self 17 | assert: ((NeoCSVReader on: input readStream) addField; upToEnd) 18 | equals: #(('foo')('bar')('foobar')). 19 | self 20 | assert: ((NeoCSVReader on: input readStream) 21 | recordClass: Dictionary; 22 | addFieldAt: #one; 23 | upToEnd) 24 | equals: { 25 | { #one->'foo' } asDictionary. 26 | { #one->'bar' } asDictionary. 27 | { #one->'foobar' } asDictionary }. 28 | "if we configure the reader for 3 fields, the last one should be nil" 29 | self 30 | assert: ((NeoCSVReader on: input readStream) addField; addIntegerField; addField; upToEnd) 31 | equals: #(('foo' 1 nil)('bar' 2 nil)('foobar' 3 nil)). 32 | "for dictionaries, it depends on the empty field value" 33 | self 34 | assert: ((NeoCSVReader on: input readStream) 35 | recordClass: Dictionary; 36 | addFieldAt: #one; 37 | addIntegerFieldAt: #two; 38 | addFieldAt: #three; 39 | upToEnd) 40 | equals: { 41 | { #one->'foo'. #two->1 } asDictionary. 42 | { #one->'bar'. #two->2 } asDictionary. 43 | { #one->'foobar'. #two->3 } asDictionary }. 44 | self 45 | assert: ((NeoCSVReader on: input readStream) 46 | recordClass: Dictionary; 47 | emptyFieldValue: #passNil; 48 | addFieldAt: #one; 49 | addIntegerFieldAt: #two; 50 | addFieldAt: #three; 51 | upToEnd) 52 | equals: { 53 | { #one->'foo'. #two->1. #three->nil } asDictionary. 54 | { #one->'bar'. #two->2. #three->nil } asDictionary. 55 | { #one->'foobar'. #two->3. #three->nil } asDictionary }. 56 | 57 | 58 | ] 59 | 60 | { #category : #testing } 61 | NeoCSVReaderTests >> testConversionErrors [ 62 | self 63 | should: [ (NeoCSVReader on: 'a' readStream) addIntegerField; upToEnd ] 64 | raise: Error. 65 | self 66 | should: [ (NeoCSVReader on: 'a' readStream) addFloatField; upToEnd ] 67 | raise: Error. 68 | ] 69 | 70 | { #category : #testing } 71 | NeoCSVReaderTests >> testEmbeddedQuotes [ 72 | self 73 | assert: (NeoCSVReader on: '1,"x""y""z",3' readStream) upToEnd 74 | equals: #(('1' 'x"y"z' '3')) 75 | ] 76 | 77 | { #category : #testing } 78 | NeoCSVReaderTests >> testEmptyConversions [ 79 | | input | 80 | input := (String crlf join: #( '1,2.5,foo' ',,' )). 81 | self 82 | assert: ((NeoCSVReader on: input readStream) 83 | addIntegerField; 84 | addFloatField; 85 | addField; 86 | upToEnd) 87 | equals: { 88 | #( 1 2.5 'foo' ). 89 | #( nil nil nil ) } 90 | ] 91 | 92 | { #category : #testing } 93 | NeoCSVReaderTests >> testEmptyConversionsTestObject [ 94 | | input | 95 | input := (String crlf join: #( '1,2.5,foo' ',,' )). 96 | self 97 | assert: ((NeoCSVReader on: input readStream) 98 | recordClass: NeoCSVTestObject; 99 | addIntegerField: #x: ; 100 | addFloatField: #y: ; 101 | addField: #z: ; 102 | upToEnd) 103 | equals: { 104 | NeoCSVTestObject x: 1 y: 2.5 z: 'foo'. 105 | NeoCSVTestObject new } 106 | ] 107 | 108 | { #category : #testing } 109 | NeoCSVReaderTests >> testEmptyFieldInput [ 110 | self 111 | assert: ((NeoCSVReader on: 'one,NULL,two\NULL,NULL,three\NULL,NULL,NULL' withCRs readStream) 112 | emptyFieldInput: [ :field | field = 'NULL' ]; upToEnd) 113 | equals: #((one nil two) (nil nil three) (nil nil nil)). 114 | self 115 | assert: ((NeoCSVReader on: 'one,NULL,two\NULL,NULL,three\NULL,NULL,NULL' withCRs readStream) 116 | emptyFieldInput: [ :field | field = 'NULL' ]; emptyFieldValue: #empty; upToEnd) 117 | equals: #((one empty two) (empty empty three) (empty empty empty)). 118 | self 119 | assert: ((NeoCSVReader on: 'one, NULL,two\" NULL","NULL ","three"\NULL,NULL,NULL' withCRs readStream) 120 | emptyFieldInput: [ :field | field trimBoth asLowercase = #null ]; emptyFieldValue: #empty; upToEnd) 121 | equals: #((one empty two) (empty empty three) (empty empty empty)). 122 | self 123 | assert: ((NeoCSVReader on: 'one,NULL,two\"nil","NULL","three"\nil,NULL,nil' withCRs readStream) 124 | emptyFieldInput: [ :field | #('NULL' 'nil') includes: field ]; emptyFieldValue: #empty; upToEnd) 125 | equals: #((one empty two) (empty empty three) (empty empty empty)). 126 | self 127 | assert: ((NeoCSVReader on: 'one,,two\,,three\,,' withCRs readStream) 128 | emptyFieldInput: [ :field | field isEmpty ]; upToEnd) 129 | equals: #((one nil two) (nil nil three) (nil nil nil)). 130 | 131 | ] 132 | 133 | { #category : #testing } 134 | NeoCSVReaderTests >> testEmptyFieldQuoted [ 135 | self 136 | assert: (NeoCSVReader on: '"1",,"3"' readStream) upToEnd 137 | equals: #(('1' nil '3')) 138 | ] 139 | 140 | { #category : #testing } 141 | NeoCSVReaderTests >> testEmptyFieldSecondRecordQuoted [ 142 | self 143 | assert: (NeoCSVReader on: '"foo","bar"\"100",' withCRs readStream) upToEnd 144 | equals: #(('foo' 'bar')('100' nil)) 145 | ] 146 | 147 | { #category : #testing } 148 | NeoCSVReaderTests >> testEmptyFieldSecondRecordUnquoted [ 149 | self 150 | assert: (NeoCSVReader on: 'foo,bar\100,' withCRs readStream) upToEnd 151 | equals: #(('foo' 'bar')('100' nil)) 152 | ] 153 | 154 | { #category : #testing } 155 | NeoCSVReaderTests >> testEmptyFieldUnquoted [ 156 | self 157 | assert: (NeoCSVReader on: '1,,3' readStream) upToEnd 158 | equals: #(('1' nil '3')) 159 | ] 160 | 161 | { #category : #testing } 162 | NeoCSVReaderTests >> testEmptyFieldValue [ 163 | self 164 | assert: ((NeoCSVReader on: '"1",,3,"","5"' readStream) 165 | emptyFieldValue: #empty; 166 | upToEnd) 167 | equals: #(('1' empty '3' empty '5')). 168 | self 169 | assert: ((NeoCSVReader on: '"1",,3,"","5"' readStream) 170 | emptyFieldValue: ''; 171 | upToEnd) 172 | equals: #(('1' '' '3' '' '5')). 173 | self 174 | assert: ((NeoCSVReader on: 'a,b,c\,,\"","",""\1,2,3\' withCRs readStream) 175 | emptyFieldValue: #empty; 176 | upToEnd) 177 | equals: #(('a' 'b' 'c')(empty empty empty)(empty empty empty)('1' '2' '3')) 178 | ] 179 | 180 | { #category : #testing } 181 | NeoCSVReaderTests >> testEmptyFieldValuePassNil [ 182 | | date1 date2 | 183 | date1 := '1900-01-01' asDate. 184 | date2 := '2000-12-31' asDate. 185 | self 186 | assert: ((NeoCSVReader on: 187 | 'date1,date2\2018-01-01,2018-02-01\2018-01-01,\,2018-02-01\\' withCRs readStream) 188 | emptyFieldValue: #passNil; 189 | addFieldConverter: [ :input | input ifNil: [ date1 ] ifNotNil: [ input asDate ] ]; 190 | addFieldConverter: [ :input | input ifNil: [ date2 ] ifNotNil: [ input asDate ] ]; 191 | skipHeader; 192 | upToEnd) 193 | equals: (Array 194 | with: (Array with: '2018-01-01' asDate with: '2018-02-01' asDate) 195 | with: (Array with: '2018-01-01' asDate with: date2) 196 | with: (Array with: date1 with: '2018-02-01' asDate) 197 | with: (Array with: date1 with: date2)). 198 | ] 199 | 200 | { #category : #testing } 201 | NeoCSVReaderTests >> testEmptyLastFieldQuoted [ 202 | self 203 | assert: (NeoCSVReader on: '"1","2",""' readStream) upToEnd 204 | equals: #(('1' '2' nil)) 205 | ] 206 | 207 | { #category : #testing } 208 | NeoCSVReaderTests >> testEmptyLastFieldUnquoted [ 209 | self 210 | assert: (NeoCSVReader on: '1,2,' readStream) upToEnd 211 | equals: #(('1' '2' nil)) 212 | ] 213 | 214 | { #category : #testing } 215 | NeoCSVReaderTests >> testEnumerating [ 216 | | numbers csv | 217 | numbers := (1 to: 10) collect: [ :each | { each asString. each asWords } ]. 218 | csv := String cr join: (numbers collect: [ :each | $, join: each ]). 219 | self 220 | assert: ((NeoCSVReader on: csv readStream) collect: [ :each | each ]) 221 | equals: numbers. 222 | self 223 | assert: ((NeoCSVReader on: csv readStream) 224 | addIntegerField; addField; 225 | select: [ :each | each first even ]) 226 | equals: ((NeoCSVReader on: csv readStream) 227 | addIntegerField; addField; 228 | reject: [ :each | each first odd ]) 229 | ] 230 | 231 | { #category : #testing } 232 | NeoCSVReaderTests >> testOneLineEmpty [ 233 | self 234 | assert: (NeoCSVReader on: '' readStream) upToEnd 235 | equals: #() 236 | ] 237 | 238 | { #category : #testing } 239 | NeoCSVReaderTests >> testOneLineOneFieldQuoted [ 240 | self 241 | assert: (NeoCSVReader on: '"1"' readStream) upToEnd 242 | equals: #(('1')) 243 | ] 244 | 245 | { #category : #testing } 246 | NeoCSVReaderTests >> testOneLineOneFieldUnquoted [ 247 | self 248 | assert: (NeoCSVReader on: '1' readStream) upToEnd 249 | equals: #(('1')) 250 | ] 251 | 252 | { #category : #testing } 253 | NeoCSVReaderTests >> testOneLineQuoted [ 254 | self 255 | assert: (NeoCSVReader on: '"1","2","3"' readStream) upToEnd 256 | equals: #(('1' '2' '3')) 257 | ] 258 | 259 | { #category : #testing } 260 | NeoCSVReaderTests >> testOneLineUnquoted [ 261 | self 262 | assert: (NeoCSVReader on: '1,2,3' readStream) upToEnd 263 | equals: #(('1' '2' '3')) 264 | ] 265 | 266 | { #category : #testing } 267 | NeoCSVReaderTests >> testReadAsByteArrays [ 268 | | input | 269 | input := (String crlf join: #( '1,2,3' '1,2,3' '1,2,3' '')). 270 | self 271 | assert: ((NeoCSVReader on: input readStream) 272 | recordClass: ByteArray; 273 | addIntegerField; 274 | addIntegerField ; 275 | addIntegerField; 276 | upToEnd) 277 | equals: { 278 | #[1 2 3]. 279 | #[1 2 3]. 280 | #[1 2 3].} 281 | ] 282 | 283 | { #category : #testing } 284 | NeoCSVReaderTests >> testReadAsIntegerArrays [ 285 | | input | 286 | input := (String crlf join: #( '100,200,300' '100,200,300' '100,200,300' '')). 287 | self 288 | assert: ((NeoCSVReader on: input readStream) 289 | recordClass: IntegerArray; 290 | addIntegerField; 291 | addIntegerField ; 292 | addIntegerField; 293 | upToEnd) 294 | equals: { 295 | #(100 200 300) asIntegerArray. 296 | #(100 200 300) asIntegerArray. 297 | #(100 200 300) asIntegerArray } 298 | ] 299 | 300 | { #category : #testing } 301 | NeoCSVReaderTests >> testReadDictionaries [ 302 | | input | 303 | input := (String crlf join: #( '"x","y","z"' '100,200,300' '100,200,300' '100,200,300' '')). 304 | self 305 | assert: ((NeoCSVReader on: input readStream) 306 | skipHeader; 307 | recordClass: Dictionary; 308 | addIntegerFieldAt: #x ; 309 | addIntegerFieldAt: #y ; 310 | addIntegerFieldAt: #z ; 311 | upToEnd) 312 | equals: { 313 | Dictionary newFromPairs: #(x 100 y 200 z 300). 314 | Dictionary newFromPairs: #(x 100 y 200 z 300). 315 | Dictionary newFromPairs: #(x 100 y 200 z 300) } 316 | ] 317 | 318 | { #category : #testing } 319 | NeoCSVReaderTests >> testReadFloatsRadixPointComma [ 320 | | input output | 321 | input := (String lf join: #( '"x";"y";"z"' '10,0;20,123;-30,5' '10,0;20,123;-30,5' '10,0;20,123;-30,5' '')). 322 | output := (NeoCSVReader on: input readStream) 323 | separator: $; ; 324 | skipHeader; 325 | addFloatFieldRadixPointComma; 326 | addFloatFieldRadixPointComma; 327 | addFloatFieldRadixPointComma; 328 | upToEnd. 329 | output do: [ :record | 330 | #(10.0 20.123 -30.5) with: record do: [ :x :y | 331 | self assert: (x closeTo: y) ] ] 332 | ] 333 | 334 | { #category : #testing } 335 | NeoCSVReaderTests >> testReadHeader [ 336 | | input | 337 | input := (String crlf join: #( '"x","y","z"' '100,200,300' '100,200,300' '100,200,300' '')). 338 | self 339 | assert: (NeoCSVReader on: input readStream) readHeader 340 | equals: #('x' 'y' 'z') 341 | ] 342 | 343 | { #category : #testing } 344 | NeoCSVReaderTests >> testReadIntegers [ 345 | | input | 346 | input := (String crlf join: #( '"x","y","z"' '100,200,300' '100,200,300' '100,200,300' '')). 347 | self 348 | assert: ((NeoCSVReader on: input readStream) 349 | skipHeader; 350 | addIntegerField; 351 | addIntegerField ; 352 | addIntegerField; 353 | upToEnd) 354 | equals: #((100 200 300)(100 200 300)(100 200 300)) 355 | ] 356 | 357 | { #category : #testing } 358 | NeoCSVReaderTests >> testReadIntegersReadingHeaderAfterFieldDefinitions [ 359 | | input | 360 | input := (String crlf join: #( '"x","y","z"' '100,200,300' '100,200,300' '100,200,300' '')). 361 | self 362 | assert: ((NeoCSVReader on: input readStream) 363 | addIntegerField; 364 | addIntegerField ; 365 | addIntegerField; 366 | skipHeader; 367 | upToEnd) 368 | equals: #((100 200 300)(100 200 300)(100 200 300)) 369 | ] 370 | 371 | { #category : #testing } 372 | NeoCSVReaderTests >> testReadTestsObjects [ 373 | | input | 374 | input := (String crlf join: #( '"x","y","z"' '100,200,300' '100,200,300' '100,200,300' '')). 375 | self 376 | assert: ((NeoCSVReader on: input readStream) 377 | skipHeader; 378 | recordClass: NeoCSVTestObject; 379 | addIntegerField: #x: ; 380 | addIntegerField: #y: ; 381 | addIntegerField: #z: ; 382 | upToEnd) 383 | equals: { 384 | NeoCSVTestObject example. 385 | NeoCSVTestObject example. 386 | NeoCSVTestObject example } 387 | ] 388 | 389 | { #category : #testing } 390 | NeoCSVReaderTests >> testReadTestsObjectsUsingBlockAccessors [ 391 | | input | 392 | input := (String crlf join: #( '"x","y","z"' '100,200,300' '100,200,300' '100,200,300' '')). 393 | self 394 | assert: ((NeoCSVReader on: input readStream) 395 | skipHeader; 396 | recordClass: NeoCSVTestObject; 397 | addIntegerField: [ :object :value | object x: value ]; 398 | addIntegerField: [ :object :value | object y: value ]; 399 | addIntegerField: [ :object :value | object z: value ]; 400 | upToEnd) 401 | equals: { 402 | NeoCSVTestObject example. 403 | NeoCSVTestObject example. 404 | NeoCSVTestObject example } 405 | ] 406 | 407 | { #category : #testing } 408 | NeoCSVReaderTests >> testReadTestsObjectsWithEmptyFieldValue [ 409 | | input | 410 | input := (String crlf join: #( '"x","y","z"' '100,200,300' '1,,3' '100,200,300' '')). 411 | self 412 | assert: ((NeoCSVReader on: input readStream) 413 | skipHeader; 414 | recordClass: NeoCSVTestObject2; 415 | emptyFieldValue: #empty; 416 | addIntegerField: #x: ; 417 | addIntegerField: #y: ; 418 | addIntegerField: #z: ; 419 | upToEnd) 420 | equals: { 421 | NeoCSVTestObject2 example. 422 | NeoCSVTestObject2 new x: 1; z: 3; yourself. "Note that y contains #y from #initialize and was NOT touched" 423 | NeoCSVTestObject2 example }. 424 | self 425 | assert: ((NeoCSVReader on: input readStream) 426 | skipHeader; 427 | recordClass: NeoCSVTestObject2; 428 | addIntegerField: #x: ; 429 | addIntegerField: #y: ; 430 | addIntegerField: #z: ; 431 | upToEnd) 432 | equals: { 433 | NeoCSVTestObject2 example. 434 | NeoCSVTestObject2 new x: 1; z: 3; yourself. "Note that y contains #y from #initialize and was NOT touched" 435 | NeoCSVTestObject2 example } 436 | ] 437 | 438 | { #category : #testing } 439 | NeoCSVReaderTests >> testReadTestsObjectsWithIgnoredField [ 440 | | input | 441 | input := (String crlf join: #( '"x","y",''-'',"z"' '100,200,a,300' '100,200,b,300' '100,200,c,300' '')). 442 | self 443 | assert: ((NeoCSVReader on: input readStream) 444 | skipHeader; 445 | recordClass: NeoCSVTestObject; 446 | addIntegerField: #x: ; 447 | addIntegerField: #y: ; 448 | addIgnoredField; 449 | addIntegerField: #z: ; 450 | upToEnd) 451 | equals: { 452 | NeoCSVTestObject example. 453 | NeoCSVTestObject example. 454 | NeoCSVTestObject example } 455 | ] 456 | 457 | { #category : #testing } 458 | NeoCSVReaderTests >> testReadWithIgnoredField [ 459 | | input | 460 | input := (String crlf join: #( '1,2,a,3' '1,2,b,3' '1,2,c,3' '')). 461 | self 462 | assert: ((NeoCSVReader on: input readStream) 463 | addIntegerField; 464 | addIntegerField; 465 | addIgnoredField; 466 | addIntegerField; 467 | upToEnd) 468 | equals: { 469 | #(1 2 3). 470 | #(1 2 3). 471 | #(1 2 3).} 472 | ] 473 | 474 | { #category : #testing } 475 | NeoCSVReaderTests >> testSimpleCrLfQuoted [ 476 | | input | 477 | input := (String crlf join: #('"1","2","3"' '"4","5","6"' '"7","8","9"' '')). 478 | self 479 | assert: (NeoCSVReader on: input readStream) upToEnd 480 | equals: #(('1' '2' '3')('4' '5' '6')('7' '8' '9')) 481 | ] 482 | 483 | { #category : #testing } 484 | NeoCSVReaderTests >> testSimpleCrLfUnquoted [ 485 | | input | 486 | input := (String crlf join: #('1,2,3' '4,5,6' '7,8,9' '')). 487 | self 488 | assert: (NeoCSVReader on: input readStream) upToEnd 489 | equals: #(('1' '2' '3')('4' '5' '6')('7' '8' '9')) 490 | ] 491 | 492 | { #category : #testing } 493 | NeoCSVReaderTests >> testSimpleCrQuoted [ 494 | | input | 495 | input := (String cr join: #('"1","2","3"' '"4","5","6"' '"7","8","9"' '')). 496 | self 497 | assert: (NeoCSVReader on: input readStream) upToEnd 498 | equals: #(('1' '2' '3')('4' '5' '6')('7' '8' '9')) 499 | ] 500 | 501 | { #category : #testing } 502 | NeoCSVReaderTests >> testSimpleCrUnquoted [ 503 | | input | 504 | input := (String cr join: #('1,2,3' '4,5,6' '7,8,9' '')). 505 | self 506 | assert: (NeoCSVReader on: input readStream) upToEnd 507 | equals: #(('1' '2' '3')('4' '5' '6')('7' '8' '9')) 508 | ] 509 | 510 | { #category : #testing } 511 | NeoCSVReaderTests >> testSimpleLfQuoted [ 512 | | input | 513 | input := (String lf join: #('"1","2","3"' '"4","5","6"' '"7","8","9"' '')). 514 | self 515 | assert: (NeoCSVReader on: input readStream) upToEnd 516 | equals: #(('1' '2' '3')('4' '5' '6')('7' '8' '9')) 517 | ] 518 | 519 | { #category : #testing } 520 | NeoCSVReaderTests >> testSimpleLfUnquoted [ 521 | | input | 522 | input := (String lf join: #('1,2,3' '4,5,6' '7,8,9' '')). 523 | self 524 | assert: (NeoCSVReader on: input readStream) upToEnd 525 | equals: #(('1' '2' '3')('4' '5' '6')('7' '8' '9')) 526 | ] 527 | 528 | { #category : #testing } 529 | NeoCSVReaderTests >> testSimpleSemiColonDelimited [ 530 | | input | 531 | input := (String crlf join: #('1;2;3' '4;5;6' '7;8;9' '')). 532 | self 533 | assert: ((NeoCSVReader on: input readStream) 534 | separator: $; ; 535 | upToEnd) 536 | equals: #(('1' '2' '3')('4' '5' '6')('7' '8' '9')) 537 | ] 538 | 539 | { #category : #testing } 540 | NeoCSVReaderTests >> testSimpleTabDelimited [ 541 | | input | 542 | input := (String crlf join: #('1 2 3' '4 5 6' '7 8 9' '')). 543 | self 544 | assert: ((NeoCSVReader on: input readStream) 545 | separator: Character tab ; 546 | upToEnd) 547 | equals: #(('1' '2' '3')('4' '5' '6')('7' '8' '9')) 548 | ] 549 | 550 | { #category : #testing } 551 | NeoCSVReaderTests >> testSkipping [ 552 | | reader | 553 | reader := NeoCSVReader on: 'A,1\B,2\C,3\D,4\E,5\F,6' withCRs readStream. 554 | reader skip. 555 | self assert: reader next equals: #('B' '2'). 556 | reader skip: 2. 557 | self assert: reader next equals: #('E' '5'). 558 | reader skip. 559 | self assert: reader atEnd. 560 | reader skip. 561 | self assert: reader atEnd. 562 | 563 | reader := NeoCSVReader on: 'LETTER,DIGIT\A,1\B,2\C,3\D,4\E,5\F,6' withCRs readStream. 564 | reader skipHeader. 565 | reader skip. 566 | self assert: reader next equals: #('B' '2'). 567 | reader skip: 2. 568 | self assert: reader next equals: #('E' '5'). 569 | reader skip. 570 | self assert: reader atEnd. 571 | 572 | ] 573 | 574 | { #category : #testing } 575 | NeoCSVReaderTests >> testSkippingEmptyRecords [ 576 | | input output | 577 | input := '1,2,3\\4,5,6\,,\7,8,9' withCRs. 578 | output := (NeoCSVReader on: input readStream) 579 | select: [ :each | each notEmpty and: [ (each allSatisfy: #isNil) not ] ]. 580 | self assert: output equals: #(#('1' '2' '3') #('4' '5' '6') #('7' '8' '9')). 581 | output := (NeoCSVReader on: input readStream) 582 | emptyFieldValue: ''; 583 | select: [ :each | each notEmpty and: [ (each allSatisfy: #isEmpty) not ] ]. 584 | self assert: output equals: #(#('1' '2' '3') #('4' '5' '6') #('7' '8' '9')) 585 | ] 586 | 587 | { #category : #testing } 588 | NeoCSVReaderTests >> testSkippingSpecial [ 589 | | reader | 590 | reader := NeoCSVReader on: 'A,1\B,\,3\,\\F,6' withCRs readStream. 591 | reader skip: 5. 592 | self assert: reader next equals: #('F' '6'). 593 | self assert: reader atEnd. 594 | 595 | reader := NeoCSVReader on: 'A,1\"\",\,"\"\,\\F,6' withCRs readStream. 596 | reader skip: 5. 597 | self assert: reader next equals: #('F' '6'). 598 | self assert: reader atEnd. 599 | ] 600 | 601 | { #category : #testing } 602 | NeoCSVReaderTests >> testStrictParsing [ 603 | { 604 | 'foo,1\bar,2\foobar,3' -> [ :reader | reader addField ]. 605 | 'foo,1\bar,2\foobar,3' -> [ :reader | reader addField; addField; addField ]. 606 | 'foo,1\bar,2\foobar,3' -> [ :reader | reader recordClass: Dictionary; addFieldAt: #a ]. 607 | 'foo,1\bar,2\foobar,3' -> [ :reader | reader recordClass: Dictionary; addFieldAt: #a; addFieldAt: #b; addFieldAt: #c ]. 608 | 'one\one,two' -> [ :reader | reader ]. 609 | 'one,two\one' -> [ :reader | reader ]. 610 | '"foo",1\"missing quote,2\"foobar",3' -> [ :reader | reader ]. 611 | '"foo",1\missing quote,"2\"foobar",3' -> [ :reader | reader ]. 612 | } do: [ :specification | | inputString configurator reader | 613 | inputString := specification key. 614 | configurator := specification value. 615 | reader := NeoCSVReader on: inputString withCRs readStream. 616 | reader beStrict. 617 | self 618 | should: [ 619 | configurator value: reader. 620 | reader upToEnd ] 621 | raise: Error ] 622 | ] 623 | -------------------------------------------------------------------------------- /repository/Neo-CSV-Tests/NeoCSVTestObject.class.st: -------------------------------------------------------------------------------- 1 | " 2 | I am NeoCSVTestObject. 3 | " 4 | Class { 5 | #name : #NeoCSVTestObject, 6 | #superclass : #Object, 7 | #instVars : [ 8 | 'x', 9 | 'y', 10 | 'z' 11 | ], 12 | #category : 'Neo-CSV-Tests' 13 | } 14 | 15 | { #category : #'instance creation' } 16 | NeoCSVTestObject class >> example [ 17 | ^ self x: 100 y: 200 z: 300 18 | ] 19 | 20 | { #category : #'instance creation' } 21 | NeoCSVTestObject class >> x: x y: y z: z [ 22 | ^ self new 23 | x: x; 24 | y: y; 25 | z: z; 26 | yourself 27 | ] 28 | 29 | { #category : #comparing } 30 | NeoCSVTestObject >> = anObject [ 31 | self == anObject 32 | ifTrue: [ ^ true ]. 33 | self class = anObject class 34 | ifFalse: [ ^ false ]. 35 | ^ x = anObject x 36 | and: [ 37 | y = anObject y 38 | and: [ 39 | z = anObject z ] ] 40 | ] 41 | 42 | { #category : #comparing } 43 | NeoCSVTestObject >> hash [ 44 | ^ x hash bitXor: (y hash bitXor: z) 45 | ] 46 | 47 | { #category : #accessing } 48 | NeoCSVTestObject >> x [ 49 | ^ x 50 | ] 51 | 52 | { #category : #accessing } 53 | NeoCSVTestObject >> x: anObject [ 54 | x := anObject 55 | ] 56 | 57 | { #category : #accessing } 58 | NeoCSVTestObject >> y [ 59 | ^ y 60 | ] 61 | 62 | { #category : #accessing } 63 | NeoCSVTestObject >> y: anObject [ 64 | y := anObject 65 | ] 66 | 67 | { #category : #accessing } 68 | NeoCSVTestObject >> z [ 69 | ^ z 70 | ] 71 | 72 | { #category : #accessing } 73 | NeoCSVTestObject >> z: anObject [ 74 | z := anObject 75 | ] 76 | -------------------------------------------------------------------------------- /repository/Neo-CSV-Tests/NeoCSVTestObject2.class.st: -------------------------------------------------------------------------------- 1 | " 2 | I am NeoCSVTestObject2. I am a NeoCSVTestObject. 3 | 4 | I initialize my fields to specific values. 5 | " 6 | Class { 7 | #name : #NeoCSVTestObject2, 8 | #superclass : #NeoCSVTestObject, 9 | #category : 'Neo-CSV-Tests' 10 | } 11 | 12 | { #category : #initalize } 13 | NeoCSVTestObject2 >> initialize [ 14 | super initialize. 15 | x := #x. 16 | y := #y. 17 | z := #z 18 | ] 19 | -------------------------------------------------------------------------------- /repository/Neo-CSV-Tests/NeoCSVWriterTests.class.st: -------------------------------------------------------------------------------- 1 | " 2 | I am NeoCSVWriterTests, a suite of unit tests for NeoCSVWriter. 3 | 4 | " 5 | Class { 6 | #name : #NeoCSVWriterTests, 7 | #superclass : #TestCase, 8 | #category : 'Neo-CSV-Tests' 9 | } 10 | 11 | { #category : #testing } 12 | NeoCSVWriterTests >> testEmptyFieldValue [ 13 | self 14 | assert: (String streamContents: [ :stream | 15 | (NeoCSVWriter on: stream) 16 | nextPut: #(one two three); 17 | nextPutAll: #( (1 2 nil) (4 nil 6) (nil 8 9)) ]) 18 | equals: (OSPlatform current lineEnding join: #( '"one","two","three"' '"1","2",""' '"4","","6"' '"","8","9"' '')). 19 | self 20 | assert: (String streamContents: [ :stream | 21 | (NeoCSVWriter on: stream) 22 | emptyFieldValue: #empty; 23 | nextPut: #(one two three); 24 | nextPutAll: #( (1 2 empty) (4 empty 6) (empty 8 9)) ]) 25 | equals: (OSPlatform current lineEnding join: #( '"one","two","three"' '"1","2",""' '"4","","6"' '"","8","9"' '')). 26 | self 27 | assert: (String streamContents: [ :stream | 28 | (NeoCSVWriter on: stream) 29 | emptyFieldValue: Object new; 30 | nextPut: #(one two three); 31 | nextPutAll: #( (1 2 nil) (4 nil 6) (nil 8 9)) ]) 32 | equals: (OSPlatform current lineEnding join: #( '"one","two","three"' '"1","2","nil"' '"4","nil","6"' '"nil","8","9"' '')) 33 | ] 34 | 35 | { #category : #testing } 36 | NeoCSVWriterTests >> testObjectFieldsTestObjects [ 37 | self 38 | assert: (String streamContents: [ :stream | 39 | (NeoCSVWriter on: stream) 40 | nextPut: #(x y z); 41 | addObjectFields: #(x y z); 42 | nextPutAll: { 43 | NeoCSVTestObject example. 44 | NeoCSVTestObject example. 45 | NeoCSVTestObject example } ]) 46 | equals: (OSPlatform current lineEnding join: #( '"x","y","z"' '100,200,300' '100,200,300' '100,200,300' '')) 47 | ] 48 | 49 | { #category : #testing } 50 | NeoCSVWriterTests >> testObjectFieldsTestObjectsExtra [ 51 | self 52 | assert: (String streamContents: [ :stream | 53 | (NeoCSVWriter on: stream) 54 | fieldWriter: #raw; 55 | nextPut: #(x empty y constant z); 56 | addObjectField: #x; 57 | addEmptyField; 58 | addObjectField: #y; 59 | addConstantField: 'X'; 60 | addObjectField: #z; 61 | nextPutAll: { 62 | NeoCSVTestObject example. 63 | NeoCSVTestObject example. 64 | NeoCSVTestObject example } ]) 65 | equals: (OSPlatform current lineEnding join: #( 66 | 'x,empty,y,constant,z' 67 | '100,,200,X,300' 68 | '100,,200,X,300' 69 | '100,,200,X,300' '')) 70 | ] 71 | 72 | { #category : #testing } 73 | NeoCSVWriterTests >> testObjectFieldsTestObjectsUsingBlockAccessors [ 74 | self 75 | assert: (String streamContents: [ :stream | 76 | (NeoCSVWriter on: stream) 77 | nextPut: #(x y z); 78 | addObjectFields: { 79 | [ :object | object x ]. 80 | [ :object | object y ]. 81 | [ :object | object z ] }; 82 | nextPutAll: { 83 | NeoCSVTestObject example. 84 | NeoCSVTestObject example. 85 | NeoCSVTestObject example } ]) 86 | equals: (OSPlatform current lineEnding join: #( '"x","y","z"' '100,200,300' '100,200,300' '100,200,300' '')) 87 | ] 88 | 89 | { #category : #testing } 90 | NeoCSVWriterTests >> testOptionalQuotedFields [ 91 | self 92 | assert: 93 | (String 94 | streamContents: [ :stream | 95 | (NeoCSVWriter on: stream) 96 | fieldWriter: #optionalQuoted; 97 | nextPut: 98 | {'one'. 99 | 't,wo'. 100 | 't"hree'. 101 | 'fo' , OSPlatform current lineEnding , 'ur'} ]) 102 | equals: 'one,"t,wo","t""hree","fo' , OSPlatform current lineEnding , 'ur"', OSPlatform current lineEnding 103 | ] 104 | 105 | { #category : #testing } 106 | NeoCSVWriterTests >> testRawFieldsDictionaries [ 107 | self 108 | assert: (String streamContents: [ :stream | 109 | (NeoCSVWriter on: stream) 110 | nextPut: #(x y z); 111 | addRawFieldsAt: #(x y z); 112 | nextPutAll: { 113 | Dictionary newFromPairs: #(x 100 y 200 z 300). 114 | Dictionary newFromPairs: #(x 400 y 500 z 600). 115 | Dictionary newFromPairs: #(x 700 y 800 z 900) } ]) 116 | equals: (OSPlatform current lineEnding join: #( '"x","y","z"' '100,200,300' '400,500,600' '700,800,900' '')) 117 | ] 118 | 119 | { #category : #testing } 120 | NeoCSVWriterTests >> testRawFieldsTestObjects [ 121 | self 122 | assert: (String streamContents: [ :stream | 123 | (NeoCSVWriter on: stream) 124 | nextPut: #(x y z); 125 | addRawFields: #(x y z); 126 | nextPutAll: { 127 | NeoCSVTestObject example. 128 | NeoCSVTestObject example. 129 | NeoCSVTestObject example } ]) 130 | equals: (OSPlatform current lineEnding join: #( '"x","y","z"' '100,200,300' '100,200,300' '100,200,300' '')) 131 | ] 132 | 133 | { #category : #testing } 134 | NeoCSVWriterTests >> testSimple [ 135 | self 136 | assert: (String streamContents: [ :stream | 137 | (NeoCSVWriter on: stream) 138 | nextPut: #(one two three); 139 | nextPutAll: #( (1 2 3) (4 5 6) (7 8 9)) ]) 140 | equals: (OSPlatform current lineEnding join: #( '"one","two","three"' '"1","2","3"' '"4","5","6"' '"7","8","9"' '')) 141 | ] 142 | 143 | { #category : #testing } 144 | NeoCSVWriterTests >> testSimpleOptionalQuoted [ 145 | self 146 | assert: (String streamContents: [ :stream | 147 | (NeoCSVWriter on: stream) 148 | fieldWriter: #optionalQuoted; 149 | nextPut: #(one two 'thr,ee'); 150 | nextPutAll: #( (1 2 3) (4 5 6) (7 8 9)) ]) 151 | equals: (OSPlatform current lineEnding join: #( 'one,two,"thr,ee"' '1,2,3' '4,5,6' '7,8,9' '')) 152 | ] 153 | 154 | { #category : #testing } 155 | NeoCSVWriterTests >> testSimpleRaw [ 156 | self 157 | assert: (String streamContents: [ :stream | 158 | (NeoCSVWriter on: stream) 159 | fieldWriter: #raw; 160 | nextPut: #(one two three); 161 | nextPutAll: #( (1 2 3) (4 5 6) (7 8 9)) ]) 162 | equals: (OSPlatform current lineEnding join: #( 'one,two,three' '1,2,3' '4,5,6' '7,8,9' '')) 163 | ] 164 | 165 | { #category : #testing } 166 | NeoCSVWriterTests >> testWideOptionalQuoted [ 167 | self 168 | assert: (String streamContents: [ :out | 169 | (NeoCSVWriter on: out) 170 | fieldWriter: #optionalQuoted; 171 | nextPut: { 1. 'foo "1" ', Character euro asString. true} ]) 172 | equals: ('1,"foo ""1"" ', Character euro asString, '",true', OSPlatform current lineEnding) 173 | ] 174 | 175 | { #category : #testing } 176 | NeoCSVWriterTests >> testWriteEmbeddedQuote [ 177 | | header | 178 | header := String streamContents: [ :out | 179 | (NeoCSVWriter on: out) 180 | nextPut: #(foo 'x"y"z') ]. 181 | self assert: header equals: '"foo","x""y""z"', OSPlatform current lineEnding 182 | ] 183 | 184 | { #category : #testing } 185 | NeoCSVWriterTests >> testWriteHeader [ 186 | | header | 187 | header := String streamContents: [ :out | 188 | (NeoCSVWriter on: out) 189 | writeHeader: #(foo bar) ]. 190 | self assert: header equals: '"foo","bar"', OSPlatform current lineEnding 191 | ] 192 | -------------------------------------------------------------------------------- /repository/Neo-CSV-Tests/NeoNumberParserTests.class.st: -------------------------------------------------------------------------------- 1 | " 2 | I am NeoNumberParserTests the unit test suite for NeoNumberParser. 3 | " 4 | Class { 5 | #name : #NeoNumberParserTests, 6 | #superclass : #TestCase, 7 | #category : 'Neo-CSV-Tests' 8 | } 9 | 10 | { #category : #testing } 11 | NeoNumberParserTests >> testBinaryIntegers [ 12 | self assert: (NeoNumberParser parse: '1111011' base: 2) equals: 123. 13 | self assert: (NeoNumberParser parse: '-1111011' base: 2) equals: -123. 14 | self assert: (NeoNumberParser parse: '0' base: 2) equals: 0. 15 | 16 | 17 | ] 18 | 19 | { #category : #testing } 20 | NeoNumberParserTests >> testDecimalIntegers [ 21 | self assert: (NeoNumberParser parse: '123') equals: 123. 22 | self assert: (NeoNumberParser parse: '-123') equals: -123. 23 | self assert: (NeoNumberParser parse: '0') equals: 0. 24 | self assert: (NeoNumberParser parse: '12345678901234567890') equals: 12345678901234567890. 25 | 26 | self 27 | assert: (NeoNumberParser parse: '12345678901234567890123456789012345678901234567890123456789012345678901234567890') 28 | equals: 12345678901234567890123456789012345678901234567890123456789012345678901234567890. 29 | 30 | self assert: (NeoNumberParser parse: '00123ABC') equals: 123. 31 | self assert: (NeoNumberParser parse: '-0') equals: 0. 32 | 33 | ] 34 | 35 | { #category : #testing } 36 | NeoNumberParserTests >> testDigitGroupSeparator [ 37 | self 38 | assert: ((NeoNumberParser on: '123,456') digitGroupSeparator: $,; next) 39 | equals: 123456. 40 | self 41 | assert: ((NeoNumberParser on: '123 456 789') digitGroupSeparator: Character space; next) 42 | equals: 123456789. 43 | self 44 | assert: ((NeoNumberParser on: '-3.14159 26535 89793 23846') digitGroupSeparator: Character space; next) 45 | closeTo: Float pi negated. 46 | self 47 | assert: ((NeoNumberParser on: '2.71828 18284 59045 23536') digitGroupSeparator: Character space; next) 48 | closeTo: Float e. 49 | self 50 | assert: ((NeoNumberParser on: '-123''456') digitGroupSeparator: $'; next) 51 | equals: -123456. 52 | self 53 | assert: ((NeoNumberParser on: '123_456.25') digitGroupSeparator: $_; next) 54 | closeTo: 123456.25. 55 | self 56 | assert: ((NeoNumberParser on: '-123.456,25') digitGroupSeparator: $.; radixPoint: $,; next) 57 | closeTo: -123456.25. 58 | ] 59 | 60 | { #category : #testing } 61 | NeoNumberParserTests >> testErrors [ 62 | self should: [ NeoNumberParser parse: nil ] raise: Error. 63 | self should: [ NeoNumberParser parse: '' ] raise: Error. 64 | self should: [ NeoNumberParser parse: '.5' ] raise: Error. 65 | 66 | self should: [ NeoNumberParser parse: '-' ] raise: Error. 67 | self should: [ NeoNumberParser parse: '+' ] raise: Error. 68 | self should: [ NeoNumberParser parse: 'x' ] raise: Error. 69 | 70 | self should: [ NeoNumberParser parse: '-a' ] raise: Error. 71 | self should: [ NeoNumberParser parse: '_' ] raise: Error. 72 | ] 73 | 74 | { #category : #testing } 75 | NeoNumberParserTests >> testFloats [ 76 | #('1.5' 1.5 '-1.5' -1.5 '0.0' 0.0 '3.14159' 3.14159 '1e3' 1000.0 '1e-2' 0.01) 77 | pairsDo: [ :string :float | 78 | self assert: ((NeoNumberParser parse: string) closeTo: float) ] 79 | ] 80 | 81 | { #category : #testing } 82 | NeoNumberParserTests >> testFloatsRadixPoint [ 83 | #('1,5' 1.5 '-1,5' -1.5 '0,0' 0.0 '3,14159' 3.14159 '1e3' 1000.0 '1e-2' 0.01) 84 | pairsDo: [ :string :float | 85 | self assert: (((NeoNumberParser on: string) radixPoint: $,; next) closeTo: float) ] 86 | ] 87 | 88 | { #category : #testing } 89 | NeoNumberParserTests >> testHexadecimalIntegers [ 90 | self assert: (NeoNumberParser parse: '7B' base: 16) equals: 123. 91 | self assert: (NeoNumberParser parse: '-7B' base: 16) equals: -123. 92 | self assert: (NeoNumberParser parse: '0' base: 16) equals: 0. 93 | "On some platforms Character>>#digitValue only handles uppercase, 94 | then NeoNumberParser cannot deal with lowercase hex characters" 95 | $a digitValue = 10 ifFalse: [ ^ self ]. 96 | self assert: (NeoNumberParser parse: '7b' base: 16) equals: 123. 97 | self assert: (NeoNumberParser parse: '-7b' base: 16) equals: -123 98 | 99 | ] 100 | 101 | { #category : #testing } 102 | NeoNumberParserTests >> testNumberExtraction [ 103 | self assert: (NeoNumberParser parse: '00123ABC') equals: 123. 104 | self assert: ((NeoNumberParser on: ' 123ABC') consumeWhitespace; next) equals: 123. 105 | self should: [ (NeoNumberParser on: ' 123ABC') consumeWhitespace; next; failIfNotAtEnd ] raise: Error. 106 | 107 | ] 108 | 109 | { #category : #testing } 110 | NeoNumberParserTests >> testOctalIntegers [ 111 | self assert: (NeoNumberParser parse: '173' base: 8) equals: 123. 112 | self assert: (NeoNumberParser parse: '-173' base: 8) equals: -123. 113 | self assert: (NeoNumberParser parse: '0' base: 8) equals: 0. 114 | 115 | 116 | ] 117 | -------------------------------------------------------------------------------- /repository/Neo-CSV-Tests/package.st: -------------------------------------------------------------------------------- 1 | Package { #name : #'Neo-CSV-Tests' } 2 | --------------------------------------------------------------------------------