├── .gitignore ├── README.md ├── index.js ├── package-lock.json ├── package.json └── script.txt /.gitignore: -------------------------------------------------------------------------------- 1 | dataset/ 2 | final.csv 3 | # Logs 4 | logs 5 | *.log 6 | npm-debug.log* 7 | yarn-debug.log* 8 | yarn-error.log* 9 | lerna-debug.log* 10 | # Diagnostic reports (https://nodejs.org/api/report.html) 11 | report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json 12 | 13 | # Runtime data 14 | pids 15 | *.pid 16 | *.seed 17 | *.pid.lock 18 | 19 | # Directory for instrumented libs generated by jscoverage/JSCover 20 | lib-cov 21 | 22 | # Coverage directory used by tools like istanbul 23 | coverage 24 | *.lcov 25 | 26 | # nyc test coverage 27 | .nyc_output 28 | 29 | # Grunt intermediate storage (https://gruntjs.com/creating-plugins#storing-task-files) 30 | .grunt 31 | 32 | # Bower dependency directory (https://bower.io/) 33 | bower_components 34 | 35 | # node-waf configuration 36 | .lock-wscript 37 | 38 | # Compiled binary addons (https://nodejs.org/api/addons.html) 39 | build/Release 40 | 41 | # Dependency directories 42 | node_modules/ 43 | jspm_packages/ 44 | 45 | # Snowpack dependency directory (https://snowpack.dev/) 46 | web_modules/ 47 | 48 | # TypeScript cache 49 | *.tsbuildinfo 50 | 51 | # Optional npm cache directory 52 | .npm 53 | 54 | # Optional eslint cache 55 | .eslintcache 56 | 57 | # Microbundle cache 58 | .rpt2_cache/ 59 | .rts2_cache_cjs/ 60 | .rts2_cache_es/ 61 | .rts2_cache_umd/ 62 | 63 | # Optional REPL history 64 | .node_repl_history 65 | 66 | # Output of 'npm pack' 67 | *.tgz 68 | 69 | # Yarn Integrity file 70 | .yarn-integrity 71 | 72 | # dotenv environment variables file 73 | .env 74 | .env.test 75 | 76 | # parcel-bundler cache (https://parceljs.org/) 77 | .cache 78 | .parcel-cache 79 | 80 | # Next.js build output 81 | .next 82 | out 83 | 84 | # Nuxt.js build / generate output 85 | .nuxt 86 | dist 87 | 88 | # Gatsby files 89 | .cache/ 90 | # Comment in the public line in if your project uses Gatsby and not Next.js 91 | # https://nextjs.org/blog/next-9-1#public-directory-support 92 | # public 93 | 94 | # vuepress build output 95 | .vuepress/dist 96 | 97 | # Serverless directories 98 | .serverless/ 99 | 100 | # FuseBox cache 101 | .fusebox/ 102 | 103 | # DynamoDB Local files 104 | .dynamodb/ 105 | 106 | # TernJS port file 107 | .tern-port 108 | 109 | # Stores VSCode versions used for testing VSCode extensions 110 | .vscode-test 111 | 112 | # yarn v2 113 | .yarn/cache 114 | .yarn/unplugged 115 | .yarn/build-state.yml 116 | .yarn/install-state.gz 117 | .pnp.* 118 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # concat-large-files 2 | 3 | This repo is an example about how to concat large files using Node.js 4 | 5 | ## Running 6 | 7 | Restore dependencies with `npm ci` 8 | 9 | 1. There are the concatener in [./index.js](index.js) that will concat all `.csv` files and generate the `final.csv` file with all entries. It will convert data from csv, on `name` field it will replace spaces by underscore ( `_` ) then convert to csv again and finally save its output on `final.csv`. 10 | 11 | ## Download files from Kaggle 12 | 13 | 2018: https://www.kaggle.com/stackoverflow/stack-overflow-2018-developer-survey?select=survey_results_public.csv 14 | 2017: https://www.kaggle.com/stackoverflow/so-survey-2017?select=survey_results_public.csv 15 | -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | // console.log(await Promise.resolve(true)) 2 | 3 | import { dirname, join } from 'path' 4 | import { promisify } from 'util' 5 | import { promises, createReadStream, createWriteStream } from 'fs' 6 | import { pipeline, Transform } from 'stream' 7 | const pipelineAsync = promisify(pipeline) 8 | 9 | import csvtojson from 'csvtojson' 10 | import jsontocsv from 'json-to-csv-stream' 11 | import StreamConcat from 'stream-concat' 12 | 13 | const { readdir } = promises 14 | import debug from 'debug' 15 | import { STATUS_CODES } from 'http' 16 | const log = debug('app:concat') 17 | 18 | 19 | const { pathname: currentFile } = new URL(import.meta.url) 20 | const cwd = dirname(currentFile) 21 | const filesDir = `${cwd}/dataset` 22 | const output = `${cwd}/final.csv` 23 | 24 | console.time('concat-data') 25 | const files = (await readdir(filesDir)) 26 | .filter(item => !(!!~item.indexOf('.zip'))) 27 | 28 | log(`processing ${files}`) 29 | const ONE_SECOND = 1000 30 | // quando os outros processos acabarem ele morre junto 31 | setInterval(() => process.stdout.write('.'), ONE_SECOND).unref() 32 | 33 | // const combinedStreams = createReadStream(join(filesDir, files[0])) 34 | const streams = files.map( 35 | item => createReadStream(join(filesDir, item)) 36 | ) 37 | const combinedStreams = new StreamConcat(streams) 38 | 39 | const finalStream = createWriteStream(output) 40 | const handleStream = new Transform({ 41 | transform: (chunk, encoding, cb) => { 42 | const data = JSON.parse(chunk) 43 | const output = { 44 | id: data.Respondent, 45 | country: data.Country 46 | } 47 | // log(`id: ${output.id}`) 48 | return cb(null, JSON.stringify(output)) 49 | } 50 | }) 51 | 52 | await pipelineAsync( 53 | combinedStreams, 54 | csvtojson(), 55 | handleStream, 56 | jsontocsv(), 57 | finalStream 58 | ) 59 | log(`${files.length} files merged! on ${output}`) 60 | console.timeEnd('concat-data') -------------------------------------------------------------------------------- /package-lock.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "concat-large-files", 3 | "version": "1.0.0", 4 | "lockfileVersion": 1, 5 | "requires": true, 6 | "dependencies": { 7 | "JSONStream": { 8 | "version": "1.1.4", 9 | "resolved": "https://registry.npmjs.org/JSONStream/-/JSONStream-1.1.4.tgz", 10 | "integrity": "sha1-vhGklZOOiC0nd3PRGYbzl0qLo3o=", 11 | "requires": { 12 | "jsonparse": "^1.2.0", 13 | "through": ">=2.2.7 <3" 14 | } 15 | }, 16 | "argparse": { 17 | "version": "1.0.10", 18 | "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz", 19 | "integrity": "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==", 20 | "requires": { 21 | "sprintf-js": "~1.0.2" 22 | } 23 | }, 24 | "bluebird": { 25 | "version": "3.7.2", 26 | "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.7.2.tgz", 27 | "integrity": "sha512-XpNj6GDQzdfW+r2Wnn7xiSAd7TM3jzkxGXBGTtWKuSXv1xUV+azxAm8jdWZN06QTQk+2N2XB9jRDkvbmQmcRtg==" 28 | }, 29 | "core-util-is": { 30 | "version": "1.0.2", 31 | "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz", 32 | "integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac=" 33 | }, 34 | "csv-write-stream": { 35 | "version": "2.0.0", 36 | "resolved": "https://registry.npmjs.org/csv-write-stream/-/csv-write-stream-2.0.0.tgz", 37 | "integrity": "sha1-/C2iGkjW6l+MF/3jnPuRHk8CkrA=", 38 | "requires": { 39 | "argparse": "^1.0.7", 40 | "generate-object-property": "^1.0.0", 41 | "ndjson": "^1.3.0" 42 | } 43 | }, 44 | "csvtojson": { 45 | "version": "2.0.10", 46 | "resolved": "https://registry.npmjs.org/csvtojson/-/csvtojson-2.0.10.tgz", 47 | "integrity": "sha512-lUWFxGKyhraKCW8Qghz6Z0f2l/PqB1W3AO0HKJzGIQ5JRSlR651ekJDiGJbBT4sRNNv5ddnSGVEnsxP9XRCVpQ==", 48 | "requires": { 49 | "bluebird": "^3.5.1", 50 | "lodash": "^4.17.3", 51 | "strip-bom": "^2.0.0" 52 | } 53 | }, 54 | "debug": { 55 | "version": "4.1.1", 56 | "resolved": "https://registry.npmjs.org/debug/-/debug-4.1.1.tgz", 57 | "integrity": "sha512-pYAIzeRo8J6KPEaJ0VWOh5Pzkbw/RetuzehGM7QRRX5he4fPHx2rdKMB256ehJCkX+XRQm16eZLqLNS8RSZXZw==", 58 | "requires": { 59 | "ms": "^2.1.1" 60 | } 61 | }, 62 | "duplexify": { 63 | "version": "3.7.1", 64 | "resolved": "https://registry.npmjs.org/duplexify/-/duplexify-3.7.1.tgz", 65 | "integrity": "sha512-07z8uv2wMyS51kKhD1KsdXJg5WQ6t93RneqRxUHnskXVtlYYkLqM0gqStQZ3pj073g687jPCHrqNfCzawLYh5g==", 66 | "requires": { 67 | "end-of-stream": "^1.0.0", 68 | "inherits": "^2.0.1", 69 | "readable-stream": "^2.0.0", 70 | "stream-shift": "^1.0.0" 71 | } 72 | }, 73 | "end-of-stream": { 74 | "version": "1.4.4", 75 | "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.4.tgz", 76 | "integrity": "sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q==", 77 | "requires": { 78 | "once": "^1.4.0" 79 | } 80 | }, 81 | "generate-object-property": { 82 | "version": "1.2.0", 83 | "resolved": "https://registry.npmjs.org/generate-object-property/-/generate-object-property-1.2.0.tgz", 84 | "integrity": "sha1-nA4cQDCM6AT0eDYYuTf6iPmdUNA=", 85 | "requires": { 86 | "is-property": "^1.0.0" 87 | } 88 | }, 89 | "inherits": { 90 | "version": "2.0.4", 91 | "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", 92 | "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==" 93 | }, 94 | "is-property": { 95 | "version": "1.0.2", 96 | "resolved": "https://registry.npmjs.org/is-property/-/is-property-1.0.2.tgz", 97 | "integrity": "sha1-V/4cTkhHTt1lsJkR8msc1Ald2oQ=" 98 | }, 99 | "is-utf8": { 100 | "version": "0.2.1", 101 | "resolved": "https://registry.npmjs.org/is-utf8/-/is-utf8-0.2.1.tgz", 102 | "integrity": "sha1-Sw2hRCEE0bM2NA6AeX6GXPOffXI=" 103 | }, 104 | "isarray": { 105 | "version": "1.0.0", 106 | "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", 107 | "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=" 108 | }, 109 | "json-stringify-safe": { 110 | "version": "5.0.1", 111 | "resolved": "https://registry.npmjs.org/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz", 112 | "integrity": "sha1-Epai1Y/UXxmg9s4B1lcB4sc1tus=" 113 | }, 114 | "json-to-csv-stream": { 115 | "version": "1.1.0", 116 | "resolved": "https://registry.npmjs.org/json-to-csv-stream/-/json-to-csv-stream-1.1.0.tgz", 117 | "integrity": "sha512-q9YcRFPYmnN7ocKveVcGhf0tFjP3g5p8tq1tcVWEGw180MeRToLRTdSyDsf8oUshhxwy0khNznd7itYF4YuC8g==", 118 | "requires": { 119 | "JSONStream": "~1.1.2", 120 | "csv-write-stream": "~2.0.0", 121 | "pumpify": "~1.3.4", 122 | "through2": "~2.0.1" 123 | } 124 | }, 125 | "jsonparse": { 126 | "version": "1.3.1", 127 | "resolved": "https://registry.npmjs.org/jsonparse/-/jsonparse-1.3.1.tgz", 128 | "integrity": "sha1-P02uSpH6wxX3EGL4UhzCOfE2YoA=" 129 | }, 130 | "lodash": { 131 | "version": "4.17.20", 132 | "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.20.tgz", 133 | "integrity": "sha512-PlhdFcillOINfeV7Ni6oF1TAEayyZBoZ8bcshTHqOYJYlrqzRK5hagpagky5o4HfCzzd1TRkXPMFq6cKk9rGmA==" 134 | }, 135 | "minimist": { 136 | "version": "1.2.5", 137 | "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.5.tgz", 138 | "integrity": "sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw==" 139 | }, 140 | "ms": { 141 | "version": "2.1.2", 142 | "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", 143 | "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" 144 | }, 145 | "ndjson": { 146 | "version": "1.5.0", 147 | "resolved": "https://registry.npmjs.org/ndjson/-/ndjson-1.5.0.tgz", 148 | "integrity": "sha1-rmA7NrE0vOw0e0UkIrC/mNWDLsg=", 149 | "requires": { 150 | "json-stringify-safe": "^5.0.1", 151 | "minimist": "^1.2.0", 152 | "split2": "^2.1.0", 153 | "through2": "^2.0.3" 154 | } 155 | }, 156 | "once": { 157 | "version": "1.4.0", 158 | "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", 159 | "integrity": "sha1-WDsap3WWHUsROsF9nFC6753Xa9E=", 160 | "requires": { 161 | "wrappy": "1" 162 | } 163 | }, 164 | "process-nextick-args": { 165 | "version": "2.0.1", 166 | "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz", 167 | "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==" 168 | }, 169 | "pump": { 170 | "version": "2.0.1", 171 | "resolved": "https://registry.npmjs.org/pump/-/pump-2.0.1.tgz", 172 | "integrity": "sha512-ruPMNRkN3MHP1cWJc9OWr+T/xDP0jhXYCLfJcBuX54hhfIBnaQmAUMfDcG4DM5UMWByBbJY69QSphm3jtDKIkA==", 173 | "requires": { 174 | "end-of-stream": "^1.1.0", 175 | "once": "^1.3.1" 176 | } 177 | }, 178 | "pumpify": { 179 | "version": "1.3.6", 180 | "resolved": "https://registry.npmjs.org/pumpify/-/pumpify-1.3.6.tgz", 181 | "integrity": "sha512-BurGAcvezsINL5US9T9wGHHcLNrG6MCp//ECtxron3vcR+Rfx5Anqq7HbZXNJvFQli8FGVsWCAvywEJFV5Hx/Q==", 182 | "requires": { 183 | "duplexify": "^3.5.3", 184 | "inherits": "^2.0.3", 185 | "pump": "^2.0.0" 186 | } 187 | }, 188 | "readable-stream": { 189 | "version": "2.3.7", 190 | "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.7.tgz", 191 | "integrity": "sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw==", 192 | "requires": { 193 | "core-util-is": "~1.0.0", 194 | "inherits": "~2.0.3", 195 | "isarray": "~1.0.0", 196 | "process-nextick-args": "~2.0.0", 197 | "safe-buffer": "~5.1.1", 198 | "string_decoder": "~1.1.1", 199 | "util-deprecate": "~1.0.1" 200 | } 201 | }, 202 | "safe-buffer": { 203 | "version": "5.1.2", 204 | "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", 205 | "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" 206 | }, 207 | "split2": { 208 | "version": "2.2.0", 209 | "resolved": "https://registry.npmjs.org/split2/-/split2-2.2.0.tgz", 210 | "integrity": "sha512-RAb22TG39LhI31MbreBgIuKiIKhVsawfTgEGqKHTK87aG+ul/PB8Sqoi3I7kVdRWiCfrKxK3uo4/YUkpNvhPbw==", 211 | "requires": { 212 | "through2": "^2.0.2" 213 | } 214 | }, 215 | "sprintf-js": { 216 | "version": "1.0.3", 217 | "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz", 218 | "integrity": "sha1-BOaSb2YolTVPPdAVIDYzuFcpfiw=" 219 | }, 220 | "stream-concat": { 221 | "version": "0.3.0", 222 | "resolved": "https://registry.npmjs.org/stream-concat/-/stream-concat-0.3.0.tgz", 223 | "integrity": "sha512-HTqtuDjG9J2mKoHYgW827oA0WVx8o997vj2GRE/wrUMSkMZSzMa1OXLPqYTxEYwcuvuT8n6CsxjyTinI6ezfmQ==" 224 | }, 225 | "stream-shift": { 226 | "version": "1.0.1", 227 | "resolved": "https://registry.npmjs.org/stream-shift/-/stream-shift-1.0.1.tgz", 228 | "integrity": "sha512-AiisoFqQ0vbGcZgQPY1cdP2I76glaVA/RauYR4G4thNFgkTqr90yXTo4LYX60Jl+sIlPNHHdGSwo01AvbKUSVQ==" 229 | }, 230 | "string_decoder": { 231 | "version": "1.1.1", 232 | "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", 233 | "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", 234 | "requires": { 235 | "safe-buffer": "~5.1.0" 236 | } 237 | }, 238 | "strip-bom": { 239 | "version": "2.0.0", 240 | "resolved": "https://registry.npmjs.org/strip-bom/-/strip-bom-2.0.0.tgz", 241 | "integrity": "sha1-YhmoVhZSBJHzV4i9vxRHqZx+aw4=", 242 | "requires": { 243 | "is-utf8": "^0.2.0" 244 | } 245 | }, 246 | "through": { 247 | "version": "2.3.8", 248 | "resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz", 249 | "integrity": "sha1-DdTJ/6q8NXlgsbckEV1+Doai4fU=" 250 | }, 251 | "through2": { 252 | "version": "2.0.5", 253 | "resolved": "https://registry.npmjs.org/through2/-/through2-2.0.5.tgz", 254 | "integrity": "sha512-/mrRod8xqpA+IHSLyGCQ2s8SPHiCDEeQJSep1jqLYeEUClOFG2Qsh+4FU6G9VeqpZnGW/Su8LQGc4YKni5rYSQ==", 255 | "requires": { 256 | "readable-stream": "~2.3.6", 257 | "xtend": "~4.0.1" 258 | } 259 | }, 260 | "util-deprecate": { 261 | "version": "1.0.2", 262 | "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", 263 | "integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8=" 264 | }, 265 | "wrappy": { 266 | "version": "1.0.2", 267 | "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", 268 | "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=" 269 | }, 270 | "xtend": { 271 | "version": "4.0.2", 272 | "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz", 273 | "integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==" 274 | } 275 | } 276 | } 277 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "concat-large-files", 3 | "version": "1.0.0", 4 | "description": "", 5 | "main": "index.js", 6 | "type": "module", 7 | "scripts": { 8 | "start": "DEBUG=app* node --harmony-top-level-await index.js", 9 | "test": "echo \"Error: no test specified\" && exit 1" 10 | }, 11 | "keywords": [], 12 | "author": "", 13 | "license": "ISC", 14 | "dependencies": { 15 | "csvtojson": "^2.0.10", 16 | "debug": "^4.1.1", 17 | "json-to-csv-stream": "^1.1.0", 18 | "stream-concat": "^0.3.0" 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /script.txt: -------------------------------------------------------------------------------- 1 | node -v = 14 2 | 3 | mkdir concat-large-files 4 | npm init -y 5 | npm i debug@4.1 6 | touch index.js 7 | 8 | console.log(await Promise.resolve(0)) 9 | 10 | package.json 11 | type: module 12 | node --harmony-top-level-await index.js 13 | 14 | kaggle 15 | -> mostrar schema 1 e schema 2 16 | -> download base 1 e base 2 17 | 18 | cat dataset/2018.csv| wc -l 19 | cat dataset/2017.csv| wc -l 20 | node -p '51393+98856' 21 | 150249 22 | 23 | node -v 24 | 25 | index.js 26 | files 27 | combinedStreams with file[0] 28 | 29 | const handleStream = Transform({ 30 | transform: (chunk, enc, cb) => { 31 | const output = chunk 32 | console.log( 33 | 'output', output.toString() 34 | ) 35 | 36 | await pipelineAsync( 37 | combinedStreams, 38 | handleStream, 39 | ) 40 | show output 41 | 42 | npm i csvtojson@2.0 43 | await pipelineAsync( 44 | combinedStreams, 45 | csvtojson(), 46 | handleStream, 47 | ) 48 | 49 | const handleStream = Transform({ 50 | transform: (chunk, enc, cb) => { 51 | 52 | const data = JSON.parse(chunk) 53 | const output = { 54 | id: data.Respondent, 55 | country: data.Country, 56 | } 57 | log(`id: ${output.id}`) 58 | return cb(null, JSON.stringify(output)); 59 | }, 60 | }); 61 | 62 | npm i json-to-csv-stream@1.1 63 | const finalStreamFile = createWriteStream(output); 64 | await pipelineAsync( 65 | combinedStreams, 66 | csvtojson(), 67 | handleStream, 68 | jsontocsv(), 69 | finalStreamFile 70 | ) 71 | 72 | rm DEBUG=* do package.json 73 | 74 | npm i stream-concat@0.3 75 | const combinedStreams = new StreamConcat(streams); 76 | 77 | cat final.csv| wc -l 78 | mostrar final.csv 79 | 80 | cp 2018.csv 2020.csv 81 | cp 2018.csv 2021.csv 82 | cp 2018.csv 2022.csv 83 | cp 2018.csv 2023.csv 84 | nt 85 | node -p '51393+(98856*6)' 86 | 644529 87 | 88 | --------------------------------------------------------------------------------