├── .gitignore ├── .travis.yml ├── LICENSE ├── README.md ├── index.js ├── package-lock.json ├── package.json └── test ├── merge.js └── split.js /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules/ 2 | *.split* 3 | output/ 4 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: node_js 2 | node_js: 3 | - "node" 4 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2013-2018 GitHub Inc. 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining 4 | a copy of this software and associated documentation files (the 5 | "Software"), to deal in the Software without restriction, including 6 | without limitation the rights to use, copy, modify, merge, publish, 7 | distribute, sublicense, and/or sell copies of the Software, and to 8 | permit persons to whom the Software is furnished to do so, subject to 9 | the following conditions: 10 | 11 | The above copyright notice and this permission notice shall be 12 | included in all copies or substantial portions of the Software. 13 | 14 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 15 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 16 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 17 | NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE 18 | LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 19 | OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 20 | WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # split-file-stream 2 | [![npm version](https://badge.fury.io/js/split-file-stream.svg)](https://badge.fury.io/js/split-file-stream) 3 | 4 | Partition your readable streams into multiple files or combine files into one merged readable stream. 5 | 6 | ### You should use this module if: 7 | * You have a readable stream and want to save to multiple files 8 | * Other solutions require you to supply a path as the source. 9 | * This means you'd have to write your readable stream first, before partitioning the data. 10 | * Faster solution as disk writes are slow 11 | * Vice Versa: You want to pipe the merge output as a stream 12 | * Pipe the merge output to the response of a web request 13 | * Pipe encrypted partitioned files merge output to a decryption stream 14 | 15 | ## Installation 16 | ```sh 17 | npm install --save split-file-stream 18 | ``` 19 | 20 | ## Usage 21 | To split a read stream into multiple files: 22 | ```javascript 23 | var splitFileStream = require("split-file-stream"); 24 | let maxFileSize = 1024; // 1024 bytes per file 25 | let outputPath = __dirname + "/outputFiles"; // file path partition's prefix 26 | 27 | splitFileStream.split(readStream, maxFileSize, outputPath, (error, filePaths) => { 28 | /* If an error occured, filePaths will still contain all files that were written */ 29 | if (error) throw error; // Alternatively you could just log the error instead of throwing: if (error) console.error(error) 30 | 31 | console.log("This is an array of my new files:", filePaths); 32 | /* stream will be saved to files in the path ∈ { ./outputFiles.split-x | x ∈ N } */ 33 | }); 34 | ``` 35 | 36 | To merge a set of files together into one output stream: 37 | ```javascript 38 | var splitFileStream = require("split-file-stream"); 39 | let filePaths = filePaths; // take this filePaths array from the output of the split method 40 | 41 | splitFileStream.mergeFilesToStream(filePaths, (outStream) => { 42 | outStream.on("data", (chunk) => { 43 | console.log(`Received chunk of ${chunk.length} bytes`); 44 | }); 45 | 46 | outStream.on("end", () => { 47 | console.log("Out stream closed. All files have been merged") 48 | }); 49 | }); 50 | ``` 51 | 52 | To merge a set of files to write to disk: 53 | ```javascript 54 | // Note: You can also do this with the mergeFilesToStream method and piping the stream to a fs writeStream. 55 | var splitFileStream = require("split-file-stream"); 56 | let filePaths = filePaths; // take this filePaths array from the output of the split method 57 | let outputPath = __dirname + "/outputFile"; 58 | 59 | splitFileStream.mergeFilesToDisk(filePaths, outputPath, () => { 60 | console.log("Finished merging files"); 61 | }); 62 | ``` 63 | 64 | Example usage of the mergeFilesToDisk method using the mergeFilesToStream method: 65 | ```javascript 66 | var fs = require("fs"); 67 | var splitFileStream = require("split-file-stream"); 68 | let filePaths = filePaths; // take this filePaths array from the output of the split method 69 | let outputPath = __dirname + "/outputFile"; 70 | 71 | splitFileStream.mergeFilesToStream(filePaths, (outStream) => { 72 | let writeStream = fs.createWriteStream(outputPath); 73 | outStream.pipe(writeStream); 74 | }); 75 | ``` 76 | 77 | To split a read stream with a custom function that determines the file name: 78 | ```javascript 79 | var splitFileStream = require("split-file-stream"); 80 | let maxFileSize = 1024; // 1024 bytes per file 81 | let outputPath = __dirname + "/outputFiles"; // file path partition's prefix 82 | var customSplit = splitFileStream.getSplitWithGenFilePath((n) => `${outputPath}-${(n + 1)}`) 83 | 84 | customSplit(readStream, maxFileSize, (error, filePaths) => { 85 | /* If an error occured, filePaths will still contain all files that were written */ 86 | if (error) throw error; // Alternatively you could just log the error instead of throwing: if (error) console.error(error) 87 | 88 | console.log("This is an array of my new files:", filePaths); 89 | }); 90 | ``` 91 | 92 | Alternatively, if you'd like a lower level API for splitting a stream, you can use `_splitToStream`. This function will split your readable stream into multiple streams. This function is what is used to implement the split function. 93 | ```javascript 94 | var stream = require("stream"); 95 | var splitFileStream = require("split-file-stream"); 96 | let partitionStreamSize = 1024; // 1024 bytes per partition 97 | const outStreamCreate = (partitionNum) => { 98 | return stream.passThrough(); 99 | }; 100 | 101 | splitFileStream._splitToStream(outStreamCreate, readStream, partitionStreamSize, (error, outStreams) => { 102 | /* If an error occured, filePaths will still contain all files that were written */ 103 | if (error) throw error; // Alternatively you could just log the error instead of throwing: if (error) console.error(error) 104 | 105 | console.log("This is an array of the created output streams:", outStreams); 106 | }); 107 | ``` 108 | -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | const fs = require("fs"); 2 | const path = require("path"); 3 | const stream = require("stream"); 4 | 5 | const _mergeFiles = (partitionIndex, partitionNames, combinationStream, callback) => { 6 | if (partitionIndex == partitionNames.length) { 7 | combinationStream.end(); 8 | return callback(); 9 | } 10 | let partitionFileStream = fs.createReadStream(partitionNames[partitionIndex]); 11 | 12 | partitionFileStream.on("data", (chunk) => combinationStream.write(chunk)); 13 | partitionFileStream.on("end", () => _mergeFiles(++partitionIndex, partitionNames, combinationStream, callback)); 14 | }; 15 | 16 | module.exports.mergeFilesToDisk = (partitionNames, outputPath, callback) => { 17 | let combinationStream = fs.createWriteStream(outputPath); 18 | _mergeFiles(0, partitionNames, combinationStream, callback); 19 | }; 20 | 21 | module.exports.mergeFilesToStream = (partitionNames, callback) => { 22 | let combinationStream = new stream.PassThrough(); 23 | _mergeFiles(0, partitionNames, combinationStream, () => { }); 24 | callback(combinationStream); 25 | }; 26 | 27 | 28 | const _splitToStream = (outStreamCreate, fileStream, partitionStreamSize, callback) => { 29 | const outStreams = [], { highWaterMark: defaultChunkSize, objectMode: isObjectMode } = fileStream._readableState; 30 | let currentOutStream, currentFileSize = 0, fileStreamEnded = false, finishedWriteStreams = 0, openStream = false, partitionNum = 0, err = null; 31 | 32 | const endCurrentWriteStream = () => { 33 | currentOutStream.end(); 34 | currentOutStream = null; 35 | currentFileSize = 0; 36 | openStream = false; 37 | }; 38 | 39 | const createNewWriteStream = () => { 40 | currentOutStream = outStreamCreate(partitionNum); 41 | currentOutStream.on("finish", writeStreamFinishHandler); 42 | outStreams.push(currentOutStream); 43 | partitionNum++; 44 | } 45 | 46 | const writeStreamFinishHandler = () => { 47 | finishedWriteStreams++; 48 | if (fileStreamEnded && partitionNum == finishedWriteStreams) { 49 | callback(err, outStreams); 50 | } 51 | }; 52 | 53 | fileStream.on("readable", () => { 54 | let chunk; 55 | while (null !== (chunk = fileStream.read(Math.min(partitionStreamSize - currentFileSize, defaultChunkSize)))) { 56 | if (openStream == false) { 57 | createNewWriteStream(); 58 | openStream = true; 59 | } 60 | // A Readable stream in object mode will always return a single item from a call to readable.read(size), regardless of the value of the size argument. 61 | const writeChunk = isObjectMode ? JSON.stringify(chunk) : chunk 62 | if (writeChunk.length > partitionStreamSize) { 63 | // In objectMode one object is read from the stream, it could be that the size is bigger than the partition size 64 | err = new RangeError("Could not fit object into maxFileSize"); 65 | break; 66 | } 67 | if ((currentFileSize + writeChunk.length) > partitionStreamSize) { 68 | endCurrentWriteStream(); 69 | createNewWriteStream(); 70 | } 71 | currentOutStream.write(writeChunk); 72 | currentFileSize += writeChunk.length; 73 | if (currentFileSize == partitionStreamSize) { 74 | endCurrentWriteStream(); 75 | } 76 | } 77 | }); 78 | 79 | fileStream.on("end", () => { 80 | if (currentOutStream) { 81 | endCurrentWriteStream(); 82 | } 83 | fileStreamEnded = true; 84 | }); 85 | }; 86 | 87 | const split = (fileStream, maxFileSize, rootFilePath, callback) => 88 | _split(fileStream, maxFileSize, (n) => `${rootFilePath}.split-${n}`, callback); 89 | 90 | const getSplitWithGenFilePath = (generateFilePath) => 91 | (f, m, callback) => _split(f, m, generateFilePath, callback); 92 | 93 | const _split = (fileStream, maxFileSize, generateFilePath, callback) => { 94 | if (maxFileSize <= 0) { 95 | throw new RangeError("maxFileSize must be greater than 0"); 96 | } 97 | const partitionNames = []; 98 | 99 | const outStreamCreate = (partitionNum) => { 100 | let filePath = generateFilePath(partitionNum); 101 | return fs.createWriteStream(filePath); 102 | }; 103 | _splitToStream(outStreamCreate, fileStream, maxFileSize, (err, fileWriteStreams) => { 104 | fileWriteStreams.forEach((fileWriteStream) => partitionNames.push(fileWriteStream["path"])); 105 | callback(err, partitionNames); 106 | }); 107 | }; 108 | 109 | module.exports.split = split; 110 | module.exports.getSplitWithGenFilePath = getSplitWithGenFilePath; 111 | module.exports._splitToStream = _splitToStream; 112 | -------------------------------------------------------------------------------- /package-lock.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "split-file-stream", 3 | "version": "1.3.0", 4 | "lockfileVersion": 1, 5 | "requires": true, 6 | "dependencies": { 7 | "balanced-match": { 8 | "version": "1.0.0", 9 | "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.0.tgz", 10 | "integrity": "sha1-ibTRmasr7kneFk6gK4nORi1xt2c=", 11 | "dev": true 12 | }, 13 | "brace-expansion": { 14 | "version": "1.1.11", 15 | "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", 16 | "integrity": "sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==", 17 | "dev": true, 18 | "requires": { 19 | "balanced-match": "^1.0.0", 20 | "concat-map": "0.0.1" 21 | } 22 | }, 23 | "browser-stdout": { 24 | "version": "1.3.1", 25 | "resolved": "https://registry.npmjs.org/browser-stdout/-/browser-stdout-1.3.1.tgz", 26 | "integrity": "sha512-qhAVI1+Av2X7qelOfAIYwXONood6XlZE/fXaBSmW/T5SzLAmCgzi+eiWE7fUvbHaeNBQH13UftjpXxsfLkMpgw==", 27 | "dev": true 28 | }, 29 | "commander": { 30 | "version": "2.15.1", 31 | "resolved": "https://registry.npmjs.org/commander/-/commander-2.15.1.tgz", 32 | "integrity": "sha512-VlfT9F3V0v+jr4yxPc5gg9s62/fIVWsd2Bk2iD435um1NlGMYdVCq+MjcXnhYq2icNOizHr1kK+5TI6H0Hy0ag==", 33 | "dev": true 34 | }, 35 | "concat-map": { 36 | "version": "0.0.1", 37 | "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", 38 | "integrity": "sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=", 39 | "dev": true 40 | }, 41 | "debug": { 42 | "version": "3.1.0", 43 | "resolved": "https://registry.npmjs.org/debug/-/debug-3.1.0.tgz", 44 | "integrity": "sha512-OX8XqP7/1a9cqkxYw2yXss15f26NKWBpDXQd0/uK/KPqdQhxbPa994hnzjcE2VqQpDslf55723cKPUOGSmMY3g==", 45 | "dev": true, 46 | "requires": { 47 | "ms": "2.0.0" 48 | } 49 | }, 50 | "diff": { 51 | "version": "3.5.0", 52 | "resolved": "https://registry.npmjs.org/diff/-/diff-3.5.0.tgz", 53 | "integrity": "sha512-A46qtFgd+g7pDZinpnwiRJtxbC1hpgf0uzP3iG89scHk0AUC7A1TGxf5OiiOUv/JMZR8GOt8hL900hV0bOy5xA==", 54 | "dev": true 55 | }, 56 | "escape-string-regexp": { 57 | "version": "1.0.5", 58 | "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-1.0.5.tgz", 59 | "integrity": "sha1-G2HAViGQqN/2rjuyzwIAyhMLhtQ=", 60 | "dev": true 61 | }, 62 | "fs.realpath": { 63 | "version": "1.0.0", 64 | "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", 65 | "integrity": "sha1-FQStJSMVjKpA20onh8sBQRmU6k8=", 66 | "dev": true 67 | }, 68 | "glob": { 69 | "version": "7.1.2", 70 | "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.2.tgz", 71 | "integrity": "sha512-MJTUg1kjuLeQCJ+ccE4Vpa6kKVXkPYJ2mOCQyUuKLcLQsdrMCpBPUi8qVE6+YuaJkozeA9NusTAw3hLr8Xe5EQ==", 72 | "dev": true, 73 | "requires": { 74 | "fs.realpath": "^1.0.0", 75 | "inflight": "^1.0.4", 76 | "inherits": "2", 77 | "minimatch": "^3.0.4", 78 | "once": "^1.3.0", 79 | "path-is-absolute": "^1.0.0" 80 | } 81 | }, 82 | "growl": { 83 | "version": "1.10.5", 84 | "resolved": "https://registry.npmjs.org/growl/-/growl-1.10.5.tgz", 85 | "integrity": "sha512-qBr4OuELkhPenW6goKVXiv47US3clb3/IbuWF9KNKEijAy9oeHxU9IgzjvJhHkUzhaj7rOUD7+YGWqUjLp5oSA==", 86 | "dev": true 87 | }, 88 | "has-flag": { 89 | "version": "3.0.0", 90 | "resolved": "https://registry.npmjs.org/has-flag/-/has-flag-3.0.0.tgz", 91 | "integrity": "sha1-tdRU3CGZriJWmfNGfloH87lVuv0=", 92 | "dev": true 93 | }, 94 | "he": { 95 | "version": "1.1.1", 96 | "resolved": "https://registry.npmjs.org/he/-/he-1.1.1.tgz", 97 | "integrity": "sha1-k0EP0hsAlzUVH4howvJx80J+I/0=", 98 | "dev": true 99 | }, 100 | "inflight": { 101 | "version": "1.0.6", 102 | "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", 103 | "integrity": "sha1-Sb1jMdfQLQwJvJEKEHW6gWW1bfk=", 104 | "dev": true, 105 | "requires": { 106 | "once": "^1.3.0", 107 | "wrappy": "1" 108 | } 109 | }, 110 | "inherits": { 111 | "version": "2.0.4", 112 | "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", 113 | "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==", 114 | "dev": true 115 | }, 116 | "minimatch": { 117 | "version": "3.0.4", 118 | "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-3.0.4.tgz", 119 | "integrity": "sha512-yJHVQEhyqPLUTgt9B83PXu6W3rx4MvvHvSUvToogpwoGDOUQ+yDrR0HRot+yOCdCO7u4hX3pWft6kWBBcqh0UA==", 120 | "dev": true, 121 | "requires": { 122 | "brace-expansion": "^1.1.7" 123 | } 124 | }, 125 | "minimist": { 126 | "version": "0.0.8", 127 | "resolved": "https://registry.npmjs.org/minimist/-/minimist-0.0.8.tgz", 128 | "integrity": "sha1-hX/Kv8M5fSYluCKCYuhqp6ARsF0=", 129 | "dev": true 130 | }, 131 | "mkdirp": { 132 | "version": "0.5.1", 133 | "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-0.5.1.tgz", 134 | "integrity": "sha1-MAV0OOrGz3+MR2fzhkjWaX11yQM=", 135 | "dev": true, 136 | "requires": { 137 | "minimist": "0.0.8" 138 | } 139 | }, 140 | "mocha": { 141 | "version": "5.2.0", 142 | "resolved": "https://registry.npmjs.org/mocha/-/mocha-5.2.0.tgz", 143 | "integrity": "sha512-2IUgKDhc3J7Uug+FxMXuqIyYzH7gJjXECKe/w43IGgQHTSj3InJi+yAA7T24L9bQMRKiUEHxEX37G5JpVUGLcQ==", 144 | "dev": true, 145 | "requires": { 146 | "browser-stdout": "1.3.1", 147 | "commander": "2.15.1", 148 | "debug": "3.1.0", 149 | "diff": "3.5.0", 150 | "escape-string-regexp": "1.0.5", 151 | "glob": "7.1.2", 152 | "growl": "1.10.5", 153 | "he": "1.1.1", 154 | "minimatch": "3.0.4", 155 | "mkdirp": "0.5.1", 156 | "supports-color": "5.4.0" 157 | } 158 | }, 159 | "ms": { 160 | "version": "2.0.0", 161 | "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", 162 | "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=", 163 | "dev": true 164 | }, 165 | "once": { 166 | "version": "1.4.0", 167 | "resolved": "https://registry.npmjs.org/once/-/once-1.4.0.tgz", 168 | "integrity": "sha1-WDsap3WWHUsROsF9nFC6753Xa9E=", 169 | "dev": true, 170 | "requires": { 171 | "wrappy": "1" 172 | } 173 | }, 174 | "path-is-absolute": { 175 | "version": "1.0.1", 176 | "resolved": "https://registry.npmjs.org/path-is-absolute/-/path-is-absolute-1.0.1.tgz", 177 | "integrity": "sha1-F0uSaHNVNP+8es5r9TpanhtcX18=", 178 | "dev": true 179 | }, 180 | "supports-color": { 181 | "version": "5.4.0", 182 | "resolved": "https://registry.npmjs.org/supports-color/-/supports-color-5.4.0.tgz", 183 | "integrity": "sha512-zjaXglF5nnWpsq470jSv6P9DwPvgLkuapYmfDm3JWOm0vkNTVF2tI4UrN2r6jH1qM/uc/WtxYY1hYoA2dOKj5w==", 184 | "dev": true, 185 | "requires": { 186 | "has-flag": "^3.0.0" 187 | } 188 | }, 189 | "wrappy": { 190 | "version": "1.0.2", 191 | "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", 192 | "integrity": "sha1-tSQ9jz7BqjXxNkYFvA0QNuMKtp8=", 193 | "dev": true 194 | } 195 | } 196 | } 197 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "split-file-stream", 3 | "version": "2.0.0", 4 | "description": "Partition your readable streams into multiple files.", 5 | "main": "index.js", 6 | "scripts": { 7 | "test": "mkdir -p test/output && mocha test" 8 | }, 9 | "keywords": [ 10 | "File Partitioning", 11 | "streams", 12 | "split stream", 13 | "merge stream", 14 | "split", 15 | "merge" 16 | ], 17 | "repository": { 18 | "type": "git", 19 | "url": "git+https://github.com/dannycho7/split-file-stream.git" 20 | }, 21 | "author": "Danny Cho", 22 | "license": "ISC", 23 | "devDependencies": { 24 | "mocha": "^5.2.0" 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /test/merge.js: -------------------------------------------------------------------------------- 1 | const assert = require("assert"); 2 | const fs = require("fs"); 3 | const stream = require("stream"); 4 | const splitFileStream = require(".."); 5 | 6 | // This will run after each test in all test modules. 7 | afterEach(function(done){ 8 | fs.readdir(__dirname + "/output", function(err, files){ 9 | for(var file of files){ 10 | fs.unlink(__dirname + "/output/" + file, function(err){ 11 | if (err) return done(err); 12 | }); 13 | } 14 | done(); 15 | }); 16 | }) 17 | 18 | describe("#mergeFilesToDisk", () => { 19 | it("Should merge 2 1mb partitions into one file", (done) => { 20 | let readStream = new stream.PassThrough(); 21 | readStream.end(new Buffer.alloc(1024 * 1024 * 2)); 22 | 23 | splitFileStream.split(readStream, 1024 * 1024 * 1, __dirname + "/output/ff", (err, filePaths) => { 24 | assert.strictEqual(null, err); 25 | assert.strictEqual(2, filePaths.length); 26 | let mergeFilePath = __dirname + "/output/ff.fullfile"; 27 | splitFileStream.mergeFilesToDisk(filePaths, mergeFilePath, () => { 28 | fs.stat(mergeFilePath, (err, stats) => { 29 | assert.strictEqual(1024 * 1024 * 2, stats.size); 30 | return done(); 31 | }); 32 | }); 33 | }); 34 | }); 35 | }); 36 | 37 | describe("#mergeFilesToStream", () => { 38 | it("Should merge 2 1mb partitions into one file stream", (done) => { 39 | let readStream = new stream.PassThrough(); 40 | readStream.end(new Buffer.alloc(1024 * 1024 * 2)); 41 | 42 | splitFileStream.split(readStream, 1024 * 1024 * 1, __dirname + "/output/ff", (err, filePaths) => { 43 | assert.strictEqual(null, err); 44 | assert.strictEqual(2, filePaths.length); 45 | let mergeFilePath = __dirname + "/output/ff.fullfile"; 46 | splitFileStream.mergeFilesToStream(filePaths, (stream) => { 47 | let dataLength = 0; 48 | stream.on("data", (chunk) => { 49 | dataLength += chunk.length; 50 | }); 51 | 52 | stream.on("finish", () => { 53 | assert.strictEqual(dataLength, 1024 * 1024 * 2); 54 | return done(); 55 | }); 56 | }); 57 | }); 58 | }); 59 | }); 60 | -------------------------------------------------------------------------------- /test/split.js: -------------------------------------------------------------------------------- 1 | const assert = require("assert"); 2 | const { AssertionError } = assert; 3 | const fs = require("fs"); 4 | const stream = require("stream"); 5 | const splitFileStream = require(".."); 6 | 7 | describe("#split", () => { 8 | it("should throw on maxFileSize <= 0", (done) => { 9 | let readStream = new stream.PassThrough(); 10 | readStream.end("abcde"); 11 | 12 | let brokenSplitCall = () => 13 | splitFileStream.split(readStream, 0, __dirname + "/output/ff", () => {}); 14 | 15 | assert.throws(brokenSplitCall, RangeError); 16 | return done(); 17 | }); 18 | 19 | it("should create 5 partitions for a 5 word string of 1 byte partitions", (done) => { 20 | let readStream = new stream.PassThrough(); 21 | readStream.end("abcde"); 22 | 23 | splitFileStream.split(readStream, 1, __dirname + "/output/ff", (err, filePaths) => { 24 | assert.strictEqual(null, err); 25 | assert.strictEqual(5, filePaths.length); 26 | return done(); 27 | }); 28 | }); 29 | 30 | it("should create 2 partitions for 100mb file of 50mb chunks", (done) => { 31 | let readStream = new stream.PassThrough(); 32 | readStream.end(new Buffer.alloc(1024 * 1024 * 100)); 33 | 34 | splitFileStream.split(readStream, 1024 * 1024 * 50, __dirname + "/output/ff", (err, filePaths) => { 35 | assert.strictEqual(null, err); 36 | assert.strictEqual(2, filePaths.length); 37 | return done(); 38 | }); 39 | }); 40 | 41 | it("should create partitions that retain the same data", (done) => { 42 | let readStream = new stream.PassThrough(), inStreamContents = "CORRECT"; 43 | readStream.end(inStreamContents); 44 | 45 | splitFileStream.split(readStream, 1, __dirname + "/output/ff", (err, filePaths) => { 46 | assert.strictEqual(null, err); 47 | let concatString = ""; 48 | filePaths.forEach((filePath) => { 49 | let fileContent = fs.readFileSync(filePath); 50 | concatString += fileContent; 51 | }); 52 | 53 | assert.strictEqual(concatString, inStreamContents); 54 | return done(); 55 | }); 56 | }); 57 | 58 | it("should create partitions using custom generateFilePath", (done) => { 59 | let readStream = new stream.PassThrough(), inStreamContents = "CORRECT"; 60 | readStream.end(inStreamContents); 61 | 62 | let outputPath = __dirname + "/output/ff"; // file path partition's prefix 63 | let expectedFilePaths = Array.apply(null, Array(7)).map((v, i) => `${outputPath}-${i + 1}`); 64 | var customSplit = splitFileStream.getSplitWithGenFilePath((n) => `${outputPath}-${(n + 1)}`) 65 | 66 | customSplit(readStream, 1, (err, filePaths) => { 67 | assert.strictEqual(null, err); 68 | assert.strictEqual(filePaths.length, 7); 69 | assert.deepStrictEqual(filePaths, expectedFilePaths); 70 | 71 | let concatString = ""; 72 | filePaths.forEach((filePath) => { 73 | let fileContent = fs.readFileSync(filePath); 74 | concatString += fileContent; 75 | }); 76 | 77 | assert.strictEqual(concatString, inStreamContents); 78 | return done(); 79 | }); 80 | }); 81 | 82 | it("should support stringifying objectMode", (done) => { 83 | let readStream = new stream.Readable({ 84 | objectMode: true, 85 | read() { } 86 | }).pipe(new stream.PassThrough({ objectMode: true })); 87 | readStream.push({ qr: 'st' }) // 1 object is 11 bytes (2 bytes per character, including { }) 88 | readStream.push({ uv: 'wx' }) 89 | readStream.push(null) // end readStream 90 | splitFileStream.split(readStream, 22, __dirname + "/output/ff", (err, filePaths) => { 91 | assert.strictEqual(null, err); 92 | assert.strictEqual(1, filePaths.length); 93 | return done(); 94 | }); 95 | }); 96 | 97 | it("should throw on objects larger than maxFileSize in stringifying objectMode", (done) => { 98 | let readStream = new stream.Readable({ 99 | objectMode: true, 100 | read() { } 101 | }).pipe(new stream.PassThrough({ objectMode: true })); 102 | readStream.push({ ab: 'cd' }) // 11 bytes (2 bytes per character, including { }) 103 | readStream.push({ ef: 'ghi' }) // 12 bytes (2 bytes per character, including { }) 104 | readStream.push(null) // end readStream 105 | 106 | splitFileStream.split(readStream, 11, __dirname + "/output/ff", (err, filePaths) => { 107 | assert.strictEqual(2, filePaths.length); 108 | const {size: fileSize} = fs.statSync(filePaths[0]); 109 | assert.strictEqual(11, fileSize); 110 | const {size: fileSize2} = fs.statSync(filePaths[1]); 111 | assert.strictEqual(0, fileSize2); 112 | assert.notStrictEqual(err, null); 113 | var passErr = function(err) { throw err } 114 | assert.throws(function() { passErr(err) }, RangeError) 115 | done(); 116 | }) 117 | }); 118 | 119 | it("should split for objects larger than maxFileSize in stringifying objectMode", (done) => { 120 | let readStream = new stream.Readable({ 121 | objectMode: true, 122 | read() { } 123 | }).pipe(new stream.PassThrough({ objectMode: true })); 124 | readStream.push({ ab: 'cd' }) // 11 bytes (2 bytes per character, including { }) 125 | readStream.push({ ef: 'ghi' }) // 12 bytes (2 bytes per character, including { }) 126 | readStream.push(null) // end readStream 127 | 128 | splitFileStream.split(readStream, 22, __dirname + "/output/ff", (err, filePaths) => { 129 | assert.strictEqual(2, filePaths.length); 130 | const {size: fileSize} = fs.statSync(filePaths[0]); 131 | assert.strictEqual(11, fileSize); 132 | const {size: fileSize2} = fs.statSync(filePaths[1]); 133 | assert.strictEqual(12, fileSize2); 134 | assert.strictEqual(err, null); 135 | done(); 136 | }) 137 | }); 138 | 139 | it("should respect maxFileSize in readable stream in objectMode with object size matching", (done) => { 140 | const myTransform = new stream.Transform({ 141 | objectMode: true, 142 | transform(chunk, encoding, callback) { 143 | // Push the data onto the readable queue. 144 | callback(null, Object.keys(chunk) + Object.values(chunk)); // concatenate key+value 145 | } 146 | }); 147 | 148 | const readStream = new stream.Readable({ objectMode: true, read: () => { } }) 149 | readStream.push({ ij: 'kl' }) // 1 object is 8 bytes (2 byter per character, as concatenated by Transform) 150 | readStream.push({ mn: 'op' }) 151 | readStream.push(null) // end readStream 152 | splitFileStream.split(readStream.pipe(myTransform), 8, __dirname + "/output/ff", (err, filePaths) => { 153 | assert.strictEqual(null, err); 154 | assert.strictEqual(2, filePaths.length); 155 | return done(); 156 | }); 157 | }); 158 | 159 | it("should respect maxFileSize in readable stream in objectMode with object size unmatched", (done) => { 160 | const myTransform = new stream.Transform({ 161 | objectMode: true, 162 | transform(chunk, encoding, callback) { 163 | // Push the data onto the readable queue. 164 | callback(null, Object.keys(chunk) + Object.values(chunk)); // concatenate key+value 165 | } 166 | }); 167 | 168 | const readStream = new stream.Readable({ objectMode: true, read: () => { } }) 169 | readStream.push({ ij: 'kl' }) // 1 object is 8 bytes (2 byter per character, as concatenated by Transform) 170 | readStream.push({ mn: 'op' }) 171 | readStream.push(null) // end readStream 172 | splitFileStream.split(readStream.pipe(myTransform), 9, __dirname + "/output/ff", (err, filePaths) => { 173 | assert.strictEqual(null, err); 174 | assert.strictEqual(2, filePaths.length); 175 | return done(); 176 | }); 177 | }); 178 | }); 179 | --------------------------------------------------------------------------------