├── .gitignore ├── .vscode └── launch.json ├── README.md ├── bufferCache.js ├── index.js ├── package.json └── test.js /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules/ 2 | -------------------------------------------------------------------------------- /.vscode/launch.json: -------------------------------------------------------------------------------- 1 | { 2 | // Use IntelliSense to learn about possible Node.js debug attributes. 3 | // Hover to view descriptions of existing attributes. 4 | // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 5 | "version": "0.2.0", 6 | "configurations": [ 7 | { 8 | "type": "node", 9 | "request": "launch", 10 | "name": "Mocha Tests", 11 | "cwd": "${workspaceRoot}", 12 | "runtimeExecutable": "${workspaceRoot}/node_modules/.bin/mocha", 13 | "windows": { 14 | "runtimeExecutable": "${workspaceRoot}/node_modules/.bin/mocha.cmd" 15 | }, 16 | "runtimeArgs": [ 17 | "-u", 18 | "tdd", 19 | "--timeout", 20 | "999999", 21 | "--colors", 22 | "${workspaceRoot}/test" 23 | ], 24 | "program": "${workspaceRoot}/test.js", 25 | "internalConsoleOptions": "openOnSessionStart" 26 | }, 27 | // { 28 | // "type": "node", 29 | // "request": "launch", 30 | // "name": "启动程序", 31 | // "program": "${workspaceRoot}/test.js", 32 | // "cwd": "${workspaceRoot}" 33 | // }, 34 | { 35 | "type": "node", 36 | "request": "attach", 37 | "name": "附加到进程", 38 | "port": 5858 39 | } 40 | ] 41 | } -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # file-stream-upload-example 2 | 文件流转存服务 源代码 3 | 4 | ### Installation 5 | ``` 6 | npm install 7 | ``` 8 | 9 | ### Run the Test 10 | 11 | ``` 12 | npm test 13 | ``` 14 | 15 | ### How to Use 16 | 17 | 1. 部署服务端： 到这个仓库下载配套server端源码 https://github.com/andycall/file-upload-example-server 18 | 2. 之后就可以读读源码，跑下测试用例了。 -------------------------------------------------------------------------------- /bufferCache.js: -------------------------------------------------------------------------------- 1 | /** 2 | * @file 视频下载缓冲区 3 | */ 4 | 5 | class BufferCache { 6 | constructor (cutSize = 2097152) { 7 | this._cache = Buffer.alloc(0); 8 | this.cutSize = cutSize; 9 | this.readyCache = []; // 缓冲区 10 | } 11 | 12 | // 放入不同大小的buffer 13 | pushBuf (buf) { 14 | let cacheLength = this._cache.length; 15 | let bufLength = buf.length; 16 | 17 | this._cache = Buffer.concat([this._cache, buf], cacheLength + bufLength); 18 | 19 | this.cut(); 20 | } 21 | 22 | /** 23 | * 切分分片,小分片拼成大分片，超大分片切成小分片 24 | */ 25 | cut () { 26 | if (this._cache.length >= this.cutSize) { 27 | let totalLen = this._cache.length; 28 | let cutCount = Math.floor(totalLen / this.cutSize); 29 | 30 | for (let i = 0; i < cutCount; i++) { 31 | let newBuf = Buffer.alloc(this.cutSize); 32 | this._cache.copy(newBuf, 0, i * this.cutSize, (i + 1) * this.cutSize); 33 | this.readyCache.push(newBuf); 34 | } 35 | 36 | this._cache = this._cache.slice(cutCount * this.cutSize); 37 | } 38 | } 39 | 40 | /** 41 | * 获取等长的分片 42 | * @returns {Array} 43 | */ 44 | getChunks () { 45 | return this.readyCache; 46 | } 47 | 48 | /** 49 | * 获取数据包的最后一小节 50 | * @returns {*} 51 | */ 52 | getRemainChunks () { 53 | if (this._cache.length <= this.cutSize) { 54 | return this._cache; 55 | } 56 | else { 57 | this.cut(); 58 | return this.getRemainChunks(); 59 | } 60 | } 61 | } 62 | 63 | module.exports = BufferCache; -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | let request = require('request'); 4 | let fs = require('fs'); 5 | let uuid = require('node-uuid'); 6 | let crypto = require('crypto'); 7 | 8 | // 引入缓存模块 9 | let BufferCache = require('./bufferCache'); 10 | const chunkSplice = 2097152; // 2MB 11 | const RETRY_COUNT = 3; 12 | 13 | function getMd5(buffer) { 14 | let md5 = crypto.createHash('md5'); 15 | md5.update(buffer); 16 | return md5.digest('hex'); 17 | } 18 | 19 | module.exports = function (url, uploadURL, filename) { 20 | let bufferCache = new BufferCache(chunkSplice); 21 | let isFinished = false; 22 | 23 | function getChunks(options, onStartDownload, onDownloading, onDownloadClose) { 24 | return new Promise((resolve, reject) => { 25 | 'use strict'; 26 | 27 | let totalLength = 0; 28 | 29 | let httpStream = request({ 30 | method: 'GET', 31 | url: options.url 32 | }); 33 | // 由于不需要获取最终的文件，所以直接丢掉 34 | // 内存版Stream有最大限制才1Mb，非常容易写爆 35 | let writeStream = fs.createWriteStream('/dev/null'); 36 | 37 | // 联接Readable和Writable 38 | httpStream.pipe(writeStream); 39 | 40 | httpStream.on('response', (response) => { 41 | onStartDownload({ 42 | headers: response.headers, 43 | filename: options.filename, 44 | onUploadFinished: (err) => { 45 | if (err) { 46 | reject(err); 47 | } 48 | resolve(); 49 | } 50 | }); 51 | }).on('data', (chunk) => { 52 | totalLength += chunk.length; 53 | onDownloading(chunk, totalLength); 54 | }); 55 | 56 | writeStream.on('close', () => { 57 | onDownloadClose(totalLength); 58 | }); 59 | }); 60 | } 61 | 62 | function upload(url, data) { 63 | return new Promise((resolve, reject) => { 64 | request.post({ 65 | url: url, 66 | formData: data 67 | }, function (err, response, body) { 68 | if (!err && response.statusCode === 200) { 69 | resolve(body); 70 | } 71 | else { 72 | reject(err); 73 | } 74 | }); 75 | }); 76 | } 77 | 78 | function sendChunks(_opt) { 79 | let chunkId = 0; 80 | let maxSending = 0; 81 | let stopSend = false; 82 | 83 | function send(options) { 84 | let readyCache = options.readyCache; 85 | let fresh = options.fresh; 86 | let retryCount = options.retry; 87 | let chunkIndex; 88 | 89 | let chunk = null; 90 | 91 | // 新的数据 92 | if (fresh) { 93 | if (readyCache.length === 0) { 94 | return Promise.resolve(); 95 | } 96 | 97 | chunk = readyCache.shift(); 98 | chunkIndex = chunkId; 99 | chunkId++; 100 | } 101 | else { 102 | chunk = options.data; 103 | chunkIndex = options.index; 104 | } 105 | 106 | console.log(`chunkIndex: ${chunkIndex}, buffer:${getMd5(chunk)}`); 107 | maxSending++; 108 | return upload(uploadURL, { 109 | chunk: { 110 | value: chunk, 111 | options: { 112 | filename: `${_opt.filename}_IDSPLIT_` + chunkIndex 113 | } 114 | } 115 | }).then((response) => { 116 | maxSending--; 117 | let json = JSON.parse(response); 118 | 119 | if (json.errno === 0 && readyCache.length > 0) { 120 | return send({ 121 | retry: RETRY_COUNT, 122 | fresh: true, 123 | readyCache: readyCache 124 | }); 125 | } 126 | 127 | return Promise.resolve(json); 128 | }).catch(err => { 129 | if (retryCount > 0) { 130 | return send({ 131 | retry: retryCount - 1, 132 | index: chunkIndex, 133 | fresh: false, 134 | data: chunk, 135 | readyCache: readyCache 136 | }); 137 | } 138 | else { 139 | console.log(`upload failed of chunkIndex: ${chunkIndex}`); 140 | stopSend = true; 141 | return Promise.reject(err); 142 | } 143 | }); 144 | } 145 | 146 | return new Promise((resolve, reject) => { 147 | let readyCache = bufferCache.getChunks(); 148 | let threadPool = []; 149 | 150 | let sendTimer = setInterval(() => { 151 | if (maxSending < 4 && readyCache.length > 0) { 152 | for (let i = 0; i < 4; i++) { 153 | let thread = send({ 154 | retry: RETRY_COUNT, 155 | fresh: true, 156 | readyCache: readyCache 157 | }); 158 | 159 | threadPool.push(thread); 160 | } 161 | } 162 | else if ((isFinished && readyCache.length === 0) || stopSend) { 163 | clearTimeout(sendTimer); 164 | 165 | if (!stopSend) { 166 | console.log('got last chunk'); 167 | let lastChunk = bufferCache.getRemainChunks(); 168 | readyCache.push(lastChunk); 169 | threadPool.push(send({ 170 | retry: RETRY_COUNT, 171 | fresh: true, 172 | readyCache: readyCache 173 | })); 174 | } 175 | 176 | Promise.all(threadPool).then(() => { 177 | console.log('send success'); 178 | resolve(); 179 | }).catch(err => { 180 | console.log('send failed'); 181 | reject(err); 182 | }); 183 | } 184 | // not ready, wait for next interval 185 | }, 200); 186 | }); 187 | } 188 | 189 | function onStart(options) { 190 | let headers = options.headers; 191 | let filename = options.filename; 192 | let onUploadFinished = options.onUploadFinished; 193 | 194 | // console.log('start downloading, headers is :', headers); 195 | sendChunks({ 196 | filename: filename 197 | }).then(() => { 198 | onUploadFinished(); 199 | }).catch(err => { 200 | onUploadFinished(err); 201 | }); 202 | } 203 | 204 | function onData(chunk, downloadedLength) { 205 | // console.log('write ' + chunk.length + 'KB into cache'); 206 | // 都写入缓存中 207 | bufferCache.pushBuf(chunk); 208 | } 209 | 210 | function onFinished(totalLength) { 211 | let chunkCount = Math.ceil(totalLength / chunkSplice); 212 | console.log('total chunk count is:' + chunkCount); 213 | isFinished = true; 214 | } 215 | 216 | return getChunks({ 217 | url: url, 218 | filename: filename 219 | }, onStart, onData, onFinished); 220 | } -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "demo", 3 | "version": "1.0.0", 4 | "description": "", 5 | "main": "index.js", 6 | "scripts": { 7 | "test": "./node_modules/.bin/mocha test.js -t 20000" 8 | }, 9 | "author": "", 10 | "license": "ISC", 11 | "dependencies": { 12 | "bluebird": "^3.4.7", 13 | "node-uuid": "^1.4.7", 14 | "request": "^2.79.0" 15 | }, 16 | "devDependencies": { 17 | "mocha": "^3.2.0" 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /test.js: -------------------------------------------------------------------------------- 1 | let getChunks = require('./index'); 2 | let request = require('request'); 3 | let fs = require('fs'); 4 | let Promise = require('bluebird'); 5 | let path = require('path'); 6 | let assert = require('assert'); 7 | let BufferCache = require('./bufferCache'); 8 | 9 | const uploadURL = 'http://localhost:3000/upload'; 10 | const getMD5URL = 'http://localhost:3000/md5'; 11 | 12 | let exampleData = [ 13 | { 14 | url: 'http://gedftnj8mkvfefuaefm.exp.bcevod.com/mda-hbtp8s8gch6xv2zb/mda-hbtp8s8gch6xv2zb.mp4', 15 | md5: '1dfa3fe9bab054a8e15821f830027ef7' 16 | }, 17 | { 18 | url: 'http://gedftnj8mkvfefuaefm.exp.bcevod.com/mda-hbtpfpkqivd6muq6/mda-hbtpfpkqivd6muq6.mp4?bcevod_channel=searchbox_feed&auth_key=1492702311-0-0-a62de94daed3350d6ac4a8b85e6491e1', 19 | md5: 'ef88a44d111d0978f61f42ea76dfd71b' 20 | }, 21 | { 22 | url: 'http://gedftnj8mkvfefuaefm.exp.bcevod.com/mda-hauzmd37jyphdxad/mda-hauzmd37jyphdxad.mp4?bcevod_channel=searchbox_feed&auth_key=1492702321-0-0-c374f1e645454ec9f11d351e92f65e7e', 23 | md5: 'ad2176c322d65def3b19450bc358c7dd' 24 | }, 25 | { 26 | url: 'http://gedftnj8mkvfefuaefm.exp.bcevod.com/mda-gmjz7ynfb9g2zj7f/mda-gmjz7ynfb9g2zj7f.mp4?bcevod_channel=searchbox_feed&auth_key=1492702329-0-0-718209eba73d4ffd003d94a7415480c5', 27 | md5: 'ac765793fd360b2dfb389202cc28b635' 28 | }, 29 | { 30 | url: 'http://gedftnj8mkvfefuaefm.exp.bcevod.com/mda-hazk5y6e0r1aff5e/mda-hazk5y6e0r1aff5e.mp4', 31 | md5: 'cf3d06c6a06c4b1dbe7a3906991c56c6' 32 | } 33 | ] 34 | 35 | function getData(url, data) { 36 | return new Promise((resolve, reject) => { 37 | request({ 38 | url: url, 39 | method: 'POST', 40 | form: data 41 | }, function (err, response, data) { 42 | if (!err && response.statusCode === 200) { 43 | resolve(data); 44 | } 45 | else { 46 | reject(data); 47 | } 48 | }); 49 | }); 50 | } 51 | 52 | describe('文件下载测试', () => { 53 | it('bufferCache Test', function (done) { 54 | let bufferCache = new BufferCache(1024 * 10); 55 | 56 | var startTime = Date.now(); 57 | var originalBuffer = []; 58 | let compiledBuffer = []; 59 | let isFinished = false; 60 | 61 | let pushTimer = setInterval(() => { 62 | var randomString = []; 63 | 64 | for (let i = 0; i < 1024; i ++) { 65 | let arr = []; 66 | for (let j = 0; j < 1024; j ++) { 67 | arr.push(j % 10); 68 | } 69 | randomString.push(arr.join('')); 70 | } 71 | 72 | let buffer = Buffer.from(randomString.join('')); 73 | let bufferCopy = Buffer.alloc(buffer.length); 74 | 75 | buffer.copy(bufferCopy); 76 | originalBuffer.push(bufferCopy); 77 | bufferCache.pushBuf(buffer); 78 | 79 | if (Date.now() - startTime > 1000) { 80 | isFinished = true; 81 | clearTimeout(pushTimer); 82 | } 83 | }, 5); 84 | 85 | let outputTimer = setInterval(() => { 86 | let readyCache = bufferCache.getChunks(); 87 | 88 | while (readyCache.length > 0) { 89 | let chunk = readyCache.shift(); 90 | compiledBuffer.push(chunk); 91 | } 92 | 93 | if (isFinished) { 94 | let lastChunk = bufferCache.getRemainChunks(); 95 | compiledBuffer.push(lastChunk); 96 | clearTimeout(outputTimer); 97 | 98 | let originBuf = originalBuffer.reduce((total, next) => { 99 | return Buffer.concat([total, next], total.length + next.length); 100 | }, Buffer.alloc(0)); 101 | let compiledBuf = compiledBuffer.reduce((total, next) => { 102 | return Buffer.concat([total, next], total.length + next.length); 103 | }, Buffer.alloc(0)); 104 | 105 | assert.equal(originBuf.length, compiledBuf.length); 106 | assert.equal(originBuf.compare(compiledBuf), 0); 107 | 108 | done(); 109 | } 110 | }, 10); 111 | }); 112 | 113 | it('upload test', function(done) { 114 | Promise.map(exampleData, (item, index) => { 115 | let md5 = item.md5; 116 | let url = item.url; 117 | return getChunks(url, uploadURL, md5); 118 | }).then(() => { 119 | done(); 120 | }).catch(err => { 121 | done(err); 122 | }); 123 | }); 124 | 125 | it('download data md5sum test', (done) => { 126 | Promise.each(exampleData, (item, index) => { 127 | let md5 = item.md5; 128 | let url = item.url; 129 | 130 | return getData(getMD5URL, { 131 | filename: md5 132 | }).then((serverResponse) => { 133 | serverResponse = JSON.parse(serverResponse); 134 | let serverMd5 = serverResponse.data; 135 | assert.equal(serverMd5, md5); 136 | }); 137 | }).then(() => { 138 | done(); 139 | }).catch(err => { 140 | done(err); 141 | }) 142 | }); 143 | }); --------------------------------------------------------------------------------