├── .gitignore ├── LICENSE ├── README.md ├── examples ├── basics │ ├── README.md │ ├── package.json │ ├── photo.jpg │ └── upload.js └── client-to-server │ ├── README.md │ ├── app.js │ └── package.json ├── lib └── aliyun-oss-upload-stream.js ├── package.json └── test └── aliyun-oss-upload-stream.test.js /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules/ 2 | .DS_Store 3 | config.js -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2015 Berwin 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | 23 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # aliyun-oss-upload-stream 2 | 3 | [![NPM Version](https://img.shields.io/npm/v/aliyun-oss-upload-stream.svg)](https://www.npmjs.com/package/aliyun-oss-upload-stream) 4 | [![NPM Downloads](https://img.shields.io/npm/dm/aliyun-oss-upload-stream.svg)](https://www.npmjs.com/package/aliyun-oss-upload-stream) 5 | 6 | 用[Aliyun oss](https://github.com/aliyun-UED/aliyun-sdk-js) 的 Multipart upload API 实现的Node.js模块，通过stream的方式上传文件。 7 | 8 | ***官方指定Nodejs模块~*** 9 | 10 | ## 为什么使用stream？ 11 | 12 | * 使用stream的方式上传文件可以很大程度上降低服务器内存开销。Aliyun官方SDK并没有对stream进行一个完美的封装，所以通常上传文件（Put Object）的流程是客户端上传文件到服务器，服务器把文件数据缓存到内存，等文件全部上传完毕后，一次性上传到Aliyun Oss服务。这样做一旦瞬间上传文件的请求过多，服务器的内存开销会直线上升。而使用stream的方式上传文件的流程是客户端在上传文件数据到服务器的过程中，服务器同时也在把文件数据往Aliyun Oss服务传送，而不需要在服务器上缓存文件数据。 13 | * 可以上传大文件，根据上传数据方式不同而不同, Put Object 方式 文件最大不能超过 5GB，而使用stream的方式，文件大小不能超过 48.8TB 14 | * 更快的速度，由于传统方式（Put Object方式）是客户端上传完毕文件后，统一上传到Aliyun Oss，而stream的方式基本上客户端上传完毕后，服务器已经把一大半的文件数据上传到Aliyun了，所以速度要快很多 15 | * 使用更简单，经过封装后，stream的方式使用起来非常的方便，1分钟就可以学会如何使用 16 | 17 | ## 例子 18 | 19 | ```javascript 20 | 21 | var ALY = require('aliyun-sdk'), 22 | fs = require('fs'); 23 | 24 | var ossStream = require('aliyun-oss-upload-stream')(new ALY.OSS({ 25 | accessKeyId: '在阿里云OSS申请的 accessKeyId', 26 | secretAccessKey: '在阿里云OSS申请的 secretAccessKey', 27 | endpoint: 'http://oss-cn-hangzhou.aliyuncs.com', 28 | apiVersion: '2013-10-15' 29 | })); 30 | 31 | var upload = ossStream.upload({ 32 | Bucket: 'Bucket', 33 | Key: 'Key (可以理解为文件名)' 34 | }); 35 | 36 | // 可选配置 37 | upload.minPartSize(1048576); // 1M，表示每块part大小至少大于1M 38 | 39 | upload.on('error', function (error) { 40 | console.log('error:', error); 41 | }); 42 | 43 | upload.on('part', function (part) { 44 | console.log('part:', part); 45 | }); 46 | 47 | upload.on('uploaded', function (details) { 48 | var s = (new Date() - startTime) / 1000; 49 | console.log('details:', details); 50 | console.log('Completed upload in %d seconds', s); 51 | }); 52 | 53 | var read = fs.createReadStream('./photo.jpg'); 54 | read.pipe(upload); 55 | 56 | var startTime = new Date(); 57 | ``` 58 | 59 | ## 使用 60 | 61 | ### 初始化 62 | 63 | ```javascript 64 | var ALY = require('aliyun-sdk'); 65 | 66 | var ossStream = require('aliyun-oss-upload-stream')(new ALY.OSS({ 67 | accessKeyId: '在阿里云OSS申请的 accessKeyId', 68 | secretAccessKey: '在阿里云OSS申请的 secretAccessKey', 69 | endpoint: 'http://oss-cn-hangzhou.aliyuncs.com', 70 | apiVersion: '2013-10-15' 71 | })); 72 | ``` 73 | 74 | ### 上传文件 75 | 76 | ```javascript 77 | var upload = ossStream.upload({ 78 | Bucket: 'Bucket-Name', 79 | Key: 'Key-Name' 80 | }); 81 | 82 | var read = fs.createReadStream('./photo.jpg'); 83 | read.pipe(upload); 84 | ``` 85 | 86 | ## 操作方法 87 | 88 | **upload.minPartSize** 89 | 90 | 用于调整每次上传一小块数据的大小，不得低于200KB，默认为200KB，如果经常上传大文件，建议用此方法把值调整大一些 91 | 92 | ``` 93 | var upload = ossStream.upload({ 94 | Bucket: 'Bucket-Name', 95 | Key: 'Key-Name' 96 | }); 97 | 98 | // 可选配置 99 | upload.minPartSize(1048576); // 1M，表示每块part大小至少大于1M 100 | 101 | var read = fs.createReadStream('./photo.jpg'); 102 | read.pipe(upload); 103 | ``` 104 | 105 | ## 事件 106 | 107 | **error** 108 | 109 | 当上传过程中发生错误，触发error事件，回调函数参数为错误信息 110 | 111 | ```javascript 112 | upload.on('error', function (error) { 113 | console.log('error:', error); 114 | }); 115 | ``` 116 | **part** 117 | 118 | 上传文件的过程中，会触发part事件，回调函数参数为当前分片的信息 119 | 120 | ```javascript 121 | upload.on('part', function (part) { 122 | console.log('part:', part); 123 | }); 124 | ``` 125 | 126 | **uploaded** 127 | 128 | 上传成功后触发该事件，回调函数参数为完整的 Object 129 | 130 | ```javascript 131 | upload.on('uploaded', function (details) { 132 | console.log('details:', details); 133 | }); 134 | 135 | ``` 136 | 137 | ## 安装 138 | 139 | ``` 140 | npm i --save aliyun-oss-upload-stream 141 | ``` 142 | 143 | PS: 如果大家使用过程中，发现什么问题或者需要添加什么功能，及时通知我哈~ 我会及时更新和发布新版本~ 144 | 145 | ## The MIT License (MIT) 146 | 147 | Copyright (c) 2015 Berwin 148 | 149 | Permission is hereby granted, free of charge, to any person obtaining a copy 150 | of this software and associated documentation files (the "Software"), to deal 151 | in the Software without restriction, including without limitation the rights 152 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 153 | copies of the Software, and to permit persons to whom the Software is 154 | furnished to do so, subject to the following conditions: 155 | 156 | The above copyright notice and this permission notice shall be included in all 157 | copies or substantial portions of the Software. 158 | 159 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 160 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 161 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 162 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 163 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 164 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 165 | SOFTWARE. 166 | -------------------------------------------------------------------------------- /examples/basics/README.md: -------------------------------------------------------------------------------- 1 | ## 修改配置信息 2 | 3 | ``` 4 | vi upload.js 5 | ``` 6 | 配置accessKeyId、secretAccessKey、endpoint和Bucket、Key 7 | 8 | ``` 9 | var ossStream = require('../lib/aliyun-oss-upload-stream.js')(new ALY.OSS({ 10 | accessKeyId: '在阿里云OSS申请的 accessKeyId', 11 | secretAccessKey: '在阿里云OSS申请的 secretAccessKey', 12 | endpoint: 'http://oss-cn-hangzhou.aliyuncs.com', 13 | apiVersion: '2013-10-15' 14 | })); 15 | 16 | var upload = ossStream.upload({ 17 | Bucket: 'Bucket-Name', 18 | Key: 'Key-Name' 19 | }); 20 | ``` 21 | 22 | ## 安装依赖 23 | 24 | ``` 25 | npm install 26 | ``` 27 | 28 | ## 运行 29 | 30 | ``` 31 | node upload.js 32 | ``` 33 | 34 | 也可以在配置完信息之后直接运行下面的命令 35 | 36 | ``` 37 | npm run upload 38 | ``` -------------------------------------------------------------------------------- /examples/basics/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "aliyun-oss-upload-stream-examples-simple", 3 | "version": "1.1.0", 4 | "description": "aliyun-oss-upload-stream examples simple", 5 | "main": "./upload.js", 6 | "scripts": { 7 | "test": "echo \"Error: no test specified\" && exit 1", 8 | "upload": "npm install && node upload.js" 9 | }, 10 | "repository": { 11 | "type": "git", 12 | "url": "git+https://github.com/berwin/aliyun-oss-upload-stream.git" 13 | }, 14 | "keywords": [ 15 | "aliyun", 16 | "oss", 17 | "upload", 18 | "pipe", 19 | "stream" 20 | ], 21 | "author": "Berwin", 22 | "license": "MIT", 23 | "bugs": { 24 | "url": "https://github.com/berwin/aliyun-oss-upload-stream/issues" 25 | }, 26 | "homepage": "https://github.com/berwin/aliyun-oss-upload-stream#readme", 27 | "dependencies": { 28 | "aliyun-sdk": "^1.7.6" 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /examples/basics/photo.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/berwin/aliyun-oss-upload-stream/2ea237651ae2bdeb27ab9229205a695896a21023/examples/basics/photo.jpg -------------------------------------------------------------------------------- /examples/basics/upload.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | var ALY = require('aliyun-sdk'), 4 | fs = require('fs'); 5 | 6 | var ossStream = require('../../lib/aliyun-oss-upload-stream.js')(new ALY.OSS({ 7 | accessKeyId: '在阿里云OSS申请的 accessKeyId', 8 | secretAccessKey: '在阿里云OSS申请的 secretAccessKey', 9 | endpoint: 'http://oss-cn-hangzhou.aliyuncs.com', 10 | apiVersion: '2013-10-15' 11 | })); 12 | 13 | var upload = ossStream.upload({ 14 | Bucket: 'Bucket-Name', 15 | Key: 'Key-Name' 16 | }); 17 | 18 | upload.on('error', function (error) { 19 | console.log('error:', error); 20 | }); 21 | 22 | upload.on('part', function (part) { 23 | console.log('part:', part); 24 | }); 25 | 26 | upload.on('uploaded', function (details) { 27 | var s = (new Date() - startTime) / 1000; 28 | console.log('details:', details); 29 | console.log('Completed upload in %d seconds', s); 30 | }); 31 | 32 | var read = fs.createReadStream('./photo.jpg'); 33 | read.pipe(upload); 34 | 35 | var startTime = new Date(); -------------------------------------------------------------------------------- /examples/client-to-server/README.md: -------------------------------------------------------------------------------- 1 | # 案例 2 | 3 | ## 修改配置信息 4 | 5 | ``` 6 | vim upload.js 7 | ``` 8 | 配置accessKeyId、secretAccessKey、endpoint和Bucket、Key 9 | 10 | ``` 11 | var ossStream = require('../lib/aliyun-oss-upload-stream.js')(new ALY.OSS({ 12 | accessKeyId: '在阿里云OSS申请的 accessKeyId', 13 | secretAccessKey: '在阿里云OSS申请的 secretAccessKey', 14 | endpoint: 'http://oss-cn-hangzhou.aliyuncs.com', 15 | apiVersion: '2013-10-15' 16 | })); 17 | 18 | var upload = ossStream.upload({ 19 | Bucket: 'Bucket-Name', 20 | Key: 'Key-Name' 21 | }); 22 | ``` 23 | 24 | ## 安装依赖 25 | 26 | ``` 27 | npm install 28 | ``` 29 | 30 | ## 运行 31 | 32 | ``` 33 | npm start 34 | ``` 35 | 36 | ## 访问 37 | 38 | ``` 39 | http://127.0.0.1:1995 40 | ``` 41 | 在页面上传图片，并查看终端，会在终端打印出上传相关信息 -------------------------------------------------------------------------------- /examples/client-to-server/app.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | var http = require('http'); 4 | var Busboy = require('busboy'); 5 | var ALY = require('aliyun-sdk'); 6 | 7 | var ossStream = require('../../lib/aliyun-oss-upload-stream.js')(new ALY.OSS({ 8 | accessKeyId: '在阿里云OSS申请的 accessKeyId', 9 | secretAccessKey: '在阿里云OSS申请的 secretAccessKey', 10 | endpoint: 'http://oss-cn-hangzhou.aliyuncs.com', 11 | apiVersion: '2013-10-15' 12 | })); 13 | 14 | http.createServer(function(req, res) { 15 | if (req.method === 'POST') { 16 | var busboy = new Busboy({ headers: req.headers }); 17 | busboy.on('file', function(fieldname, file, filename, encoding, mimetype) { 18 | var upload = ossStream.upload({ 19 | Bucket: 'Bucket', 20 | Key: filename 21 | }); 22 | 23 | upload.on('error', function (error) { 24 | console.log('error:', error); 25 | }); 26 | 27 | upload.on('part', function (part) { 28 | console.log('part:', part); 29 | }); 30 | 31 | upload.on('uploaded', function (details) { 32 | console.log('details:', details); 33 | res.writeHead(303, { Connection: 'close', Location: '/' }); 34 | res.end(); 35 | }); 36 | 37 | file.pipe(upload); 38 | }); 39 | 40 | req.pipe(busboy); 41 | } else if (req.method === 'GET') { 42 | res.writeHead(200, { Connection: 'close' }); 43 | res.end('\ 44 |
\ 45 |
\ 46 |
\ 47 | \ 48 |
\ 49 | '); 50 | } 51 | }).listen(1995, function() { 52 | console.log('Listening for requests'); 53 | }); -------------------------------------------------------------------------------- /examples/client-to-server/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "aliyun-oss-upload-stream-examples-client-to-server", 3 | "version": "1.1.0", 4 | "description": "aliyun-oss-upload-stream examples client-to-server", 5 | "main": "./app.js", 6 | "scripts": { 7 | "test": "echo \"Error: no test specified\" && exit 1", 8 | "start": "node app.js" 9 | }, 10 | "repository": { 11 | "type": "git", 12 | "url": "git+https://github.com/berwin/aliyun-oss-upload-stream.git" 13 | }, 14 | "keywords": [ 15 | "aliyun", 16 | "oss", 17 | "upload", 18 | "pipe", 19 | "stream" 20 | ], 21 | "author": "Berwin", 22 | "license": "MIT", 23 | "bugs": { 24 | "url": "https://github.com/berwin/aliyun-oss-upload-stream/issues" 25 | }, 26 | "homepage": "https://github.com/berwin/aliyun-oss-upload-stream#readme", 27 | "dependencies": { 28 | "aliyun-sdk": "^1.9.1", 29 | "busboy": "^0.2.13" 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /lib/aliyun-oss-upload-stream.js: -------------------------------------------------------------------------------- 1 | /**! 2 | * Aliyun-oss-upload-stream - lib/index.js 3 | * 4 | * 使用 stream 的方式上传文件 5 | * 6 | * Authors: 7 | * Berwin (https://github.com/berwin) 8 | */ 9 | 10 | 'use strict'; 11 | 12 | var Writable = require('stream').Writable; 13 | var events = require('events'); 14 | 15 | function Client(client) { 16 | if (!client) throw new Error('Must configure an oss client before attempting to create an oss upload stream.'); 17 | 18 | // 安全监测 19 | if (!(this instanceof Client)) { 20 | return new Client(client); 21 | } 22 | 23 | this.cachedClient = client; 24 | } 25 | 26 | Client.prototype.upload = function (destinationDetails) { 27 | 28 | if (!arguments.length || Object.prototype.toString.call(destinationDetails) !== '[object Object]') throw new Error('Parameter is not correct'); 29 | 30 | var e = new events.EventEmitter(); 31 | var cachedClient = this.cachedClient; 32 | var multipartUploadID = null; 33 | var multipartUploadResult = null; 34 | 35 | // 缓存的buffer数据 36 | var receivedBuffers = []; 37 | 38 | // 缓存的buffer数据长度 39 | var receivedBuffersLength = 0; 40 | 41 | // Part 列表，用于数据全部上传成功后，验证Part有效性 42 | var partIds = []; 43 | 44 | // Part 索引 45 | var localPartNumber = 0; 46 | 47 | // 用于判断是否 Parts 全部上传完毕（服务器端->阿里云） 48 | var pendingParts = 0; 49 | // 判断是否上传完毕（客户端->服务器端） 50 | var completed = false; 51 | 52 | // 每个part的最小值 53 | var minPartSize = 204800; 54 | 55 | var ws = new Writable({ 56 | highWaterMark: 4194304 // 4 MB 57 | }); 58 | 59 | // 设置每个part的最小值 60 | ws.minPartSize = function (partSize) { 61 | if (partSize > minPartSize) { 62 | minPartSize = partSize; 63 | } 64 | return ws; 65 | }; 66 | 67 | ws.getMinPartSize = function () { 68 | return minPartSize; 69 | }; 70 | 71 | ws._write = function (chunk, encoding, next) { 72 | // 缓存buffer 73 | absorbBuffer(chunk); 74 | 75 | // 缓存区数据大小 76 | var size = Buffer.byteLength(Buffer.concat(receivedBuffers, receivedBuffersLength)); 77 | 78 | /* 79 | * 如果初始化完成，并且当前Part大小大于minPartSize（默认200KB），执行flushPart操作 80 | * Multipart Upload要求除最后一个Part以外，其他的Part大小都要大于100KB 81 | * 所以如果当前write的大小不够minPartSize的情况下只缓存，不处理数据，直到缓存大小超过minPartSize，在从缓存中取出所有缓存的part数据执行flushPart操作 82 | */ 83 | if (multipartUploadID && size > minPartSize) { 84 | flushPart(); 85 | } 86 | 87 | // 上传数据前，必须先初始化 88 | if (!multipartUploadID && !e.listeners('ready').length) { 89 | /* 90 | * 注册ready事件 91 | * 初始化完成后，只有当 completed 为真并且缓存区有数据时，执行 flushPart 操作 92 | * 93 | * 1. 为什么只在completed为真并且缓存区有数据时，执行 flushPart 操作？ 94 | * 答：因为初始化完成之前，是不会执行 flushPart 操作的， 95 | * 所以如果 客户端->服务端 的传输在初始化之前结束， 96 | * 那么初始化结束后，需要一次性将数据上传到阿里云（一般这种情况发生在小文件） 97 | * 98 | * 但还有一种情况，就是在触发ready事件的时候，状态已经改变，但回调函数还没执行的这一瞬间， 99 | * 客户端->服务端 的传输结束了，此时状态是“初始化完成”，那么 ws.end 与 ws._write 做最后一次 flushPart 操作 100 | * 因为ready事件已经触发，所以又触发了一次多余的，不必要的flushPart操作，所以需要判断缓存区是否有内容 101 | * 102 | * 2. 为什么只有completed为真时，执行flushPart操作？ 103 | * 答：如果客户端网比较慢，而服务端与阿里云之间通信比较快， 104 | * 那么会出现 初始化完成后，缓存区中的chunk大小 小于 100KB， 105 | * 如果直接上传，会触发阿里云 “除最后一个Part以外，其他的Part大小都要大于100KB” 的限制，从而报错， 106 | * 所以不需要 flushPart 操作，只改变状态就可以，当下一次触发 ws._write 时，如果条件满足会自动进行flushPart操作 107 | */ 108 | e.once('ready', function () { 109 | if (completed === true && Buffer.byteLength(Buffer.concat(receivedBuffers, receivedBuffersLength)) > 0) { 110 | flushPart(); 111 | } 112 | }); 113 | 114 | // 初始化MultipartUpload 115 | createMultipartUpload(destinationDetails); 116 | } 117 | 118 | next(); 119 | }; 120 | 121 | ws.end = function () { 122 | completed = true; 123 | 124 | // 缓存区数据大小 125 | var size = Buffer.byteLength(Buffer.concat(receivedBuffers, receivedBuffersLength)); 126 | 127 | /* 128 | * 由于_write只操作缓存区大于 minPartSize（默认200KB） 的数据，所以最后一次或最后几次加在一起的数据不够 minPartSize 的时候，_write并不会做任何处理 129 | * 所以需要在end的时候，检查当前缓存区是否有没处理完的数据，如果有，则执行最后一次flushPart操作 130 | */ 131 | if (multipartUploadID && size > 0) { 132 | flushPart(); 133 | } 134 | }; 135 | 136 | /* 137 | * 初始化 Multipart Upload 138 | * 使用 Multipart Upload 模式传输数据前,必须先调用该接口来通知 OSS 初始 化一个 Multipart Upload 事件 139 | * 140 | * 初始化结束后，触发ready事件 141 | * @param {Object} Bucket && Key 142 | */ 143 | function createMultipartUpload(details) { 144 | cachedClient.createMultipartUpload(details, function (err, data) { 145 | if (err) return abortUpload(err); 146 | multipartUploadID = data.UploadId; 147 | multipartUploadResult = data; 148 | e.emit('ready'); 149 | }); 150 | } 151 | 152 | /* 153 | * 分块(Part)上传数据 154 | * flushPart 会从缓存区中取出所有缓存数据，并清空缓存区的数据 155 | */ 156 | var flushPart = function () { 157 | var chunk = preparePartBuffer(); 158 | localPartNumber = localPartNumber + 1; 159 | pendingParts = pendingParts + 1; 160 | 161 | (function (localPartNumber) { 162 | cachedClient.uploadPart({ 163 | Body: chunk, 164 | Bucket: multipartUploadResult.Bucket, 165 | Key: multipartUploadResult.Key, 166 | UploadId: multipartUploadID, 167 | PartNumber: localPartNumber 168 | }, function (err, result) { 169 | if (err) return abortUpload(err); 170 | 171 | pendingParts = pendingParts - 1; 172 | 173 | var part = { 174 | ETag: result.ETag, 175 | PartNumber: localPartNumber 176 | }; 177 | 178 | partIds[localPartNumber - 1] = part; 179 | 180 | ws.emit('part', part); 181 | 182 | if (!pendingParts && completed) completeUpload(); 183 | }); 184 | })(localPartNumber); 185 | }; 186 | 187 | /* 188 | * 上传成功 189 | * 190 | * 在将所有数据 Part 都上传完成后,必须调用 Complete Multipart Upload API 来完成整个文件的 Multipart Upload。在执行该操作时,用户必须 供所有有效 的数据 Part 的列表(包括 part 号码和 ETAG);OSS 收到用户 交的 Part 列表后, 会逐一验证每个数据 Part 的有效性。当所有的数据 Part 验证通过后,OSS 将把 这些数据 part 组合成一个完整的 Object。 191 | * 192 | * @return {Object} ossObject 193 | */ 194 | var completeUpload = function () { 195 | cachedClient.completeMultipartUpload({ 196 | Bucket: multipartUploadResult.Bucket, 197 | Key: multipartUploadResult.Key, 198 | UploadId: multipartUploadID, 199 | CompleteMultipartUpload: { 200 | Parts: partIds 201 | } 202 | }, function (err, result) { 203 | if (err) return abortUpload(err); 204 | ws.emit('uploaded', result); 205 | }); 206 | }; 207 | 208 | /* 209 | * 缓存Buffer 210 | */ 211 | function absorbBuffer(incomingBuffer) { 212 | receivedBuffers.push(incomingBuffer); 213 | receivedBuffersLength += incomingBuffer.length; 214 | } 215 | 216 | /* 217 | * 生成 chunk 218 | * 根据缓存的buffer数据，生成一个新buffer并返回 219 | * 220 | * @return {Buffer} chunk 221 | */ 222 | function preparePartBuffer() { 223 | var combinedBuffer = Buffer.concat(receivedBuffers, receivedBuffersLength); 224 | receivedBuffers = []; 225 | receivedBuffersLength = 0; 226 | 227 | return combinedBuffer; 228 | } 229 | 230 | /* 231 | * 终止Multipart Upload 事件 232 | * 233 | * 当一个Multipart Upload事件被中止后，就不能再使用这个Upload ID做任何操作，已经上传的 Part 数据也会被删除 234 | * 235 | * @param {error} 错误信息 236 | */ 237 | function abortUpload(rootError) { 238 | 239 | // 初始化MultipartUpload的时候报错，则直接触发error事件 240 | if (multipartUploadResult === null && multipartUploadID === null) { 241 | return ws.emit('error', rootError); 242 | } 243 | 244 | cachedClient.abortMultipartUpload({ 245 | Bucket: multipartUploadResult.Bucket, 246 | Key: multipartUploadResult.Key, 247 | UploadId: multipartUploadID 248 | }, function (err) { 249 | if (err) { 250 | ws.emit('error', rootError + '\n Additionally failed to abort the multipart upload on OSS: ' + err); 251 | } else { 252 | ws.emit('error', rootError); 253 | } 254 | }); 255 | } 256 | 257 | return ws; 258 | }; 259 | 260 | module.exports = Client; -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "aliyun-oss-upload-stream", 3 | "version": "1.3.0", 4 | "description": "A Node.js module for streaming data to Aliyun oss via the multipart upload API", 5 | "main": "./lib/aliyun-oss-upload-stream.js", 6 | "scripts": { 7 | "test": "mocha -t 10000 --recursive test/" 8 | }, 9 | "repository": { 10 | "type": "git", 11 | "url": "git+https://github.com/berwin/aliyun-oss-upload-stream.git" 12 | }, 13 | "keywords": [ 14 | "aliyun", 15 | "oss", 16 | "upload", 17 | "pipe", 18 | "stream" 19 | ], 20 | "author": "Berwin", 21 | "license": "MIT", 22 | "bugs": { 23 | "url": "https://github.com/berwin/aliyun-oss-upload-stream/issues" 24 | }, 25 | "homepage": "https://github.com/berwin/aliyun-oss-upload-stream#readme", 26 | "devDependencies": { 27 | "aliyun-sdk": "^1.9.1" 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /test/aliyun-oss-upload-stream.test.js: -------------------------------------------------------------------------------- 1 | /**! 2 | * Aliyun-oss-upload-stream - test/aliyun-oss-upload-stream.test.js 3 | * 4 | * Copyright(c) Berwin and other contributors. 5 | * MIT Licensed 6 | * 7 | * Authors: 8 | * Berwin (https://github.com/berwin) 9 | */ 10 | 11 | 'use strict'; 12 | 13 | var ALY = require('aliyun-sdk'); 14 | var fs = require('fs'); 15 | 16 | var ossConfig = { 17 | accessKeyId: '在阿里云OSS申请的 accessKeyId', 18 | secretAccessKey: '在阿里云OSS申请的 secretAccessKey', 19 | endpoint: 'http://oss-cn-hangzhou.aliyuncs.com', 20 | apiVersion: '2013-10-15' 21 | }; 22 | 23 | var uploadConfig = { 24 | "Bucket": "Test-Bucket-Name", 25 | "Key": "Test-Key-Name" 26 | }; 27 | 28 | describe('Piping data into the writable upload stream', function () { 29 | var ossStream = require('../lib/aliyun-oss-upload-stream.js')(new ALY.OSS(ossConfig)); 30 | var uploadStream = ossStream.upload(uploadConfig); 31 | 32 | before(function (done) { 33 | uploadStream.on('error', function () { 34 | throw "Did not expect to receive an error"; 35 | }); 36 | 37 | done(); 38 | }); 39 | 40 | it('should emit valid part and uploaded events', function (done) { 41 | var file = fs.createReadStream(process.cwd() + '/examples/basics/photo.jpg'); 42 | var part = false, uploaded = false; 43 | 44 | uploadStream.on('part', function (details) { 45 | part = true; 46 | if (part && uploaded) done(); 47 | }); 48 | 49 | uploadStream.on('uploaded', function (details) { 50 | uploaded = true; 51 | if (part && uploaded) done(); 52 | }); 53 | 54 | file.pipe(uploadStream); 55 | file.on('error', function () { 56 | throw 'Error! Unable to open the file for reading'; 57 | }); 58 | }); 59 | }); 60 | 61 | describe('OSS Error catching', function () { 62 | describe('Bucket Not correct', function () { 63 | var ossStream = require('../lib/aliyun-oss-upload-stream.js')(new ALY.OSS(ossConfig)); 64 | var uploadStream = ossStream.upload({"Bucket": "NotcorrectBucket", "Key": "test.photo.jpg"}); 65 | 66 | it('should emit an error', function (done) { 67 | var file = fs.createReadStream(process.cwd() + '/examples/basics/photo.jpg'); 68 | 69 | uploadStream.on('error', function (err) { 70 | done(); 71 | }); 72 | 73 | file.pipe(uploadStream); 74 | }); 75 | }); 76 | 77 | describe('Config Not correct', function () { 78 | var ossStream = require('../lib/aliyun-oss-upload-stream.js')(new ALY.OSS({ 79 | accessKeyId: 'NotcorrectAccessKeyId', 80 | secretAccessKey: 'NotcorrectSecretAccessKey', 81 | endpoint: 'http://oss-cn-hangzhou.aliyuncs.com', 82 | apiVersion: '2013-10-15' 83 | })); 84 | var uploadStream = ossStream.upload(uploadConfig); 85 | 86 | it('should emit an error', function (done) { 87 | var file = fs.createReadStream(process.cwd() + '/examples/basics/photo.jpg'); 88 | 89 | uploadStream.on('error', function (err) { 90 | done(); 91 | }); 92 | 93 | file.pipe(uploadStream); 94 | }); 95 | }); 96 | }); --------------------------------------------------------------------------------