├── .gitignore ├── src ├── index.ts ├── models │ ├── accept-ranges.ts │ ├── operation.ts │ ├── start-options.ts │ ├── operation-factory.ts │ ├── partial-request-query.ts │ ├── partial-download.ts │ ├── default-operation.ts │ ├── buffer-operation.ts │ ├── file-operation.ts │ └── multipart-download.ts └── utilities │ ├── path-formatter.ts │ ├── url-parser.ts │ ├── validation.ts │ └── file-segmentation.ts ├── .npmignore ├── .travis.yml ├── tsconfig.json ├── .vscode └── tasks.json ├── tslint.json ├── test ├── url-parser-test.ts ├── path-formatter-test.ts ├── test-config.ts ├── partial-download-test.ts ├── file-segmentation-test.ts ├── partial-request-query-test.ts ├── operation-factory-test.ts ├── default-operation-test.ts ├── validation-test.ts ├── buffer-operation-test.ts ├── multipart-download-test.ts └── file-operation-test.ts ├── LICENSE ├── package.json └── README.md /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | 3 | dist 4 | 5 | .nyc_output 6 | -------------------------------------------------------------------------------- /src/index.ts: -------------------------------------------------------------------------------- 1 | import {MultipartDownload} from './models/multipart-download'; 2 | 3 | export = MultipartDownload; -------------------------------------------------------------------------------- /.npmignore: -------------------------------------------------------------------------------- 1 | .vscode 2 | node_modules 3 | test 4 | 5 | /dist/test 6 | /src 7 | 8 | .gitignore 9 | .travis.yml 10 | tsconfig.json 11 | tslint.json 12 | .nyc_output 13 | -------------------------------------------------------------------------------- /src/models/accept-ranges.ts: -------------------------------------------------------------------------------- 1 | export class AcceptRanges { 2 | public static readonly None: string = 'none'; 3 | public static readonly Bytes: string = 'bytes'; 4 | } 5 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: node_js 2 | node_js: 3 | - node 4 | - lts/* 5 | 6 | dist: trusty 7 | 8 | install: 9 | - npm install 10 | 11 | script: 12 | - npm run lint:dry 13 | - npm run build 14 | - npm test -------------------------------------------------------------------------------- /src/models/operation.ts: -------------------------------------------------------------------------------- 1 | import events = require('events'); 2 | import request = require('request'); 3 | 4 | export interface Operation { 5 | start(url: string, contentLength: number, numOfConnections: number, headers?: request.Headers): events.EventEmitter; 6 | } 7 | -------------------------------------------------------------------------------- /src/models/start-options.ts: -------------------------------------------------------------------------------- 1 | import {Headers} from 'request'; 2 | 3 | export interface StartOptions { 4 | numOfConnections?: number; 5 | writeToBuffer?: boolean; 6 | saveDirectory?: string; 7 | fileName?: string; 8 | headers?: Headers; 9 | } 10 | -------------------------------------------------------------------------------- /src/utilities/path-formatter.ts: -------------------------------------------------------------------------------- 1 | import path = require('path'); 2 | 3 | export class PathFormatter { 4 | public static format(directory: string, filename: string): string { 5 | const fullPath: string = `${directory}${path.sep}${filename}`; 6 | 7 | return fullPath; 8 | } 9 | } 10 | -------------------------------------------------------------------------------- /tsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "module": "commonjs", 4 | "target": "es6", 5 | "noImplicitAny": false, 6 | "sourceMap": false, 7 | "outDir": "dist", 8 | "declaration": true 9 | }, 10 | "exclude": [ 11 | ".vscode", 12 | "node_modules" 13 | ] 14 | } -------------------------------------------------------------------------------- /.vscode/tasks.json: -------------------------------------------------------------------------------- 1 | { 2 | // See https://go.microsoft.com/fwlink/?LinkId=733558 3 | // for the documentation about the tasks.json format 4 | "version": "0.1.0", 5 | "command": "tsc", 6 | "isShellCommand": true, 7 | "args": ["-w", "-p", "."], 8 | "showOutput": "silent", 9 | "isBackground": true, 10 | "problemMatcher": "$tsc-watch" 11 | } -------------------------------------------------------------------------------- /src/utilities/url-parser.ts: -------------------------------------------------------------------------------- 1 | import url = require('url'); 2 | 3 | export class UrlParser { 4 | public static getFilename(fileUrl: string): string { 5 | const parsedUrl: url.Url = url.parse(fileUrl); 6 | 7 | const filename: RegExpExecArray = new RegExp(/(?:\/.+)?\/(.+)/, '').exec(parsedUrl.path); 8 | 9 | return filename[1]; 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /tslint.json: -------------------------------------------------------------------------------- 1 | { 2 | "defaultSeverity": "error", 3 | "extends": [ 4 | "tslint:recommended" 5 | ], 6 | "jsRules": {}, 7 | "rules": { 8 | "quotemark": [true, "single"], 9 | "interface-name": [true, "never-prefix"], 10 | "object-literal-sort-keys": [false], 11 | "max-line-length": [false] 12 | }, 13 | "rulesDirectory": [] 14 | } -------------------------------------------------------------------------------- /test/url-parser-test.ts: -------------------------------------------------------------------------------- 1 | import {expect} from 'chai'; 2 | 3 | import {UrlParser} from '../src/utilities/url-parser'; 4 | 5 | describe('Url parser', () => { 6 | it('get filename from url', () => { 7 | const filename: string = 'cat.png'; 8 | 9 | const result: string = UrlParser.getFilename('https://homepages.cae.wisc.edu/~ece533/images/cat.png'); 10 | 11 | expect(result).to.equal(filename); 12 | }); 13 | }); -------------------------------------------------------------------------------- /test/path-formatter-test.ts: -------------------------------------------------------------------------------- 1 | import os = require('os'); 2 | import path = require('path'); 3 | 4 | import {expect} from 'chai'; 5 | 6 | import {PathFormatter} from '../src/utilities/path-formatter'; 7 | 8 | describe('Path formatter', () => { 9 | it('is correct path format', () => { 10 | const filePath: string = `${os.tmpdir()}${path.sep}test.txt`; 11 | 12 | const result: string = PathFormatter.format(os.tmpdir(), 'test.txt'); 13 | 14 | expect(result).to.equal(filePath); 15 | }); 16 | }); -------------------------------------------------------------------------------- /test/test-config.ts: -------------------------------------------------------------------------------- 1 | export interface ContentLengthUrlPair { 2 | readonly contentLength: number; 3 | readonly url: string; 4 | } 5 | 6 | export class TestConfig { 7 | public static readonly AcceptRangesSupportedUrl: ContentLengthUrlPair = {contentLength: 992514, url: 'https://upload.wikimedia.org/wikipedia/commons/a/a3/June_odd-eyed-cat.jpg'}; 8 | public static readonly AcceptRangesUnsupportedUrl: ContentLengthUrlPair = {contentLength: 51567, url: 'https://s-media-cache-ak0.pinimg.com/736x/92/9d/3d/929d3d9f76f406b5ac6020323d2d32dc.jpg'}; 9 | 10 | public static readonly Timeout: number = 10000; 11 | } -------------------------------------------------------------------------------- /test/partial-download-test.ts: -------------------------------------------------------------------------------- 1 | import {expect} from 'chai'; 2 | 3 | import {TestConfig} from './test-config'; 4 | 5 | import {PartialDownload} from '../src/models/partial-download'; 6 | 7 | describe('Partial download', () => { 8 | it('download a segment of a file', (done) => { 9 | let segmentSize: number = 0; 10 | 11 | new PartialDownload() 12 | .start(TestConfig.AcceptRangesSupportedUrl.url, {start: 0, end: 199}) 13 | .on('data', (data, offset) => { 14 | segmentSize += data.length; 15 | }) 16 | .on('end', () => { 17 | expect(segmentSize).to.equal(200); 18 | done(); 19 | }); 20 | }).timeout(TestConfig.Timeout);; 21 | }); -------------------------------------------------------------------------------- /src/models/operation-factory.ts: -------------------------------------------------------------------------------- 1 | import {BufferOperation} from './buffer-operation'; 2 | import {DefaultOperation} from './default-operation'; 3 | import {FileOperation} from './file-operation'; 4 | import {Operation} from './operation'; 5 | import {StartOptions} from './start-options'; 6 | 7 | export class OperationFactory { 8 | public static getOperation(options: StartOptions): Operation { 9 | 10 | let operation: Operation; 11 | if (options.writeToBuffer) { 12 | operation = new BufferOperation(); 13 | } else if (options.saveDirectory) { 14 | operation = new FileOperation(options.saveDirectory, options.fileName); 15 | } else { 16 | operation = new DefaultOperation(); 17 | } 18 | 19 | return operation; 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /test/file-segmentation-test.ts: -------------------------------------------------------------------------------- 1 | import {expect} from 'chai'; 2 | 3 | import {FileSegmentation} from '../src/utilities/file-segmentation'; 4 | 5 | import {PartialDownloadRange} from '../src/models/partial-download'; 6 | 7 | describe('File download segmentation', () => { 8 | it('is correct size segmentation', () => { 9 | const downloadRanges: PartialDownloadRange[] = []; 10 | downloadRanges.push({start: 0, end: 199}); 11 | downloadRanges.push({start: 200, end: 399}); 12 | downloadRanges.push({start: 400, end: 599}); 13 | downloadRanges.push({start: 600, end: 799}); 14 | downloadRanges.push({start: 800, end: 999}); 15 | 16 | const result: PartialDownloadRange[] = FileSegmentation.getSegmentsRange(1000, 5); 17 | 18 | expect(result).to.deep.equal(downloadRanges); 19 | }); 20 | }); -------------------------------------------------------------------------------- /src/utilities/validation.ts: -------------------------------------------------------------------------------- 1 | import fs = require('fs'); 2 | import validFilename = require('valid-filename'); 3 | import validator = require('validator'); 4 | 5 | export class Validation { 6 | public static isUrl(url: string): boolean { 7 | return validator.isURL(url); 8 | } 9 | 10 | public static isValidNumberOfConnections(numOfConnections: number): boolean { 11 | const isValid: boolean = numOfConnections > 0; 12 | 13 | return isValid; 14 | } 15 | 16 | public static isDirectory(directory: string): boolean { 17 | let isDirectory: boolean; 18 | 19 | try { 20 | const stat: fs.Stats = fs.lstatSync(directory); 21 | isDirectory = stat.isDirectory(); 22 | } catch (err) { 23 | isDirectory = false; 24 | } 25 | 26 | return isDirectory; 27 | } 28 | 29 | public static isValidFileName(fileName: string): boolean { 30 | return validFilename(fileName); 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /src/models/partial-request-query.ts: -------------------------------------------------------------------------------- 1 | import request = require('request'); 2 | 3 | export interface PartialRequestMetadata { 4 | readonly acceptRanges: string; 5 | readonly contentLength: number; 6 | } 7 | 8 | export class PartialRequestQuery { 9 | public getMetadata(url: string, headers?: request.Headers): Promise { 10 | 11 | return new Promise((resolve, reject) => { 12 | const options: request.CoreOptions = {}; 13 | 14 | options.headers = headers || null; 15 | 16 | request.head(url, options, (err, res, body) => { 17 | if (err) { 18 | return reject(err); 19 | } 20 | 21 | const metadata = { 22 | acceptRanges: res.headers['accept-ranges'], 23 | contentLength: parseInt(res.headers['content-length'], 10), 24 | }; 25 | 26 | return resolve(metadata); 27 | }); 28 | }); 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2017 Zulhilmi Mohamed Zainuddin 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /src/models/partial-download.ts: -------------------------------------------------------------------------------- 1 | import events = require('events'); 2 | import request = require('request'); 3 | 4 | import {AcceptRanges} from './accept-ranges'; 5 | 6 | export interface PartialDownloadRange { 7 | readonly start: number; 8 | readonly end: number; 9 | } 10 | 11 | export class PartialDownload extends events.EventEmitter { 12 | 13 | public start(url: string, range: PartialDownloadRange, headers?: request.Headers): PartialDownload { 14 | const options: request.CoreOptions = {}; 15 | 16 | options.headers = headers || {}; 17 | options.headers.Range = `${AcceptRanges.Bytes}=${range.start}-${range.end}`; 18 | 19 | let offset: number = range.start; 20 | request 21 | .get(url, options) 22 | .on('error', (err) => { 23 | this.emit('error', err); 24 | }) 25 | .on('data', (data) => { 26 | this.emit('data', data, offset); 27 | offset += data.length; 28 | }) 29 | .on('end', () => { 30 | this.emit('end'); 31 | }); 32 | 33 | return this; 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /test/partial-request-query-test.ts: -------------------------------------------------------------------------------- 1 | import {expect} from 'chai'; 2 | 3 | import {TestConfig} from './test-config'; 4 | 5 | import {AcceptRanges} from '../src/models/accept-ranges'; 6 | import {PartialRequestQuery} from '../src/models/partial-request-query'; 7 | 8 | describe('Partial request query', () => { 9 | it('with Accept-Ranges header', (done) => { 10 | const partialRequestQuery: PartialRequestQuery = new PartialRequestQuery(); 11 | 12 | partialRequestQuery 13 | .getMetadata(TestConfig.AcceptRangesSupportedUrl.url) 14 | .then((metadata) => { 15 | expect(metadata.acceptRanges).to.equal(AcceptRanges.Bytes); 16 | expect(metadata.contentLength).to.not.be.NaN; 17 | done(); 18 | }); 19 | }).timeout(TestConfig.Timeout);; 20 | 21 | xit('without Accept-Ranges header', (done) => { 22 | const partialRequestQuery: PartialRequestQuery = new PartialRequestQuery(); 23 | 24 | partialRequestQuery 25 | .getMetadata(TestConfig.AcceptRangesUnsupportedUrl.url) 26 | .then((metadata) => { 27 | expect(metadata.acceptRanges).to.not.exist; 28 | expect(metadata.contentLength).to.not.be.NaN; 29 | done(); 30 | }); 31 | }).timeout(TestConfig.Timeout);; 32 | }); -------------------------------------------------------------------------------- /src/utilities/file-segmentation.ts: -------------------------------------------------------------------------------- 1 | import {PartialDownloadRange} from '../models/partial-download'; 2 | 3 | export class FileSegmentation { 4 | public static getSegmentsRange(fileSize: number, numOfSegments: number): PartialDownloadRange[] { 5 | const segmentSizes: number[] = FileSegmentation.getSegmentsSize(fileSize, numOfSegments); 6 | 7 | let startRange: number = 0; 8 | const segmentRanges: PartialDownloadRange[] = segmentSizes.map((value, index, array) => { 9 | 10 | const sizes: number[] = array.slice(0, index + 1); 11 | const sum: number = sizes.reduce((accumulator, currentValue) => { 12 | return accumulator + currentValue; 13 | }, 0); 14 | 15 | const range: PartialDownloadRange = {start: startRange, end: sum - 1}; 16 | 17 | startRange = sum; 18 | 19 | return range; 20 | }); 21 | 22 | return segmentRanges; 23 | } 24 | 25 | private static getSegmentsSize(fileSize: number, numOfSegments: number): number[] { 26 | const segmentSize: number = Math.floor(fileSize / numOfSegments); 27 | const remainder: number = fileSize % numOfSegments; 28 | 29 | const segmentSizes: number[] = new Array(numOfSegments).fill(segmentSize); 30 | segmentSizes[segmentSizes.length - 1] += remainder; 31 | 32 | return segmentSizes; 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "multipart-download", 3 | "version": "1.2.5", 4 | "description": "Speed up download of a single file with multiple HTTP GET connections running in parallel", 5 | "main": "dist/src/index.js", 6 | "types": "dist/src/index.d.ts", 7 | "scripts": { 8 | "build": "rm -rfv ./dist && tsc", 9 | "lint:dry": "tslint -c tslint.json src/**/*.ts", 10 | "lint:fix": "tslint -c tslint.json --fix src/**/*.ts", 11 | "test": "nyc mocha --reporter spec ./dist/test" 12 | }, 13 | "author": "Zulhilmi Mohamed Zainuddin", 14 | "license": "MIT", 15 | "repository": { 16 | "type": "git", 17 | "url": "https://github.com/zulhilmizainuddin/multipart-download.git" 18 | }, 19 | "keywords": [ 20 | "accept", 21 | "connection", 22 | "download", 23 | "get", 24 | "http", 25 | "multiple", 26 | "parallel", 27 | "range", 28 | "speed", 29 | "accelerate", 30 | "multi", 31 | "threaded" 32 | ], 33 | "dependencies": { 34 | "request": "^2.88.0", 35 | "valid-filename": "^2.0.1", 36 | "validator": "^13.7.0" 37 | }, 38 | "devDependencies": { 39 | "@types/chai": "^4.2.0", 40 | "@types/mocha": "^5.2.7", 41 | "@types/node": "^10.14.15", 42 | "@types/request": "^2.48.2", 43 | "chai": "^4.2.0", 44 | "mocha": "^10.1.0", 45 | "nyc": "^14.1.1", 46 | "tslint": "^5.18.0", 47 | "typescript": "^3.5.3" 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /src/models/default-operation.ts: -------------------------------------------------------------------------------- 1 | import events = require('events'); 2 | import request = require('request'); 3 | 4 | import {FileSegmentation} from '../utilities/file-segmentation'; 5 | 6 | import {Operation} from './operation'; 7 | import {PartialDownload, PartialDownloadRange} from './partial-download'; 8 | 9 | export class DefaultOperation implements Operation { 10 | 11 | private readonly emitter: events.EventEmitter = new events.EventEmitter(); 12 | 13 | public start(url: string, contentLength: number, numOfConnections: number, headers?: request.Headers): events.EventEmitter { 14 | let endCounter: number = 0; 15 | 16 | const segmentsRange: PartialDownloadRange[] = FileSegmentation.getSegmentsRange(contentLength, numOfConnections); 17 | for (const segmentRange of segmentsRange) { 18 | 19 | new PartialDownload() 20 | .start(url, segmentRange, headers) 21 | .on('error', (err) => { 22 | this.emitter.emit('error', err); 23 | }) 24 | .on('data', (data, offset) => { 25 | this.emitter.emit('data', data, offset); 26 | }) 27 | .on('end', () => { 28 | if (++endCounter === numOfConnections) { 29 | this.emitter.emit('end', null); 30 | } 31 | }); 32 | } 33 | 34 | return this.emitter; 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /test/operation-factory-test.ts: -------------------------------------------------------------------------------- 1 | import os = require('os'); 2 | 3 | import {expect} from 'chai'; 4 | 5 | import {TestConfig} from './test-config'; 6 | 7 | import {BufferOperation} from '../src/models/buffer-operation'; 8 | import {DefaultOperation} from '../src/models/default-operation'; 9 | import {FileOperation} from '../src/models/file-operation'; 10 | import {Operation} from '../src/models/operation'; 11 | import {OperationFactory} from '../src/models/operation-factory'; 12 | import {StartOptions} from '../src/models/start-options'; 13 | 14 | describe('Operation factory', () => { 15 | it('default operation', () => { 16 | const options: StartOptions = {}; 17 | 18 | const operation: Operation = OperationFactory.getOperation(options); 19 | 20 | expect(operation).to.be.instanceof(DefaultOperation); 21 | }); 22 | 23 | it('buffer operation', () => { 24 | const options: StartOptions = { 25 | writeToBuffer: true 26 | }; 27 | 28 | const operation: Operation = OperationFactory.getOperation(options); 29 | 30 | expect(operation).to.be.instanceof(BufferOperation); 31 | }); 32 | 33 | it('file operation', () => { 34 | const options: StartOptions = { 35 | saveDirectory: os.tmpdir() 36 | }; 37 | 38 | const operation: Operation = OperationFactory.getOperation(options); 39 | 40 | expect(operation).to.be.instanceof(FileOperation); 41 | }); 42 | }); -------------------------------------------------------------------------------- /src/models/buffer-operation.ts: -------------------------------------------------------------------------------- 1 | import events = require('events'); 2 | import request = require('request'); 3 | 4 | import {FileSegmentation} from '../utilities/file-segmentation'; 5 | 6 | import {Operation} from './operation'; 7 | import {PartialDownload, PartialDownloadRange} from './partial-download'; 8 | 9 | export class BufferOperation implements Operation { 10 | 11 | private readonly emitter: events.EventEmitter = new events.EventEmitter(); 12 | 13 | public start(url: string, contentLength: number, numOfConnections: number, headers?: request.Headers): events.EventEmitter { 14 | const buffer = Buffer.allocUnsafe(contentLength); 15 | 16 | let endCounter: number = 0; 17 | 18 | const segmentsRange: PartialDownloadRange[] = FileSegmentation.getSegmentsRange(contentLength, numOfConnections); 19 | for (const segmentRange of segmentsRange) { 20 | 21 | new PartialDownload() 22 | .start(url, segmentRange, headers) 23 | .on('error', (err) => { 24 | this.emitter.emit('error', err); 25 | }) 26 | .on('data', (data, offset) => { 27 | this.emitter.emit('data', data, offset); 28 | 29 | data.copy(buffer, offset); 30 | }) 31 | .on('end', () => { 32 | if (++endCounter === numOfConnections) { 33 | this.emitter.emit('end', buffer); 34 | } 35 | }); 36 | } 37 | 38 | return this.emitter; 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /test/default-operation-test.ts: -------------------------------------------------------------------------------- 1 | import {expect} from 'chai'; 2 | 3 | import {TestConfig} from './test-config'; 4 | 5 | import {DefaultOperation} from '../src/models/default-operation'; 6 | 7 | describe('Default operation', () => { 8 | it('single connection download', (done) => { 9 | const numOfConnections: number = 1; 10 | let fileContentLengthCounter: number = 0; 11 | 12 | new DefaultOperation() 13 | .start(TestConfig.AcceptRangesSupportedUrl.url, TestConfig.AcceptRangesSupportedUrl.contentLength, numOfConnections) 14 | .on('data', (data, offset) => { 15 | fileContentLengthCounter += data.length; 16 | }) 17 | .on('end', () => { 18 | expect(fileContentLengthCounter).to.equal(TestConfig.AcceptRangesSupportedUrl.contentLength); 19 | 20 | done(); 21 | }); 22 | }).timeout(TestConfig.Timeout); 23 | 24 | it('multi connection download', (done) => { 25 | const numOfConnections: number = 5; 26 | let fileContentLengthCounter: number = 0; 27 | 28 | new DefaultOperation() 29 | .start(TestConfig.AcceptRangesSupportedUrl.url, TestConfig.AcceptRangesSupportedUrl.contentLength, numOfConnections) 30 | .on('data', (data, offset) => { 31 | fileContentLengthCounter += data.length; 32 | }) 33 | .on('end', () => { 34 | expect(fileContentLengthCounter).to.equal(TestConfig.AcceptRangesSupportedUrl.contentLength); 35 | 36 | done(); 37 | }); 38 | }).timeout(TestConfig.Timeout); 39 | }); -------------------------------------------------------------------------------- /test/validation-test.ts: -------------------------------------------------------------------------------- 1 | import {expect} from 'chai'; 2 | 3 | import {Validation} from '../src/utilities/validation'; 4 | 5 | describe('Input validation', () => { 6 | it('is valid url', () => { 7 | const url: string = 'http://github.com'; 8 | const result: boolean = Validation.isUrl(url); 9 | 10 | expect(result).to.be.true; 11 | }); 12 | 13 | it('is valid number of connections', () => { 14 | const numOfConnections: number = 1; 15 | const result: boolean = Validation.isValidNumberOfConnections(numOfConnections); 16 | 17 | expect(result).to.be.true; 18 | }); 19 | 20 | it('is invalid number of connections', () => { 21 | const numOfConnections: number = 0; 22 | const result: boolean = Validation.isValidNumberOfConnections(numOfConnections); 23 | 24 | expect(result).to.not.be.true; 25 | }); 26 | 27 | it('is valid directory', () => { 28 | const directory: string = __dirname; 29 | const result: boolean = Validation.isDirectory(directory); 30 | 31 | expect(result).to.be.true; 32 | }); 33 | 34 | it('is invalid directory', () => { 35 | const directory: string = '/invalid/directory'; 36 | const result: boolean = Validation.isDirectory(directory); 37 | 38 | expect(result).to.not.be.true; 39 | }); 40 | 41 | it('is valid file name', () => { 42 | const fileName: string = 'grumpy_cat.jpg'; 43 | const result: boolean = Validation.isValidFileName(fileName); 44 | 45 | expect(result).to.be.true; 46 | }); 47 | 48 | it('is invalid file name', () => { 49 | const fileName: string = 'grumpy*cat.jpg'; 50 | const result: boolean = Validation.isValidFileName(fileName); 51 | 52 | expect(result).to.not.be.true; 53 | }); 54 | }); -------------------------------------------------------------------------------- /test/buffer-operation-test.ts: -------------------------------------------------------------------------------- 1 | import {expect} from 'chai'; 2 | 3 | import {TestConfig} from './test-config'; 4 | 5 | import {BufferOperation} from '../src/models/buffer-operation'; 6 | 7 | describe('Buffer operation', () => { 8 | it('single connection download', (done) => { 9 | const numOfConnections: number = 1; 10 | let fileContentLengthCounter: number = 0; 11 | 12 | new BufferOperation() 13 | .start(TestConfig.AcceptRangesSupportedUrl.url, TestConfig.AcceptRangesSupportedUrl.contentLength, numOfConnections) 14 | .on('data', (data, offset) => { 15 | fileContentLengthCounter += data.length; 16 | }) 17 | .on('end', (buffer) => { 18 | expect(buffer.length).to.be.equal(TestConfig.AcceptRangesSupportedUrl.contentLength); 19 | expect(fileContentLengthCounter).to.equal(TestConfig.AcceptRangesSupportedUrl.contentLength); 20 | 21 | done(); 22 | }); 23 | }).timeout(TestConfig.Timeout); 24 | 25 | it('multi connection download', (done) => { 26 | const numOfConnections: number = 5; 27 | let fileContentLengthCounter: number = 0; 28 | 29 | new BufferOperation() 30 | .start(TestConfig.AcceptRangesSupportedUrl.url, TestConfig.AcceptRangesSupportedUrl.contentLength, numOfConnections) 31 | .on('data', (data, offset) => { 32 | fileContentLengthCounter += data.length; 33 | }) 34 | .on('end', (buffer) => { 35 | expect(buffer.length).to.be.equal(TestConfig.AcceptRangesSupportedUrl.contentLength); 36 | expect(fileContentLengthCounter).to.equal(TestConfig.AcceptRangesSupportedUrl.contentLength); 37 | 38 | done(); 39 | }); 40 | }).timeout(TestConfig.Timeout); 41 | }); -------------------------------------------------------------------------------- /test/multipart-download-test.ts: -------------------------------------------------------------------------------- 1 | import fs = require('fs'); 2 | import os = require('os'); 3 | 4 | import {expect} from 'chai'; 5 | 6 | import {TestConfig} from './test-config'; 7 | 8 | import {MultipartDownload} from '../src/models/multipart-download'; 9 | import {StartOptions} from '../src/models/start-options'; 10 | 11 | describe('Multipart download', () => { 12 | it('download with Accept-Ranges header without passing start options', (done) => { 13 | let fileContentLengthCounter: number = 0; 14 | 15 | new MultipartDownload() 16 | .start(TestConfig.AcceptRangesSupportedUrl.url) 17 | .on('data', (data, offset) => { 18 | fileContentLengthCounter += data.length; 19 | }) 20 | .on('end', () => { 21 | expect(fileContentLengthCounter).to.equal(TestConfig.AcceptRangesSupportedUrl.contentLength); 22 | done(); 23 | }); 24 | }).timeout(TestConfig.Timeout);; 25 | 26 | it('download with Accept-Ranges header with start options', (done) => { 27 | const options: StartOptions = { 28 | numOfConnections: 5 29 | }; 30 | 31 | let fileContentLengthCounter: number = 0; 32 | 33 | new MultipartDownload() 34 | .start(TestConfig.AcceptRangesSupportedUrl.url, options) 35 | .on('data', (data, offset) => { 36 | fileContentLengthCounter += data.length; 37 | }) 38 | .on('end', () => { 39 | expect(fileContentLengthCounter).to.equal(TestConfig.AcceptRangesSupportedUrl.contentLength); 40 | done(); 41 | }); 42 | }).timeout(TestConfig.Timeout);; 43 | 44 | it('download without Accept-Ranges header with start options', (done) => { 45 | const options: StartOptions = { 46 | numOfConnections: 5 47 | }; 48 | 49 | let fileContentLengthCounter: number = 0; 50 | 51 | new MultipartDownload() 52 | .start(TestConfig.AcceptRangesUnsupportedUrl.url, options) 53 | .on('data', (data, offset) => { 54 | fileContentLengthCounter += data.length; 55 | }) 56 | .on('end', () => { 57 | expect(fileContentLengthCounter).to.equal(TestConfig.AcceptRangesUnsupportedUrl.contentLength); 58 | done(); 59 | }); 60 | }).timeout(TestConfig.Timeout);; 61 | }); -------------------------------------------------------------------------------- /src/models/file-operation.ts: -------------------------------------------------------------------------------- 1 | import events = require('events'); 2 | import fs = require('fs'); 3 | import request = require('request'); 4 | 5 | import {FileSegmentation} from '../utilities/file-segmentation'; 6 | import {PathFormatter} from '../utilities/path-formatter'; 7 | import {UrlParser} from '../utilities/url-parser'; 8 | 9 | import {Operation} from './operation'; 10 | import {PartialDownload, PartialDownloadRange} from './partial-download'; 11 | 12 | export class FileOperation implements Operation { 13 | 14 | private readonly emitter: events.EventEmitter = new events.EventEmitter(); 15 | 16 | public constructor(private saveDirectory: string, private fileName?: string) { } 17 | 18 | public start(url: string, contentLength: number, numOfConnections: number, headers?: request.Headers): events.EventEmitter { 19 | const file: string = this.fileName ? this.fileName : UrlParser.getFilename(url); 20 | const filePath: string = PathFormatter.format(this.saveDirectory, file); 21 | 22 | let endCounter: number = 0; 23 | 24 | fs.open(filePath, 'w+', 0o644, (err, fd) => { 25 | if (err) { 26 | this.emitter.emit('error', err); 27 | return; 28 | } 29 | 30 | const segmentsRange: PartialDownloadRange[] = FileSegmentation.getSegmentsRange(contentLength, numOfConnections); 31 | 32 | for (const segmentRange of segmentsRange) { 33 | 34 | new PartialDownload() 35 | .start(url, segmentRange, headers) 36 | .on('error', (error) => { 37 | this.emitter.emit('error', error); 38 | }) 39 | .on('data', (data, offset) => { 40 | fs.write(fd, data, 0, data.length, offset, (error) => { 41 | if (error) { 42 | this.emitter.emit('error', error); 43 | } else { 44 | this.emitter.emit('data', data, offset); 45 | } 46 | }); 47 | }) 48 | .on('end', () => { 49 | if (++endCounter === numOfConnections) { 50 | fs.close(fd, (error) => { 51 | if (error) { 52 | this.emitter.emit('error', error); 53 | } else { 54 | this.emitter.emit('end', filePath); 55 | } 56 | }); 57 | } 58 | }); 59 | } 60 | 61 | }); 62 | 63 | return this.emitter; 64 | } 65 | 66 | } 67 | -------------------------------------------------------------------------------- /src/models/multipart-download.ts: -------------------------------------------------------------------------------- 1 | import events = require('events'); 2 | 3 | import {Validation} from '../utilities/validation'; 4 | 5 | import {AcceptRanges} from './accept-ranges'; 6 | import {Operation} from './operation'; 7 | import {OperationFactory} from './operation-factory'; 8 | import {PartialRequestMetadata, PartialRequestQuery} from './partial-request-query'; 9 | import {StartOptions} from './start-options'; 10 | 11 | export interface MultipartOperation { 12 | start(url: string, options?: StartOptions): MultipartOperation; 13 | } 14 | 15 | export class MultipartDownload extends events.EventEmitter implements MultipartOperation { 16 | private static readonly SINGLE_CONNECTION: number = 1; 17 | 18 | public start(url: string, options: StartOptions = { numOfConnections: MultipartDownload.SINGLE_CONNECTION }): MultipartDownload { 19 | options.numOfConnections = options.numOfConnections || MultipartDownload.SINGLE_CONNECTION; 20 | 21 | const validationError: Error = this.validateInputs(url, options); 22 | if (validationError) { 23 | this.emit('error', validationError); 24 | } 25 | 26 | this.execute(url, options); 27 | 28 | return this; 29 | } 30 | 31 | private execute(url: string, options: StartOptions): void { 32 | new PartialRequestQuery() 33 | .getMetadata(url, options.headers) 34 | .then((metadata) => { 35 | 36 | const metadataError: Error = this.validateMetadata(url, metadata); 37 | if (metadataError) { 38 | this.emit('error', metadataError); 39 | } 40 | 41 | if (metadata.acceptRanges !== AcceptRanges.Bytes) { 42 | options.numOfConnections = MultipartDownload.SINGLE_CONNECTION; 43 | } 44 | 45 | const operation: Operation = OperationFactory.getOperation(options); 46 | operation 47 | .start(url, metadata.contentLength, options.numOfConnections, options.headers) 48 | .on('error', (err) => { 49 | this.emit('error', err); 50 | }) 51 | .on('data', (data, offset) => { 52 | this.emit('data', data, offset); 53 | }) 54 | .on('end', (output) => { 55 | this.emit('end', output); 56 | }); 57 | }) 58 | .catch((err) => { 59 | this.emit('error', err); 60 | }); 61 | } 62 | 63 | private validateInputs(url: string, options: StartOptions): Error { 64 | if (!Validation.isUrl(url)) { 65 | return new Error('Invalid URL provided'); 66 | } 67 | 68 | if (!Validation.isValidNumberOfConnections(options.numOfConnections)) { 69 | return new Error('Invalid number of connections provided'); 70 | } 71 | 72 | if (options.saveDirectory && !Validation.isDirectory(options.saveDirectory)) { 73 | return new Error('Invalid save directory provided'); 74 | } 75 | 76 | if (options.fileName && !Validation.isValidFileName(options.fileName)) { 77 | return new Error('Invalid file name provided'); 78 | } 79 | 80 | return null; 81 | } 82 | 83 | private validateMetadata(url: string, metadata: PartialRequestMetadata): Error { 84 | if (isNaN(metadata.contentLength)) { 85 | return new Error(`Failed to query Content-Length of ${url}`); 86 | } 87 | 88 | return null; 89 | } 90 | } 91 | -------------------------------------------------------------------------------- /test/file-operation-test.ts: -------------------------------------------------------------------------------- 1 | import fs = require('fs'); 2 | import os = require('os'); 3 | 4 | import {expect} from 'chai'; 5 | 6 | import {TestConfig} from './test-config'; 7 | 8 | import {FileOperation} from '../src/models/file-operation'; 9 | 10 | describe('File operation', () => { 11 | it('single connection download', (done) => { 12 | const numOfConnections: number = 1; 13 | let fileContentLengthCounter: number = 0; 14 | 15 | new FileOperation(os.tmpdir()) 16 | .start(TestConfig.AcceptRangesSupportedUrl.url, TestConfig.AcceptRangesSupportedUrl.contentLength, numOfConnections) 17 | .on('data', (data, offset) => { 18 | fileContentLengthCounter += data.length; 19 | }) 20 | .on('end', (filePath) => { 21 | expect(fileContentLengthCounter).to.equal(TestConfig.AcceptRangesSupportedUrl.contentLength); 22 | 23 | expect(filePath).to.exist; 24 | 25 | // check downloaded file exist 26 | fs.lstat(filePath, (err, stats) => { 27 | expect(err).to.be.null; 28 | 29 | // delete downloaded file 30 | fs.unlink(filePath, (err) => { 31 | expect(err).to.be.null; 32 | 33 | done(); 34 | }); 35 | }); 36 | }); 37 | }).timeout(TestConfig.Timeout);; 38 | 39 | it('multi connection download and save file with name from url', (done) => { 40 | const numOfConnections: number = 5; 41 | let fileContentLengthCounter: number = 0; 42 | 43 | new FileOperation(os.tmpdir()) 44 | .start(TestConfig.AcceptRangesSupportedUrl.url, TestConfig.AcceptRangesSupportedUrl.contentLength, numOfConnections) 45 | .on('data', (data, offset) => { 46 | fileContentLengthCounter += data.length; 47 | }) 48 | .on('end', (filePath) => { 49 | expect(fileContentLengthCounter).to.equal(TestConfig.AcceptRangesSupportedUrl.contentLength); 50 | 51 | expect(filePath).to.exist; 52 | 53 | // check downloaded file exist 54 | fs.lstat(filePath, (err, stats) => { 55 | expect(err).to.be.null; 56 | 57 | // delete downloaded file 58 | fs.unlink(filePath, (err) => { 59 | expect(err).to.be.null; 60 | 61 | done(); 62 | }); 63 | }); 64 | }); 65 | }).timeout(TestConfig.Timeout);; 66 | 67 | it('multi connection download and save file with name with specified name', (done) => { 68 | const numOfConnections: number = 5; 69 | let fileContentLengthCounter: number = 0; 70 | 71 | new FileOperation(os.tmpdir(), 'kittycat.png') 72 | .start(TestConfig.AcceptRangesSupportedUrl.url, TestConfig.AcceptRangesSupportedUrl.contentLength, numOfConnections) 73 | .on('data', (data, offset) => { 74 | fileContentLengthCounter += data.length; 75 | }) 76 | .on('end', (filePath) => { 77 | expect(fileContentLengthCounter).to.equal(TestConfig.AcceptRangesSupportedUrl.contentLength); 78 | 79 | expect(filePath).to.exist; 80 | 81 | // check downloaded file exist 82 | fs.lstat(filePath, (err, stats) => { 83 | expect(err).to.be.null; 84 | 85 | // delete downloaded file 86 | fs.unlink(filePath, (err) => { 87 | expect(err).to.be.null; 88 | 89 | done(); 90 | }); 91 | }); 92 | }); 93 | }).timeout(TestConfig.Timeout);; 94 | }); -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # multipart-download [![Build Status](https://travis-ci.org/zulhilmizainuddin/multipart-download.svg?branch=master)](https://travis-ci.org/zulhilmizainuddin/multipart-download) 2 | 3 | [![NPM](https://nodei.co/npm/multipart-download.png?downloads=true&downloadRank=true&stars=true)](https://nodei.co/npm/multipart-download/) 4 | 5 | Speed up download of a single file with multiple HTTP GET connections running in parallel 6 | 7 | ## Class: MultipartDownload 8 | 9 | MultipartDownload is an `EventEmitter`. 10 | 11 | ### start(url[, options]) 12 | - `url` <string> Url of file to be downloaded 13 | - `options` <StartOptions> Download options (Optional) 14 | - `numOfConnections` <number> Number of HTTP GET connections to use for performing the download (Optional) 15 | - `writeToBuffer` <boolean> Store downloaded data to buffer (Optional) 16 | - `saveDirectory` <string> Directory to save the downloaded file (Optional) 17 | - `fileName` <string> Set name of the downloaded file (Optional) 18 | - `headers` <Object> Set custom HTTP headers (Optional) 19 | 20 | Starts the download operation from the `url`. 21 | 22 | Multiple HTTP GET connections will only be used if the target server supports partial requests. 23 | If the target server does not support partial requests, only a single HTTP GET connection will be used regardless of what the `numOfConnections` is set to. 24 | 25 | If the `numOfConnections` parameter is not provided, a single connection will be used. 26 | 27 | If the `writeToBuffer` parameter is set to `true`, the downloaded file will be written into a buffer. 28 | 29 | If the `saveDirectory` parameter is provided, the downloaded file will be saved to the `saveDirectory`. 30 | 31 | If the `fileName` parameter is provided, the downloaded file will be renamed to `fileName`. 32 | If the `fileName` parameter is not provided, the downloaded file will maintain its original file name. 33 | 34 | If the `headers` parameter is provided, the headers will be included in the HTTP request. 35 | 36 | #### Event: 'error' 37 | - `err` <Error> Emitted error 38 | 39 | #### Event: 'data' 40 | - `data` <string> | <Buffer> Chunk of data received 41 | - `offset` <number> Offset for the chunk of data received 42 | 43 | The file being downloaded can be manually constructed and manipulated using the `data` and `offset` received. 44 | 45 | #### Event: 'end' 46 | - `output` <string> Downloaded file buffer or downloaded file saved path 47 | 48 | `output` is the buffer of the downloaded file if the `writeToBuffer` parameter is set to `true`. 49 | 50 | `output` is the location of the saved file if the `saveDirectory` parameter is provided. 51 | 52 | `output` will be `null` if `writeToBuffer` is not set to `true` or `saveDirectory` parameter is not provided. 53 | 54 | ### ~~start(url[, numOfConnections, saveDirectory])~~ :exclamation: DEPRECATED and REMOVED in v1.0.0 55 | 56 | ### Example 57 | 58 | #### Download without writing to buffer or saving to file 59 | 60 | ```javascript 61 | const MultipartDownload = require('multipart-download'); 62 | 63 | new MultipartDownload() 64 | .start('https://homepages.cae.wisc.edu/~ece533/images/cat.png', { 65 | numOfConnections: 5 66 | }) 67 | .on('error', (err) => { 68 | // handle error here 69 | }) 70 | .on('data', (data, offset) => { 71 | // manipulate data here 72 | }) 73 | .on('end', () => { 74 | 75 | }); 76 | ``` 77 | 78 | #### Download and write to buffer 79 | 80 | ```javascript 81 | const MultipartDownload = require('multipart-download'); 82 | 83 | new MultipartDownload() 84 | .start('https://homepages.cae.wisc.edu/~ece533/images/cat.png', { 85 | numOfConnections: 5, 86 | writeToBuffer: true 87 | }) 88 | .on('error', (err) => { 89 | // handle error here 90 | }) 91 | .on('data', (data, offset) => { 92 | // manipulate data here 93 | }) 94 | .on('end', (buffer) => { 95 | console.log(`Downloaded file buffer: ${buffer}`); 96 | }); 97 | ``` 98 | 99 | #### Download and save to file 100 | 101 | ```javascript 102 | const os = require('os'); 103 | 104 | const MultipartDownload = require('multipart-download'); 105 | 106 | new MultipartDownload() 107 | .start('https://homepages.cae.wisc.edu/~ece533/images/cat.png', { 108 | numOfConnections: 5, 109 | saveDirectory: os.tmpdir(), 110 | fileName: 'kitty.png' 111 | }) 112 | .on('error', (err) => { 113 | // handle error here 114 | }) 115 | .on('data', (data, offset) => { 116 | // manipulate data here 117 | }) 118 | .on('end', (filePath) => { 119 | console.log(`Downloaded file path: ${filePath}`); 120 | }); 121 | ``` 122 | --------------------------------------------------------------------------------