├── .gitignore
├── package.json
├── README.md
├── server.js
├── frontend
├── index.html
├── index.js
└── uploader.js
├── LICENSE
└── backend
└── uploadByChank.js
/.gitignore:
--------------------------------------------------------------------------------
1 | files/*
2 | .idea/*
3 | node_modules/*
--------------------------------------------------------------------------------
/package.json:
--------------------------------------------------------------------------------
1 | {
2 | "name": "multithreaded-uploader",
3 | "version": "1.0.0",
4 | "description": "",
5 | "main": "server.js",
6 | "author": "Michael Pilov",
7 | "license": "MIT",
8 | "dependencies": {
9 | "express": "4.16.2"
10 | }
11 | }
12 |
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | # Multithreaded uploader
2 | This project have developed to show you how to create multithreaded file uploading in javascript.
3 |
4 | More information about multithreaded uploading you can find in my article [here](https://medium.com/@pilovm/multithreaded-file-uploading-with-javascript-dafabce34ccd).
5 | ## How to test it
6 | You have to clone the repository and start the server
7 | ```
8 | git clone https://github.com/pilovm/multithreaded-uploader.git
9 | ```
10 | ```
11 | npm i
12 | ```
13 | ```
14 | npm start
15 | ```
16 | Then you can open `localhost:3000` and try to upload files. They will store in `files` folder.
17 |
--------------------------------------------------------------------------------
/server.js:
--------------------------------------------------------------------------------
1 | const express = require("express");
2 | const path = require("path");
3 | const fs = require("fs");
4 |
5 | const {loadingByChunks, initUploading} = require("./backend/uploadByChank");
6 |
7 | const app = express();
8 |
9 | app.post("/upload", loadingByChunks);
10 |
11 | app.post("/upload/init", initUploading);
12 |
13 | app.get("*", function (request, response) {
14 | const fullPath = request.params[0];
15 |
16 | if (fullPath === "/") {
17 | response.sendFile(path.join('frontend/', 'index.html'), { root: __dirname });
18 | return;
19 | }
20 |
21 | response.sendFile(path.join('frontend/', fullPath), { root: __dirname });
22 | });
23 |
24 | app.listen(3000, () => {
25 | console.log("Server started on port 3000");
26 | });
--------------------------------------------------------------------------------
/frontend/index.html:
--------------------------------------------------------------------------------
1 |
2 |
3 | Multithreaded file uploader
4 |
9 |
10 |
11 |
12 |
30 |
31 |
32 |
33 |
34 |
--------------------------------------------------------------------------------
/LICENSE:
--------------------------------------------------------------------------------
1 | MIT License
2 |
3 | Copyright (c) 2018 pilovm
4 |
5 | Permission is hereby granted, free of charge, to any person obtaining a copy
6 | of this software and associated documentation files (the "Software"), to deal
7 | in the Software without restriction, including without limitation the rights
8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9 | copies of the Software, and to permit persons to whom the Software is
10 | furnished to do so, subject to the following conditions:
11 |
12 | The above copyright notice and this permission notice shall be included in all
13 | copies or substantial portions of the Software.
14 |
15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21 | SOFTWARE.
22 |
--------------------------------------------------------------------------------
/frontend/index.js:
--------------------------------------------------------------------------------
1 | !(function () {
2 | const form = document.getElementById("form");
3 | const fileInput = document.getElementById("file");
4 | const uploadButton = document.getElementById("upload");
5 | const continueButton = document.getElementById("continue");
6 | const resetButton = document.getElementById("reset");
7 | const progressNode = document.getElementById("progress");
8 | const timeNode = document.getElementById("uploadTime");
9 | const threadsNode = document.getElementById("threads");
10 | const chunkSizeNode = document.getElementById("chunkSize");
11 |
12 | form.addEventListener("submit", (event) => {
13 | event.preventDefault();
14 |
15 | fileInput.setAttribute("disabled", "disabled");
16 | uploadButton.setAttribute("disabled", "disabled");
17 | timeNode.innerText = "";
18 |
19 | const file = fileInput.files[0];
20 | let aborted = false;
21 | const endTimer = getTimeCounter();
22 |
23 | const chunksUploader = uploader()
24 | .onProgress(({loaded, total}) => {
25 | const percent = Math.round(loaded / total * 100 * 100) / 100;
26 |
27 | progressNode.innerText = `${percent}%`;
28 | })
29 | .options({
30 | chunkSize: Number(chunkSizeNode.value) * 1024 * 1024,
31 | threadsQuantity: Number(threadsNode.value)
32 | })
33 | .send(file)
34 | .end((error, data) => {
35 | if (error) {
36 | if (!aborted) {
37 | continueButton.removeAttribute("disabled");
38 | continueButton.addEventListener("click", () => {
39 | chunksUploader.continue();
40 | continueButton.setAttribute("disabled", "disabled");
41 | }, {once: true});
42 | }
43 |
44 | console.log("Error", error);
45 | return;
46 | }
47 |
48 | const timeSpent = endTimer();
49 |
50 | fileInput.removeAttribute("disabled");
51 | uploadButton.removeAttribute("disabled");
52 | continueButton.setAttribute("disabled", "disabled");
53 | timeNode.innerText = `(${timeSpent / 1000} sec)`;
54 | });
55 |
56 | resetButton.addEventListener("click", () => {
57 | aborted = true;
58 | chunksUploader.abort();
59 | fileInput.removeAttribute("disabled");
60 | uploadButton.removeAttribute("disabled");
61 | continueButton.setAttribute("disabled", "disabled");
62 | progressNode.innerText = "";
63 | timeNode.innerText = "";
64 |
65 | }, {once: true});
66 | });
67 |
68 | function getTimeCounter() {
69 | const start = + new Date();
70 |
71 | return () => {
72 | return + new Date() - start;
73 | }
74 | }
75 | })();
--------------------------------------------------------------------------------
/backend/uploadByChank.js:
--------------------------------------------------------------------------------
1 | const fs = require("fs");
2 | const crypto = require("crypto");
3 |
4 | const fileStorage = {};
5 |
6 | function UploadedFile(name, size, chunksQuantity) {
7 | this.contentSizes = [];
8 | this.chunks = [];
9 | this.name = name;
10 | this.size = size;
11 | this.chunksQuantity = chunksQuantity;
12 | this.chunksDone = 0;
13 | }
14 |
15 | UploadedFile.prototype.getChunkLength = function (id) {
16 | if (!this.chunks[id]) {
17 | return 0;
18 | }
19 |
20 | return this.chunks[id].length;
21 | };
22 |
23 | UploadedFile.prototype.pushChunk = function(id, chunk, contentLength) {
24 | const completeChunk = Buffer.concat(chunk);
25 |
26 | if (completeChunk.length !== contentLength) {
27 | return false;
28 | }
29 |
30 | this.chunks[id] = completeChunk;
31 | this.chunksDone += 1;
32 |
33 | return true;
34 | };
35 |
36 | UploadedFile.prototype.isCompleted = function() {
37 | return this.chunksQuantity === this.chunksDone;
38 | };
39 |
40 | UploadedFile.prototype.getContent = function () {
41 | return Buffer.concat(this.chunks);
42 | };
43 |
44 | function initUploading(request, response) {
45 | if (!"x-content-name" in request.headers) {
46 | sendBadRequest(response, "Can't initialize file uploading: request has no content name header");
47 | return;
48 | }
49 |
50 | if (!"x-content-length" in request.headers) {
51 | sendBadRequest(response, "Can't initialize file uploading: request has no content length header");
52 | return;
53 | }
54 |
55 | if (!"x-chunks-quantity" in request.headers) {
56 | sendBadRequest(response, "Can't initialize file uploading: request has no chunks quantity header");
57 | return;
58 | }
59 |
60 | const name = request.headers["x-content-name"];
61 | const size = Number(request.headers["x-content-length"]);
62 | const chunksQuantity = Number(request.headers["x-chunks-quantity"]);
63 | const fileId = crypto.randomBytes(128).toString("hex");
64 |
65 | fileStorage[fileId] = new UploadedFile(name, size, chunksQuantity);
66 |
67 | response.write(JSON.stringify({
68 | status: 200,
69 | fileId
70 | }));
71 | response.end();
72 | }
73 |
74 | function loadingByChunks(request, response) {
75 | if (!"x-content-id" in request.headers) {
76 | sendBadRequest(response, "Request has no content id header");
77 | return;
78 | }
79 |
80 | if (!"x-chunk-id" in request.headers) {
81 | sendBadRequest(response, "Request has no chunk id header");
82 | return;
83 | }
84 |
85 | const fileId = request.headers["x-content-id"];
86 | const chunkId = request.headers["x-chunk-id"];
87 | const chunkSize = Number(request.headers["content-length"]);
88 | const file = fileStorage[fileId];
89 | const chunk = [];
90 |
91 | if (!file) {
92 | sendBadRequest(response, "Wrong content id header");
93 | return;
94 | }
95 |
96 | request.on("data", (part) => {
97 | chunk.push(part);
98 | }).on("end", () => {
99 | const chunkComplete = file.pushChunk(chunkId, chunk, chunkSize);
100 |
101 | if (!chunkComplete) {
102 | sendBadRequest(response, "Chunk uploading was not completed");
103 | return;
104 | }
105 |
106 | const size = file.getChunkLength(chunkId);
107 |
108 | if (file.isCompleted()) {
109 | const fstream = fs.createWriteStream(__dirname + '/../files/' + file.name);
110 |
111 | fstream.write(file.getContent());
112 | fstream.end();
113 |
114 | delete fileStorage[fileId];
115 | }
116 |
117 | response.setHeader("Content-Type", "application/json");
118 | response.write(JSON.stringify({
119 | status: 200,
120 | size
121 | }));
122 | response.end();
123 | });
124 | }
125 |
126 | function sendBadRequest(response, error) {
127 | response.write(JSON.stringify({
128 | status: 400,
129 | error
130 | }));
131 | response.end();
132 | }
133 |
134 | exports.loadingByChunks = loadingByChunks;
135 | exports.initUploading = initUploading;
--------------------------------------------------------------------------------
/frontend/uploader.js:
--------------------------------------------------------------------------------
1 | const uploader = function () {
2 | function Uploader() {
3 | this.chunkSize = 1024 * 1024;
4 | this.threadsQuantity = 2;
5 |
6 | this.file = null;
7 | this.aborted = false;
8 | this.uploadedSize = 0;
9 | this.progressCache = {};
10 | this.activeConnections = {};
11 | }
12 |
13 | Uploader.prototype.setOptions = function(options = {}) {
14 | this.chunkSize = options.chunkSize;
15 | this.threadsQuantity = options.threadsQuantity;
16 | }
17 |
18 | Uploader.prototype.setupFile = function(file) {
19 | if (!file) {
20 | return;
21 | }
22 |
23 | this.file = file;
24 | }
25 |
26 | Uploader.prototype.start = function() {
27 | if (!this.file) {
28 | throw new Error("Can't start uploading: file have not chosen");
29 | }
30 |
31 | const chunksQuantity = Math.ceil(this.file.size / this.chunkSize);
32 | this.chunksQueue = new Array(chunksQuantity).fill().map((_, index) => index).reverse();
33 |
34 | const xhr = new XMLHttpRequest();
35 |
36 | xhr.open("post", "/upload/init");
37 |
38 | xhr.setRequestHeader("X-Content-Length", this.file.size);
39 | xhr.setRequestHeader("X-Content-Name", this.file.name);
40 | xhr.setRequestHeader("X-Chunks-Quantity", chunksQuantity);
41 |
42 | xhr.onreadystatechange = () => {
43 | if (xhr.readyState === 4 && xhr.status === 200) {
44 | const response = JSON.parse(xhr.responseText);
45 |
46 | if (!response.fileId || response.status !== 200) {
47 | this.complete(new Error("Can't create file id"));
48 | return;
49 | }
50 |
51 | this.fileId = response.fileId;
52 | this.sendNext();
53 | }
54 | };
55 |
56 | xhr.onerror = (error) => {
57 | this.complete(error);
58 | };
59 |
60 | xhr.send();
61 | }
62 |
63 | Uploader.prototype.sendNext = function() {
64 | const activeConnections = Object.keys(this.activeConnections).length;
65 |
66 | if (activeConnections >= this.threadsQuantity) {
67 | return;
68 | }
69 |
70 | if (!this.chunksQueue.length) {
71 | if (!activeConnections) {
72 | this.complete(null);
73 | }
74 |
75 | return;
76 | }
77 |
78 | const chunkId = this.chunksQueue.pop();
79 | const sentSize = chunkId * this.chunkSize;
80 | const chunk = this.file.slice(sentSize, sentSize + this.chunkSize);
81 |
82 | this.sendChunk(chunk, chunkId)
83 | .then(() => {
84 | this.sendNext();
85 | })
86 | .catch((error) => {
87 | this.chunksQueue.push(chunkId);
88 |
89 | this.complete(error);
90 | });
91 |
92 | this.sendNext();
93 | }
94 |
95 | Uploader.prototype.complete = function(error) {
96 | if (error && !this.aborted) {
97 | this.end(error);
98 | return;
99 | }
100 |
101 | setTimeout(() => init());
102 |
103 | this.end(error);
104 | }
105 |
106 | Uploader.prototype.sendChunk = function(chunk, id) {
107 | return new Promise(async (resolve, reject) => {
108 | try {
109 | const response = await this.upload(chunk, id);
110 | const {status, size} = JSON.parse(response);
111 |
112 | if (status !== 200 || size !== chunk.size) {
113 | reject(new Error("Failed chunk upload"));
114 | return;
115 | }
116 | } catch (error) {
117 | reject(error);
118 | return;
119 | }
120 |
121 | resolve();
122 | })
123 | }
124 |
125 | Uploader.prototype.handleProgress = function(chunkId, event) {
126 | if (event.type === "progress" || event.type === "error" || event.type === "abort") {
127 | this.progressCache[chunkId] = event.loaded;
128 | }
129 |
130 | if (event.type === "loadend") {
131 | this.uploadedSize += this.progressCache[chunkId] || 0;
132 | delete this.progressCache[chunkId];
133 | }
134 |
135 | const inProgress = Object.keys(this.progressCache).reduce((memo, id) => memo += this.progressCache[id], 0);
136 |
137 | const sendedLength = Math.min(this.uploadedSize + inProgress, this.file.size);
138 |
139 | this.onProgress({
140 | loaded: sendedLength,
141 | total: this.file.size
142 | })
143 | }
144 |
145 | Uploader.prototype.upload = function(file, id) {
146 | return new Promise((resolve, reject) => {
147 | const xhr = this.activeConnections[id] = new XMLHttpRequest();
148 | const progressListener = this.handleProgress.bind(this, id);
149 |
150 | xhr.upload.addEventListener("progress", progressListener);
151 |
152 | xhr.addEventListener("error", progressListener);
153 | xhr.addEventListener("abort", progressListener);
154 | xhr.addEventListener("loadend", progressListener);
155 |
156 | xhr.open("post", "/upload");
157 |
158 | xhr.setRequestHeader("Content-Type", "application/octet-stream");
159 | xhr.setRequestHeader("Content-Length", file.size);
160 | xhr.setRequestHeader("X-Content-Id", this.fileId);
161 | xhr.setRequestHeader("X-Chunk-Id", id);
162 |
163 | xhr.onreadystatechange = (event) => {
164 | if (xhr.readyState === 4 && xhr.status === 200) {
165 | resolve(xhr.responseText);
166 | delete this.activeConnections[id];
167 | }
168 | };
169 |
170 | xhr.onerror = (error) => {
171 | reject(error);
172 | delete this.activeConnections[id];
173 | };
174 |
175 | xhr.onabort = () => {
176 | reject(new Error("Upload canceled by user"));
177 | delete this.activeConnections[id];
178 | };
179 |
180 | xhr.send(file);
181 | })
182 | }
183 |
184 | Uploader.prototype.on = function(method, callback) {
185 | if (typeof callback !== "function") {
186 | callback = () => {};
187 | }
188 |
189 | this[method] = callback;
190 | }
191 |
192 | Uploader.prototype.abort = function() {
193 | Object.keys(this.activeConnections).forEach((id) => {
194 | this.activeConnections[id].abort();
195 | });
196 |
197 | this.aborted = true;
198 | }
199 |
200 | const multithreadedUploader = new Uploader();
201 |
202 | return {
203 | options: function (options) {
204 | multithreadedUploader.setOptions(options);
205 |
206 | return this;
207 | },
208 | send: function (file) {
209 | multithreadedUploader.setupFile(file);
210 |
211 | return this;
212 | },
213 | continue: function () {
214 | multithreadedUploader.sendNext();
215 | },
216 | onProgress: function (callback) {
217 | multithreadedUploader.on("onProgress", callback);
218 |
219 | return this;
220 | },
221 | end: function (callback) {
222 | multithreadedUploader.on("end", callback);
223 | multithreadedUploader.start();
224 |
225 | return this;
226 | },
227 | abort: function () {
228 | multithreadedUploader.abort();
229 | }
230 | }
231 | };
--------------------------------------------------------------------------------