├── .editorconfig ├── .eslintrc ├── .github └── workflows │ ├── build.yml │ └── publish.yml ├── .gitignore ├── LICENSE.md ├── README.md ├── bin ├── actions.js ├── index.js ├── runner.js └── utils.js ├── dist └── client │ └── storacle.client.js ├── package-lock.json ├── package.json ├── src ├── browser │ └── client │ │ ├── .babelrc │ │ ├── index.js │ │ └── mock │ │ └── crypto.js ├── client.js ├── db │ └── transports │ │ └── loki │ │ └── index.js ├── errors.js ├── index.js ├── node.js ├── schema.js ├── server │ └── transports │ │ └── express │ │ ├── api │ │ ├── butler │ │ │ ├── controllers.js │ │ │ └── routes.js │ │ ├── master │ │ │ ├── controllers.js │ │ │ └── routes.js │ │ ├── node │ │ │ ├── controllers.js │ │ │ └── routes.js │ │ ├── routes.js │ │ └── slave │ │ │ ├── controllers.js │ │ │ └── routes.js │ │ ├── client │ │ ├── controllers.js │ │ └── routes.js │ │ ├── index.js │ │ ├── midds.js │ │ └── routes.js └── utils.js ├── test ├── .eslintrc ├── client.js ├── db │ └── loki.js ├── group.js ├── index.js ├── node.js ├── routes.js ├── server │ └── express.js ├── services.js ├── tools.js └── utils.js └── webpack.client.js /.editorconfig: -------------------------------------------------------------------------------- 1 | root = true 2 | 3 | [*] 4 | end_of_line = lf 5 | insert_final_newline = false 6 | indent_style = space 7 | indent_size = 2 8 | -------------------------------------------------------------------------------- /.eslintrc: -------------------------------------------------------------------------------- 1 | { 2 | "parser": "@babel/eslint-parser", 3 | "parserOptions": { 4 | "ecmaVersion": 2021, 5 | "sourceType": "module", 6 | "requireConfigFile": false, 7 | "babelOptions": { 8 | "plugins": [ 9 | "@babel/plugin-syntax-import-assertions" 10 | ] 11 | } 12 | }, 13 | "env": { 14 | "browser": true, 15 | "node": true, 16 | "es6": true 17 | }, 18 | "extends": "eslint:recommended", 19 | "rules": { 20 | "no-console": "warn" 21 | }, 22 | "globals": {} 23 | } 24 | -------------------------------------------------------------------------------- /.github/workflows/build.yml: -------------------------------------------------------------------------------- 1 | name: build 2 | on: 3 | push: 4 | branches: [ master ] 5 | pull_request: 6 | branches: [ master ] 7 | jobs: 8 | build: 9 | runs-on: ubuntu-latest 10 | strategy: 11 | matrix: 12 | node-version: [20] 13 | steps: 14 | - uses: actions/checkout@v4 15 | - name: Run tests with ${{ matrix.node-version }} 16 | uses: actions/setup-node@v4 17 | with: 18 | node-version: ${{ matrix.node-version }} 19 | - run: npm i -g npm@latest 20 | - run: npm ci 21 | - run: npm ddp 22 | - run: npm run build-ci -------------------------------------------------------------------------------- /.github/workflows/publish.yml: -------------------------------------------------------------------------------- 1 | name: publish 2 | on: 3 | push: 4 | branches: [ master ] 5 | jobs: 6 | publish: 7 | runs-on: ubuntu-latest 8 | steps: 9 | - uses: actions/checkout@v4 10 | - uses: actions/setup-node@v4 11 | with: 12 | node-version: 20 13 | - run: npm i -g npm@latest 14 | - run: npm ci 15 | - run: npm ddp 16 | - run: npm run build-ci 17 | - uses: JS-DevTools/npm-publish@v1 18 | with: 19 | token: ${{ secrets.NPM_TOKEN }} -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /node_modules 2 | *~ 3 | 4 | *.log 5 | *.bat 6 | *.git 7 | *.suo 8 | *.sln 9 | *.swp 10 | *.swo 11 | .vscode 12 | .idea -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | The MIT License 2 | 3 | Copyright (c) Alexander Balasyan 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated 6 | documentation files (the "Software"), to deal in the Software without restriction, including without limitation 7 | the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, 8 | and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 9 | 10 | The above copyright notice and this permission notice shall be included in all copies or substantial portions 11 | of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED 14 | TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 15 | THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF 16 | CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 17 | DEALINGS IN THE SOFTWARE. -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # [Storacle](https://github.com/ortexx/storacle/) [alpha] [![npm version](https://badge.fury.io/js/storacle.svg)](https://badge.fury.io/js/storacle) [![Build status](https://github.com/ortexx/storacle/workflows/build/badge.svg)](https://github.com/ortexx/storacle/actions) 2 | 3 | Storacle is a decentralized file storage based on [the spreadable protocol](https://github.com/ortexx/spreadable/). 4 | 5 | There is [an article here](https://ortex.medium.com/storacle-a-decentralized-file-storage-3f0c5c57591c) with an explanation. 6 | 7 | ```javascript 8 | import { Node } from 'storacle'; 9 | 10 | try { 11 | const node = new Node({ 12 | port: 4000, 13 | hostname: 'localhost' 14 | }); 15 | await node.init(); 16 | } 17 | catch(err) { 18 | console.error(err.stack); 19 | process.exit(1); 20 | } 21 | ``` 22 | 23 | ```javascript 24 | import { Client } from 'storacle'; 25 | 26 | try { 27 | const client = new Client({ 28 | address: 'localhost:4000' 29 | }); 30 | await client.init(); 31 | 32 | // Store our file 33 | const hash = await client.storeFile('./my-file'); 34 | 35 | // Get the direct file link 36 | const link = await client.getFileLink(hash); 37 | 38 | // Create the requested file link 39 | const requestedLink = client.createRequestedFileLink(hash); 40 | 41 | // Remove the file 42 | await client.removeFile(hash); 43 | } 44 | catch(err) { 45 | console.error(err.stack); 46 | process.exit(1); 47 | } 48 | ``` 49 | 50 | The example above shows the simplest usage of the library. But the server can be flexibly configured. 51 | 52 | ## Browser client 53 | You can also use the client in a browser. Look at the description of [the spreadable library](https://github.com/ortexx/spreadable/#how-to-use-the-client-in-a-browser). In window you have __window.ClientStoracle__ instead of __window.ClientSpreadable__. The prepared file name is __storacle.client.js__. 54 | 55 | ## How to use it via the command line 56 | Look at the description of [the spreadable library](https://github.com/ortexx/spreadable/#how-to-use-it-via-the-command-line). You only need to change everywhere **spreadable** word to **storacle**. 57 | 58 | ## How it works 59 | 60 | Nodes interact via [the spreadable mechanism](https://github.com/ortexx/spreadable/). The file can be added to the network through any node. Files are saved entirely, not splitted into parts. After saving you get a hash of the file. With this hash you can later get it again, delete it or do something else. If possible, links to files are cached so if you work with the same file and in some other cases you will receive it immediately without re-traversing the network. For better reliability files can be duplicated. How exactly you can customize yourself. By default, each file tends to have its copies in amount of __Math.ceil(Math.sqrt(networkSize))__. 61 | 62 | ## What are the limitations 63 | The number of files on one node is limited by the file system. The speed of receiving / saving in the network files is limited by spreadable protocol. 64 | 65 | ## What are the requirements 66 | Look at [the spreadable requirements](https://github.com/ortexx/spreadable/#what-are-the-requirements). 67 | 68 | ## Where to use it 69 | 70 | ### 1. Wherever files need to be stored decentralized 71 | For example, you can create a network that stores books, invite new members, gather a large collection of books, and share links with others. Instead of books there could be anything else. 72 | 73 | ### 2. For own needs 74 | Storing files of your own projects, websites, etc. The network can be made private and you will not need to figure out how to organize the file storage. 75 | 76 | ### 3. Serverless solutions 77 | Since the library is written in javascript, you can receive / send / work with files in the browser and do not use server code at all. In some cases, it can be very convenient. 78 | 79 | ## Node configuration 80 | 81 | When you create an instance of the node you can pass options below. Only specific options of this library are described here, without considering the options of the parent classes. 82 | 83 | * {integer} __[request.clientStoringConcurrency=20]__ - the maximum number of simultaneous client storing requests per endpoint. 84 | 85 | * {number|string} __[request.fileStoringNodeTimeout="2h"]__ - file storing timeout. 86 | 87 | * {number|string} __[request.cacheTimeout=250]__ - file cache link check timeout. 88 | 89 | * {number|string} __[storage.dataSize="45%"]__ - amount of space for storing files. If indicated in percent, the calculation will be based on the maximum available disk space. 90 | 91 | * {number|string} __[storage.tempSize="45%"]__ - amount of space for temporary files. If indicated in percent, the calculation will be based on the maximum available disk space. 92 | 93 | * {number|string} __[storage.tempLifetime="2h"]__ - temporary files holding period. 94 | 95 | * {integer} __[storage.autoCleanSize=0]__ - amount of space that should always be free. If indicated in percent, the calculation will be based on storage.dataSize. 96 | 97 | * {object} __[file]__ - section that responds for a single file settings. 98 | 99 | * {number|string} __[file.maxSize="40%"]__ - maximum size of one file. If indicated in percent, the calculation will be based on the maximum available disk space. 100 | 101 | * {number|string} __[file.minSize=0]__ - minimum size of one file. If indicated in percent, the calculation will be based on the maximum available disk space. 102 | 103 | * {integer|string} __[file.preferredDuplicates="auto"]__ - preferred number of file copies on the network. If indicated in percent, the calculation will be based on the network size. If the option is "auto" it will be calculated as `Math.ceil(Math.sqrt(networkSize))`. 104 | 105 | * {number|string} __[file.responseCacheLifetime="7d"]__ - period of file caching after giving it to the client. 106 | 107 | * {string[]} __[file.mimeWhitelist=[]]__ - whitelist for filtering a file by mime type. 108 | 109 | * {string[]} __[file.mimeBlacklist=[]]__ - blacklist for filtering a file by mime type. 110 | 111 | * {string[]} __[file.extWhitelist=[]]__ - whitelist for filtering a file by its extension. 112 | 113 | * {string[]} __[file.extBlacklist=[]]__ - blacklist for filtering a file by its extension. 114 | 115 | * {object|false} __[file.linkCache]__ - file link caching transport options. 116 | 117 | * {integer} __[file.linkCache.limit=50000]__ - maximum cache links. 118 | 119 | * {number|string} __[file.linkCache.lifetime="1d"]__ - cache link holding period. 120 | 121 | * {number|string} __[task.cleanUpStorageInterval="30s"]__ - storage cleanup task interval. 122 | 123 | * {number|string} __[task.cleanUpTempDirInterval="20s"]__ - temporary folder cleanup task interval. 124 | 125 | * {number|string} __[task.calculateStorageInfoInterval="3s"]__ - storage information calculaion task interval. 126 | 127 | ## Client configuration 128 | 129 | When you create an instance of the client you can pass options below. Only specific options of this library are described here, without considering the options of the parent classes. 130 | 131 | * {number|string} __[request.fileStoringTimeout="2.05h"]__ - file storing timeout. 132 | 133 | * {number|string} __[request.fileGettingTimeout="1h"]__ - file getting timeout. 134 | 135 | * {number|string} __[request.fileRemovalTimeout="10s"]__ - file removal timeout. 136 | 137 | * {number|string} __[request.fileLinkGettingTimeout="10s"]__ - file link getting timeout. 138 | 139 | ## Client interface 140 | 141 | async __Client.prototype.storeFile()__ - add file to the network. 142 | * {string|fse.ReadStream|Buffer|Blob} __file__ - any file 143 | * {object} __[options]__ - storing options 144 | * {number} __[options.timeout]__ - storing timeout 145 | 146 | async __Client.prototype.getFileLink()__ - get the file link by the hash. 147 | * {string} __hash__ - file hash 148 | * {object} __[options]__ - getting options 149 | * {number} __[options.timeout]__ - getting timeout 150 | 151 | async __Client.prototype.getFileLinks()__ - get all file links by the hash. 152 | * {string} __hash__ - file hash 153 | * {object} __[options]__ - getting options 154 | * {number} __[options.timeout]__ - getting timeout 155 | 156 | async __Client.prototype.getFileToBuffer()__ - download the file and return the buffer. 157 | * {string} __hash__ - file hash 158 | * {object} __[options]__ - getting options 159 | * {number} __[options.timeout]__ - getting timeout 160 | 161 | async __Client.prototype.getFileToPath()__ - download the file and write it to the specified path. 162 | * {string} __hash__ - file hash 163 | * {object} __[options]__ - getting options 164 | * {number} __[options.timeout]__ - getting timeout 165 | 166 | async __Client.prototype.getFileToBlob()__ - download the file and return the blob. For browser client only. 167 | * {string} __hash__ - file hash 168 | * {object} __[options]__ - getting options 169 | * {number} __[options.timeout]__ - getting timeout 170 | 171 | async __Client.prototype.removeFile()__ - Remove the file by hash. 172 | * {string} __hash__ - file hash 173 | * {object} __[options]__ - removal options 174 | * {number} __[options.timeout]__ - removal timeout 175 | 176 | __Client.prototype.createRequestedFileLink()__ - сreate a requested file link. This is convenient if you need to get the link without doing any asynchronous operations at the moment. 177 | * {string} __hash__ - file hash 178 | * {object} __[options]__ - options 179 | 180 | ## Contribution 181 | 182 | If you face a bug or have an idea how to improve the library, create an issue on github. In order to fix something or add new code yourself fork the library, make changes and create a pull request to the master branch. Don't forget about tests in this case. Also you can join [the project on github](https://github.com/ortexx/storacle/projects/1). 183 | -------------------------------------------------------------------------------- /bin/actions.js: -------------------------------------------------------------------------------- 1 | import chalk from "chalk"; 2 | import yargs from "yargs"; 3 | import srcUtils from "../src/utils.js"; 4 | import utils from "./utils.js"; 5 | import _actions from "spreadable/bin/actions.js"; 6 | 7 | const argv = yargs(process.argv).argv; 8 | const actions = Object.assign({}, _actions); 9 | 10 | /** 11 | * Normalize the files info 12 | */ 13 | actions.normalizeFilesInfo = async (node) => { 14 | await node.normalizeFilesInfo(); 15 | //eslint-disable-next-line no-console 16 | console.log(chalk.cyan('The files info has been normalized')); 17 | }; 18 | 19 | /** 20 | * Clean up the storage 21 | */ 22 | actions.cleanUpStorage = async (node) => { 23 | await node.cleanUpStorage(); 24 | //eslint-disable-next-line no-console 25 | console.log(chalk.cyan('The storage has been cleaned up')); 26 | }; 27 | 28 | /** 29 | * Export all files to another node 30 | */ 31 | actions.exportFiles = async (node) => { 32 | await node.exportFiles(argv.address || argv.n); 33 | //eslint-disable-next-line no-console 34 | console.log(chalk.cyan('The files have been exported')); 35 | }; 36 | 37 | /** 38 | * Store the file 39 | */ 40 | actions.storeFile = async (node) => { 41 | const filePath = utils.getAbsolutePath(argv.filePath || argv.f); 42 | const fileText = argv.fileText || argv.t; 43 | const hash = await node.storeFile(filePath || new Buffer(fileText)); 44 | //eslint-disable-next-line no-console 45 | console.log(chalk.cyan(`The file with hash "${hash}" has been stored`)); 46 | }; 47 | 48 | /** 49 | * Get the file link 50 | */ 51 | actions.getFileLink = async (node) => { 52 | const hash = argv.hash || argv.h; 53 | const link = await node.getFileLink(hash); 54 | 55 | if (!link) { 56 | throw new Error(`There is no file with the hash ${hash}`); 57 | } 58 | 59 | //eslint-disable-next-line no-console 60 | console.log(chalk.cyan(`The file link is "${link}"`)); 61 | }; 62 | 63 | /** 64 | * Get the file to the path 65 | */ 66 | actions.getFileToPath = async (node) => { 67 | const hash = argv.hash || argv.h; 68 | const filePath = utils.getAbsolutePath(argv.filePath || argv.f); 69 | const link = await node.getFileLink(hash); 70 | 71 | if (!link) { 72 | throw new Error(`There is no file with the hash ${hash}`); 73 | } 74 | 75 | await srcUtils.fetchFileToPath(filePath, link, node.createDefaultRequestOptions()); 76 | //eslint-disable-next-line no-console 77 | console.log(chalk.cyan(`The file "${hash}" has been saved to "${filePath}"`)); 78 | }; 79 | 80 | /** 81 | * Remove the file 82 | */ 83 | actions.removeFile = async (node) => { 84 | const hash = argv.hash || argv.h; 85 | await node.removeFile(hash); 86 | //eslint-disable-next-line no-console 87 | console.log(chalk.cyan(`The file "${hash}" has been removed`)); 88 | }; 89 | 90 | /** 91 | * Flush the files cache 92 | */ 93 | actions.flushFilesCache = async (node) => { 94 | await node.cacheFile.flush(); 95 | //eslint-disable-next-line no-console 96 | console.log(chalk.cyan(`The files cache has been flushed`)); 97 | }; 98 | 99 | export default actions; 100 | -------------------------------------------------------------------------------- /bin/index.js: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | import runner from "./runner.js"; 3 | import Node from "../src/index.js"; 4 | import actions from "./actions.js"; 5 | 6 | runner('storacle', Node, actions); 7 | -------------------------------------------------------------------------------- /bin/runner.js: -------------------------------------------------------------------------------- 1 | export { default } from "spreadable/bin/runner.js"; 2 | -------------------------------------------------------------------------------- /bin/utils.js: -------------------------------------------------------------------------------- 1 | export { default } from "spreadable/bin/utils.js"; 2 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "storacle", 3 | "version": "0.3.6", 4 | "description": "Decentralized file storage", 5 | "main": "./src/index.js", 6 | "type": "module", 7 | "bin": { 8 | "storacle": "./bin/index.js" 9 | }, 10 | "author": { 11 | "name": "Alexander Balasyan", 12 | "email": "mywebstreet@gmail.com" 13 | }, 14 | "homepage": "https://github.com/ortexx/storacle", 15 | "scripts": { 16 | "eslint": "eslint src bin test", 17 | "test": "mocha ./test/index.js --timeout=30000", 18 | "build-client": "webpack --config=webpack.client.js", 19 | "build-client-prod": "cross-env NODE_ENV=production webpack --config=webpack.client.js", 20 | "build-ci": "npm run eslint && npm run test && npm run build-client-prod" 21 | }, 22 | "husky": { 23 | "hooks": { 24 | "pre-commit": "npm run build-ci && git add ./dist/*" 25 | } 26 | }, 27 | "keywords": [ 28 | "storacle", 29 | "file", 30 | "files", 31 | "storage", 32 | "network", 33 | "distributed", 34 | "decentralized", 35 | "decentralization", 36 | "distribution", 37 | "information", 38 | "data" 39 | ], 40 | "license": "MIT", 41 | "devDependencies": { 42 | "@babel/core": "^7.23.7", 43 | "@babel/eslint-parser": "^7.23.10", 44 | "@babel/plugin-syntax-import-assertions": "^7.23.3", 45 | "@babel/plugin-transform-runtime": "^7.23.7", 46 | "@babel/preset-env": "^7.23.8", 47 | "babel-loader": "^9.1.3", 48 | "chai": "^5.0.0", 49 | "cross-env": "^7.0.3", 50 | "css-minimizer-webpack-plugin": "^5.0.1", 51 | "eslint": "^8.56.0", 52 | "eslint-webpack-plugin": "^4.0.1", 53 | "husky": "^4.3.8", 54 | "mini-css-extract-plugin": "^2.7.7", 55 | "mocha": "^10.2.0", 56 | "node-polyfill-webpack-plugin": "^3.0.0", 57 | "terser-webpack-plugin": "^5.3.10", 58 | "webpack": "^5.89.0", 59 | "webpack-cli": "^5.1.4" 60 | }, 61 | "dependencies": { 62 | "bytes": "^3.1.2", 63 | "chalk": "^5.3.0", 64 | "create-hash": "^1.2.0", 65 | "detect-file-type": "^0.2.8", 66 | "express-form-data": "^2.0.23", 67 | "fs-extra": "^11.2.0", 68 | "hasha": "5.0.0", 69 | "lodash-es": "^4.17.21", 70 | "mime": "^4.0.1", 71 | "node-fetch": "^2.7.0", 72 | "splaytree": "^3.1.2", 73 | "spreadable": "~0.3.10", 74 | "yargs": "^17.7.2" 75 | }, 76 | "repository": { 77 | "type": "git", 78 | "url": "https://github.com/ortexx/storacle" 79 | }, 80 | "engines": { 81 | "node": ">=20.0.0" 82 | } 83 | } 84 | -------------------------------------------------------------------------------- /src/browser/client/.babelrc: -------------------------------------------------------------------------------- 1 | { 2 | "presets": ["@babel/preset-env"], 3 | "plugins": ["@babel/plugin-transform-runtime"] 4 | } -------------------------------------------------------------------------------- /src/browser/client/index.js: -------------------------------------------------------------------------------- 1 | import client from "../../client.js"; 2 | export default client(); 3 | -------------------------------------------------------------------------------- /src/browser/client/mock/crypto.js: -------------------------------------------------------------------------------- 1 | import createHash from "create-hash/browser.js"; 2 | const crypto = { createHash }; 3 | export default crypto; 4 | -------------------------------------------------------------------------------- /src/client.js: -------------------------------------------------------------------------------- 1 | import merge from "lodash-es/merge.js"; 2 | import client from "spreadable/src/client.js"; 3 | import fse from "fs-extra"; 4 | import utils from "./utils.js"; 5 | import errors from "./errors.js"; 6 | import pack from "../package.json" assert { type: "json" } 7 | 8 | const Client = client(); 9 | 10 | export default (Parent) => { 11 | /** 12 | * Class to manage client requests to the network 13 | */ 14 | return class ClientStoracle extends (Parent || Client) { 15 | static get version() { return pack.version; } 16 | static get codename() { return pack.name; } 17 | static get utils() { return utils; } 18 | static get errors() { return errors; } 19 | constructor(options = {}) { 20 | options = merge({ 21 | request: { 22 | fileStoringTimeout: '2.05h', 23 | fileGettingTimeout: '1h', 24 | fileRemovalTimeout: '10s', 25 | fileLinkGettingTimeout: '10s' 26 | }, 27 | }, options); 28 | super(options); 29 | } 30 | 31 | /** 32 | * Get the network files count 33 | * 34 | * @async 35 | * @param {object} [options] 36 | * @returns {number} 37 | */ 38 | async getNetworkFilesCount(options = {}) { 39 | return (await this.request('get-network-files-count', options)).count; 40 | } 41 | 42 | /** 43 | * Get the file link 44 | * 45 | * @async 46 | * @param {string} hash 47 | * @param {object} [options] 48 | * @returns {string} 49 | */ 50 | async getFileLink(hash, options = {}) { 51 | return (await this.request('get-file-link', Object.assign({}, options, { 52 | body: { hash }, 53 | timeout: options.timeout || this.options.request.fileLinkGettingTimeout 54 | }))).link; 55 | } 56 | 57 | /** 58 | * Get the file links array 59 | * 60 | * @async 61 | * @param {string} hash 62 | * @param {object} [options] 63 | * @returns {string} 64 | */ 65 | async getFileLinks(hash, options = {}) { 66 | return (await this.request('get-file-links', Object.assign({}, options, { 67 | body: { hash }, 68 | timeout: options.timeout || this.options.request.fileLinkGettingTimeout 69 | }))).links; 70 | } 71 | 72 | /** 73 | * Get the file to a buffer 74 | * 75 | * @async 76 | * @param {string} hash 77 | * @param {object} [options] 78 | * @returns {Buffer} 79 | */ 80 | async getFileToBuffer(hash, options = {}) { 81 | this.envTest(false, 'getFileToBuffer'); 82 | const { result, timer } = await this.getFileLinkAndTimer(hash, options); 83 | return await utils.fetchFileToBuffer(result.link, this.createDefaultRequestOptions({ timeout: timer() })); 84 | } 85 | 86 | /** 87 | * Get the file and save it to the path 88 | * 89 | * @async 90 | * @param {string} hash 91 | * @param {string} filePath 92 | * @param {object} [options] 93 | */ 94 | async getFileToPath(hash, filePath, options = {}) { 95 | this.envTest(false, 'getFileToPath'); 96 | const { result, timer } = await this.getFileLinkAndTimer(hash, options); 97 | await utils.fetchFileToPath(filePath, result.link, this.createDefaultRequestOptions({ timeout: timer() })); 98 | } 99 | 100 | /** 101 | * Get file to a blob 102 | * 103 | * @param {string} hash 104 | * @param {object} [options] 105 | * @returns {Blob} 106 | */ 107 | async getFileToBlob(hash, options = {}) { 108 | this.envTest(true, 'getFileToBlob'); 109 | const { result, timer } = await this.getFileLinkAndTimer(hash, options); 110 | return utils.fetchFileToBlob(result.link, this.createDefaultRequestOptions({ timeout: timer() })); 111 | } 112 | 113 | /** 114 | * Get the file link and timer 115 | * 116 | * @param {string} hash 117 | * @param {object} options 118 | * @returns {Object} 119 | */ 120 | async getFileLinkAndTimer(hash, options) { 121 | const timeout = options.timeout || this.options.request.fileGettingTimeout; 122 | const timer = this.createRequestTimer(timeout); 123 | const result = await this.request('get-file-link', Object.assign({}, options, { 124 | body: { hash }, 125 | timeout: timer(this.options.request.fileLinkGettingTimeout) 126 | })); 127 | if (!result.link) { 128 | throw new errors.WorkError(`Link for hash "${hash}" is not found`, 'ERR_STORACLE_NOT_FOUND_LINK'); 129 | } 130 | return { 131 | result, 132 | timer 133 | }; 134 | } 135 | 136 | /** 137 | * Store the file to the storage 138 | * 139 | * @async 140 | * @param {string|Buffer|fse.ReadStream|Blob|File} file 141 | * @param {object} [options] 142 | */ 143 | async storeFile(file, options = {}) { 144 | const destroyFileStream = () => utils.isFileReadStream(file) && file.destroy(); 145 | try { 146 | const info = await utils.getFileInfo(file); 147 | if (typeof file == 'string') { 148 | file = fse.createReadStream(file); 149 | } 150 | const result = await this.request('store-file', Object.assign({}, options, { 151 | formData: { 152 | file: { 153 | value: file, 154 | options: { 155 | filename: info.hash + (info.ext ? '.' + info.ext : ''), 156 | contentType: info.mime 157 | } 158 | } 159 | }, 160 | timeout: options.timeout || this.options.request.fileStoringTimeout 161 | })); 162 | destroyFileStream(); 163 | return result.hash; 164 | } 165 | catch (err) { 166 | destroyFileStream(); 167 | throw err; 168 | } 169 | } 170 | 171 | /** 172 | * Remove the file 173 | * 174 | * @async 175 | * @param {string} hash 176 | * @param {object} [options] 177 | * @returns {object} 178 | */ 179 | async removeFile(hash, options = {}) { 180 | return await this.request('remove-file', Object.assign({}, options, { 181 | body: { hash }, 182 | timeout: options.timeout || this.options.request.fileRemovalTimeout 183 | })); 184 | } 185 | 186 | /** 187 | * Create a deferred file link 188 | * 189 | * @param {string} hash 190 | * @param {object} options 191 | * @returns {string} 192 | */ 193 | createRequestedFileLink(hash, options = {}) { 194 | return this.createRequestUrl(`request-file/${hash}`, options); 195 | } 196 | 197 | /** 198 | * Prepare the options 199 | */ 200 | prepareOptions() { 201 | super.prepareOptions(); 202 | this.options.request.fileGettingTimeout = utils.getMs(this.options.request.fileGettingTimeout); 203 | this.options.request.fileStoringTimeout = utils.getMs(this.options.request.fileStoringTimeout); 204 | this.options.request.fileRemovalTimeout = utils.getMs(this.options.request.fileRemovalTimeout); 205 | this.options.request.fileLinkGettingTimeout = utils.getMs(this.options.request.fileLinkGettingTimeout); 206 | } 207 | }; 208 | }; 209 | -------------------------------------------------------------------------------- /src/db/transports/loki/index.js: -------------------------------------------------------------------------------- 1 | import loki from "spreadable/src/db/transports/loki/index.js"; 2 | 3 | const DatabaseLoki = loki(); 4 | 5 | export default (Parent) => { 6 | /** 7 | * Lokijs storacle database transport 8 | */ 9 | return class DatabaseLokiStoracle extends (Parent || DatabaseLoki) { 10 | /** 11 | * @see DatabaseLoki.prototype.initCollectionData 12 | */ 13 | initCollectionData() { 14 | super.initCollectionData.apply(this, arguments); 15 | const filesTotalSize = this.col.data.findOne({ name: 'filesTotalSize' }); 16 | const filesCount = this.col.data.findOne({ name: 'filesCount' }); 17 | 18 | if (!filesTotalSize) { 19 | this.col.data.insert({ name: 'filesTotalSize', value: 0 }); 20 | } 21 | 22 | if (!filesCount) { 23 | this.col.data.insert({ name: 'filesCount', value: 0 }); 24 | } 25 | } 26 | }; 27 | }; 28 | -------------------------------------------------------------------------------- /src/errors.js: -------------------------------------------------------------------------------- 1 | import * as errors from "spreadable/src/errors.js"; 2 | export default errors; 3 | -------------------------------------------------------------------------------- /src/index.js: -------------------------------------------------------------------------------- 1 | import node from "./node.js"; 2 | import client from "./client.js"; 3 | 4 | const Node = node(); 5 | const Client = client(); 6 | 7 | export { Client }; 8 | export { Node }; 9 | export default { Client, Node }; 10 | -------------------------------------------------------------------------------- /src/node.js: -------------------------------------------------------------------------------- 1 | import path from "path"; 2 | import url from "url"; 3 | import fse from "fs-extra"; 4 | import fetch from "node-fetch"; 5 | import assign from "lodash-es/assign.js"; 6 | import merge from "lodash-es/merge.js"; 7 | import bytes from "bytes"; 8 | import SplayTree from "splaytree"; 9 | import node from "spreadable/src/node.js"; 10 | import cacheDatabase from "spreadable/src/cache/transports/database/index.js"; 11 | import databaseLoki from "./db/transports/loki/index.js"; 12 | import serverExpress from "./server/transports/express/index.js"; 13 | import utils from "./utils.js"; 14 | import errors from "./errors.js"; 15 | import schema from "./schema.js"; 16 | import pack from "../package.json" assert { type: "json" } 17 | 18 | const Node = node(); 19 | const CacheDatabaseStoracle = cacheDatabase(); 20 | const DatabaseLokiStoracle = databaseLoki(); 21 | const ServerExpressStoracle = serverExpress(); 22 | 23 | export default (Parent) => { 24 | /** 25 | * Class to manage the storacle node 26 | */ 27 | return class NodeStoracle extends (Parent || Node) { 28 | static get version() { return pack.version; } 29 | static get codename() { return pack.name; } 30 | static get DatabaseTransport() { return DatabaseLokiStoracle; } 31 | static get ServerTransport() { return ServerExpressStoracle; } 32 | static get CacheFileTransport() { return CacheDatabaseStoracle; } 33 | 34 | /** 35 | * @see Node 36 | */ 37 | constructor(options = {}) { 38 | options = merge({ 39 | request: { 40 | clientStoringConcurrency: 20, 41 | fileStoringNodeTimeout: '2h', 42 | cacheTimeout: 250 43 | }, 44 | storage: { 45 | dataSize: '45%', 46 | tempSize: '45%', 47 | tempLifetime: '2h', 48 | autoCleanSize: 0 49 | }, 50 | file: { 51 | maxSize: '40%', 52 | minSize: 0, 53 | preferredDuplicates: 'auto', 54 | responseCacheLifetime: '7d', 55 | mimeWhitelist: [], 56 | mimeBlacklist: [], 57 | extWhitelist: [], 58 | extBlacklist: [], 59 | linkCache: { 60 | limit: 50000, 61 | lifetime: '2h' 62 | } 63 | }, 64 | task: { 65 | cleanUpStorageInterval: '30s', 66 | cleanUpTempDirInterval: '20s', 67 | calculateStorageInfoInterval: '3s' 68 | } 69 | }, options); 70 | super(options); 71 | this.storageDataSize = 0; 72 | this.storageTempSize = 0; 73 | this.storageAutoCleanSize = 0; 74 | this.fileMaxSize = 0; 75 | this.fileMinSize = 0; 76 | this.CacheFileTransport = this.constructor.CacheFileTransport; 77 | this.__dirNestingSize = 2; 78 | this.__dirNameLength = 1; 79 | this.__blockQueue = {}; 80 | } 81 | 82 | /** 83 | * @see Node.prototype.initBeforeSync 84 | */ 85 | async initBeforeSync() { 86 | await this.createFolders(); 87 | await this.calculateStorageInfo(); 88 | return await super.initBeforeSync.apply(this, arguments); 89 | } 90 | 91 | /** 92 | * @see Node.prototype.prepareServices 93 | */ 94 | async prepareServices() { 95 | await super.prepareServices.apply(this, arguments); 96 | await this.prepareCache(); 97 | } 98 | 99 | /** 100 | * Prepare the cache service 101 | * 102 | * @async 103 | */ 104 | async prepareCache() { 105 | if (this.options.file.linkCache) { 106 | this.cacheFile = await this.addService('file', new this.CacheFileTransport(this.options.file.linkCache)); 107 | } 108 | } 109 | 110 | /** 111 | * Prepare the task service 112 | * 113 | * @async 114 | */ 115 | async prepareTask() { 116 | await super.prepareTask.apply(this, arguments); 117 | 118 | if (!this.task) { 119 | return; 120 | } 121 | 122 | if (this.options.task.cleanUpStorageInterval) { 123 | await this.task.add('cleanUpStorage', this.options.task.cleanUpStorageInterval, () => this.cleanUpStorage()); 124 | } 125 | 126 | if (this.options.task.cleanUpTempDirInterval) { 127 | await this.task.add('cleanUpTempDir', this.options.task.cleanUpTempDirInterval, () => this.cleanUpTempDir()); 128 | } 129 | 130 | if (this.options.task.calculateStorageInfoInterval) { 131 | await this.task.add('calculateStorageInfo', this.options.task.calculateStorageInfoInterval, () => this.calculateStorageInfo()); 132 | } 133 | } 134 | 135 | /** 136 | * @see Node.prototype.sync 137 | */ 138 | async sync() { 139 | await super.sync.apply(this, arguments); 140 | this.cacheFile && await this.cacheFile.normalize(); 141 | } 142 | 143 | /** 144 | * @see Node.prototype.getStatusInfo 145 | */ 146 | async getStatusInfo(pretty = false) { 147 | const storage = await this.getStorageInfo(); 148 | 149 | if (pretty) { 150 | for (let key in storage) { 151 | storage[key] = bytes(storage[key]); 152 | } 153 | } 154 | 155 | return merge(await super.getStatusInfo(pretty), storage, { 156 | filesCount: await this.db.getData('filesCount') 157 | }); 158 | } 159 | 160 | /** 161 | * Create the necessary folders 162 | * 163 | * @async 164 | */ 165 | async createFolders() { 166 | this.filesPath = path.join(this.storagePath, 'files'); 167 | this.tempPath = path.join(this.storagePath, 'tmp'); 168 | await fse.ensureDir(this.filesPath); 169 | await fse.ensureDir(this.tempPath); 170 | } 171 | 172 | /** 173 | * Calculate the storage info 174 | * 175 | * @async 176 | */ 177 | async calculateStorageInfo() { 178 | const info = await utils.getDiskInfo(this.filesPath); 179 | const used = await this.getStorageTotalSize(); 180 | const tempDirInfo = await this.getTempDirInfo(); 181 | this.storageDataSize = this.options.storage.dataSize; 182 | this.storageTempSize = this.options.storage.tempSize; 183 | this.storageAutoCleanSize = this.options.storage.autoCleanSize; 184 | this.fileMaxSize = this.options.file.maxSize; 185 | this.fileMinSize = this.options.file.minSize; 186 | const available = info.available + used + tempDirInfo.size; 187 | 188 | if (typeof this.storageDataSize == 'string') { 189 | const arr = this.storageDataSize.split(' - '); 190 | this.storageDataSize = Math.floor(available * parseFloat(arr[0]) / 100); 191 | arr[1] && (this.storageDataSize -= utils.getBytes(arr[1])); 192 | } 193 | 194 | if (typeof this.storageTempSize == 'string') { 195 | const arr = this.storageTempSize.split(' - '); 196 | this.storageTempSize = Math.floor(available * parseFloat(this.storageTempSize) / 100); 197 | arr[1] && (this.storageTempSize -= utils.getBytes(arr[1])); 198 | } 199 | 200 | if (this.storageDataSize > available) { 201 | throw new Error(`"storage.dataSize" is greater than available disk space`); 202 | } 203 | 204 | if (this.storageTempSize > available) { 205 | throw new Error(`"storage.tempSize" is greater than available disk space`); 206 | } 207 | 208 | if (this.storageDataSize + this.storageTempSize > available) { 209 | throw new Error(`"storage.dataSize" + "storage.tempSize" is greater than available disk space`); 210 | } 211 | 212 | if (typeof this.storageAutoCleanSize == 'string') { 213 | this.storageAutoCleanSize = Math.floor(this.storageDataSize * parseFloat(this.storageAutoCleanSize) / 100); 214 | } 215 | 216 | if (this.storageAutoCleanSize > this.storageDataSize) { 217 | throw new Error(`"storage.autoCleanSize" is greater than "storage.dataSize"`); 218 | } 219 | 220 | if (typeof this.fileMaxSize == 'string') { 221 | this.fileMaxSize = Math.floor(available * parseFloat(this.fileMaxSize) / 100); 222 | } 223 | 224 | if (typeof this.fileMinSize == 'string') { 225 | this.fileMinSize = Math.floor(available * parseFloat(this.fileMinSize) / 100); 226 | } 227 | 228 | if (this.fileMaxSize > this.storageDataSize) { 229 | throw new Error(`"file.maxSize" is greater than "storage.dataSize"`); 230 | } 231 | 232 | if (this.fileMaxSize < this.fileMinSize) { 233 | throw new Error(`"file.maxSize" is less than "file.minSize"`); 234 | } 235 | 236 | if (this.calculateTempFileMinSize(this.fileMaxSize) > this.storageTempSize) { 237 | throw new Error(`Minimum temp file size is greater than "storage.tempSize"`); 238 | } 239 | } 240 | 241 | /** 242 | * Get the file storing filter options 243 | * 244 | * @async 245 | * @param {object} info 246 | * @returns {object} 247 | */ 248 | async getFileStoringFilterOptions(info) { 249 | return { 250 | uniq: 'address', 251 | fnCompare: await this.createSuspicionComparisonFunction('storeFile', await this.createFileStoringComparisonFunction()), 252 | fnFilter: c => !c.existenceInfo && c.isAvailable, 253 | schema: schema.getFileStoringInfoSlaveResponse(), 254 | limit: await this.getFileDuplicatesCount(info) 255 | }; 256 | } 257 | 258 | /** 259 | * Create a file storing comparison function 260 | * 261 | * @async 262 | * @returns {function} 263 | */ 264 | async createFileStoringComparisonFunction() { 265 | return (a, b) => b.free - a.free; 266 | } 267 | 268 | /** 269 | * Get the file links filter options 270 | * 271 | * @async 272 | * @returns {object} 273 | */ 274 | async getFileLinksFilterOptions() { 275 | return { 276 | uniq: 'link', 277 | fnFilter: c => utils.isValidFileLink(c.link) 278 | }; 279 | } 280 | 281 | /** 282 | * Store the file to the network 283 | * 284 | * @async 285 | * @param {string|Buffer|fse.ReadStream} file 286 | * @param {object} [options] 287 | * @returns {string} 288 | */ 289 | async storeFile(file, options = {}) { 290 | const destroyFileStream = () => utils.isFileReadStream(file) && file.destroy(); 291 | 292 | try { 293 | const timer = this.createRequestTimer(options.timeout); 294 | 295 | if (typeof file == 'string') { 296 | file = fse.createReadStream(file); 297 | } 298 | 299 | const info = await utils.getFileInfo(file); 300 | 301 | if (!info.size || !info.hash) { 302 | throw new errors.WorkError('This file cannot be added to the network', 'ERR_STORACLE_INVALID_FILE'); 303 | } 304 | 305 | const masterRequestTimeout = await this.getRequestMasterTimeout(); 306 | let results = await this.requestNetwork('get-file-storing-info', { 307 | body: { info }, 308 | timeout: timer([masterRequestTimeout, this.options.request.fileStoringNodeTimeout], { min: masterRequestTimeout, grabFree: true }), 309 | responseSchema: schema.getFileStoringInfoMasterResponse() 310 | }); 311 | const existing = results.reduce((p, c) => p.concat(c.existing), []); 312 | const duplicates = await this.getFileDuplicatesCount(info); 313 | const limit = duplicates - existing.length; 314 | 315 | if (limit <= 0) { 316 | destroyFileStream(); 317 | return info.hash; 318 | } 319 | 320 | const filterOptions = Object.assign(await this.getFileStoringFilterOptions(info), { limit }); 321 | const candidates = await this.filterCandidatesMatrix(results.map(r => r.candidates), filterOptions); 322 | 323 | if (!candidates.length && !existing.length) { 324 | throw new errors.WorkError('Not found a suitable server to store the file', 'ERR_STORACLE_NOT_FOUND_STORAGE'); 325 | } 326 | 327 | if (candidates.length) { 328 | await this.db.addBehaviorCandidate('storeFile', candidates[0].address); 329 | } 330 | else { 331 | destroyFileStream(); 332 | return info.hash; 333 | } 334 | const servers = candidates.map(c => c.address).sort(await this.createAddressComparisonFunction()); 335 | const dupOptions = Object.assign({}, options, { timeout: timer() }); 336 | const result = await this.duplicateFile(servers, file, info, dupOptions); 337 | 338 | if (!result && !existing.length) { 339 | throw new errors.WorkError('Not found an available server to store the file', 'ERR_STORACLE_NOT_FOUND_STORAGE'); 340 | } 341 | 342 | if (!result) { 343 | destroyFileStream(); 344 | return info.hash; 345 | } 346 | 347 | destroyFileStream(); 348 | return result.hash; 349 | } 350 | catch (err) { 351 | destroyFileStream(); 352 | throw err; 353 | } 354 | } 355 | 356 | /** 357 | * Duplicate the file 358 | * 359 | * @async 360 | * @param {string[]} servers 361 | * @param {fse.ReadStream|Buffer} file 362 | * @param {object} info 363 | * @param {object} [options] 364 | * @returns {object} 365 | */ 366 | async duplicateFile(servers, file, info, options = {}) { 367 | options = assign({ 368 | responseSchema: schema.getFileStoringResponse(), 369 | cache: true 370 | }, options); 371 | let tempFile; 372 | const streams = []; 373 | const isStream = utils.isFileReadStream(file); 374 | 375 | if (isStream) { 376 | streams.push(file); 377 | const name = path.basename(file.path); 378 | await fse.pathExists(path.join(this.tempPath, name)) && (tempFile = name); 379 | } 380 | 381 | options.serverOptions = address => { 382 | if (isStream) { 383 | file = fse.createReadStream(file.path); 384 | streams.push(file); 385 | } 386 | 387 | return { 388 | timeout: this.options.request.fileStoringNodeTimeout, 389 | formData: merge({}, options.formData, { 390 | file: address == (this.address && tempFile) || { 391 | value: file, 392 | options: { 393 | filename: info.hash + (info.ext ? '.' + info.ext : ''), 394 | contentType: info.mim 395 | } 396 | } 397 | }) 398 | }; 399 | }; 400 | 401 | try { 402 | const result = await this.duplicateData(options.action || `store-file/${info.hash}`, servers, options); 403 | result && options.cache && await this.updateFileCache(result.hash, { link: result.link }); 404 | streams.forEach(s => s.destroy()); 405 | return result; 406 | } 407 | catch (err) { 408 | streams.forEach(s => s.destroy()); 409 | throw err; 410 | } 411 | } 412 | 413 | /** 414 | * Get the file links array 415 | * 416 | * @async 417 | * @param {string} hash 418 | * @param {object} [options] 419 | * @returns {string[]} 420 | */ 421 | async getFileLinks(hash, options = {}) { 422 | let results = await this.requestNetwork('get-file-links', { 423 | body: { hash }, 424 | timeout: options.timeout, 425 | responseSchema: schema.getFileLinksMasterResponse({ networkOptimum: await this.getNetworkOptimum() }) 426 | }); 427 | const filterOptions = merge(await this.getFileLinksFilterOptions()); 428 | const links = await this.filterCandidatesMatrix(results.map(r => r.links), filterOptions); 429 | return links.map(c => c.link); 430 | } 431 | 432 | /** 433 | * Get the file link 434 | * 435 | * @async 436 | * @param {string} hash 437 | * @param {object} [options] 438 | * @returns {string} 439 | */ 440 | async getFileLink(hash, options = {}) { 441 | options = merge({ 442 | cache: true 443 | }, options); 444 | 445 | if (await this.hasFile(hash)) { 446 | return await this.createFileLink(hash); 447 | } 448 | 449 | if (this.cacheFile && options.cache) { 450 | const cache = await this.cacheFile.get(hash); 451 | 452 | if (cache) { 453 | const link = cache.value.link; 454 | 455 | if (await this.checkCacheLink(link)) { 456 | return link; 457 | } 458 | 459 | await this.cacheFile.remove(hash); 460 | } 461 | } 462 | 463 | const links = await this.getFileLinks(hash, options); 464 | 465 | if (links.length) { 466 | const link = utils.getRandomElement(links); 467 | options.cache && await this.updateFileCache(hash, { link }); 468 | return link; 469 | } 470 | 471 | return ''; 472 | } 473 | 474 | /** 475 | * Remove the file 476 | * 477 | * @async 478 | * @param {string} hash 479 | * @param {object} [options] 480 | * @returns {object} 481 | */ 482 | async removeFile(hash, options = {}) { 483 | const result = await this.requestNetwork('remove-file', { 484 | body: { hash }, 485 | options: options.timeout, 486 | responseSchema: schema.getFileRemovalMasterResponse() 487 | }); 488 | return { removed: result.reduce((p, c) => p + c.removed, 0) }; 489 | } 490 | 491 | /** 492 | * Get the network files count 493 | * 494 | * @async 495 | * @param {object} [options] 496 | * @returns {number} 497 | */ 498 | async getNetworkFilesCount(options = {}) { 499 | const result = await this.requestNetwork('get-network-files-count', { 500 | timeout: options.timeout, 501 | responseSchema: schema.getNetworkFilesCountMasterResponse() 502 | }); 503 | return result.reduce((p, c) => p + c.count, 0); 504 | } 505 | 506 | /** 507 | * Update the file cache 508 | * 509 | * @async 510 | * @param {string} title 511 | * @param {object} value 512 | * @param {string} value.link 513 | */ 514 | async updateFileCache(title, value) { 515 | if (!this.cacheFile || !utils.isValidFileLink(value.link) || url.parse(value.link).host == this.address) { 516 | return; 517 | } 518 | 519 | await this.cacheFile.set(title, value); 520 | } 521 | 522 | /** 523 | * Get the file duplicates count 524 | * 525 | * @async 526 | * @param {object} info 527 | * @param {integer} info.size 528 | * @param {string} info.hash 529 | * @returns {number} 530 | */ 531 | async getFileDuplicatesCount() { 532 | return this.getValueGivenNetworkSize(this.options.file.preferredDuplicates); 533 | } 534 | 535 | /** 536 | * Get the disk usage information 537 | * 538 | * @async 539 | * @param {object} data 540 | * @returns {object} 541 | */ 542 | async getStorageInfo(data = {}) { 543 | data = Object.assign({ 544 | total: true, 545 | available: true, 546 | allowed: true, 547 | used: true, 548 | free: true, 549 | clean: true, 550 | tempAllowed: true, 551 | tempUsed: true, 552 | tempFree: true, 553 | fileMaxSize: true, 554 | fileMinSize: true 555 | }, data); 556 | const diskInfo = await utils.getDiskInfo(this.filesPath); 557 | const info = {}; 558 | let used; 559 | let tempUsed; 560 | 561 | if (data.used || data.available || data.free) { 562 | used = await this.getStorageTotalSize(); 563 | } 564 | 565 | if (data.tempUsed || data.tempFree) { 566 | tempUsed = (await this.getTempDirInfo()).size; 567 | } 568 | 569 | data.total && (info.total = diskInfo.total); 570 | data.available && (info.available = diskInfo.available + used); 571 | data.allowed && (info.allowed = this.storageDataSize); 572 | data.used && (info.used = used); 573 | data.free && (info.free = this.storageDataSize - used); 574 | data.clean && (info.clean = this.storageAutoCleanSize); 575 | data.tempAllowed && (info.tempAllowed = this.storageTempSize); 576 | data.tempUsed && (info.tempUsed = tempUsed); 577 | data.tempFree && (info.tempFree = this.storageTempSize - tempUsed); 578 | data.fileMaxSize && (info.fileMaxSize = this.fileMaxSize); 579 | data.fileMinSize && (info.fileMinSize = this.fileMinSize); 580 | return info; 581 | } 582 | 583 | /** 584 | * Iterate all files 585 | * 586 | * @async 587 | * @param {function} fn 588 | * @param {object} [options] 589 | */ 590 | async iterateFiles(fn, options = {}) { 591 | options = assign({ ignoreFolders: true }, options); 592 | const iterate = async (dir, level) => { 593 | if (options.maxLevel && level > options.maxLevel) { 594 | return; 595 | } 596 | 597 | const files = await fse.readdir(dir); 598 | 599 | for (let i = 0; i < files.length; i++) { 600 | try { 601 | const filePath = path.join(dir, files[i]); 602 | const stat = await fse.stat(filePath); 603 | 604 | if (stat.isDirectory()) { 605 | await iterate(filePath, level + 1); 606 | 607 | if (options.ignoreFolders) { 608 | continue; 609 | } 610 | } 611 | 612 | await fn(filePath, stat); 613 | } 614 | catch (err) { 615 | if (!['ENOENT', 'EINVAL'].includes(err.code)) { 616 | throw err; 617 | } 618 | } 619 | } 620 | }; 621 | await iterate(this.filesPath, 1); 622 | } 623 | 624 | /** 625 | * Get the storage cleaning up tree 626 | * 627 | * @async 628 | * @returns {SplayTree} - node data must be like { size: 1, path: '' } 629 | */ 630 | async getStorageCleaningUpTree() { 631 | const tree = new SplayTree((a, b) => a.atimeMs - b.atimeMs); 632 | await this.iterateFiles((filePath, stat) => { 633 | tree.insert({ atimeMs: stat.atimeMs }, { size: stat.size, path: filePath }); 634 | }); 635 | return tree; 636 | } 637 | 638 | /** 639 | * Clean up the storage 640 | * 641 | * @async 642 | */ 643 | async cleanUpStorage() { 644 | if (!this.storageAutoCleanSize) { 645 | return; 646 | } 647 | 648 | const storageInfoData = { tempUsed: false, tempFree: false }; 649 | const storage = await this.getStorageInfo(storageInfoData); 650 | const needSize = this.storageAutoCleanSize - storage.free; 651 | 652 | if (needSize <= 0) { 653 | return; 654 | } 655 | 656 | this.logger.info(`It is necessary to clean ${needSize} byte(s)`); 657 | const tree = await this.getStorageCleaningUpTree(); 658 | let node = tree.minNode(); 659 | 660 | while (node) { 661 | const obj = node.data; 662 | try { 663 | await this.removeFileFromStorage(path.basename(obj.path)); 664 | if ((await this.getStorageInfo(storageInfoData)).free >= this.storageAutoCleanSize) { 665 | break; 666 | } 667 | } 668 | catch (err) { 669 | this.logger.warn(err.stack); 670 | } 671 | node = tree.next(node); 672 | } 673 | 674 | if ((await this.getStorageInfo(storageInfoData)).free < this.storageAutoCleanSize) { 675 | this.logger.warn('Unable to free up space on the disk completely'); 676 | } 677 | } 678 | 679 | /** 680 | * Clean up the temp dir 681 | * 682 | * @async 683 | */ 684 | async cleanUpTempDir() { 685 | const files = await fse.readdir(this.tempPath); 686 | 687 | for (let i = 0; i < files.length; i++) { 688 | try { 689 | const filePath = path.join(this.tempPath, files[i]); 690 | const stat = await fse.stat(filePath); 691 | 692 | if (Date.now() - stat.mtimeMs <= this.options.storage.tempLifetime) { 693 | continue; 694 | } 695 | 696 | await fse.remove(filePath); 697 | } 698 | catch (err) { 699 | this.logger.warn(err.stack); 700 | } 701 | } 702 | } 703 | 704 | /** 705 | * Normalize the files info 706 | * 707 | * @async 708 | */ 709 | async normalizeFilesInfo() { 710 | let size = 0; 711 | let count = 0; 712 | await this.iterateFiles((fp, stat) => (count++, size += stat.size)); 713 | await this.db.setData('filesTotalSize', size); 714 | await this.db.setData('filesCount', count); 715 | } 716 | 717 | /** 718 | * Export all files to another server 719 | * 720 | * @async 721 | * @param {string} address 722 | * @param {object} [options] 723 | * @param {boolean} [options.strict] - all files must be exported or to throw an error 724 | * @param {number} [options.timeout] 725 | */ 726 | async exportFiles(address, options = {}) { 727 | options = merge({ 728 | strict: false 729 | }, options); 730 | let success = 0; 731 | let fail = 0; 732 | const timer = this.createRequestTimer(options.timeout); 733 | await this.requestServer(address, `/ping`, { 734 | method: 'GET', 735 | timeout: timer(this.options.request.pingTimeout) || this.options.request.pingTimeout 736 | }); 737 | await this.iterateFiles(async (filePath) => { 738 | const info = await utils.getFileInfo(filePath); 739 | let file; 740 | 741 | try { 742 | file = fse.createReadStream(filePath); 743 | await this.duplicateFile([address], file, info, { timeout: timer() }); 744 | success++; 745 | file.destroy(); 746 | this.logger.info(`File "${info.hash}" has been exported`); 747 | } 748 | catch (err) { 749 | file.destroy(); 750 | if (options.strict) { 751 | throw err; 752 | } 753 | fail++; 754 | this.logger.warn(err.stack); 755 | this.logger.info(`File "${info.hash}" has been failed`); 756 | } 757 | }); 758 | 759 | if (!success && !fail) { 760 | this.logger.info(`There haven't been files to export`); 761 | } 762 | else if (!fail) { 763 | this.logger.info(`${success} file(s) have been exported`); 764 | } 765 | else { 766 | this.logger.info(`${success} file(s) have been exported, ${fail} file(s) have been failed`); 767 | } 768 | } 769 | 770 | /** 771 | * Create the file link 772 | * 773 | * @async 774 | * @param {string} hash 775 | */ 776 | async createFileLink(hash) { 777 | const info = await utils.getFileInfo(this.getFilePath(hash), { hash: false }); 778 | return `${this.getRequestProtocol()}://${this.address}/file/${hash}${info.ext ? '.' + info.ext : ''}`; 779 | } 780 | 781 | /** 782 | * Check the node has the file 783 | * 784 | * @async 785 | * @param {string} hash 786 | * @returns {boolean} 787 | */ 788 | async hasFile(hash) { 789 | return await fse.pathExists(this.getFilePath(hash)); 790 | } 791 | 792 | /** 793 | * Add the file to the storage 794 | * 795 | * @async 796 | * @param {fse.ReadStream|string} file 797 | * @param {string} hash 798 | * @param {string} [options] 799 | * @returns {boolean} 800 | */ 801 | async addFileToStorage(file, hash, options = {}) { 802 | await this.withBlockingFile(hash, async () => { 803 | const sourcePath = file.path || file; 804 | const stat = await fse.stat(sourcePath); 805 | const destPath = this.getFilePath(hash); 806 | const dir = path.dirname(destPath); 807 | const exists = await fse.pathExists(destPath); 808 | 809 | try { 810 | await fse.ensureDir(dir); 811 | await fse[options.copy ? 'copy' : 'move'](sourcePath, destPath, { overwrite: true }); 812 | if (!exists) { 813 | await this.db.setData('filesTotalSize', row => row.value + stat.size); 814 | await this.db.setData('filesCount', row => row.value + 1); 815 | } 816 | } 817 | catch (err) { 818 | await this.normalizeDir(dir); 819 | throw err; 820 | } 821 | }); 822 | } 823 | 824 | /** 825 | * Remove the file from the storage 826 | * 827 | * @async 828 | * @param {string} hash 829 | */ 830 | async removeFileFromStorage(hash) { 831 | await this.withBlockingFile(hash, async () => { 832 | const filePath = this.getFilePath(hash); 833 | const stat = await fse.stat(filePath); 834 | let dir = path.dirname(filePath); 835 | 836 | try { 837 | await fse.remove(filePath); 838 | await this.db.setData('filesTotalSize', row => row.value - stat.size); 839 | await this.db.setData('filesCount', row => row.value - 1); 840 | await this.normalizeDir(dir); 841 | } 842 | catch (err) { 843 | await this.normalizeDir(dir); 844 | 845 | if (err.code != 'ENOENT') { 846 | throw err; 847 | } 848 | } 849 | }); 850 | } 851 | 852 | /** 853 | * Normalize the storage directory 854 | * 855 | * @param {string} dir 856 | */ 857 | async normalizeDir(dir) { 858 | const filesPath = path.normalize(this.filesPath); 859 | 860 | while (dir.length > filesPath.length) { 861 | try { 862 | const files = await fse.readdir(dir); 863 | !files.length && await fse.remove(dir); 864 | dir = path.dirname(dir); 865 | } 866 | catch (err) { 867 | if (['ENOENT', 'EINVAL'].includes(err.code)) { 868 | return; 869 | } 870 | 871 | throw err; 872 | } 873 | } 874 | } 875 | 876 | /** 877 | * Empty the storage 878 | * 879 | * @async 880 | */ 881 | async emptyStorage() { 882 | await fse.emptyDir(this.filesPath); 883 | await fse.emptyDir(this.tempPath); 884 | await this.db.setData('filesTotalSize', 0); 885 | await this.db.setData('filesCount', 0); 886 | } 887 | 888 | /** 889 | * Run the function blocking the file 890 | * 891 | * @async 892 | * @param {string} hash 893 | * @param {function} fn 894 | * @returns {*} 895 | */ 896 | async withBlockingFile(hash, fn) { 897 | !this.__blockQueue[hash] && (this.__blockQueue[hash] = []); 898 | const queue = this.__blockQueue[hash]; 899 | return new Promise((resolve, reject) => { 900 | const handler = async () => { 901 | let err; 902 | let res; 903 | 904 | try { 905 | res = await fn(); 906 | } 907 | catch (e) { 908 | err = e; 909 | } 910 | 911 | err ? reject(err) : resolve(res); 912 | queue.shift(); 913 | queue.length ? queue[0]() : delete this.__blockQueue[hash]; 914 | }; 915 | queue.push(handler); 916 | queue.length <= 1 && handler(); 917 | }); 918 | } 919 | 920 | /** 921 | * Get the storage total size 922 | * 923 | * @async 924 | * @returns {integer} 925 | */ 926 | async getStorageTotalSize() { 927 | let filesSize = await this.db.getData('filesTotalSize'); 928 | let foldersSize = 0; 929 | await this.iterateFiles((fp, stat) => foldersSize += stat.size, { 930 | maxLevel: this.__dirNestingSize, 931 | ignoreFolders: false 932 | }); 933 | return filesSize + foldersSize; 934 | } 935 | 936 | /** 937 | * Get the temp folder total size 938 | * 939 | * @async 940 | * @returns {integer} 941 | */ 942 | async getTempDirInfo() { 943 | let size = 0; 944 | let count = 0; 945 | const files = await fse.readdir(this.tempPath); 946 | 947 | for (let i = 0; i < files.length; i++) { 948 | try { 949 | const filePath = path.join(this.tempPath, files[i]); 950 | size += (await fse.stat(filePath)).size; 951 | count += 1; 952 | } 953 | catch (err) { 954 | if (err.code != 'ENOENT') { 955 | throw err; 956 | } 957 | } 958 | } 959 | 960 | return { size, count }; 961 | } 962 | 963 | /** 964 | * Get the file existence info 965 | * 966 | * @see NodeStoracle.prototype.fileAvailabilityTest 967 | * @returns {boolean} 968 | */ 969 | async getFileExistenceInfo(info) { 970 | return await this.hasFile(info.hash) ? info : null; 971 | } 972 | 973 | /** 974 | * Check the file availability 975 | * 976 | * @see NodeStoracle.prototype.fileAvailabilityTest 977 | * @return {boolean} 978 | */ 979 | async checkFileAvailability() { 980 | try { 981 | await this.fileAvailabilityTest(...arguments); 982 | return true; 983 | } 984 | catch (err) { 985 | if (err instanceof errors.WorkError) { 986 | return false; 987 | } 988 | 989 | throw err; 990 | } 991 | } 992 | 993 | /** 994 | * Test the file availability 995 | * 996 | * @async 997 | * @param {object} info 998 | * @param {integer} info.size 999 | * @param {string} info.hash 1000 | * @param {string} [info.mime] 1001 | * @param {string} [info.ext] 1002 | * @param {object} [info.storage] 1003 | */ 1004 | async fileAvailabilityTest(info = {}) { 1005 | const storage = info.storage || await this.getStorageInfo(); 1006 | const mimeWhite = this.options.file.mimeWhitelist || []; 1007 | const mimeBlack = this.options.file.mimeBlacklist || []; 1008 | const extWhite = this.options.file.extWhitelist || []; 1009 | const extBlack = this.options.file.extBlacklist || []; 1010 | 1011 | if (!info.size || !info.hash) { 1012 | throw new errors.WorkError('Wrong file', 'ERR_STORACLE_WRONG_FILE'); 1013 | } 1014 | 1015 | if (info.size > storage.free) { 1016 | throw new errors.WorkError('Not enough space to store', 'ERR_STORACLE_NOT_ENOUGH_SPACE'); 1017 | } 1018 | 1019 | if (this.calculateTempFileMinSize(info.size) > storage.tempFree) { 1020 | throw new errors.WorkError('Not enough space in the temp folder', 'ERR_STORACLE_NOT_ENOUGH_SPACE_TEMP'); 1021 | } 1022 | 1023 | if (info.size > this.fileMaxSize) { 1024 | throw new errors.WorkError('File is too big', 'ERR_STORACLE_FILE_MAX_SIZE'); 1025 | } 1026 | 1027 | if (info.size < this.fileMinSize) { 1028 | throw new errors.WorkError('File is too small', 'ERR_STORACLE_FILE_MIN_SIZE'); 1029 | } 1030 | 1031 | if (mimeWhite.length && (!info.mime || mimeWhite.indexOf(info.mime) == -1)) { 1032 | throw new errors.WorkError('File mime type is denied', 'ERR_STORACLE_FILE_MIME_TYPE'); 1033 | } 1034 | 1035 | if (mimeBlack.length && (mimeBlack.indexOf(info.mime) != -1)) { 1036 | throw new errors.WorkError('File mime type is denied', 'ERR_STORACLE_FILE_MIME_TYPE'); 1037 | } 1038 | 1039 | if (extWhite.length && (!info.ext || extWhite.indexOf(info.ext) == -1)) { 1040 | throw new errors.WorkError('File extension is denied', 'ERR_STORACLE_FILE_EXTENSION'); 1041 | } 1042 | 1043 | if (extBlack.length && (extBlack.indexOf(info.ext) != -1)) { 1044 | throw new errors.WorkError('File extension is denied', 'ERR_STORACLE_FILE_EXTENSION'); 1045 | } 1046 | } 1047 | 1048 | /** 1049 | * Check the cache link is available 1050 | * 1051 | * @async 1052 | * @param {string} link 1053 | * @returns {boolean} 1054 | */ 1055 | async checkCacheLink(link) { 1056 | if (!link || typeof link != 'string') { 1057 | return false; 1058 | } 1059 | try { 1060 | const res = await fetch(link, this.createDefaultRequestOptions({ 1061 | method: 'HEAD', 1062 | headers: { 'storacle-cache-check': 'true' }, 1063 | timeout: this.options.request.cacheTimeout 1064 | })); 1065 | return res.status >= 200 && res.status < 300; 1066 | } 1067 | catch (err) { 1068 | return false; 1069 | } 1070 | } 1071 | 1072 | /** 1073 | * @see Node.prototype.getAvailabilityParts 1074 | */ 1075 | async getAvailabilityParts() { 1076 | return (await super.getAvailabilityParts()).concat([ 1077 | await this.getAvailabilityTempDir() 1078 | ]); 1079 | } 1080 | 1081 | /** 1082 | * Get the node temp folder availability 1083 | * 1084 | * @async 1085 | * @returns {float} 0-1 1086 | */ 1087 | async getAvailabilityTempDir() { 1088 | const info = await this.getTempDirInfo(); 1089 | return 1 - info.size / this.storageTempSize; 1090 | } 1091 | 1092 | /** 1093 | * Calculate a minimum temp file size 1094 | * 1095 | * @param {number} size 1096 | * @returns {number} 1097 | */ 1098 | calculateTempFileMinSize(size) { 1099 | return size; 1100 | } 1101 | 1102 | /** 1103 | * Prepare the options 1104 | */ 1105 | prepareOptions() { 1106 | super.prepareOptions(); 1107 | this.options.storage.dataSize = utils.getBytes(this.options.storage.dataSize); 1108 | this.options.storage.tempSize = utils.getBytes(this.options.storage.tempSize); 1109 | this.options.storage.autoCleanSize = utils.getBytes(this.options.storage.autoCleanSize); 1110 | this.options.file.maxSize = utils.getBytes(this.options.file.maxSize); 1111 | this.options.file.minSize = utils.getBytes(this.options.file.minSize); 1112 | this.options.file.responseCacheLifetime = utils.getMs(this.options.file.responseCacheLifetime); 1113 | this.options.storage.tempLifetime = utils.getMs(this.options.storage.tempLifetime); 1114 | this.options.request.fileStoringNodeTimeout = utils.getMs(this.options.request.fileStoringNodeTimeout); 1115 | } 1116 | 1117 | /** 1118 | * Get the file path 1119 | * 1120 | * @param {string} hash 1121 | * @returns {string} 1122 | */ 1123 | getFilePath(hash) { 1124 | const subs = []; 1125 | 1126 | for (let i = 0, o = 0; i < this.__dirNestingSize; i++, o += this.__dirNameLength) { 1127 | subs.push(hash.slice(o, o + this.__dirNameLength)); 1128 | } 1129 | 1130 | return path.join(this.filesPath, ...subs, hash); 1131 | } 1132 | 1133 | /** 1134 | * Test the hash 1135 | * 1136 | * @param {string} hash 1137 | */ 1138 | hashTest(hash) { 1139 | if (!hash || typeof hash != 'string') { 1140 | throw new errors.WorkError('Invalid hash', 'ERR_STORACLE_INVALID_HASH'); 1141 | } 1142 | } 1143 | }; 1144 | }; 1145 | -------------------------------------------------------------------------------- /src/schema.js: -------------------------------------------------------------------------------- 1 | import merge from "lodash-es/merge.js"; 2 | import utils from "./utils.js"; 3 | import _schema from "spreadable/src/schema.js"; 4 | 5 | const schema = Object.assign({}, _schema); 6 | 7 | schema.getStatusResponse = function () { 8 | return merge(_schema.getStatusResponse(), { 9 | props: { 10 | total: 'number', 11 | available: 'number', 12 | allowed: 'number', 13 | used: 'number', 14 | free: 'number', 15 | clean: 'number', 16 | tempAllowed: 'number', 17 | tempUsed: 'number', 18 | tempFree: 'number', 19 | fileMaxSize: 'number', 20 | fileMinSize: 'number', 21 | filesCount: 'number' 22 | } 23 | }); 24 | }; 25 | 26 | schema.getStatusPrettyResponse = function () { 27 | return merge(this.getStatusResponse(), _schema.getStatusPrettyResponse(), { 28 | props: { 29 | total: 'string', 30 | available: 'string', 31 | allowed: 'string', 32 | used: 'string', 33 | free: 'string', 34 | clean: 'string', 35 | tempAllowed: 'string', 36 | tempUsed: 'string', 37 | tempFree: 'string', 38 | fileMaxSize: 'string', 39 | fileMinSize: 'string' 40 | } 41 | }); 42 | }; 43 | 44 | schema.getFileExistenceInfo = function () { 45 | return { 46 | type: 'object', 47 | props: { 48 | hash: 'string', 49 | size: 'number', 50 | mime: 'string', 51 | ext: 'string', 52 | storage: 'object' 53 | }, 54 | required: ['hash', 'size'], 55 | expected: true, 56 | canBeNull: true 57 | }; 58 | }; 59 | 60 | schema.getFileLink = function () { 61 | return { 62 | type: 'string', 63 | value: val => val == '' || utils.isValidFileLink(val) 64 | }; 65 | }; 66 | 67 | schema.getFileStoringResponse = function () { 68 | return { 69 | type: 'object', 70 | props: { 71 | address: this.getAddress(), 72 | hash: 'string', 73 | link: this.getFileLink() 74 | }, 75 | strict: true 76 | }; 77 | }; 78 | 79 | schema.getFileStoringInfoMasterResponse = function () { 80 | return this.getFileStoringInfoButlerResponse(); 81 | }; 82 | 83 | schema.getFileStoringInfoButlerResponse = function () { 84 | const address = this.getAddress(); 85 | return { 86 | type: 'object', 87 | props: { 88 | address, 89 | candidates: { 90 | type: 'array', 91 | uniq: 'address', 92 | items: this.getFileStoringInfoSlaveResponse() 93 | }, 94 | existing: { 95 | type: 'array', 96 | uniq: 'address', 97 | items: { 98 | type: 'object', 99 | props: { 100 | address, 101 | existenceInfo: this.getFileExistenceInfo() 102 | }, 103 | strict: true 104 | } 105 | } 106 | }, 107 | strict: true 108 | }; 109 | }; 110 | 111 | schema.getFileStoringInfoSlaveResponse = function () { 112 | return { 113 | type: 'object', 114 | props: { 115 | address: this.getAddress(), 116 | free: 'number', 117 | isAvailable: 'boolean', 118 | existenceInfo: this.getFileExistenceInfo() 119 | }, 120 | strict: true 121 | }; 122 | }; 123 | 124 | schema.getFileLinksMasterResponse = function () { 125 | return this.getFileLinksButlerResponse(); 126 | }; 127 | 128 | schema.getFileLinksButlerResponse = function () { 129 | return { 130 | type: 'object', 131 | props: { 132 | address: this.getAddress(), 133 | links: { 134 | type: 'array', 135 | items: this.getFileLinksSlaveResponse() 136 | } 137 | }, 138 | strict: true 139 | }; 140 | }; 141 | 142 | schema.getFileLinksSlaveResponse = function () { 143 | return { 144 | type: 'object', 145 | props: { 146 | address: this.getAddress(), 147 | link: this.getFileLink() 148 | }, 149 | strict: true 150 | }; 151 | }; 152 | 153 | schema.getFileRemovalMasterResponse = function () { 154 | return this.getFileRemovalButlerResponse(); 155 | }; 156 | 157 | schema.getFileRemovalButlerResponse = function () { 158 | return { 159 | type: 'object', 160 | props: { 161 | address: this.getAddress(), 162 | removed: 'number' 163 | }, 164 | strict: true 165 | }; 166 | }; 167 | 168 | schema.getFileRemovalSlaveResponse = function () { 169 | return { 170 | type: 'object', 171 | props: { 172 | address: this.getAddress(), 173 | removed: 'number' 174 | }, 175 | strict: true 176 | }; 177 | }; 178 | 179 | schema.getNetworkFilesCountMasterResponse = function () { 180 | return this.getNetworkFilesCountButlerResponse(); 181 | }; 182 | 183 | schema.getNetworkFilesCountButlerResponse = function () { 184 | return this.getNetworkFilesCountSlaveResponse(); 185 | }; 186 | 187 | schema.getNetworkFilesCountSlaveResponse = function () { 188 | return { 189 | type: 'object', 190 | props: { 191 | count: 'number', 192 | address: this.getAddress() 193 | }, 194 | strict: true 195 | }; 196 | }; 197 | 198 | export default schema; 199 | -------------------------------------------------------------------------------- /src/server/transports/express/api/butler/controllers.js: -------------------------------------------------------------------------------- 1 | import schema from "../../../../../schema.js"; 2 | import pick from "lodash-es/pick.js"; 3 | 4 | export const getFileStoringInfo = node => { 5 | return async (req, res, next) => { 6 | try { 7 | const info = req.body.info || {}; 8 | node.hashTest(info.hash); 9 | const options = node.createRequestNetworkOptions(req.body, { 10 | responseSchema: schema.getFileStoringInfoSlaveResponse() 11 | }); 12 | const results = await node.requestNetwork('get-file-storing-info', options); 13 | const existing = results.filter(c => c.existenceInfo).map(c => pick(c, ['address', 'existenceInfo'])); 14 | const candidates = await node.filterCandidates(results, await node.getFileStoringFilterOptions(info)); 15 | res.send({ candidates, existing }); 16 | } 17 | catch (err) { 18 | next(err); 19 | } 20 | }; 21 | }; 22 | 23 | export const getFileLinks = node => { 24 | return async (req, res, next) => { 25 | try { 26 | node.hashTest(req.body.hash); 27 | const options = node.createRequestNetworkOptions(req.body, { 28 | responseSchema: schema.getFileLinksSlaveResponse() 29 | }); 30 | const results = await node.requestNetwork('get-file-links', options); 31 | const links = await node.filterCandidates(results, await node.getFileLinksFilterOptions()); 32 | return res.send({ links }); 33 | } 34 | catch (err) { 35 | next(err); 36 | } 37 | }; 38 | }; 39 | 40 | export const removeFile = node => { 41 | return async (req, res, next) => { 42 | try { 43 | node.hashTest(req.body.hash); 44 | const options = node.createRequestNetworkOptions(req.body, { 45 | responseSchema: schema.getFileRemovalSlaveResponse() 46 | }); 47 | const results = await node.requestNetwork('remove-file', options); 48 | const removed = results.reduce((p, c) => p + c.removed, 0); 49 | return res.send({ removed }); 50 | } 51 | catch (err) { 52 | next(err); 53 | } 54 | }; 55 | }; 56 | 57 | export const getNetworkFilesCount = node => { 58 | return async (req, res, next) => { 59 | try { 60 | const options = node.createRequestNetworkOptions(req.body, { 61 | responseSchema: schema.getNetworkFilesCountSlaveResponse() 62 | }); 63 | const results = await node.requestNetwork('get-network-files-count', options); 64 | const count = results.reduce((p, c) => p + c.count, 0); 65 | return res.send({ count }); 66 | } 67 | catch (err) { 68 | next(err); 69 | } 70 | }; 71 | }; 72 | -------------------------------------------------------------------------------- /src/server/transports/express/api/butler/routes.js: -------------------------------------------------------------------------------- 1 | import * as controllers from "./controllers.js"; 2 | 3 | export default [ 4 | /** 5 | * Get candidates to store the file 6 | * 7 | * @api {post} /api/butler/get-file-storing-info 8 | * @apiParam {object} info 9 | * @apiParam {string} info.size 10 | * @apiParam {string} info.hash 11 | * @apiSuccess {object} - { candidates: ... } 12 | */ 13 | { 14 | name: 'getFileStoringInfo', 15 | method: 'post', 16 | url: '/get-file-storing-info', 17 | fn: controllers.getFileStoringInfo 18 | }, 19 | 20 | /** 21 | * Get the file links 22 | * 23 | * @api {post} /api/butler/get-file-links 24 | * @apiParam {string} hash - file hash 25 | */ 26 | { 27 | name: 'getFileLinks', 28 | method: 'post', 29 | url: '/get-file-links', 30 | fn: controllers.getFileLinks 31 | }, 32 | 33 | /** 34 | * Remove the file 35 | * 36 | * @api {post} /api/butler/remove-file 37 | * @apiParam {string} hash - file hash 38 | */ 39 | { 40 | name: 'removeFile', 41 | method: 'post', 42 | url: '/remove-file', 43 | fn: controllers.removeFile 44 | }, 45 | 46 | /** 47 | * Get the network files count 48 | * 49 | * @api {post} /api/butler/get-network-files-count 50 | */ 51 | { 52 | name: 'getNetworkFilesCount', 53 | method: 'post', 54 | url: '/get-network-files-count', 55 | fn: controllers.getNetworkFilesCount 56 | } 57 | ]; 58 | -------------------------------------------------------------------------------- /src/server/transports/express/api/master/controllers.js: -------------------------------------------------------------------------------- 1 | import schema from "../../../../../schema.js"; 2 | 3 | export const getFileStoringInfo = node => { 4 | return async (req, res, next) => { 5 | try { 6 | const info = req.body.info || {}; 7 | node.hashTest(info.hash); 8 | const options = node.createRequestNetworkOptions(req.body, { 9 | responseSchema: schema.getFileStoringInfoButlerResponse() 10 | }); 11 | const results = await node.requestNetwork('get-file-storing-info', options); 12 | const existing = results.reduce((p, c) => p.concat(c.existing), []); 13 | const opts = await node.getFileStoringFilterOptions(info); 14 | const candidates = await node.filterCandidatesMatrix(results.map(r => r.candidates), opts); 15 | res.send({ candidates, existing }); 16 | } 17 | catch (err) { 18 | next(err); 19 | } 20 | }; 21 | }; 22 | 23 | export const getFileLinks = node => { 24 | return async (req, res, next) => { 25 | try { 26 | node.hashTest(req.body.hash); 27 | const options = node.createRequestNetworkOptions(req.body, { 28 | responseSchema: schema.getFileLinksButlerResponse() 29 | }); 30 | const results = await node.requestNetwork('get-file-links', options); 31 | const opts = await node.getFileLinksFilterOptions(); 32 | const links = await node.filterCandidatesMatrix(results.map(r => r.links), opts); 33 | return res.send({ links }); 34 | } 35 | catch (err) { 36 | next(err); 37 | } 38 | }; 39 | }; 40 | 41 | export const removeFile = node => { 42 | return async (req, res, next) => { 43 | try { 44 | node.hashTest(req.body.hash); 45 | const options = node.createRequestNetworkOptions(req.body, { 46 | responseSchema: schema.getFileRemovalButlerResponse() 47 | }); 48 | const results = await node.requestNetwork('remove-file', options); 49 | const removed = results.reduce((p, c) => p + c.removed, 0); 50 | return res.send({ removed }); 51 | } 52 | catch (err) { 53 | next(err); 54 | } 55 | }; 56 | }; 57 | 58 | export const getNetworkFilesCount = node => { 59 | return async (req, res, next) => { 60 | try { 61 | const options = node.createRequestNetworkOptions(req.body, { 62 | responseSchema: schema.getNetworkFilesCountButlerResponse() 63 | }); 64 | const results = await node.requestNetwork('get-network-files-count', options); 65 | const count = results.reduce((p, c) => p + c.count, 0); 66 | return res.send({ count }); 67 | } 68 | catch (err) { 69 | next(err); 70 | } 71 | }; 72 | }; 73 | -------------------------------------------------------------------------------- /src/server/transports/express/api/master/routes.js: -------------------------------------------------------------------------------- 1 | import * as controllers from "./controllers.js"; 2 | 3 | export default [ 4 | /** 5 | * Get candidates to store the file 6 | * 7 | * @api {post} /api/master/get-file-storing-info 8 | * @apiParam {object} info 9 | * @apiParam {string} info.size 10 | * @apiParam {string} info.hash 11 | * @apiSuccess {object} - { candidates: ... } 12 | */ 13 | { 14 | name: 'getFileStoringInfo', 15 | method: 'post', 16 | url: '/get-file-storing-info', 17 | fn: controllers.getFileStoringInfo 18 | }, 19 | 20 | /** 21 | * Get the file links 22 | * 23 | * @api {post} /api/master/get-file-links 24 | * @apiParam {string} hash - file hash 25 | */ 26 | { 27 | name: 'getFileLinks', 28 | method: 'post', 29 | url: '/get-file-links', 30 | fn: controllers.getFileLinks 31 | }, 32 | 33 | /** 34 | * Remove the file 35 | * 36 | * @api {post} /api/master/remove-file 37 | * @apiParam {string} hash - file hash 38 | */ 39 | { 40 | name: 'removeFile', 41 | method: 'post', 42 | url: '/remove-file', 43 | fn: controllers.removeFile 44 | }, 45 | 46 | /** 47 | * Get the network files count 48 | * 49 | * @api {post} /api/master/get-network-files-count 50 | */ 51 | { 52 | name: 'getNetworkFilesCount', 53 | method: 'post', 54 | url: '/get-network-files-count', 55 | fn: controllers.getNetworkFilesCount 56 | } 57 | ]; 58 | -------------------------------------------------------------------------------- /src/server/transports/express/api/node/controllers.js: -------------------------------------------------------------------------------- 1 | import utils from "../../../../../utils.js"; 2 | import fse from "fs-extra"; 3 | 4 | export const storeFile = node => { 5 | return async (req, res, next) => { 6 | let file; 7 | 8 | try { 9 | file = req.body.file; 10 | const duplicates = req.body.duplicates || []; 11 | const info = await utils.getFileInfo(file); 12 | await node.fileAvailabilityTest(info); 13 | 14 | if (!await node.hasFile(info.hash)) { 15 | await node.addFileToStorage(file, info.hash); 16 | } 17 | 18 | file.destroy(); 19 | const link = await node.createFileLink(info.hash); 20 | 21 | if (duplicates.length) { 22 | file = fse.createReadStream(node.getFilePath(info.hash)); 23 | node.duplicateFile(duplicates, file, info) 24 | .then(() => { 25 | file.destroy(); 26 | }) 27 | .catch((err) => { 28 | file.destroy(); 29 | node.logger.error(err.stack); 30 | }); 31 | } 32 | 33 | res.send({ hash: info.hash, link }); 34 | } 35 | catch (err) { 36 | utils.isFileReadStream(file) && file.destroy(); 37 | next(err); 38 | } 39 | }; 40 | }; 41 | -------------------------------------------------------------------------------- /src/server/transports/express/api/node/routes.js: -------------------------------------------------------------------------------- 1 | import * as controllers from "./controllers.js"; 2 | import midds from "../../midds.js"; 3 | 4 | export default [ 5 | /** 6 | * Store the file 7 | * 8 | * @api {post} /api/node/store-file/:hash 9 | * @apiParam {fse.ReadStream|string} file 10 | */ 11 | { 12 | name: 'storeFile', 13 | method: 'post', 14 | url: '/store-file/:hash', 15 | fn: [ 16 | midds.requestQueueFileHash, 17 | midds.filesFormData, 18 | midds.prepareFileToStore, 19 | controllers.storeFile 20 | ] 21 | } 22 | ]; 23 | -------------------------------------------------------------------------------- /src/server/transports/express/api/routes.js: -------------------------------------------------------------------------------- 1 | export default []; 2 | -------------------------------------------------------------------------------- /src/server/transports/express/api/slave/controllers.js: -------------------------------------------------------------------------------- 1 | export const getFileStoringInfo = node => { 2 | return async (req, res, next) => { 3 | try { 4 | const info = req.body.info || {}; 5 | node.hashTest(info.hash); 6 | const testInfo = Object.assign({}, info); 7 | testInfo.storage = await node.getStorageInfo(); 8 | res.send({ 9 | free: testInfo.storage.free, 10 | existenceInfo: await node.getFileExistenceInfo(testInfo), 11 | isAvailable: await node.checkFileAvailability(testInfo) 12 | }); 13 | } 14 | catch (err) { 15 | next(err); 16 | } 17 | }; 18 | }; 19 | 20 | export const getFileLinks = node => { 21 | return async (req, res, next) => { 22 | try { 23 | const hash = req.body.hash; 24 | node.hashTest(hash); 25 | return res.send({ link: await node.hasFile(hash) ? await node.createFileLink(hash) : '' }); 26 | } 27 | catch (err) { 28 | next(err); 29 | } 30 | }; 31 | }; 32 | 33 | export const removeFile = node => { 34 | return async (req, res, next) => { 35 | try { 36 | const hash = req.body.hash; 37 | node.hashTest(hash); 38 | const hasFile = await node.hasFile(hash); 39 | if (hasFile) { 40 | await node.removeFileFromStorage(hash); 41 | } 42 | res.send({ removed: +hasFile }); 43 | } 44 | catch (err) { 45 | next(err); 46 | } 47 | }; 48 | }; 49 | 50 | export const getNetworkFilesCount = node => { 51 | return async (req, res, next) => { 52 | try { 53 | res.send({ count: await node.db.getData('filesCount') }); 54 | } 55 | catch (err) { 56 | next(err); 57 | } 58 | }; 59 | }; 60 | -------------------------------------------------------------------------------- /src/server/transports/express/api/slave/routes.js: -------------------------------------------------------------------------------- 1 | import * as controllers from "./controllers.js"; 2 | import midds from "../../midds.js"; 3 | 4 | export default [ 5 | /** 6 | * Get the file storing info 7 | * 8 | * @api {post} /api/slave/get-file-storing-info 9 | * @apiParam {object} info 10 | * @apiParam {string} info.size 11 | * @apiParam {string} info.hash 12 | * @apiSuccess {object} - { candidates: ... } 13 | */ 14 | { 15 | name: 'getFileStoringInfo', 16 | method: 'post', 17 | url: '/get-file-storing-info', 18 | fn: controllers.getFileStoringInfo 19 | }, 20 | 21 | /** 22 | * Get the file links 23 | * 24 | * @api {post} /api/slave/get-file-links 25 | * @apiParam {string} hash - file hash 26 | */ 27 | { 28 | name: 'getFileLinks', 29 | method: 'post', 30 | url: '/get-file-links', 31 | fn: controllers.getFileLinks 32 | }, 33 | 34 | /** 35 | * Remove the file 36 | * 37 | * @api {post} /api/slave/remove-file 38 | * @apiParam {string} hash - file hash 39 | */ 40 | { 41 | name: 'removeFile', 42 | method: 'post', 43 | url: '/remove-file', 44 | fn: [ 45 | midds.requestQueueFileHash, 46 | controllers.removeFile 47 | ] 48 | }, 49 | 50 | /** 51 | * Get the network files count 52 | * 53 | * @api {post} /api/slave/get-network-files-count 54 | */ 55 | { 56 | name: 'getNetworkFilesCount', 57 | method: 'post', 58 | url: '/get-network-files-count', 59 | fn: controllers.getNetworkFilesCount 60 | } 61 | ]; 62 | -------------------------------------------------------------------------------- /src/server/transports/express/client/controllers.js: -------------------------------------------------------------------------------- 1 | import utils from "../../../../utils.js"; 2 | import errors from "../../../../errors.js"; 3 | 4 | export const requestFile = node => { 5 | return async (req, res, next) => { 6 | try { 7 | const hash = req.params.hash; 8 | 9 | if (!hash) { 10 | throw new errors.WorkError('"hash" field is invalid', 'ERR_STORACLE_INVALID_HASH_FIELD'); 11 | } 12 | 13 | const link = await node.getFileLink(hash); 14 | 15 | if (!link) { 16 | throw new errors.NotFoundError('File not found'); 17 | } 18 | 19 | res.redirect(link); 20 | } 21 | catch (err) { 22 | next(err); 23 | } 24 | }; 25 | }; 26 | 27 | export const storeFile = node => { 28 | return async (req, res, next) => { 29 | try { 30 | const file = req.body.file; 31 | 32 | if (!utils.isFileReadStream(file)) { 33 | throw new errors.WorkError('"file" field is invalid', 'ERR_STORACLE_INVALID_FILE_FIELD'); 34 | } 35 | 36 | const hash = await node.storeFile(file, node.prepareClientMessageOptions(req.body)); 37 | res.send({ hash }); 38 | } 39 | catch (err) { 40 | next(err); 41 | } 42 | }; 43 | }; 44 | 45 | export const getFileLink = node => { 46 | return async (req, res, next) => { 47 | try { 48 | const hash = req.body.hash; 49 | 50 | if (!hash) { 51 | throw new errors.WorkError('"hash" field is invalid', 'ERR_STORACLE_INVALID_HASH_FIELD'); 52 | } 53 | 54 | const link = await node.getFileLink(hash, node.prepareClientMessageOptions(req.body)); 55 | res.send({ link }); 56 | } 57 | catch (err) { 58 | next(err); 59 | } 60 | }; 61 | }; 62 | 63 | export const getFileLinks = node => { 64 | return async (req, res, next) => { 65 | try { 66 | const hash = req.body.hash; 67 | 68 | if (!hash) { 69 | throw new errors.WorkError('"hash" field is invalid', 'ERR_STORACLE_INVALID_HASH_FIELD'); 70 | } 71 | 72 | const links = await node.getFileLinks(hash, node.prepareClientMessageOptions(req.body)); 73 | res.send({ links }); 74 | } 75 | catch (err) { 76 | next(err); 77 | } 78 | }; 79 | }; 80 | 81 | export const removeFile = node => { 82 | return async (req, res, next) => { 83 | try { 84 | const hash = req.body.hash; 85 | 86 | if (!hash) { 87 | throw new errors.WorkError('"hash" field is invalid', 'ERR_STORACLE_INVALID_HASH_FIELD'); 88 | } 89 | 90 | const result = await node.removeFile(hash, node.prepareClientMessageOptions(req.body)); 91 | res.send(result); 92 | } 93 | catch (err) { 94 | next(err); 95 | } 96 | }; 97 | }; 98 | 99 | export const getNetworkFilesCount = node => { 100 | return async (req, res, next) => { 101 | try { 102 | res.send({ count: await node.getNetworkFilesCount(node.prepareClientMessageOptions(req.body)) }); 103 | } 104 | catch (err) { 105 | next(err); 106 | } 107 | }; 108 | }; 109 | -------------------------------------------------------------------------------- /src/server/transports/express/client/routes.js: -------------------------------------------------------------------------------- 1 | import * as controllers from "./controllers.js"; 2 | import midds from "../midds.js"; 3 | 4 | export default [ 5 | /** 6 | * Request the file 7 | * 8 | * @api {get} /client/request-file/:hash 9 | * @apiParam {string} hash - file hash 10 | */ 11 | { 12 | name: 'requestFile', 13 | method: 'get', 14 | url: '/request-file/:hash', 15 | fn: [ 16 | midds.requestQueueClient, 17 | controllers.requestFile 18 | ] 19 | }, 20 | 21 | /** 22 | * Store a file 23 | * 24 | * @api {post} /client/store-file/ 25 | * @apiParam {fse.ReadStream|string} file 26 | * @apiSuccess {object} - { hash: '' } 27 | */ 28 | { 29 | name: 'storeFile', 30 | method: 'post', 31 | url: '/store-file', 32 | fn: (node) => [ 33 | midds.requestQueueClient(node, { limit: node.options.request.clientStoringConcurrency }), 34 | midds.filesFormData(node), 35 | controllers.storeFile(node) 36 | ] 37 | }, 38 | 39 | /** 40 | * Get the file link 41 | * 42 | * @api {post} /client/get-file-link 43 | * @apiParam {string} hash - file hash 44 | * @apiSuccess {object} - { link: '' } 45 | */ 46 | { 47 | name: 'getFileLink', 48 | method: 'post', 49 | url: '/get-file-link', 50 | fn: [ 51 | midds.requestQueueClient, 52 | controllers.getFileLink 53 | ] 54 | }, 55 | 56 | /** 57 | * Get the file links array 58 | * 59 | * @api {post} /client/get-file-links 60 | * @apiParam {string} hash - file hash 61 | * @apiSuccess {object} - { links: [''] } 62 | */ 63 | { 64 | name: 'getFileLinks', 65 | method: 'post', 66 | url: '/get-file-links', 67 | fn: [ 68 | midds.requestQueueClient, 69 | controllers.getFileLinks 70 | ] 71 | }, 72 | 73 | /** 74 | * Remove the file 75 | * 76 | * @api {post} /client/remove-file 77 | * @apiParam {string} hash - file hash 78 | * @apiSuccess {object} - { removed: 0 } 79 | */ 80 | { 81 | name: 'removeFile', 82 | method: 'post', 83 | url: '/remove-file', 84 | fn: [ 85 | midds.requestQueueClient, 86 | controllers.removeFile 87 | ] 88 | }, 89 | 90 | /** 91 | * Get the network files count 92 | * 93 | * @api {post} /client/get-network-files-count 94 | */ 95 | { 96 | name: 'getNetworkFilesCount', 97 | method: 'post', 98 | url: '/get-network-files-count', 99 | fn: controllers.getNetworkFilesCount 100 | } 101 | ]; 102 | -------------------------------------------------------------------------------- /src/server/transports/express/index.js: -------------------------------------------------------------------------------- 1 | import express from "spreadable/src/server/transports/express/index.js"; 2 | import routes from "./routes.js"; 3 | import routesClient from "./client/routes.js"; 4 | import routesApi from "./api/routes.js"; 5 | import routesApiMaster from "./api/master/routes.js"; 6 | import routesApiButler from "./api/butler/routes.js"; 7 | import routesApiSlave from "./api/slave/routes.js"; 8 | import routesApiNode from "./api/node/routes.js"; 9 | 10 | const ServerExpress = express(); 11 | 12 | export default (Parent) => { 13 | return class ServerExpressStoracle extends (Parent || ServerExpress) { 14 | /** 15 | * @see ServerExpress.prototype.getMainRoutes 16 | */ 17 | getMainRoutes() { 18 | const arr = super.getMainRoutes(); 19 | arr.splice(arr.findIndex(r => r.name == 'bodyParser'), 0, ...routes.slice()); 20 | return arr; 21 | } 22 | 23 | /** 24 | * @see ServerExpress.prototype.getClientRoutes 25 | */ 26 | getClientRoutes() { 27 | return super.getClientRoutes().concat(routesClient); 28 | } 29 | 30 | /** 31 | * @see ServerExpress.prototype.getApiRoutes 32 | */ 33 | getApiRoutes() { 34 | return super.getApiRoutes().concat(routesApi); 35 | } 36 | 37 | /** 38 | * @see ServerExpress.prototype.getApiMasterRoutes 39 | */ 40 | getApiMasterRoutes() { 41 | return super.getApiMasterRoutes().concat(routesApiMaster); 42 | } 43 | 44 | /** 45 | * @see ServerExpress.prototype.getApiButlerRoutes 46 | */ 47 | getApiButlerRoutes() { 48 | return super.getApiButlerRoutes().concat(routesApiButler); 49 | } 50 | 51 | /** 52 | * @see ServerExpress.prototype.getApiSlaveRoutes 53 | */ 54 | getApiSlaveRoutes() { 55 | return super.getApiSlaveRoutes().concat(routesApiSlave); 56 | } 57 | 58 | /** 59 | * @see ServerExpress.prototype.getApiNodeRoutes 60 | */ 61 | getApiNodeRoutes() { 62 | return super.getApiNodeRoutes().concat(routesApiNode); 63 | } 64 | }; 65 | }; 66 | -------------------------------------------------------------------------------- /src/server/transports/express/midds.js: -------------------------------------------------------------------------------- 1 | import errors from "../../../errors.js"; 2 | import utils from "../../../utils.js"; 3 | import formData from "express-form-data"; 4 | import fse from "fs-extra"; 5 | import path from "path"; 6 | import midds from "spreadable/src/server/transports/express/midds.js"; 7 | 8 | /** 9 | * Provide files receiving 10 | */ 11 | midds.file = node => { 12 | return async (req, res, next) => { 13 | try { 14 | const hash = req.params.hash.split('.')[0]; 15 | 16 | if (!await node.hasFile(hash)) { 17 | throw new errors.NotFoundError('File not found'); 18 | } 19 | 20 | if (req.headers['storacle-cache-check']) { 21 | return res.send(''); 22 | } 23 | const cache = Math.ceil(node.options.file.responseCacheLifetime / 1000); 24 | const filePath = node.getFilePath(hash); 25 | const info = await utils.getFileInfo(filePath, { hash: false }); 26 | info.mime && res.setHeader("Content-Type", info.mime); 27 | cache && res.set('Cache-Control', `public, max-age=${cache}`); 28 | res.setHeader("Content-Length", info.size); 29 | const stream = fse.createReadStream(filePath); 30 | stream.on('error', next).pipe(res); 31 | } 32 | catch (err) { 33 | next(err); 34 | } 35 | }; 36 | }; 37 | 38 | /** 39 | * Prepare file for storing 40 | */ 41 | midds.prepareFileToStore = node => { 42 | return async (req, res, next) => { 43 | let file; 44 | 45 | try { 46 | file = req.body.file; 47 | const invalidFileErr = new errors.WorkError('"file" field is invalid', 'ERR_STORACLE_INVALID_FILE_FIELD'); 48 | 49 | if (file && !utils.isFileReadStream(file)) { 50 | if (!utils.isIpEqual(req.clientIp, node.ip)) { 51 | throw invalidFileErr; 52 | } 53 | 54 | try { 55 | file = fse.createReadStream(path.join(node.tempPath, file)); 56 | } 57 | catch (err) { 58 | throw invalidFileErr; 59 | } 60 | } 61 | 62 | if (!file || !utils.isFileReadStream(file)) { 63 | throw invalidFileErr; 64 | } 65 | 66 | req.body.file = file; 67 | next(); 68 | } 69 | catch (err) { 70 | utils.isFileReadStream(file) && file.destroy(); 71 | next(err); 72 | } 73 | }; 74 | }; 75 | 76 | /** 77 | * Provide files storing 78 | */ 79 | midds.filesFormData = node => { 80 | return [ 81 | async (req, res, next) => { 82 | try { 83 | let info = await node.getTempDirInfo(); 84 | let maxSize = node.storageTempSize - info.size; 85 | let length = +req.headers['content-length']; 86 | 87 | if (length > node.fileMaxSize) { 88 | throw new errors.WorkError('The file is too big', 'ERR_STORACLE_FILE_BIG'); 89 | } 90 | 91 | if (length < node.fileMinSize) { 92 | throw new errors.WorkError('The file is too small', 'ERR_STORACLE_FILE_SMALL'); 93 | } 94 | 95 | if (node.calculateTempFileMinSize(length) > maxSize) { 96 | throw new errors.WorkError('Not enough place in the temp folder', 'ERR_STORACLE_REQUEST_TEMP_NOT_ENOUGH'); 97 | } 98 | 99 | formData.parse({ 100 | autoClean: true, 101 | maxFilesSize: node.fileMaxSize, 102 | uploadDir: node.tempPath 103 | })(req, res, next); 104 | } 105 | catch (err) { 106 | next(err); 107 | } 108 | }, 109 | formData.format(), 110 | formData.stream(), 111 | formData.union() 112 | ]; 113 | }; 114 | 115 | /** 116 | * Control file requests limit by the file hash 117 | */ 118 | midds.requestQueueFileHash = (node) => { 119 | const options = { limit: 1 }; 120 | return (req, res, next) => { 121 | return midds.requestQueue(node, `fileHash=${req.params.hash || req.body.hash}`, options)(req, res, next); 122 | }; 123 | }; 124 | 125 | export default midds; 126 | -------------------------------------------------------------------------------- /src/server/transports/express/routes.js: -------------------------------------------------------------------------------- 1 | import midds from "./midds.js"; 2 | 3 | export default [ 4 | { 5 | name: 'file', 6 | method: 'get', 7 | url: '/file/:hash', 8 | fn: node => ([ 9 | midds.networkAccess(node), 10 | midds.file(node) 11 | ]) 12 | } 13 | ]; 14 | -------------------------------------------------------------------------------- /src/utils.js: -------------------------------------------------------------------------------- 1 | import mime from "mime"; 2 | import hasha from 'hasha'; 3 | import detectMime from "detect-file-type"; 4 | import fse from "fs-extra"; 5 | import stream from "stream"; 6 | import fetch from "node-fetch"; 7 | import urlib from "url"; 8 | import errors from "./errors.js"; 9 | import _utils from "spreadable/src/utils.js"; 10 | 11 | const utils = Object.assign({}, _utils); 12 | 13 | /** 14 | * Fetch the file to a buffer 15 | * 16 | * @async 17 | * @param {string} link 18 | * @param {object} [options] 19 | * @returns {Promise} 20 | */ 21 | utils.fetchFileToBuffer = async function (link, options = {}) { 22 | options = Object.assign({}, options, { method: 'GET' }); 23 | 24 | try { 25 | let result = await fetch(link, options); 26 | return result.buffer(); 27 | } 28 | catch (err) { 29 | throw utils.isRequestTimeoutError(err)? utils.createRequestTimeoutError(): err; 30 | } 31 | }; 32 | 33 | /** 34 | * Fetch the file to a blob 35 | * 36 | * @async 37 | * @param {string} link 38 | * @param {object} [options] 39 | * @returns {Blob} 40 | */ 41 | utils.fetchFileToBlob = async function (link, options = {}) { 42 | const controller = new AbortController(); 43 | options = Object.assign({}, options, { 44 | method: 'GET', 45 | signal: controller.signal 46 | }); 47 | const timer = this.getRequestTimer(options.timeout); 48 | let timeIsOver = false; 49 | 50 | try { 51 | let result = await fetch(link, options); 52 | let timeoutObj; 53 | const timeout = timer(); 54 | 55 | if (timeout) { 56 | timeoutObj = setTimeout(() => { 57 | timeIsOver = true; 58 | controller.abort(); 59 | }, timeout); 60 | } 61 | 62 | const blob = await result.blob(); 63 | timeoutObj && clearTimeout(timeoutObj); 64 | return blob; 65 | } 66 | catch (err) { 67 | throw utils.isRequestTimeoutError(err) || timeIsOver? utils.createRequestTimeoutError(): err; 68 | } 69 | }; 70 | 71 | /** 72 | * Fetch the file to the path 73 | * 74 | * @async 75 | * @param {string} filePath 76 | * @param {string} link 77 | * @param {object} [options] 78 | */ 79 | utils.fetchFileToPath = async function (filePath, link, options = {}) { 80 | options = Object.assign({}, options, { method: 'GET' }); 81 | const timer = this.getRequestTimer(options.timeout); 82 | let result; 83 | 84 | try { 85 | result = await fetch(link, options); 86 | } 87 | catch (err) { 88 | throw utils.isRequestTimeoutError(err)? utils.createRequestTimeoutError(): err; 89 | } 90 | 91 | return await new Promise((resolve, reject) => { 92 | const stream = fse.createWriteStream(filePath); 93 | const timeout = timer(); 94 | let timeIsOver = false; 95 | let timeoutObj; 96 | if (timeout) { 97 | timeoutObj = setTimeout(() => { 98 | timeIsOver = true; 99 | stream.close(); 100 | }, timeout); 101 | } 102 | result.body 103 | .pipe(stream) 104 | .on('error', reject) 105 | .on('finish', () => { 106 | clearTimeout(timeoutObj); 107 | timeIsOver? reject(utils.createRequestTimeoutError()): resolve(); 108 | }); 109 | }); 110 | }; 111 | 112 | /** 113 | * Check the file is fse.ReadStream or fse.ReadStream 114 | * 115 | * @param {*} obj 116 | * @returns {boolean} 117 | */ 118 | utils.isFileReadStream = function (obj) { 119 | return stream && typeof stream == 'function' && stream.Readable && (obj instanceof stream.Readable); 120 | }; 121 | 122 | /** 123 | * Get the disk info 124 | * 125 | * @async 126 | * @param {string} dir 127 | * @returns {object} 128 | */ 129 | utils.getDiskInfo = async function (dir) { 130 | const stats = await fse.promises.statfs(dir); 131 | return { 132 | available: stats.bsize * stats.bavail, 133 | free: stats.bsize * stats.bfree, 134 | total: stats.bsize * stats.blocks 135 | }; 136 | }; 137 | 138 | /** 139 | * Get the file info 140 | * 141 | * @async 142 | * @param {string|Buffer|fse.ReadStream|Blob} file 143 | * @param {object} data 144 | * @returns {object} 145 | */ 146 | utils.getFileInfo = async function (file, data = {}) { 147 | data = Object.assign({ 148 | size: true, 149 | mime: true, 150 | ext: true, 151 | hash: true 152 | }, data); 153 | let info = {}; 154 | 155 | if (typeof Blob == 'function' && file instanceof Blob) { 156 | data.size && (info.size = file.size); 157 | data.mime && (info.mime = file.type); 158 | (data.mime && data.ext) && (info.ext = mime.getExtension(info.mime)); 159 | data.hash && (info.hash = await this.getFileHash(file)); 160 | } 161 | else if (this.isFileReadStream(file) || typeof file == 'string') { 162 | const filePath = file.path || file; 163 | data.size && (info.size = (await fse.stat(filePath)).size); 164 | data.mime && (info.mime = await this.getFileMimeType(filePath)); 165 | (data.mime && data.ext) && (info.ext = mime.getExtension(info.mime)); 166 | data.hash && (info.hash = await this.getFileHash(filePath)); 167 | } 168 | else if (typeof Buffer == 'function' && Buffer.isBuffer(file)) { 169 | data.size && (info.size = file.length); 170 | data.mime && (info.mime = await this.getFileMimeType(file)); 171 | (data.mime && data.ext) && (info.ext = mime.getExtension(info.mime)); 172 | data.hash && (info.hash = await this.getFileHash(file)); 173 | } 174 | else { 175 | throw new errors.WorkError('Wrong file format', 'ERR_STORACLE_WRONG_FILE'); 176 | } 177 | 178 | return info; 179 | }; 180 | 181 | /** 182 | * Get the file hash 183 | * 184 | * @async 185 | * @param {string|Buffer|fse.ReadStream|Blob} file 186 | * @returns {string} 187 | */ 188 | utils.getFileHash = async function (file) { 189 | if (typeof Blob == 'function' && file instanceof Blob) { 190 | return await hasha(await this.blobToBuffer(file), { algorithm: 'md5' }); 191 | } 192 | else if (this.isFileReadStream(file) || typeof file == 'string') { 193 | return await hasha.fromFile(file.path || file, { algorithm: 'md5' }); 194 | } 195 | else if (typeof Buffer == 'function' && Buffer.isBuffer(file)) { 196 | return await hasha(file, { algorithm: 'md5' }); 197 | } 198 | 199 | throw new errors.WorkError('Wrong file format', 'ERR_STORACLE_WRONG_FILE'); 200 | }; 201 | 202 | /** 203 | * Get the file mime type 204 | * 205 | * @async 206 | * @param {string|fse.ReadStream|Buffer} content 207 | * @returns {string} 208 | */ 209 | utils.getFileMimeType = async function (content) { 210 | return await new Promise((resolve, reject) => { 211 | this.isFileReadStream(content) && (content = content.path); 212 | detectMime[Buffer.isBuffer(content)? 'fromBuffer': 'fromFile'](content, (err, result) => { 213 | if (err) { 214 | return reject(err); 215 | } 216 | resolve(result ? result.mime : 'text/plain'); 217 | }); 218 | }); 219 | }; 220 | 221 | /** 222 | * Convert the blob to a Buffer 223 | * 224 | * @async 225 | * @param {Blob} blob 226 | * @returns {Buffer} 227 | */ 228 | utils.blobToBuffer = async function (blob) { 229 | return await new Promise((resolve, reject) => { 230 | const reader = new FileReader(); 231 | const fn = result => { 232 | reader.removeEventListener('loadend', fn); 233 | if (result.error) { 234 | return reject(result.error); 235 | } 236 | resolve(Buffer.from(reader.result)); 237 | }; 238 | reader.addEventListener('loadend', fn); 239 | reader.readAsArrayBuffer(blob); 240 | }); 241 | }; 242 | 243 | /** 244 | * Check the file link is valid 245 | * 246 | * @param {string} link 247 | * @param {object} [options] 248 | * @returns {boolean} 249 | */ 250 | utils.isValidFileLink = function (link, options = {}) { 251 | if (typeof link != 'string') { 252 | return false; 253 | } 254 | 255 | const info = urlib.parse(link); 256 | 257 | if (!info.hostname || !this.isValidHostname(info.hostname)) { 258 | return false; 259 | } 260 | 261 | if (!info.port || !this.isValidPort(info.port)) { 262 | return false; 263 | } 264 | 265 | if (!info.protocol.match(/^https?:?$/)) { 266 | return false; 267 | } 268 | 269 | if (!info.pathname || !info.pathname.match(new RegExp(`\\/${options.action || 'file'}\\/[a-z0-9_-]+(\\.[\\w\\d]+)*$`, 'i'))) { 270 | return false; 271 | } 272 | 273 | return true; 274 | }; 275 | 276 | export default utils; 277 | -------------------------------------------------------------------------------- /test/.eslintrc: -------------------------------------------------------------------------------- 1 | { 2 | "env": { 3 | "mocha": true 4 | }, 5 | "rules": { 6 | "no-empty": [2, { "allowEmptyCatch": true }] 7 | } 8 | } -------------------------------------------------------------------------------- /test/client.js: -------------------------------------------------------------------------------- 1 | import { assert } from "chai"; 2 | import path from "path"; 3 | import fse from "fs-extra"; 4 | import node from "../src/node.js"; 5 | import client from "../src/client.js"; 6 | import utils from "../src/utils.js"; 7 | import tools from "./tools.js"; 8 | 9 | const Node = node(); 10 | const Client = client(); 11 | 12 | export default function () { 13 | describe('Client', () => { 14 | let client; 15 | let node; 16 | 17 | before(async function () { 18 | node = new Node(await tools.createNodeOptions()); 19 | await node.init(); 20 | }); 21 | 22 | after(async function () { 23 | await node.deinit(); 24 | }); 25 | 26 | describe('instance creation', function () { 27 | it('should create an instance', async function () { 28 | const options = await tools.createClientOptions({ address: node.address }); 29 | assert.doesNotThrow(() => client = new Client(options)); 30 | }); 31 | }); 32 | 33 | describe('.init()', function () { 34 | it('should not throw an exception', async function () { 35 | await client.init(); 36 | }); 37 | }); 38 | 39 | describe('.storeFile()', function () { 40 | it('should not store the wrong file type', async () => { 41 | try { 42 | await client.storeFile({}); 43 | throw new Error('Fail'); 44 | } 45 | catch (err) { 46 | assert.isOk(err.message.match('Wrong file')); 47 | } 48 | }); 49 | 50 | it('should store the file from a buffer', async () => { 51 | const hash = await client.storeFile(Buffer.from('client-1')); 52 | assert.isTrue(await node.hasFile(hash)); 53 | }); 54 | 55 | it('should store the file from the path', async () => { 56 | const filePath = path.join(tools.tmpPath, '1.txt'); 57 | await fse.writeFile(filePath, 'client-2'); 58 | const hash = await client.storeFile(filePath); 59 | assert.isTrue(await node.hasFile(hash)); 60 | }); 61 | 62 | it('should store the file from fse.ReadStream', async () => { 63 | const filePath = path.join(tools.tmpPath, '1.txt'); 64 | await fse.writeFile(filePath, 'client-3'); 65 | const hash = await client.storeFile(filePath); 66 | assert.isTrue(await node.hasFile(hash)); 67 | }); 68 | }); 69 | 70 | describe('.getFileLink()', () => { 71 | it('should not return the wrong file hash link', async () => { 72 | assert.isFalse(utils.isValidFileLink(await client.getFileLink('wrong'))); 73 | }); 74 | 75 | it('should return the right link', async () => { 76 | const hash = await client.storeFile(Buffer.from('hello')); 77 | assert.isTrue(utils.isValidFileLink(await client.getFileLink(hash))); 78 | }); 79 | }); 80 | 81 | describe('.getFileLinks()', () => { 82 | it('should return an empty array', async () => { 83 | const links = await client.getFileLinks('wrong'); 84 | assert.isOk(Array.isArray(links) && !links.length); 85 | }); 86 | 87 | it('should return the right link in an array', async () => { 88 | const hash = await client.storeFile(Buffer.from('hello')); 89 | const links = await client.getFileLinks(hash); 90 | assert.isTrue(utils.isValidFileLink(links[0])); 91 | }); 92 | }); 93 | 94 | describe('.getFileToBuffer()', () => { 95 | it('should throw an exception because of a wrong file hash', async () => { 96 | try { 97 | await client.getFileToBuffer('wrong'); 98 | throw new Error('Fail'); 99 | } 100 | catch (err) { 101 | assert.isOk(err.message.match('Link for hash')); 102 | } 103 | }); 104 | 105 | it('should return the buffer', async () => { 106 | const buffer = Buffer.from('hello'); 107 | const json = JSON.stringify([...buffer]); 108 | const hash = await client.storeFile(buffer); 109 | const result = await client.getFileToBuffer(hash); 110 | assert.instanceOf(result, Buffer, 'check the intance'); 111 | assert.equal(json, JSON.stringify([...result]), 'check the data'); 112 | }); 113 | }); 114 | 115 | describe('.getFileToPath()', () => { 116 | it('should throw an exception because of a wrong file hash', async () => { 117 | try { 118 | await client.getFileToBuffer('wrong'); 119 | throw new Error('Fail'); 120 | } 121 | catch (err) { 122 | assert.isOk(err.message.match('Link for hash')); 123 | } 124 | }); 125 | 126 | it('should save the file', async () => { 127 | const text = 'hello'; 128 | const filePath = path.join(tools.tmpPath, '1.txt'); 129 | const hash = await client.storeFile(Buffer.from(text)); 130 | await client.getFileToPath(hash, filePath); 131 | assert.equal(await fse.readFile(filePath), text); 132 | }); 133 | }); 134 | 135 | describe('.removeFile()', () => { 136 | it('should remove the file', async () => { 137 | const hash = await client.storeFile(Buffer.from('hello')); 138 | const res = await client.removeFile(hash); 139 | assert.equal(res.removed, 1, 'check the result'); 140 | assert.isFalse(await node.hasFile(hash), 'check the file'); 141 | }); 142 | }); 143 | 144 | describe('.getNetworkFilesCount()', function () { 145 | it('should get the right count', async function () { 146 | const count = await client.getNetworkFilesCount(); 147 | assert.equal(count, await node.db.getData('filesCount')); 148 | }); 149 | }); 150 | describe('.createRequestedFileLink()', () => { 151 | it('should return the right link', async () => { 152 | const hash = 'hash'; 153 | const link = client.createRequestedFileLink(hash); 154 | assert.equal(link, `${client.getRequestProtocol()}://${client.workerAddress}/client/request-file/${hash}`); 155 | }); 156 | }); 157 | 158 | describe('.deinit()', function () { 159 | it('should not throw an exception', async function () { 160 | await client.deinit(); 161 | }); 162 | }); 163 | }); 164 | } -------------------------------------------------------------------------------- /test/db/loki.js: -------------------------------------------------------------------------------- 1 | import { assert } from "chai"; 2 | import tools from "../tools.js"; 3 | import loki from "../../src/db/transports/loki/index.js"; 4 | import fse from "fs-extra"; 5 | 6 | const DatabaseLokiStoracle = loki(); 7 | 8 | export default function () { 9 | describe('DatabaseLokiStoracle', () => { 10 | let loki; 11 | 12 | describe('instance creation', function () { 13 | it('should create an instance', function () { 14 | assert.doesNotThrow(() => loki = new DatabaseLokiStoracle({ 15 | filename: tools.getDbFilePath(this.node) 16 | })); 17 | loki.node = this.node; 18 | }); 19 | }); 20 | 21 | describe('.init()', function () { 22 | it('should not throw an exception', async function () { 23 | await loki.init(); 24 | }); 25 | 26 | it('should create the db file', async function () { 27 | assert.isTrue(await fse.pathExists(tools.getDbFilePath(this.node))); 28 | }); 29 | }); 30 | 31 | describe('.deinit()', function () { 32 | it('should not throw an exception', async function () { 33 | await loki.deinit(); 34 | }); 35 | }); 36 | 37 | describe('reinitialization', () => { 38 | it('should not throw an exception', async function () { 39 | await loki.init(); 40 | }); 41 | }); 42 | 43 | describe('.destroy()', function () { 44 | it('should not throw an exception', async function () { 45 | await loki.destroy(); 46 | }); 47 | 48 | it('should remove the db file', async function () { 49 | assert.isFalse(await fse.pathExists(tools.getDbFilePath(this.node))); 50 | }); 51 | }); 52 | }); 53 | } -------------------------------------------------------------------------------- /test/group.js: -------------------------------------------------------------------------------- 1 | import { assert } from "chai"; 2 | import node from "../src/node.js"; 3 | import client from "../src/client.js"; 4 | import utils from "../src/utils.js"; 5 | import tools from "./tools.js"; 6 | 7 | const Node = node(); 8 | const Client = client(); 9 | 10 | export default function () { 11 | describe('group communication', () => { 12 | let nodes; 13 | let client; 14 | let buffer; 15 | let duplicates; 16 | let fileStoringNodeTimeout; 17 | 18 | before(async () => { 19 | nodes = []; 20 | fileStoringNodeTimeout = 800; 21 | 22 | for (let i = 0; i < 4; i++) { 23 | const node = new Node(await tools.createNodeOptions({ request: { fileStoringNodeTimeout } })); 24 | await node.init(); 25 | nodes.push(node); 26 | node.initialNetworkAddress = nodes[0].address; 27 | } 28 | 29 | client = new Client(await tools.createClientOptions({ address: nodes[0].address })); 30 | await client.init(); 31 | await tools.nodesSync(nodes, nodes.length * 3); 32 | buffer = Buffer.from('hello'); 33 | duplicates = await nodes[0].getFileDuplicatesCount(); 34 | }); 35 | 36 | after(async () => { 37 | for (let i = 0; i < nodes.length; i++) { 38 | await nodes[i].deinit(); 39 | } 40 | }); 41 | 42 | it('should get the right network size', async () => { 43 | for (let i = 0; i < nodes.length; i++) { 44 | assert.equal(await nodes[i].getNetworkSize(), nodes.length); 45 | } 46 | }); 47 | 48 | it('should store the file', async () => { 49 | const hash = await client.storeFile(buffer); 50 | await tools.wait(fileStoringNodeTimeout); 51 | let count = 0; 52 | 53 | for (let i = 0; i < nodes.length; i++) { 54 | (await nodes[i].hasFile(hash)) && count++; 55 | } 56 | 57 | assert.equal(count, duplicates); 58 | }); 59 | 60 | it('should not store the existent files again', async () => { 61 | const hash = await client.storeFile(buffer); 62 | await tools.wait(fileStoringNodeTimeout); 63 | let count = 0; 64 | 65 | for (let i = 0; i < nodes.length; i++) { 66 | (await nodes[i].hasFile(hash)) && count++; 67 | } 68 | 69 | assert.equal(count, duplicates); 70 | }); 71 | 72 | it('should store the necessary count of duplicates', async () => { 73 | const hash = await utils.getFileHash(buffer); 74 | 75 | for (let i = 0; i < nodes.length; i++) { 76 | if (await nodes[i].hasFile(hash)) { 77 | await nodes[i].removeFileFromStorage(hash); 78 | break; 79 | } 80 | } 81 | 82 | await client.storeFile(buffer); 83 | await tools.wait(fileStoringNodeTimeout); 84 | let count = 0; 85 | 86 | for (let i = 0; i < nodes.length; i++) { 87 | (await nodes[i].hasFile(hash)) && count++; 88 | } 89 | 90 | assert.equal(count, duplicates); 91 | }); 92 | 93 | it('should return the right links', async () => { 94 | const hash = await utils.getFileHash(buffer); 95 | const links = await client.getFileLinks(hash); 96 | assert.equal(links.length, duplicates); 97 | }); 98 | 99 | it('should remove the file', async () => { 100 | const hash = await utils.getFileHash(buffer); 101 | await client.removeFile(hash); 102 | let count = 0; 103 | 104 | for (let i = 0; i < nodes.length; i++) { 105 | (await nodes[i].hasFile(hash)) && count++; 106 | } 107 | 108 | assert.equal(count, 0); 109 | }); 110 | 111 | it('should store files in parallel', async () => { 112 | const length = 10; 113 | const p = []; 114 | let count = 0; 115 | 116 | for (let i = 0; i < length; i++) { 117 | p.push(client.storeFile(Buffer.from(i + ''))); 118 | } 119 | 120 | await Promise.all(p); 121 | await tools.wait(fileStoringNodeTimeout * 2); 122 | 123 | for (let i = 0; i < nodes.length; i++) { 124 | count += await nodes[i].db.getData('filesCount'); 125 | } 126 | 127 | assert.isOk(count >= length * duplicates); 128 | }); 129 | 130 | it('should get the network files count', async () => { 131 | let count = 0; 132 | let res = await client.getNetworkFilesCount(); 133 | 134 | for (let i = 0; i < nodes.length; i++) { 135 | const node = nodes[i]; 136 | count += await node.db.getData('filesCount'); 137 | } 138 | 139 | assert.equal(count, res); 140 | }); 141 | }); 142 | } -------------------------------------------------------------------------------- /test/index.js: -------------------------------------------------------------------------------- 1 | import fse from "fs-extra"; 2 | import tools from "./tools.js"; 3 | import utils from "./utils.js"; 4 | import node from "./node.js"; 5 | import client from "./client.js"; 6 | import services from "./services.js"; 7 | import routes from "./routes.js"; 8 | import group from "./group.js"; 9 | 10 | describe('storacle', () => { 11 | before(() => fse.ensureDir(tools.tmpPath)); 12 | after(() => fse.remove(tools.tmpPath)); 13 | 14 | describe('utils', utils.bind(this)); 15 | describe('node', node.bind(this)); 16 | describe('client', client.bind(this)); 17 | describe('services', services.bind(this)); 18 | describe('routes', routes.bind(this)); 19 | describe('group', group.bind(this)); 20 | }); 21 | -------------------------------------------------------------------------------- /test/node.js: -------------------------------------------------------------------------------- 1 | import { assert } from "chai"; 2 | import fse from "fs-extra"; 3 | import path from "path"; 4 | import merge from "lodash-es/merge.js"; 5 | import node from "../src/node.js"; 6 | import utils from "../src/utils.js"; 7 | import tools from "./tools.js"; 8 | 9 | const Node = node(); 10 | 11 | export default function () { 12 | describe('Node', () => { 13 | let node; 14 | 15 | describe('instance creation', () => { 16 | it('should create an instance', async () => { 17 | const options = await tools.createNodeOptions(); 18 | assert.doesNotThrow(() => node = new Node(options)); 19 | }); 20 | }); 21 | 22 | describe('.init()', () => { 23 | it('should not throw an exception', async () => { 24 | await node.init(); 25 | }); 26 | 27 | it('should create the storage', async () => { 28 | assert.isTrue(await fse.pathExists(tools.getStoragePath(node.port))); 29 | }); 30 | }); 31 | 32 | describe('.calculateStorageInfo()', () => { 33 | let data; 34 | let defaultOptions; 35 | 36 | before(async () => { 37 | defaultOptions = merge({}, node.options); 38 | data = await node.getStorageInfo(); 39 | }); 40 | 41 | after(async () => { 42 | node.options = defaultOptions; 43 | await node.calculateStorageInfo(); 44 | }); 45 | 46 | it('should define the necessary variables as expected', async () => { 47 | node.options.storage.dataSize = 1024; 48 | node.options.storage.tempSize = 1024; 49 | node.options.storage.autoCleanSize = 8; 50 | node.options.file.maxSize = 128; 51 | node.options.file.minSize = 1; 52 | await node.calculateStorageInfo(); 53 | assert.equal(node.storageDataSize, node.options.storage.dataSize, 'check "storage.dataSize"'); 54 | assert.equal(node.storageTempSize, node.options.storage.tempSize, 'check "storage.tempSize"'); 55 | assert.equal(node.storageAutoCleanSize, node.options.storage.autoCleanSize, 'check "storage.autoCleanSize"'); 56 | assert.equal(node.fileMaxSize, node.options.file.maxSize, 'check "file.minSize"'); 57 | }); 58 | 59 | it('should define the necessary variables as expected using a percentage', async () => { 60 | node.options.storage.dataSize = '50%'; 61 | node.options.storage.tempSize = '50% - 1b'; 62 | node.options.storage.autoCleanSize = '10%'; 63 | node.options.file.maxSize = '20%'; 64 | node.options.file.minSize = '10%'; 65 | await node.calculateStorageInfo(); 66 | assert.equal(node.storageDataSize, Math.floor(data.available / 2), 'check "storage.dataSize"'); 67 | assert.equal(Math.floor(node.storageTempSize), Math.floor(node.storageDataSize) - 1, 'check "storage.tempSize"'); 68 | assert.equal(Math.floor(node.storageAutoCleanSize), Math.floor(node.storageDataSize * 0.1), 'check "storage.autoCleanSize"'); 69 | assert.equal(Math.floor(node.fileMaxSize), Math.floor(data.available * 0.2), 'check "file.maxSize"'); 70 | assert.equal(Math.floor(node.fileMinSize), Math.floor(data.available * 0.1), 'check "file.minSize"'); 71 | }); 72 | 73 | it('should throw "storage.dataSize" error', async () => { 74 | node.options.storage.dataSize = '110%'; 75 | node.options.storage.tempSize = '80%'; 76 | 77 | try { 78 | await node.calculateStorageInfo(); 79 | throw new Error('Fail'); 80 | } 81 | catch (err) { 82 | assert.isOk(err.message.includes('"storage.dataSize"')); 83 | } 84 | }); 85 | 86 | it('should throw "storage.tempSize" error', async () => { 87 | node.options.storage.dataSize = '80%'; 88 | node.options.storage.tempSize = '110%'; 89 | 90 | try { 91 | await node.calculateStorageInfo(); 92 | throw new Error('Fail'); 93 | } 94 | catch (err) { 95 | assert.isOk(err.message.includes('"storage.tempSize"')); 96 | } 97 | }); 98 | 99 | it('should throw "storage.autoCleanSize" error', async () => { 100 | node.options.storage.dataSize = '80%'; 101 | node.options.storage.tempSize = '10%'; 102 | node.options.storage.autoCleanSize = '110%'; 103 | 104 | try { 105 | await node.calculateStorageInfo(); 106 | throw new Error('Fail'); 107 | } 108 | catch (err) { 109 | assert.isOk(err.message.includes('"storage.autoCleanSize"')); 110 | } 111 | node.options.storage.autoCleanSize = 0; 112 | }); 113 | 114 | it('should throw "file.maxSize" error because of the data size', async () => { 115 | node.options.storage.dataSize = '10%'; 116 | node.options.storage.tempSize = '30%'; 117 | node.options.file.maxSize = '20%'; 118 | 119 | try { 120 | await node.calculateStorageInfo(); 121 | throw new Error('Fail'); 122 | } 123 | catch (err) { 124 | assert.isOk(err.message.includes('"file.maxSize"')); 125 | } 126 | }); 127 | 128 | it('should throw "file.maxSize" error because of the temp size', async () => { 129 | node.options.storage.dataSize = '30%'; 130 | node.options.storage.tempSize = '10%'; 131 | node.options.file.maxSize = '20%'; 132 | 133 | try { 134 | await node.calculateStorageInfo(); 135 | throw new Error('Fail'); 136 | } 137 | catch (err) { 138 | assert.isOk(err.message.includes('Minimum temp file')); 139 | } 140 | }); 141 | 142 | it('should throw "file.maxSize" error because of "file.minSize"', async () => { 143 | node.options.storage.dataSize = '30%'; 144 | node.options.storage.tempSize = '30%'; 145 | node.options.file.maxSize = '20%'; 146 | node.options.file.minSize = '25%'; 147 | 148 | try { 149 | await node.calculateStorageInfo(); 150 | throw new Error('Fail'); 151 | } 152 | catch (err) { 153 | assert.isOk(err.message.includes('file.minSize')); 154 | } 155 | }); 156 | 157 | it('should throw "dataSize + tempSize" error', async () => { 158 | node.options.storage.dataSize = '50%'; 159 | node.options.storage.tempSize = '80%'; 160 | 161 | try { 162 | await node.calculateStorageInfo(); 163 | throw new Error('Fail'); 164 | } 165 | catch (err) { 166 | assert.isOk(err.message.includes('"storage.dataSize" + "storage.tempSize"')); 167 | } 168 | }); 169 | }); 170 | 171 | assert.hasAllKeys({ foo: 1, bar: 2, baz: 3 }, ['foo', 'bar', 'baz']); 172 | 173 | describe('.getStorageInfo()', () => { 174 | let keys; 175 | 176 | before(() => { 177 | keys = [ 178 | 'total', 'available', 'allowed', 'used', 179 | 'free', 'clean', 'fileMaxSize', 'fileMinSize', 180 | 'tempAllowed', 'tempUsed', 'tempFree' 181 | ]; 182 | }); 183 | 184 | it('should have all the keys', async () => { 185 | assert.hasAllKeys(await node.getStorageInfo(), keys); 186 | }); 187 | 188 | it('should get number values', async () => { 189 | const info = await node.getStorageInfo(); 190 | for (let i = 0; i < keys.length; i++) { 191 | assert.isNumber(info[keys[i]]); 192 | } 193 | }); 194 | 195 | it('should exclude the necessary props', async () => { 196 | for (let i = 0; i < keys.length; i++) { 197 | const key = keys[i]; 198 | const info = await node.getStorageInfo({ [key]: false }); 199 | assert.isUndefined(info[key]); 200 | } 201 | }); 202 | }); 203 | 204 | describe('.getFilePath()', () => { 205 | it('should return the right format', () => { 206 | const filePath = node.getFilePath('d131dd02c5e6eec4693d9a0698aff95c'); 207 | assert.equal(filePath, path.join(node.filesPath, 'd', '1', 'd131dd02c5e6eec4693d9a0698aff95c')); 208 | }); 209 | }); 210 | 211 | describe('.storeFile()', () => { 212 | let filesCount; 213 | 214 | before(() => { 215 | filesCount = 0; 216 | }); 217 | 218 | it('should not store the wrong file type', async () => { 219 | try { 220 | await node.storeFile({}); 221 | throw new Error('Fail'); 222 | } 223 | catch (err) { 224 | assert.isOk(err.message.match('Wrong file')); 225 | } 226 | }); 227 | 228 | it('should store the file from a buffer', async () => { 229 | const hash = await node.storeFile(Buffer.from('hello')); 230 | filesCount++; 231 | assert.isTrue(await fse.pathExists(node.getFilePath(hash)), 'check the file'); 232 | assert.equal(await node.db.getData('filesCount'), filesCount), 'check the count'; 233 | }); 234 | 235 | it('should overwrite the same file', async () => { 236 | await node.storeFile(Buffer.from('hello')); 237 | assert.equal(await node.db.getData('filesCount'), filesCount); 238 | }); 239 | 240 | it('should store the file from the path', async () => { 241 | const filePath = path.join(tools.tmpPath, '1.txt'); 242 | await fse.writeFile(filePath, 'bye'); 243 | const hash = await node.storeFile(filePath); 244 | filesCount++; 245 | assert.isTrue(await fse.pathExists(node.getFilePath(hash)), 'check the file'); 246 | assert.equal(await node.db.getData('filesCount'), filesCount), 'check the count'; 247 | }); 248 | 249 | it('should store the file from fse.ReadStream', async () => { 250 | const filePath = path.join(tools.tmpPath, '1.txt'); 251 | await fse.writeFile(filePath, 'goodbye'); 252 | const hash = await node.storeFile(fse.createReadStream(filePath)); 253 | filesCount++; 254 | assert.isTrue(await fse.pathExists(node.getFilePath(hash)), 'check the file'); 255 | assert.equal(await node.db.getData('filesCount'), filesCount), 'check the count'; 256 | }); 257 | 258 | it('should store the file with the temp path', async () => { 259 | const filePath = path.join(node.tempPath, '1.txt'); 260 | await fse.writeFile(filePath, 'morning'); 261 | const hash = await node.storeFile(await fse.createReadStream(filePath)); 262 | filesCount++; 263 | assert.isTrue(await fse.pathExists(node.getFilePath(hash)), 'check the file'); 264 | assert.equal(await node.db.getData('filesCount'), filesCount), 'check the count'; 265 | }); 266 | }); 267 | 268 | describe('.hasFile()', () => { 269 | it('should return false', async () => { 270 | assert.isFalse(await node.hasFile('wrong')); 271 | }); 272 | 273 | it('should return true', async () => { 274 | const hash = await node.storeFile(Buffer.from('hello')); 275 | assert.isTrue(await node.hasFile(hash)); 276 | }); 277 | }); 278 | 279 | describe('.createFileLink()', () => { 280 | it('should create a right file link', async () => { 281 | const hash = await node.storeFile(Buffer.from('hello')); 282 | assert.equal(await node.createFileLink(hash), `http://${node.address}/file/${hash}.txt`); 283 | }); 284 | }); 285 | 286 | describe('.getFileLink()', () => { 287 | it('should not return the wrong file hash link', async () => { 288 | assert.isFalse(utils.isValidFileLink(await node.getFileLink('wrong'))); 289 | }); 290 | 291 | it('should return the right link', async () => { 292 | const hash = await node.storeFile(Buffer.from('hello')); 293 | assert.isTrue(utils.isValidFileLink(await node.getFileLink(hash))); 294 | }); 295 | }); 296 | 297 | describe('.getFileLinks()', () => { 298 | it('should return an empty array', async () => { 299 | const links = await node.getFileLinks('wrong'); 300 | assert.isOk(Array.isArray(links) && !links.length); 301 | }); 302 | 303 | it('should return the right link in an array', async () => { 304 | const hash = await node.storeFile(Buffer.from('hello')); 305 | const links = await node.getFileLinks(hash); 306 | assert.isTrue(utils.isValidFileLink(links[0])); 307 | }); 308 | }); 309 | 310 | describe('.removeFile()', () => { 311 | it('should remove the file', async () => { 312 | const hash = await node.storeFile(Buffer.from('hello')); 313 | const filesCount = await node.db.getData('filesCount'); 314 | const res = await node.removeFile(hash); 315 | assert.equal(res.removed, 1, 'check the result'); 316 | assert.isFalse(await fse.pathExists(node.getFilePath(hash)), 'check the file'); 317 | assert.equal(await node.db.getData('filesCount'), filesCount - 1, 'check the count'); 318 | }); 319 | }); 320 | 321 | describe('.getNetworkFilesCount()', function () { 322 | it('should get the right count', async function () { 323 | const count = await node.getNetworkFilesCount(); 324 | assert.equal(count, await node.db.getData('filesCount')); 325 | }); 326 | }); 327 | 328 | describe('.emptyStorage()', () => { 329 | it('should remove all files', async () => { 330 | await node.storeFile(Buffer.from('hello')); 331 | let files = await fse.readdir(node.filesPath); 332 | assert.isOk(files.length > 0, 'check before'); 333 | await node.emptyStorage(); 334 | files = await fse.readdir(node.filesPath); 335 | assert.equal(files.length, 0, 'check after'); 336 | }); 337 | }); 338 | 339 | describe('.addFileToStorage()', () => { 340 | it('should add the file', async () => { 341 | await node.emptyStorage(); 342 | const filePath = path.join(node.tempPath, '1.txt'); 343 | await fse.writeFile(filePath, 'hello'); 344 | const stat = await fse.stat(filePath); 345 | const file = fse.createReadStream(filePath); 346 | const hash = await utils.getFileHash(file); 347 | await node.addFileToStorage(file, hash); 348 | assert.isTrue(await fse.pathExists(node.getFilePath(hash)), 'check the file'); 349 | assert.equal(await node.db.getData('filesCount'), 1, 'check the count'); 350 | assert.equal(await node.db.getData('filesTotalSize'), stat.size, 'check the size'); 351 | }); 352 | }); 353 | 354 | describe('.removeFileFromStorage()', () => { 355 | it('should remove the file', async () => { 356 | const hash = await utils.getFileHash(Buffer.from('hello')); 357 | await node.removeFileFromStorage(hash); 358 | assert.isFalse(await fse.pathExists(node.getFilePath(hash)), 'check the file'); 359 | assert.equal(await node.db.getData('filesCount'), 0, 'check the count'); 360 | assert.equal(await node.db.getData('filesTotalSize'), 0, 'check the size'); 361 | }); 362 | }); 363 | 364 | describe('.normalizeDir()', () => { 365 | let dir; 366 | let hash; 367 | 368 | it('should not remove the directory', async () => { 369 | await node.emptyStorage(); 370 | hash = await node.storeFile(Buffer.from('hello')); 371 | dir = path.dirname(node.getFilePath(hash)); 372 | await node.normalizeDir(dir); 373 | assert.isTrue(await fse.pathExists(dir)); 374 | }); 375 | 376 | it('should remove the directory', async () => { 377 | await fse.remove(node.getFilePath(hash)); 378 | await node.normalizeDir(dir); 379 | assert.isFalse(await fse.pathExists(dir), 'check the directory'); 380 | assert.isNotOk((await fse.readdir(node.filesPath)).length, 'check the files path'); 381 | }); 382 | }); 383 | 384 | describe('.getStorageTotalSize()', () => { 385 | it('should get the right size', async () => { 386 | await node.emptyStorage(); 387 | assert.equal(0, await node.getStorageTotalSize(), 'check the empty folder'); 388 | const buffer = Buffer.from('hello'); 389 | const hash = await node.storeFile(buffer); 390 | const folder = path.dirname(await node.getFilePath(hash)); 391 | const size = (await fse.stat(folder)).size * node.__dirNestingSize + buffer.length; 392 | assert.equal(size, await node.getStorageTotalSize(), 'check with the files'); 393 | }); 394 | }); 395 | describe('.getTempDirInfo()', () => { 396 | let size; 397 | let count; 398 | 399 | before(async () => { 400 | size = 0; 401 | count = 0; 402 | await fse.emptyDir(node.tempPath); 403 | }); 404 | 405 | after(async () => { 406 | await fse.emptyDir(node.tempPath); 407 | }); 408 | 409 | it('should get a zero', async () => { 410 | let info = await node.getTempDirInfo(); 411 | assert.equal(count, info.count, 'check the folder files count'); 412 | assert.equal(size, info.size, 'check the folder size'); 413 | }); 414 | 415 | it('should return count as 1 and the actual file size', async () => { 416 | const filePath = path.join(node.tempPath, '1.txt'); 417 | await fse.writeFile(filePath, 'hello'); 418 | size += (await fse.stat(filePath)).size; 419 | count++; 420 | let info = await node.getTempDirInfo(); 421 | assert.equal(count, info.count, 'check the folder files count'); 422 | assert.equal(size, info.size, 'check the folder size'); 423 | }); 424 | 425 | it('should return count as 2 and the actual files size', async () => { 426 | const filePath = path.join(node.tempPath, '2.txt'); 427 | await fse.writeFile(filePath, 'goodbye'); 428 | size += (await fse.stat(filePath)).size; 429 | count++; 430 | let info = await node.getTempDirInfo(); 431 | assert.equal(count, info.count, 'check the folder files count'); 432 | assert.equal(size, info.size, 'check the folder size'); 433 | }); 434 | }); 435 | 436 | describe('.cleanUpStorage()', () => { 437 | let lastStorageAutoCleanSize; 438 | 439 | before(() => { 440 | lastStorageAutoCleanSize = node.storageAutoCleanSize; 441 | }); 442 | 443 | after(() => { 444 | node.storageAutoCleanSize = lastStorageAutoCleanSize; 445 | }); 446 | 447 | it('should not remove anything', async () => { 448 | await node.emptyStorage(); 449 | await node.storeFile(Buffer.from('hello')); 450 | await node.calculateStorageInfo(); 451 | const size = await node.getStorageTotalSize(); 452 | node.storageAutoCleanSize = node.storageDataSize - size; 453 | await node.cleanUpStorage(); 454 | assert.equal(await node.getStorageTotalSize(), size); 455 | }); 456 | 457 | it('should remove the file', async () => { 458 | await node.calculateStorageInfo(); 459 | const size = await node.getStorageTotalSize(); 460 | node.storageAutoCleanSize = node.storageDataSize - size / 2; 461 | await node.cleanUpStorage(); 462 | assert.equal(await node.getStorageTotalSize(), 0); 463 | }); 464 | 465 | it('should remove only one file', async () => { 466 | const buffer = Buffer.from('hello'); 467 | await node.storeFile(buffer); 468 | await node.storeFile(Buffer.from('goodbye')); 469 | await node.calculateStorageInfo(); 470 | const size = await node.getStorageTotalSize(); 471 | node.storageAutoCleanSize = node.storageDataSize - size + buffer.length; 472 | await node.cleanUpStorage(); 473 | assert.equal(await node.db.getData('filesCount'), 1); 474 | }); 475 | }); 476 | 477 | describe('.cleanUpTempDir()', () => { 478 | let options; 479 | 480 | before(async () => { 481 | options = merge({}, node.options); 482 | await fse.emptyDir(node.tempPath); 483 | }); 484 | 485 | after(() => { 486 | node.options = options; 487 | }); 488 | 489 | it('should not remove anything', async () => { 490 | node.options.storage.tempLifetime = Infinity; 491 | const filePath = path.join(node.tempPath, '1.txt'); 492 | await fse.writeFile(filePath, 'hello'); 493 | await node.cleanUpTempDir(); 494 | const files = await fse.readdir(node.tempPath); 495 | assert.equal(files.length, 1); 496 | }); 497 | 498 | it('should remove only one file', async () => { 499 | await tools.wait(500); 500 | const filePath = path.join(node.tempPath, '2.txt'); 501 | await fse.writeFile(filePath, 'hi'); 502 | const stat = await fse.stat(filePath); 503 | node.options.storage.tempLifetime = Date.now() - stat.atimeMs + 100; 504 | await node.cleanUpTempDir(); 505 | const files = await fse.readdir(node.tempPath); 506 | assert.equal(files.length, 1); 507 | }); 508 | 509 | it('should remove all files', async () => { 510 | node.options.storage.tempLifetime = 1; 511 | const filePath = path.join(node.tempPath, '3.txt'); 512 | await fse.writeFile(filePath, 'hey'); 513 | await tools.wait(1); 514 | await node.cleanUpTempDir(); 515 | const files = await fse.readdir(node.tempPath); 516 | assert.equal(files.length, 0); 517 | }); 518 | }); 519 | 520 | describe('.normalizeFilesInfo()', () => { 521 | it('should fix "filesCount"', async () => { 522 | await node.emptyStorage(); 523 | await node.storeFile(Buffer.from('hello')); 524 | const count = await node.db.getData('filesCount'); 525 | await node.db.setData('filesCount', 10); 526 | await node.normalizeFilesInfo(); 527 | assert.equal(await node.db.getData('filesCount'), count); 528 | }); 529 | 530 | it('should fix "filesTotalSize"', async () => { 531 | const size = await node.db.getData('filesTotalSize'); 532 | await node.db.setData('filesTotalSize', Infinity); 533 | await node.normalizeFilesInfo(); 534 | assert.equal(await node.db.getData('filesTotalSize'), size); 535 | }); 536 | }); 537 | 538 | describe('.exportFiles()', () => { 539 | let importNode; 540 | 541 | before(async () => { 542 | importNode = new Node(await tools.createNodeOptions()); 543 | await importNode.init(); 544 | }); 545 | 546 | after(async () => { 547 | await importNode.deinit(); 548 | }); 549 | 550 | it('should export the file', async () => { 551 | await node.emptyStorage(); 552 | const hash = await node.storeFile(Buffer.from('hello')); 553 | await node.exportFiles(importNode.address); 554 | assert.isTrue(await importNode.hasFile(hash)); 555 | }); 556 | }); 557 | 558 | describe('.fileAvailabilityTest()', () => { 559 | let options; 560 | 561 | before(async () => { 562 | options = merge({}, node.options); 563 | }); 564 | 565 | after(() => { 566 | node.options = options; 567 | }); 568 | 569 | it('should throw an error because of wrong size', async () => { 570 | try { 571 | await node.fileAvailabilityTest({ hash: '1' }); 572 | throw new Error('Fail'); 573 | } 574 | catch (err) { 575 | assert.isOk(err.message.match('Wrong file')); 576 | } 577 | }); 578 | 579 | it('should throw an error because of storage size', async () => { 580 | try { 581 | await node.fileAvailabilityTest({ hash: '1', size: (await node.getStorageInfo()).free + 1 }); 582 | throw new Error('Fail'); 583 | } 584 | catch (err) { 585 | assert.isOk(err.message.match('Not enough space')); 586 | } 587 | }); 588 | 589 | it('should throw an error because of temp size', async () => { 590 | try { 591 | await node.fileAvailabilityTest({ hash: '1', size: 2, storage: { tempFree: 1, free: 3 } }); 592 | throw new Error('Fail'); 593 | } 594 | catch (err) { 595 | assert.isOk(err.message.match('space in the temp')); 596 | } 597 | }); 598 | 599 | it('should throw an error because of max size', async () => { 600 | try { 601 | node.fileMaxSize = 1; 602 | await node.fileAvailabilityTest({ hash: '1', size: node.fileMaxSize + 1 }); 603 | throw new Error('Fail'); 604 | } 605 | catch (err) { 606 | node.fileMaxSize = Infinity; 607 | assert.isOk(err.message.match('too big')); 608 | } 609 | }); 610 | 611 | it('should throw an error because of min size', async () => { 612 | try { 613 | node.fileMinSize = 2; 614 | await node.fileAvailabilityTest({ hash: '1', size: node.fileMinSize - 1 }); 615 | throw new Error('Fail'); 616 | } 617 | catch (err) { 618 | node.fileMinSize = 0; 619 | assert.isOk(err.message.match('too small')); 620 | } 621 | }); 622 | 623 | it('should throw an error because of hash', async () => { 624 | try { 625 | await node.fileAvailabilityTest({ size: 1 }); 626 | throw new Error('Fail'); 627 | } 628 | catch (err) { 629 | assert.isOk(err.message.match('Wrong file')); 630 | } 631 | }); 632 | 633 | it('should throw an error because of mime whitelist', async () => { 634 | node.options.file.mimeWhitelist = ['image/jpeg']; 635 | try { 636 | await node.fileAvailabilityTest({ size: 1, hash: '1', mime: 'text/plain' }); 637 | throw new Error('Fail'); 638 | } 639 | catch (err) { 640 | assert.isOk(err.message.match('mime type')); 641 | } 642 | }); 643 | 644 | it('should throw an error because of mime blacklist', async () => { 645 | node.options.file.mimeBlacklist = ['image/jpeg']; 646 | node.options.file.mimeWhitelist = []; 647 | 648 | try { 649 | await node.fileAvailabilityTest({ size: 1, hash: '1', mime: 'image/jpeg' }); 650 | throw new Error('Fail'); 651 | } 652 | catch (err) { 653 | assert.isOk(err.message.match('mime type')); 654 | } 655 | }); 656 | 657 | it('should throw an error because of extension whitelist', async () => { 658 | node.options.file.extWhitelist = ['jpeg']; 659 | 660 | try { 661 | await node.fileAvailabilityTest({ size: 1, hash: '1', ext: 'txt' }); 662 | throw new Error('Fail'); 663 | } 664 | catch (err) { 665 | assert.isOk(err.message.match('extension')); 666 | } 667 | }); 668 | 669 | it('should throw an error because of extension blacklist', async () => { 670 | node.options.file.extBlacklist = ['jpeg']; 671 | node.options.file.extWhitelist = []; 672 | 673 | try { 674 | await node.fileAvailabilityTest({ size: 1, hash: '1', ext: 'jpeg' }); 675 | throw new Error('Fail'); 676 | } 677 | catch (err) { 678 | assert.isOk(err.message.match('extension')); 679 | } 680 | }); 681 | 682 | it('should not throw an error', async () => { 683 | await node.fileAvailabilityTest({ size: 1, hash: '1' }); 684 | await node.fileAvailabilityTest({ size: 1, hash: '1', mime: 'audio/mpeg', ext: 'mp3' }); 685 | }); 686 | }); 687 | 688 | describe('.checkFileAvailability()', () => { 689 | it('should return true', async () => { 690 | await node.checkFileAvailability({ size: 1, hash: '1' }); 691 | }); 692 | 693 | it('should return false', async () => { 694 | await node.checkFileAvailability({ size: 1 }, 'check the size'); 695 | await node.checkFileAvailability({ hash: '1' }, 'check the hash'); 696 | }); 697 | }); 698 | 699 | describe('.checkCacheLink()', () => { 700 | it('should return false', async () => { 701 | await node.checkCacheLink(`http://${node.address}/file/wrong`); 702 | }); 703 | 704 | it('should return true', async () => { 705 | const hash = await node.storeFile(Buffer.from('hello')); 706 | await node.checkCacheLink(`http://${node.address}/file/${hash}`); 707 | }); 708 | }); 709 | 710 | describe('.deinit()', () => { 711 | it('should not throw an exception', async () => { 712 | await node.deinit(); 713 | }); 714 | 715 | it('should not remove the storage', async () => { 716 | assert.isTrue(await fse.pathExists(tools.getStoragePath(node.port))); 717 | }); 718 | }); 719 | 720 | describe('reinitialization', () => { 721 | it('should not throw an exception', async () => { 722 | await node.init(); 723 | }); 724 | 725 | it('should create the storage', async () => { 726 | assert.isTrue(await fse.pathExists(tools.getStoragePath(node.port))); 727 | }); 728 | }); 729 | 730 | describe('.destroy()', () => { 731 | it('should not throw an exception', async () => { 732 | await node.destroy(); 733 | }); 734 | 735 | it('should remove the storage', async () => { 736 | assert.isFalse(await fse.pathExists(tools.getStoragePath(node.port))); 737 | }); 738 | }); 739 | }); 740 | } -------------------------------------------------------------------------------- /test/routes.js: -------------------------------------------------------------------------------- 1 | import { assert } from "chai"; 2 | import fetch from "node-fetch"; 3 | import path from "path"; 4 | import fse from "fs-extra"; 5 | import node from "../src/node.js"; 6 | import client from "../src/client.js"; 7 | import utils from "../src/utils.js"; 8 | import schema from "../src/schema.js"; 9 | import tools from "./tools.js"; 10 | 11 | const Node = node(); 12 | const Client = client(); 13 | 14 | export default function () { 15 | describe('routes', () => { 16 | let node; 17 | let client; 18 | 19 | before(async function () { 20 | node = new Node(await tools.createNodeOptions({ 21 | network: { 22 | auth: { username: 'username', password: 'password' } 23 | } 24 | })); 25 | await node.init(); 26 | client = new Client(await tools.createClientOptions({ 27 | address: node.address, 28 | auth: { username: 'username', password: 'password' } 29 | })); 30 | await client.init(); 31 | }); 32 | 33 | after(async function () { 34 | await node.deinit(); 35 | await client.deinit(); 36 | }); 37 | 38 | describe('/status', function () { 39 | it('should return an auth error', async function () { 40 | const res = await fetch(`http://${node.address}/status`); 41 | assert.equal(await res.status, 401); 42 | }); 43 | 44 | it('should return the status', async function () { 45 | const options = client.createDefaultRequestOptions({ method: 'get' }); 46 | const res = await fetch(`http://${node.address}/status`, options); 47 | const json = await res.json(); 48 | assert.doesNotThrow(() => { 49 | utils.validateSchema(schema.getStatusResponse(), json); 50 | }); 51 | }); 52 | 53 | it('should return the pretty status', async function () { 54 | const options = client.createDefaultRequestOptions({ method: 'get' }); 55 | const res = await fetch(`http://${node.address}/status?pretty`, options); 56 | const json = await res.json(); 57 | assert.doesNotThrow(() => { 58 | utils.validateSchema(schema.getStatusPrettyResponse(), json); 59 | }); 60 | }); 61 | }); 62 | 63 | describe('/file/:hash', function () { 64 | it('should return an auth error', async function () { 65 | const res = await fetch(`http://${node.address}/file/hash`); 66 | assert.equal(await res.status, 401); 67 | }); 68 | 69 | it('should return 404', async function () { 70 | const options = client.createDefaultRequestOptions({ method: 'get' }); 71 | const res = await fetch(`http://${node.address}/file/wrong-hash`, options); 72 | assert.equal(res.status, 404); 73 | }); 74 | 75 | it('should return the file', async function () { 76 | const text = 'route-file-check'; 77 | const filePath = path.join(tools.tmpPath, '1.txt'); 78 | const hash = await node.storeFile(Buffer.from(text)); 79 | const options = client.createDefaultRequestOptions({ method: 'get' }); 80 | const res = await fetch(`http://${node.address}/file/${hash}`, options); 81 | await tools.saveResponseToFile(res, filePath); 82 | assert.equal(await fse.readFile(filePath), text); 83 | }); 84 | }); 85 | 86 | describe('/client/request-file/:hash', function () { 87 | it('should return an auth error', async function () { 88 | const res = await fetch(`http://${node.address}/client/request-file/hash`, { method: 'get' }); 89 | assert.equal(await res.status, 401); 90 | }); 91 | 92 | it('should return 404', async function () { 93 | const options = client.createDefaultRequestOptions({ method: 'get' }); 94 | const res = await fetch(`http://${node.address}/client/request-file/wrong-hash`, options); 95 | assert.equal(res.status, 404); 96 | }); 97 | 98 | it('should return the file', async function () { 99 | const text = 'route-request-file-check'; 100 | const filePath = path.join(tools.tmpPath, '1.txt'); 101 | const hash = await node.storeFile(Buffer.from(text)); 102 | const options = client.createDefaultRequestOptions({ method: 'get' }); 103 | const res = await fetch(`http://${node.address}/client/request-file/${hash}`, options); 104 | await tools.saveResponseToFile(res, filePath); 105 | assert.equal(await fse.readFile(filePath), text); 106 | }); 107 | }); 108 | 109 | describe('/client/store-file', function () { 110 | it('should return an auth error', async function () { 111 | const res = await fetch(`http://${node.address}/client/store-file`, { method: 'post' }); 112 | assert.equal(await res.status, 401); 113 | }); 114 | 115 | it('should return an error', async function () { 116 | const options = client.createDefaultRequestOptions(); 117 | const res = await fetch(`http://${node.address}/client/store-file`, options); 118 | assert.equal(res.status, 422); 119 | }); 120 | 121 | it('should save the file', async function () { 122 | const text = 'check-client-store-file'; 123 | const fileOptions = { contentType: 'text/plain', filename: `${text}.txt` }; 124 | const body = tools.createRequestFormData({ 125 | file: { value: Buffer.from(text), options: fileOptions } 126 | }); 127 | const options = client.createDefaultRequestOptions({ body }); 128 | const res = await fetch(`http://${node.address}/client/store-file`, options); 129 | const json = await res.json(); 130 | assert.isTrue(await node.hasFile(json.hash)); 131 | }); 132 | }); 133 | 134 | describe('/client/get-file-link/', function () { 135 | it('should return an auth error', async function () { 136 | const res = await fetch(`http://${node.address}/client/get-file-link`, { method: 'post' }); 137 | assert.equal(await res.status, 401); 138 | }); 139 | 140 | it('should return a data error', async function () { 141 | const options = client.createDefaultRequestOptions(); 142 | const res = await fetch(`http://${node.address}/client/get-file-link`, options); 143 | assert.equal(res.status, 422); 144 | }); 145 | 146 | it('should return the link', async function () { 147 | const hash = await node.storeFile(Buffer.from('hello')); 148 | const options = client.createDefaultRequestOptions(tools.createJsonRequestOptions({ body: { hash } })); 149 | const res = await fetch(`http://${node.address}/client/get-file-link`, options); 150 | const json = await res.json(); 151 | assert.equal(json.link, await node.createFileLink(hash)); 152 | }); 153 | }); 154 | 155 | describe('/client/get-file-links/', function () { 156 | it('should return an auth error', async function () { 157 | const res = await fetch(`http://${node.address}/client/get-file-links`, { method: 'post' }); 158 | assert.equal(await res.status, 401); 159 | }); 160 | 161 | it('should return a data error', async function () { 162 | const options = client.createDefaultRequestOptions(); 163 | const res = await fetch(`http://${node.address}/client/get-file-links`, options); 164 | assert.equal(res.status, 422); 165 | }); 166 | 167 | it('should return the link', async function () { 168 | const hash = await node.storeFile(Buffer.from('hello')); 169 | const options = client.createDefaultRequestOptions(tools.createJsonRequestOptions({ body: { hash } })); 170 | const res = await fetch(`http://${node.address}/client/get-file-links`, options); 171 | const json = await res.json(); 172 | assert.equal(json.links[0], await node.createFileLink(hash)); 173 | }); 174 | }); 175 | 176 | describe('/client/remove-file/', function () { 177 | it('should return an auth error', async function () { 178 | const res = await fetch(`http://${node.address}/client/remove-file/`, { method: 'post' }); 179 | assert.equal(await res.status, 401); 180 | }); 181 | 182 | it('should return a data error', async function () { 183 | const options = client.createDefaultRequestOptions(); 184 | const res = await fetch(`http://${node.address}/client/remove-file/`, options); 185 | assert.equal(res.status, 422); 186 | }); 187 | 188 | it('should remove the file', async function () { 189 | const hash = await node.storeFile(Buffer.from('hello')); 190 | const options = client.createDefaultRequestOptions(tools.createJsonRequestOptions({ body: { hash } })); 191 | const res = await fetch(`http://${node.address}/client/remove-file/`, options); 192 | const json = await res.json(); 193 | assert.equal(json.removed, 1, 'check the response'); 194 | assert.isFalse(await node.hasFile(hash), 'check the file'); 195 | }); 196 | }); 197 | 198 | describe('/client/get-network-files-count/', function () { 199 | it('should return an auth error', async function () { 200 | const res = await fetch(`http://${node.address}/client/get-network-files-count/`, { method: 'post' }); 201 | assert.equal(await res.status, 401); 202 | }); 203 | 204 | it('should get the right count', async function () { 205 | const res = await fetch(`http://${node.address}/client/get-network-files-count/`, client.createDefaultRequestOptions()); 206 | const json = tools.createServerResponse(node.address, await res.json()); 207 | assert.equal(json.count, await node.db.getData('filesCount')); 208 | }); 209 | }); 210 | 211 | describe('/api/master/get-network-files-count/', function () { 212 | it('should return an auth error', async function () { 213 | const res = await fetch(`http://${node.address}/api/master/get-network-files-count/`, { method: 'post' }); 214 | assert.equal(await res.status, 401); 215 | }); 216 | 217 | it('should return the right schema', async function () { 218 | const options = node.createDefaultRequestOptions(); 219 | const res = await fetch(`http://${node.address}/api/master/get-network-files-count/`, options); 220 | const json = tools.createServerResponse(node.address, await res.json()); 221 | assert.doesNotThrow(() => { 222 | utils.validateSchema(schema.getNetworkFilesCountMasterResponse(), json); 223 | }); 224 | }); 225 | }); 226 | 227 | describe('/api/butler/get-network-files-count/', function () { 228 | it('should return an auth error', async function () { 229 | const res = await fetch(`http://${node.address}/api/butler/get-network-files-count/`, { method: 'post' }); 230 | assert.equal(await res.status, 401); 231 | }); 232 | 233 | it('should return the right schema', async function () { 234 | const options = node.createDefaultRequestOptions(); 235 | const res = await fetch(`http://${node.address}/api/butler/get-network-files-count/`, options); 236 | const json = tools.createServerResponse(node.address, await res.json()); 237 | assert.doesNotThrow(() => { 238 | utils.validateSchema(schema.getNetworkFilesCountButlerResponse(), json); 239 | }); 240 | }); 241 | }); 242 | 243 | describe('/api/slave/get-network-files-count/', function () { 244 | it('should return an auth error', async function () { 245 | const res = await fetch(`http://${node.address}/api/slave/get-network-files-count/`, { method: 'post' }); 246 | assert.equal(await res.status, 401); 247 | }); 248 | 249 | it('should return the right schema', async function () { 250 | const options = node.createDefaultRequestOptions(); 251 | const res = await fetch(`http://${node.address}/api/slave/get-network-files-count/`, options); 252 | const json = tools.createServerResponse(node.address, await res.json()); 253 | assert.doesNotThrow(() => { 254 | utils.validateSchema(schema.getNetworkFilesCountSlaveResponse(), json); 255 | }); 256 | }); 257 | }); 258 | 259 | describe('/api/master/get-file-storing-info/', function () { 260 | it('should return an auth error', async function () { 261 | const res = await fetch(`http://${node.address}/api/master/get-file-storing-info/`, { method: 'post' }); 262 | assert.equal(await res.status, 401); 263 | }); 264 | 265 | it('should return a data error', async function () { 266 | const options = node.createDefaultRequestOptions(tools.createJsonRequestOptions()); 267 | const res = await fetch(`http://${node.address}/api/master/get-file-storing-info/`, options); 268 | assert.equal(res.status, 422); 269 | }); 270 | 271 | it('should return the right schema', async function () { 272 | const body = { 273 | level: 2, 274 | info: { 275 | size: 1, 276 | hash: 'hash' 277 | } 278 | }; 279 | const options = node.createDefaultRequestOptions(tools.createJsonRequestOptions({ body })); 280 | const res = await fetch(`http://${node.address}/api/master/get-file-storing-info/`, options); 281 | const json = tools.createServerResponse(node.address, await res.json()); 282 | assert.doesNotThrow(() => { 283 | utils.validateSchema(schema.getFileStoringInfoMasterResponse(), json); 284 | }); 285 | }); 286 | }); 287 | 288 | describe('/api/master/get-file-links/', function () { 289 | it('should return an auth error', async function () { 290 | const res = await fetch(`http://${node.address}/api/master/get-file-links/`, { method: 'post' }); 291 | assert.equal(await res.status, 401); 292 | }); 293 | 294 | it('should return a data error', async function () { 295 | const options = node.createDefaultRequestOptions(tools.createJsonRequestOptions()); 296 | const res = await fetch(`http://${node.address}/api/master/get-file-links/`, options); 297 | assert.equal(res.status, 422); 298 | }); 299 | 300 | it('should return the right schema', async function () { 301 | const body = { 302 | level: 2, 303 | hash: 'hash' 304 | }; 305 | const options = node.createDefaultRequestOptions(tools.createJsonRequestOptions({ body })); 306 | const res = await fetch(`http://${node.address}/api/master/get-file-links/`, options); 307 | const json = tools.createServerResponse(node.address, await res.json()); 308 | assert.doesNotThrow(() => { 309 | utils.validateSchema(schema.getFileLinksMasterResponse(), json); 310 | }); 311 | }); 312 | }); 313 | 314 | describe('/api/master/remove-file/', function () { 315 | it('should return an auth error', async function () { 316 | const res = await fetch(`http://${node.address}/api/master/remove-file/`, { method: 'post' }); 317 | assert.equal(await res.status, 401); 318 | }); 319 | 320 | it('should return a data error', async function () { 321 | const options = node.createDefaultRequestOptions(tools.createJsonRequestOptions()); 322 | const res = await fetch(`http://${node.address}/api/master/remove-file/`, options); 323 | assert.equal(res.status, 422); 324 | }); 325 | 326 | it('should return the right schema', async function () { 327 | const body = { 328 | level: 2, 329 | hash: 'hash' 330 | }; 331 | const options = node.createDefaultRequestOptions(tools.createJsonRequestOptions({ body })); 332 | const res = await fetch(`http://${node.address}/api/master/remove-file/`, options); 333 | const json = tools.createServerResponse(node.address, await res.json()); 334 | assert.doesNotThrow(() => { 335 | utils.validateSchema(schema.getFileRemovalMasterResponse(), json); 336 | }); 337 | }); 338 | }); 339 | 340 | describe('/api/butler/get-file-storing-info/', function () { 341 | it('should return an auth error', async function () { 342 | const res = await fetch(`http://${node.address}/api/butler/get-file-storing-info/`, { method: 'post' }); 343 | assert.equal(await res.status, 401); 344 | }); 345 | 346 | it('should return a data error', async function () { 347 | const options = node.createDefaultRequestOptions(tools.createJsonRequestOptions()); 348 | const res = await fetch(`http://${node.address}/api/butler/get-file-storing-info/`, options); 349 | assert.equal(res.status, 422); 350 | }); 351 | 352 | it('should return the right schema', async function () { 353 | const body = { 354 | level: 1, 355 | info: { 356 | size: 1, 357 | hash: 'hash' 358 | } 359 | }; 360 | const options = node.createDefaultRequestOptions(tools.createJsonRequestOptions({ body })); 361 | const res = await fetch(`http://${node.address}/api/butler/get-file-storing-info/`, options); 362 | const json = tools.createServerResponse(node.address, await res.json()); 363 | assert.doesNotThrow(() => { 364 | utils.validateSchema(schema.getFileStoringInfoButlerResponse(), json); 365 | }); 366 | }); 367 | }); 368 | 369 | describe('/api/butler/get-file-links/', function () { 370 | it('should return an auth error', async function () { 371 | const res = await fetch(`http://${node.address}/api/butler/get-file-links/`, { method: 'post' }); 372 | assert.equal(await res.status, 401); 373 | }); 374 | 375 | it('should return a data error', async function () { 376 | const options = node.createDefaultRequestOptions(tools.createJsonRequestOptions()); 377 | const res = await fetch(`http://${node.address}/api/butler/get-file-links/`, options); 378 | assert.equal(res.status, 422); 379 | }); 380 | 381 | it('should return the right schema', async function () { 382 | const body = { 383 | level: 1, 384 | hash: 'hash' 385 | }; 386 | const options = node.createDefaultRequestOptions(tools.createJsonRequestOptions({ body })); 387 | const res = await fetch(`http://${node.address}/api/butler/get-file-links/`, options); 388 | const json = tools.createServerResponse(node.address, await res.json()); 389 | assert.doesNotThrow(() => { 390 | utils.validateSchema(schema.getFileLinksButlerResponse(), json); 391 | }); 392 | }); 393 | }); 394 | 395 | describe('/api/butler/remove-file/', function () { 396 | it('should return an auth error', async function () { 397 | const res = await fetch(`http://${node.address}/api/butler/remove-file/`, { method: 'post' }); 398 | assert.equal(await res.status, 401); 399 | }); 400 | 401 | it('should return a data error', async function () { 402 | const options = node.createDefaultRequestOptions(tools.createJsonRequestOptions()); 403 | const res = await fetch(`http://${node.address}/api/butler/remove-file/`, options); 404 | assert.equal(res.status, 422); 405 | }); 406 | 407 | it('should return the right schema', async function () { 408 | const body = { 409 | level: 1, 410 | hash: 'hash' 411 | }; 412 | const options = node.createDefaultRequestOptions(tools.createJsonRequestOptions({ body })); 413 | const res = await fetch(`http://${node.address}/api/butler/remove-file/`, options); 414 | const json = tools.createServerResponse(node.address, await res.json()); 415 | assert.doesNotThrow(() => { 416 | utils.validateSchema(schema.getFileRemovalButlerResponse(), json); 417 | }); 418 | }); 419 | }); 420 | 421 | describe('/api/slave/get-file-storing-info/', function () { 422 | it('should return an auth error', async function () { 423 | const res = await fetch(`http://${node.address}/api/slave/get-file-storing-info/`, { method: 'post' }); 424 | assert.equal(await res.status, 401); 425 | }); 426 | 427 | it('should return a data error', async function () { 428 | const options = node.createDefaultRequestOptions(); 429 | const res = await fetch(`http://${node.address}/api/slave/get-file-storing-info/`, options); 430 | assert.equal(res.status, 422); 431 | }); 432 | 433 | it('should return the right schema', async function () { 434 | const body = { 435 | level: 0, 436 | info: { 437 | size: 1, 438 | hash: 'hash' 439 | } 440 | }; 441 | const options = node.createDefaultRequestOptions(tools.createJsonRequestOptions({ body })); 442 | const res = await fetch(`http://${node.address}/api/slave/get-file-storing-info/`, options); 443 | const json = tools.createServerResponse(node.address, await res.json()); 444 | assert.doesNotThrow(() => { 445 | utils.validateSchema(schema.getFileStoringInfoSlaveResponse(), json); 446 | }); 447 | }); 448 | }); 449 | 450 | describe('/api/slave/get-file-links/', function () { 451 | it('should return an auth error', async function () { 452 | const res = await fetch(`http://${node.address}/api/slave/get-file-links/`, { method: 'post' }); 453 | assert.equal(await res.status, 401); 454 | }); 455 | 456 | it('should return a data error', async function () { 457 | const options = node.createDefaultRequestOptions(); 458 | const res = await fetch(`http://${node.address}/api/slave/get-file-links/`, options); 459 | assert.equal(res.status, 422); 460 | }); 461 | 462 | it('should return the right schema for empty link', async function () { 463 | const body = { 464 | level: 0, 465 | hash: 'hash' 466 | }; 467 | const options = node.createDefaultRequestOptions(tools.createJsonRequestOptions({ body })); 468 | const res = await fetch(`http://${node.address}/api/slave/get-file-links/`, options); 469 | const json = tools.createServerResponse(node.address, await res.json()); 470 | assert.doesNotThrow(() => { 471 | utils.validateSchema(schema.getFileLinksSlaveResponse(), json); 472 | }); 473 | }); 474 | 475 | it('should return the right schema for the existent link', async function () { 476 | const hash = await node.storeFile(Buffer.from('hello')); 477 | const body = { 478 | level: 0, 479 | hash 480 | }; 481 | const options = node.createDefaultRequestOptions(tools.createJsonRequestOptions({ body })); 482 | const res = await fetch(`http://${node.address}/api/slave/get-file-links/`, options); 483 | const json = tools.createServerResponse(node.address, await res.json()); 484 | assert.equal(json.link, await node.createFileLink(hash), 'check the string'); 485 | assert.doesNotThrow(() => { 486 | utils.validateSchema(schema.getFileLinksSlaveResponse(), json, 'check the validation'); 487 | }); 488 | }); 489 | }); 490 | 491 | describe('/api/slave/remove-file/', function () { 492 | it('should return an auth error', async function () { 493 | const res = await fetch(`http://${node.address}/api/slave/remove-file/`, { method: 'post' }); 494 | assert.equal(await res.status, 401); 495 | }); 496 | 497 | it('should return a data error', async function () { 498 | const options = node.createDefaultRequestOptions(); 499 | const res = await fetch(`http://${node.address}/api/slave/remove-file/`, options); 500 | assert.equal(res.status, 422); 501 | }); 502 | 503 | it('should return the right schema', async function () { 504 | const body = { 505 | level: 0, 506 | hash: 'hash' 507 | }; 508 | const options = node.createDefaultRequestOptions(tools.createJsonRequestOptions({ body })); 509 | const res = await fetch(`http://${node.address}/api/slave/remove-file/`, options); 510 | const json = tools.createServerResponse(node.address, await res.json()); 511 | assert.doesNotThrow(() => { 512 | utils.validateSchema(schema.getFileRemovalSlaveResponse(), json); 513 | }); 514 | }); 515 | }); 516 | 517 | describe('/api/node/store-file/:hash', function () { 518 | it('should return an auth error', async function () { 519 | const res = await fetch(`http://${node.address}/api/node/store-file/hash`, { method: 'post' }); 520 | assert.equal(await res.status, 401); 521 | }); 522 | 523 | it('should return an error', async function () { 524 | const options = node.createDefaultRequestOptions(); 525 | const res = await fetch(`http://${node.address}/api/node/store-file/hash`, options); 526 | assert.equal(res.status, 422); 527 | }); 528 | 529 | it('should return the right schema for a common situation', async function () { 530 | const fileOptions = { contentType: 'text/plain', filename: `hello.txt` }; 531 | const buffer = Buffer.from('hello'); 532 | const hash = await utils.getFileHash(buffer); 533 | const body = tools.createRequestFormData({ 534 | file: { value: Buffer.from('hello'), options: fileOptions } 535 | }); 536 | const options = node.createDefaultRequestOptions({ body }); 537 | const res = await fetch(`http://${node.address}/api/node/store-file/${hash}`, options); 538 | const json = tools.createServerResponse(node.address, await res.json()); 539 | assert.doesNotThrow(() => { 540 | utils.validateSchema(schema.getFileStoringResponse(), json); 541 | }); 542 | }); 543 | 544 | it('should return the right schema for a temp file', async function () { 545 | const name = '1.txt'; 546 | const filePath = path.join(node.tempPath, name); 547 | await fse.writeFile(filePath, 'hello'); 548 | const hash = await utils.getFileHash(fse.createReadStream(filePath)); 549 | const body = { file: name }; 550 | const options = node.createDefaultRequestOptions(tools.createJsonRequestOptions({ body })); 551 | const res = await fetch(`http://${node.address}/api/node/store-file/${hash}`, options); 552 | const json = tools.createServerResponse(node.address, await res.json()); 553 | assert.doesNotThrow(() => { 554 | utils.validateSchema(schema.getFileStoringResponse(), json); 555 | }); 556 | }); 557 | }); 558 | }); 559 | } -------------------------------------------------------------------------------- /test/server/express.js: -------------------------------------------------------------------------------- 1 | import { assert } from "chai"; 2 | import express from "../../src/server/transports/express/index.js"; 3 | 4 | const ServerExpressStoracle = express(); 5 | 6 | export default function () { 7 | describe('ServerExpressStoracle', () => { 8 | let server; 9 | let nodeServer; 10 | 11 | describe('instance creation', function () { 12 | it('should create an instance', function () { 13 | assert.doesNotThrow(() => server = new ServerExpressStoracle()); 14 | server.node = this.node; 15 | nodeServer = this.node.server; 16 | this.node.server = server; 17 | }); 18 | }); 19 | 20 | describe('.init()', function () { 21 | it('should not throw an exception', async function () { 22 | await server.init(); 23 | }); 24 | }); 25 | 26 | describe('.deinit()', function () { 27 | it('should not throw an exception', async function () { 28 | await server.deinit(); 29 | }); 30 | }); 31 | 32 | describe('reinitialization', () => { 33 | it('should not throw an exception', async function () { 34 | await server.init(); 35 | }); 36 | }); 37 | 38 | describe('.destroy()', function () { 39 | it('should not throw an exception', async function () { 40 | await server.destroy(); 41 | this.node.server = nodeServer; 42 | }); 43 | }); 44 | }); 45 | } -------------------------------------------------------------------------------- /test/services.js: -------------------------------------------------------------------------------- 1 | import node from "../src/node.js"; 2 | import tools from "./tools.js"; 3 | import loki from "./db/loki.js"; 4 | import express from "./server/express.js"; 5 | 6 | const Node = node(); 7 | 8 | export default function () { 9 | describe('services', () => { 10 | before(async function () { 11 | this.node = new Node(await tools.createNodeOptions({ server: false })); 12 | await this.node.init(); 13 | }); 14 | 15 | after(async function () { 16 | await this.node.destroy(); 17 | }); 18 | 19 | describe('db', () => { 20 | describe('loki', loki.bind(this)); 21 | 22 | }); 23 | 24 | describe('server', () => { 25 | describe('express', express.bind(this)); 26 | }); 27 | }); 28 | } -------------------------------------------------------------------------------- /test/tools.js: -------------------------------------------------------------------------------- 1 | import _tools from "spreadable/test/tools.js"; 2 | import path from "path"; 3 | 4 | const tools = Object.assign({}, _tools); 5 | 6 | /** 7 | * Get the storage path 8 | * 9 | * @param {number} port 10 | * @returnss {string} 11 | */ 12 | tools.getStoragePath = function (port) { 13 | return path.join(this.tmpPath, `storage-${port}`); 14 | }; 15 | 16 | /** 17 | * Create the node options 18 | * 19 | * @async 20 | * @param {object} [options] 21 | * @returns {object} 22 | */ 23 | tools.createNodeOptions = async function (options = {}) { 24 | options = await _tools.createNodeOptions(options); 25 | delete options.db; 26 | options.storage = { 27 | path: this.getStoragePath(options.port) 28 | }; 29 | return options; 30 | }; 31 | 32 | export default tools; 33 | -------------------------------------------------------------------------------- /test/utils.js: -------------------------------------------------------------------------------- 1 | import { assert } from "chai"; 2 | import fse from "fs-extra"; 3 | import path from "path"; 4 | import utils from "../src/utils.js"; 5 | import tools from "./tools.js"; 6 | import { URL } from 'url'; 7 | 8 | const __dirname = new URL('.', import.meta.url).pathname; 9 | 10 | export default function () { 11 | describe('utils', () => { 12 | describe('.getDiskInfo()', () => { 13 | it('should return the necessary keys', async () => { 14 | assert.containsAllKeys(await utils.getDiskInfo(__dirname), ['available', 'total']); 15 | }); 16 | }); 17 | 18 | describe('.isValidFileLink()', () => { 19 | it('should return true', () => { 20 | assert.isTrue(utils.isValidFileLink('http://localhost:80/file/hash'), 'check http'); 21 | assert.isTrue(utils.isValidFileLink('https://192.0.0.1:3000/file/hash'), 'check https'); 22 | }); 23 | 24 | it('should return false', () => { 25 | assert.isFalse(utils.isValidFileLink('http://localhost/file/hash'), 'check without a port'); 26 | assert.isFalse(utils.isValidFileLink('ftp://localhost/file/hash'), 'check wrong a protocol'); 27 | assert.isFalse(utils.isValidFileLink('http://192.0.0.1:80/files/hash'), 'check the wrong path'); 28 | }); 29 | }); 30 | 31 | describe('files info', () => { 32 | let filePath; 33 | let buffer; 34 | 35 | before(() => { 36 | filePath = path.join(tools.tmpPath, '1.txt'); 37 | buffer = Buffer.from('hello'); 38 | }); 39 | 40 | describe('.getFileInfo()', () => { 41 | describe('fse.ReadStream', () => { 42 | it('should get all info', async () => { 43 | await fse.writeFile(filePath, 'hello'); 44 | const stat = await fse.stat(filePath); 45 | const info = await utils.getFileInfo(fse.createReadStream(filePath)); 46 | assert.equal(info.size, stat.size, 'check the size'); 47 | assert.equal(info.mime, 'text/plain', 'check the mime'); 48 | assert.equal(info.ext, 'txt', 'check the extension'); 49 | assert.equal(info.hash, await utils.getFileHash(filePath), 'check the hash'); 50 | }); 51 | 52 | it('should get info without hash', async () => { 53 | const info = await utils.getFileInfo(fse.createReadStream(filePath), { hash: false }); 54 | assert.hasAllKeys(info, ['size', 'mime', 'ext']); 55 | }); 56 | 57 | it('should get info without mime', async () => { 58 | const info = await utils.getFileInfo(fse.createReadStream(filePath), { mime: false }); 59 | assert.hasAllKeys(info, ['size', 'hash']); 60 | }); 61 | 62 | it('should get info without ext', async () => { 63 | const info = await utils.getFileInfo(fse.createReadStream(filePath), { ext: false }); 64 | assert.hasAllKeys(info, ['size', 'hash', 'mime']); 65 | }); 66 | 67 | it('should get info without size', async () => { 68 | const info = await utils.getFileInfo(fse.createReadStream(filePath), { size: false }); 69 | assert.hasAllKeys(info, ['ext', 'hash', 'mime']); 70 | }); 71 | }); 72 | 73 | describe('string path', () => { 74 | let filePath; 75 | 76 | before(() => { 77 | filePath = path.join(tools.tmpPath, '1.txt'); 78 | }); 79 | 80 | it('should get all info', async () => { 81 | const stat = await fse.stat(filePath); 82 | const info = await utils.getFileInfo(filePath); 83 | assert.equal(info.size, stat.size, 'check the size'); 84 | assert.equal(info.mime, 'text/plain', 'check the mime'); 85 | assert.equal(info.ext, 'txt', 'check the extension'); 86 | assert.equal(info.hash, await utils.getFileHash(filePath), 'check the hash'); 87 | }); 88 | 89 | it('should get info without hash', async () => { 90 | const info = await utils.getFileInfo(filePath, { hash: false }); 91 | assert.hasAllKeys(info, ['size', 'mime', 'ext']); 92 | }); 93 | 94 | it('should get info without mime', async () => { 95 | const info = await utils.getFileInfo(filePath, { mime: false }); 96 | assert.hasAllKeys(info, ['size', 'hash']); 97 | }); 98 | 99 | it('should get info without ext', async () => { 100 | const info = await utils.getFileInfo(filePath, { ext: false }); 101 | assert.hasAllKeys(info, ['size', 'hash', 'mime']); 102 | }); 103 | 104 | it('should get info without size', async () => { 105 | const info = await utils.getFileInfo(filePath, { size: false }); 106 | assert.hasAllKeys(info, ['ext', 'hash', 'mime']); 107 | }); 108 | }); 109 | 110 | describe('Buffer', () => { 111 | it('should get all info', async () => { 112 | const info = await utils.getFileInfo(buffer); 113 | assert.equal(info.size, buffer.length, 'check the size'); 114 | assert.equal(info.mime, 'text/plain', 'check the mime'); 115 | assert.equal(info.ext, 'txt', 'check the extension'); 116 | assert.equal(info.hash, await utils.getFileHash(buffer), 'check the hash'); 117 | }); 118 | 119 | it('should get info without hash', async () => { 120 | const info = await utils.getFileInfo(buffer, { hash: false }); 121 | assert.hasAllKeys(info, ['size', 'mime', 'ext']); 122 | }); 123 | 124 | it('should get info without mime', async () => { 125 | const info = await utils.getFileInfo(buffer, { mime: false }); 126 | assert.hasAllKeys(info, ['size', 'hash']); 127 | }); 128 | 129 | it('should get info without ext', async () => { 130 | const info = await utils.getFileInfo(buffer, { ext: false }); 131 | assert.hasAllKeys(info, ['size', 'hash', 'mime']); 132 | }); 133 | 134 | it('should get info without size', async () => { 135 | const info = await utils.getFileInfo(buffer, { size: false }); 136 | assert.hasAllKeys(info, ['ext', 'hash', 'mime']); 137 | }); 138 | }); 139 | }); 140 | 141 | describe('.getFileHash()', () => { 142 | let hash; 143 | 144 | before(() => { 145 | hash = []; 146 | }); 147 | 148 | it('should return for fse.ReadStream', async () => { 149 | hash.push(await utils.getFileHash(fse.createReadStream(filePath))); 150 | assert.isString(hash[0]); 151 | }); 152 | 153 | it('should return for string', async () => { 154 | hash.push(await utils.getFileHash(filePath)); 155 | assert.isString(hash[1]); 156 | }); 157 | 158 | it('should return for buffer', async () => { 159 | hash.push(await utils.getFileHash(buffer)); 160 | assert.isString(hash[2]); 161 | }); 162 | 163 | it('should check all hashes are equal', async () => { 164 | assert.equal(hash[0], hash[1]); 165 | assert.equal(hash[1], hash[2]); 166 | }); 167 | }); 168 | 169 | describe('.getFileMimeType()', () => { 170 | it('should return for fse.ReadStream', async () => { 171 | assert.equal('text/plain', await utils.getFileMimeType(fse.createReadStream(filePath))); 172 | }); 173 | 174 | it('should return for string', async () => { 175 | assert.equal('text/plain', await utils.getFileMimeType((filePath))); 176 | }); 177 | 178 | it('should return for buffer', async () => { 179 | assert.equal('text/plain', await utils.getFileMimeType(buffer)); 180 | }); 181 | }); 182 | }); 183 | }); 184 | } -------------------------------------------------------------------------------- /webpack.client.js: -------------------------------------------------------------------------------- 1 | import path from "path"; 2 | import merge from "lodash-es/merge.js"; 3 | import spWebpackConfig from "spreadable/webpack.client.js"; 4 | 5 | const __dirname = new URL('.', import.meta.url).pathname; 6 | 7 | export default (options = {}, wp) => { 8 | options = merge({ 9 | include: [], 10 | mock: { 11 | "stream": false, 12 | "detect-file-type": true, 13 | "diskusage": true, 14 | "fs-extra": true, 15 | "fs": true, 16 | "crypto": path.join(__dirname, "/src/browser/client/mock/crypto.js") 17 | } 18 | }, options); 19 | options.include.push([path.resolve(__dirname, 'src/browser/client')]); 20 | return wp ? spWebpackConfig(options, wp) : options; 21 | }; 22 | --------------------------------------------------------------------------------