├── .dockerignore ├── .editorconfig ├── .eslintrc ├── .gitignore ├── .npmignore ├── .npmrc ├── .travis.yml ├── Dockerfile ├── LICENSE ├── bake.sh ├── bin └── cli.js ├── changelog.md ├── examples ├── auth.js ├── file-server.nginx └── logs │ ├── access.log │ └── error.log ├── lib └── utils.js ├── package.json ├── readme.md ├── src ├── filestore.js ├── fs-blob-storage.js ├── index.js ├── middleware.js ├── mongodb-data-storage.js └── nedb-data-storage.js ├── test ├── fs-blob-storage.spec.js ├── mongodb-data-storage.spec.js ├── nedb-data-storage.spec.js ├── server.spec.js └── test.spec.js ├── todo.md └── var └── systemd └── file-store.service /.dockerignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | .git 3 | tmp 4 | var 5 | .eslint 6 | .editorconfig 7 | .gitignore 8 | .npmrc 9 | .travis.yml 10 | bake.sh 11 | changelog.sh 12 | todo.md 13 | example 14 | -------------------------------------------------------------------------------- /.editorconfig: -------------------------------------------------------------------------------- 1 | # editorconfig.org 2 | root = true 3 | 4 | [*] 5 | charset = utf-8 6 | end_of_line = lf 7 | indent_size = 4 8 | indent_style = space 9 | insert_final_newline = true 10 | trim_trailing_whitespace = true 11 | 12 | [{package.json, .travis.yml}] 13 | indent_size = 2 14 | 15 | [*.md] 16 | trim_trailing_whitespace = false 17 | -------------------------------------------------------------------------------- /.eslintrc: -------------------------------------------------------------------------------- 1 | { 2 | "env": { 3 | "node": true, 4 | "es6": true 5 | }, 6 | "rules": { 7 | "block-scoped-var": 2, 8 | "brace-style": [2, "stroustrup"], 9 | "camelcase": 0, 10 | "curly": 2, 11 | "eol-last": 2, 12 | "eqeqeq": [2, "smart"], 13 | "max-depth": [1, 3], 14 | "max-statements": [1, 25], 15 | "max-len": [1, 80], 16 | "new-cap": 0, 17 | "no-extend-native": 2, 18 | "no-mixed-spaces-and-tabs": 2, 19 | "no-trailing-spaces": 2, 20 | "no-use-before-define": [2, "nofunc"], 21 | "no-unused-vars": 1, 22 | "quotes": [2, "single", "avoid-escape"], 23 | "semi": [2, "always"], 24 | "keyword-spacing": [2, {"before": true, "after": true}], 25 | "object-curly-spacing": [2, "never"], 26 | "array-bracket-spacing": [2, "never"], 27 | "computed-property-spacing": 1, 28 | "space-unary-ops": 0, 29 | "valid-jsdoc": 1, 30 | "no-nested-ternary": 2, 31 | "no-underscore-dangle": 0, 32 | "comma-dangle": [1, "always-multiline"], 33 | "no-shadow": 1 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | npm-debug.log 3 | coverage 4 | *.pid 5 | 6 | 7 | **/.DS_Store 8 | 9 | tmp 10 | local 11 | build 12 | -------------------------------------------------------------------------------- /.npmignore: -------------------------------------------------------------------------------- 1 | var 2 | tmp 3 | .dockerignore 4 | .editorconfig 5 | .eslintrc 6 | .travis.yml 7 | bake.sh 8 | Dockerfile 9 | todo.md 10 | example 11 | -------------------------------------------------------------------------------- /.npmrc: -------------------------------------------------------------------------------- 1 | save=true 2 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | sudo: false 2 | language: node_js 3 | node_js: 4 | - "node" 5 | - "6" 6 | services: 7 | - mongodb 8 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM mhart/alpine-node:latest 2 | 3 | WORKDIR "/app" 4 | 5 | COPY . "/app" 6 | 7 | RUN npm install . 8 | 9 | ENV VERBOSE=1 10 | 11 | EXPOSE 8080 12 | 13 | CMD ["node", "bin/cli.js", "/data"] 14 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | This software is released under the MIT license: 2 | 3 | Copyright (c) 2016 Rumkin 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy of 6 | this software and associated documentation files (the "Software"), to deal in 7 | the Software without restriction, including without limitation the rights to 8 | use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of 9 | the Software, and to permit persons to whom the Software is furnished to do so, 10 | subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS 17 | FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 18 | COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER 19 | IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 20 | CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 21 | -------------------------------------------------------------------------------- /bake.sh: -------------------------------------------------------------------------------- 1 | task:init() { 2 | task:initial_deps 3 | npm init 4 | } 5 | 6 | task:initial_deps() { 7 | set -e 8 | bake dev mocha 9 | bake dev istanbul 10 | } 11 | 12 | # Install node package 13 | task:i() { 14 | npm i $@ 15 | } 16 | 17 | # Install dev dependency 18 | task:dev() { 19 | npm i --save-dev $@ 20 | } 21 | 22 | task:cov() { 23 | npm run cov 24 | } 25 | 26 | task:try() { 27 | local DIR=`mktemp -d` 28 | ./bin/cli.js $DIR & 29 | PID=$! 30 | 31 | finish() { 32 | kill -s 9 $PID 33 | } 34 | 35 | trap finish EXIT 36 | 37 | sleep 1 38 | 39 | task:test_upload; echo "" 40 | task:test_upload; echo "" 41 | 42 | curl -s "http://localhost:8080/storage/dump" ; echo "" 43 | 44 | rm -rf $DIR 45 | } 46 | 47 | task:test_upload() { 48 | local PORT=${1:-8080} 49 | local UUID=${2:-`uuidgen`} 50 | 51 | echo 'hello' | curl -X POST \ 52 | --data-binary @- \ 53 | -s \ 54 | -H 'Content-Type: text/plain' \ 55 | -H 'Content-Length: 5' \ 56 | -H 'Content-Disposition: attachment; filename=hello.txt' \ 57 | "http://localhost:$PORT/files/$UUID" 58 | } 59 | 60 | task:build:docker() { 61 | # bake build:docker IMAGE [...VERSION] 62 | local IMAGE=$1 63 | 64 | shift 1 65 | 66 | docker build -t "$IMAGE" . || exit 1 67 | 68 | for V in "$@" 69 | do 70 | docker tag "$IMAGE:latest" "$IMAGE:$V" 71 | done 72 | } 73 | -------------------------------------------------------------------------------- /bin/cli.js: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | 3 | 'use strict'; 4 | 5 | const connect = require('connect'); 6 | const hall = require('hall'); 7 | const {middleware, FileStore, NedbDataStorage, FsBlobStorage} = require('..'); 8 | const nedb = require('nedb'); 9 | const argentum = require('argentum'); 10 | const fs = require('fs'); 11 | const path = require('path'); 12 | 13 | const argv = process.argv.slice(2); 14 | const config = argentum.parse(argv, { 15 | aliases: { 16 | d: 'debug', 17 | v: 'verbose', 18 | }, 19 | defaults: { 20 | port: process.env.PORT || 8080, 21 | debug: process.env.DEBUG === '1', 22 | verbose: process.env.VERBOSE === '1', 23 | pidFile: process.env.PID_FILE || '/var/run/file-store.pid', 24 | }, 25 | }); 26 | 27 | const DEBUG = config.debug; 28 | const VERBOSE = config.verbose; 29 | const port = config.port; 30 | const dir = path.resolve(process.cwd(), argv[0] || '.'); 31 | const PID_FILE = path.resolve(process.cwd(), config.pidFile); 32 | 33 | if (fs.existsSync(PID_FILE)) { 34 | onError(`Pid file "${PID_FILE}" already exists`); 35 | } 36 | 37 | fs.writeFileSync(PID_FILE, process.pid); 38 | 39 | process.on('beforeExit', () => onExit); 40 | process.on('exit', () => onExit); 41 | process.on('SIGINT', () => { 42 | onExit() 43 | process.exit(); 44 | }); 45 | 46 | const storage = new FileStore({ 47 | dataStore: new NedbDataStorage({ 48 | db: new nedb({ 49 | filename: dir + '/files.db', 50 | autoload: true, 51 | }), 52 | }), 53 | blobStore: new FsBlobStorage({dir}), 54 | }); 55 | 56 | const router = hall(); 57 | const logger = VERBOSE 58 | ? console 59 | : null; 60 | 61 | connect() 62 | .use((req, res, next) => { 63 | VERBOSE && console.log(req.method, req.url); 64 | next(); 65 | }) 66 | .use(middleware(router, storage, logger, DEBUG)) 67 | .use((err, req, res, next) => { 68 | if (! err) { 69 | res.statusCode = 404; 70 | res.statusText = 'Nothing found'; 71 | res.end(); 72 | } 73 | else { 74 | res.statusCode = 500; 75 | res.statusText = 'Internal error'; 76 | res.end(err.message); 77 | DEBUG && console.log(err); 78 | } 79 | }) 80 | .listen(port); 81 | 82 | VERBOSE && console.log('Listening on localhost:%s', port); 83 | 84 | function onError(error) { 85 | console.error(error); 86 | process.exit(1); 87 | } 88 | 89 | function onExit() { 90 | fs.existsSync(PID_FILE) && fs.unlinkSync(PID_FILE); 91 | } 92 | -------------------------------------------------------------------------------- /changelog.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | 3 | ### v0.5.0 4 | 5 | * Add deduplication and md5 checksum support. 6 | 7 | ### v0.4.0 8 | 9 | * Update `put` order. Now it's write BLOB first. 10 | 11 | ### v0.3.0 12 | 13 | * Add tags support with `x-tag` headers. 14 | -------------------------------------------------------------------------------- /examples/auth.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | const http = require('http'); 4 | const {inspect} = require('util'); 5 | const fs = require('fs'); 6 | 7 | const VERBOSE = process.env.VERBOSE === '1'; 8 | const DEBUG = process.env.DEBUG === '1'; 9 | 10 | // Socket path or port number 11 | const sock = process.env.SOCK || '/tmp/auth.sock'; 12 | // Socket file owner 13 | const UID = process.env.UID || process.getuid(); 14 | // Socket file group (www-data or 33 for ubuntu/xenial) 15 | const GID = process.env.GID || 33; 16 | 17 | 18 | if (fs.existsSync(sock)) { 19 | fs.unlinkSync(sock); 20 | } 21 | 22 | http.createServer((req, res) => { 23 | DEBUG && console.log(inspect(req.method, {colors: true})); 24 | DEBUG && console.log(inspect(req.headers, {colors: true})); 25 | 26 | if (req.headers.authorization === 'token secret') { 27 | res.end('ok'); 28 | } 29 | else { 30 | res.statusCode = 403; 31 | res.end('access denied'); 32 | } 33 | }) 34 | .listen(sock, () => { 35 | fs.chownSync(sock, UID, GID); 36 | VERBOSE && console.log('Listening at %s', sock); 37 | }); 38 | -------------------------------------------------------------------------------- /examples/file-server.nginx: -------------------------------------------------------------------------------- 1 | server { 2 | listen 80; 3 | server_name localhost; 4 | 5 | access_log /projects/filestore/var/logs/access.log; 6 | error_log /projects/filestore/var/logs/error.log; 7 | 8 | location /auth { 9 | internal; 10 | 11 | proxy_pass "http://unix:/tmp/auth.sock"; 12 | proxy_method GET; 13 | proxy_set_header Host $host; 14 | proxy_pass_request_body off; 15 | proxy_set_header Content-Length ""; 16 | proxy_set_header X-Original-Uri $request_uri; 17 | proxy_set_header X-Original-Method $request_method; 18 | } 19 | 20 | location / { 21 | auth_request /auth; 22 | 23 | # Use unix socket to avoid direct access from network 24 | proxy_pass "http://localhost:3333"; 25 | } 26 | } 27 | -------------------------------------------------------------------------------- /examples/logs/access.log: -------------------------------------------------------------------------------- 1 | 127.0.0.1 - - [26/Sep/2016:22:32:13 +0000] "GET / HTTP/1.1" 500 202 "-" "curl/7.47.0" 2 | 127.0.0.1 - - [26/Sep/2016:22:33:06 +0000] "GET / HTTP/1.1" 500 202 "-" "curl/7.47.0" 3 | 127.0.0.1 - - [26/Sep/2016:22:34:15 +0000] "GET / HTTP/1.1" 500 202 "-" "curl/7.47.0" 4 | 127.0.0.1 - - [26/Sep/2016:22:35:07 +0000] "GET / HTTP/1.1" 500 202 "-" "curl/7.47.0" 5 | 127.0.0.1 - - [26/Sep/2016:22:39:50 +0000] "GET / HTTP/1.1" 200 612 "-" "curl/7.47.0" 6 | 127.0.0.1 - - [26/Sep/2016:22:43:45 +0000] "GET / HTTP/1.1" 500 202 "-" "curl/7.47.0" 7 | 127.0.0.1 - - [26/Sep/2016:22:45:01 +0000] "GET / HTTP/1.1" 200 612 "-" "curl/7.47.0" 8 | 127.0.0.1 - - [26/Sep/2016:22:47:39 +0000] "GET / HTTP/1.1" 200 612 "-" "curl/7.47.0" 9 | 127.0.0.1 - - [26/Sep/2016:22:48:27 +0000] "GET /files/store HTTP/1.1" 404 178 "-" "curl/7.47.0" 10 | 127.0.0.1 - - [26/Sep/2016:22:53:35 +0000] "GET /files/store HTTP/1.1" 404 178 "-" "curl/7.47.0" 11 | 127.0.0.1 - - [26/Sep/2016:22:53:47 +0000] "POST /files/store HTTP/1.1" 404 178 "-" "curl/7.47.0" 12 | 127.0.0.1 - - [26/Sep/2016:22:54:55 +0000] "POST /files/store HTTP/1.1" 404 178 "-" "curl/7.47.0" 13 | 127.0.0.1 - - [26/Sep/2016:23:12:53 +0000] "POST /files/a38f477d-989c-40c3-9027-6fa8cb8b24cc HTTP/1.1" 200 12 "-" "curl/7.47.0" 14 | 127.0.0.1 - - [26/Sep/2016:23:13:22 +0000] "GET /storage/dump HTTP/1.1" 200 180 "-" "curl/7.47.0" 15 | 127.0.0.1 - - [26/Sep/2016:23:13:27 +0000] "POST /files/b0b767ea-b447-4f97-8c7a-6d43065aca68 HTTP/1.1" 200 12 "-" "curl/7.47.0" 16 | 127.0.0.1 - - [26/Sep/2016:23:13:33 +0000] "GET /storage/dump HTTP/1.1" 200 227 "-" "curl/7.47.0" 17 | 127.0.0.1 - - [26/Sep/2016:23:31:09 +0000] "POST /files/store HTTP/1.1" 403 178 "-" "curl/7.47.0" 18 | 127.0.0.1 - - [26/Sep/2016:23:31:21 +0000] "POST /files/store HTTP/1.1" 200 12 "-" "curl/7.47.0" 19 | 127.0.0.1 - - [26/Sep/2016:23:31:54 +0000] "GET /storage/dump HTTP/1.1" 200 248 "-" "curl/7.47.0" 20 | 127.0.0.1 - - [26/Sep/2016:23:32:09 +0000] "GET /storage/dump HTTP/1.1" 200 248 "-" "curl/7.47.0" 21 | 127.0.0.1 - - [26/Sep/2016:23:32:15 +0000] "GET /storage/dump HTTP/1.1" 200 248 "-" "curl/7.47.0" 22 | 127.0.0.1 - - [26/Sep/2016:23:32:37 +0000] "GET /storage/dump HTTP/1.1" 200 248 "-" "curl/7.47.0" 23 | -------------------------------------------------------------------------------- /examples/logs/error.log: -------------------------------------------------------------------------------- 1 | 2 | 2016/09/26 22:35:07 [crit] 25086#25086: *1 connect() to unix:/tmp/auth.sock failed (13: Permission denied) while connecting to upstream, client: 127.0.0.1, server: localhost, request: "GET / HTTP/1.1", subrequest: "/auth", upstream: "http://unix:/tmp/auth.sock:/auth", host: "localhost" 3 | 2016/09/26 22:35:07 [error] 25086#25086: *1 auth request unexpected status: 502 while sending to client, client: 127.0.0.1, server: localhost, request: "GET / HTTP/1.1", host: "localhost" 4 | 2016/09/26 22:43:45 [error] 25086#25086: *6 connect() to unix:/tmp/auth.sock failed (111: Connection refused) while connecting to upstream, client: 127.0.0.1, server: localhost, request: "GET / HTTP/1.1", subrequest: "/auth", upstream: "http://unix:/tmp/auth.sock:/auth", host: "localhost" 5 | 2016/09/26 22:43:45 [error] 25086#25086: *6 auth request unexpected status: 502 while sending to client, client: 127.0.0.1, server: localhost, request: "GET / HTTP/1.1", host: "localhost" 6 | 2016/09/26 22:48:27 [error] 25713#25713: *4 open() "/usr/share/nginx/html/files/store" failed (2: No such file or directory) while sending to client, client: 127.0.0.1, server: localhost, request: "GET /files/store HTTP/1.1", host: "localhost" 7 | 2016/09/26 22:53:35 [error] 25761#25761: *1 open() "/usr/share/nginx/html/files/store" failed (2: No such file or directory) while sending to client, client: 127.0.0.1, server: localhost, request: "GET /files/store HTTP/1.1", host: "localhost" 8 | 2016/09/26 22:53:47 [error] 25761#25761: *3 open() "/usr/share/nginx/html/files/store" failed (2: No such file or directory) while sending to client, client: 127.0.0.1, server: localhost, request: "POST /files/store HTTP/1.1", host: "localhost" 9 | 2016/09/26 22:54:55 [error] 25788#25788: *1 open() "/usr/share/nginx/html/files/store" failed (2: No such file or directory) while sending to client, client: 127.0.0.1, server: localhost, request: "POST /files/store HTTP/1.1", host: "localhost" 10 | -------------------------------------------------------------------------------- /lib/utils.js: -------------------------------------------------------------------------------- 1 | function defineConst(target, name, value) { 2 | Object.defineProperty(target, name, { 3 | value, 4 | enumerable: false, 5 | configurable: false, 6 | }); 7 | } 8 | 9 | function promiseSeries(test, step) { 10 | return new Promise((resolve, reject) => { 11 | function spin() { 12 | if (test()) { 13 | Promise.resolve(step()) 14 | .then(spin) 15 | .catch(reject); 16 | } 17 | else { 18 | resolve(); 19 | } 20 | } 21 | 22 | spin(); 23 | }); 24 | } 25 | 26 | exports.defineConst = defineConst; 27 | exports.promiseSeries = promiseSeries; 28 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "main": "src/index.js", 3 | "scripts": { 4 | "cov": "istanbul cover node_modules/mocha/bin/_mocha -- -u exports -t ${TIMEOUT:-2000} -R spec test/**.spec.js", 5 | "test": "mocha -t ${TIMEOUT:-2000} test/**.spec.js" 6 | }, 7 | "license": "MIT", 8 | "name": "filestore", 9 | "version": "0.6.0", 10 | "description": "Extensible filestorage ", 11 | "bin": { 12 | "filestore": "cli.js" 13 | }, 14 | "directories": { 15 | "test": "test" 16 | }, 17 | "repository": { 18 | "type": "git", 19 | "url": "git+https://github.com/rumkin/filestorage.git" 20 | }, 21 | "keywords": [ 22 | "filestore", 23 | "filestorage", 24 | "files", 25 | "microservice", 26 | "service", 27 | "http", 28 | "api" 29 | ], 30 | "author": "rumkin", 31 | "bugs": { 32 | "url": "https://github.com/rumkin/filestorage/issues" 33 | }, 34 | "homepage": "https://github.com/rumkin/filestorage#readme", 35 | "dependencies": { 36 | "argentum": "^0.6.0", 37 | "connect": "^3.5.0", 38 | "hall": "^0.3.0", 39 | "lodash": "^4.16.1", 40 | "mkdirp": "^0.5.1", 41 | "mongodb": "^2.2.27", 42 | "nedb": "^1.8.0", 43 | "pify": "^2.3.0" 44 | }, 45 | "devDependencies": { 46 | "istanbul": "^0.4.5", 47 | "mocha": "^3.0.2", 48 | "node-fetch": "^1.6.1", 49 | "node-uuid": "^1.4.7" 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /readme.md: -------------------------------------------------------------------------------- 1 | # File storage 2 | 3 | ![Build](https://img.shields.io/travis/rumkin/file-storage.svg) 4 | 5 | Multi-backend file storage with REST interface and synchronization. This file 6 | storage supports full REST only files manipulation put/delete and backup 7 | features. 8 | 9 | ## Usage 10 | 11 | File storage could be used via docker container `rumkin/file-storage` or 12 | with manual installation. But note that public access should be prevented 13 | with iptables or authorization proxy server. 14 | 15 | Run with docker: 16 | 17 | ``` 18 | $ docker run -d --name fs -p 4444:8080 -v /var/data/file-store:/data rumkin/file-store 19 | ``` 20 | 21 | ## REST API 22 | 23 | ### POST /files/:id 24 | 25 | Add file to storage. 26 | 27 | Params: 28 | 29 | * HTTP-header 'Content-Type' sets file mime type. 30 | * HTTP-header 'Content-Length' sets file length. 31 | * HTTP-header 'Content-Disposition' sets filename. 32 | * HTTP-header 'X-Tags' sets file data tags. 33 | * HTTP body sets file content. 34 | 35 | Request: 36 | 37 | ``` 38 | POST /files/7211bef2-6856-4a18-9359-5a0373e5b8c1 39 | Content-Type: text/plain 40 | Content-Length: 5 41 | Content-Disposition: attachment; filename=hello.txt 42 | X-Tags: test, file 43 | 44 | Hello 45 | ``` 46 | 47 | Response: 48 | 49 | ``` 50 | HTTP/1.1 200 OK 51 | 52 | OK 53 | ``` 54 | 55 | ### GET /files/:id 56 | 57 | Get file from storage. If file was deleted returns 413. 58 | 59 | Params: 60 | 61 | * URL query param 'download' adds content-disposition header into response. Optional. 62 | 63 | ``` 64 | GET /files/7211bef2-6856-4a18-9359-5a0373e5b8c1?download 65 | ``` 66 | 67 | Response: 68 | ``` 69 | HTTP/1.1 200 OK 70 | Content-Type: text/plain 71 | Content-Length: 5 72 | Content-Disposition: attachment; filename=hello.txt 73 | X-Tags: test, file 74 | 75 | Hello 76 | ``` 77 | 78 | ### DELETE /files/:id 79 | 80 | Mark file as removed and delete it's binary data. 81 | 82 | Request: 83 | 84 | ``` 85 | DELETE /files/7211bef2-6856-4a18-9359-5a0373e5b8c1 86 | ``` 87 | 88 | 89 | Response: 90 | ``` 91 | HTTP/1.1 200 OK 92 | 93 | OK 94 | ``` 95 | 96 | 97 | ### HEAD /files/:id 98 | 99 | Check file exists or not. This method return's no body but meta data only. 100 | 101 | Request: 102 | 103 | ``` 104 | HEAD /files/7211bef2-6856-4a18-9359-5a0373e5b8c1 105 | ``` 106 | 107 | Response: 108 | ``` 109 | HTTP/1.1 200 OK 110 | Content-Length: 5 111 | Content-Type: text/plain 112 | X-Tags: test, file 113 | ``` 114 | 115 | 116 | ### GET /storage/dump 117 | 118 | List stored all items sorted by update date in descending order. 119 | 120 | Response is a gzipped JSON array. 121 | 122 | ``` 123 | GET /storage/dump 124 | ``` 125 | 126 | 127 | ### GET /storage/updates 128 | 129 | List storage updates since special date or from storage creation: 130 | 131 | Params: 132 | 133 | * URL query params 'after' specifies date to filter changes. Could be 134 | ISO 8601 or unix timestamp. Optional. Default is 0. 135 | 136 | Response is a JSON array. 137 | 138 | ``` 139 | GET /storage/updates?after=2016-09-26T00:00:00Z 140 | ``` 141 | 142 | Response _body_: 143 | ``` 144 | [ 145 | { 146 | "_id": "7211bef2-6856-4a18-9359-5a0373e5b8c1", 147 | "isDeleted": false, 148 | "updateDate": "2016-09-26T00:18:48.848Z", 149 | "name":"hello.txt" 150 | } 151 | ] 152 | ``` 153 | 154 | 155 | ### GET /storage/updates/count 156 | 157 | Get count of updated files in db since specified date. 158 | 159 | Params: 160 | 161 | * URL query params 'after' specifies date to filter changes. Could be 162 | ISO 8601 or unix timestamp. Optional. Default is 0. 163 | 164 | Response is a JSON number. 165 | 166 | ``` 167 | GET /storage/updates/count?after=2016-09-26T00:00:00Z 168 | ``` 169 | 170 | Response _body_: 171 | ``` 172 | 1 173 | ``` 174 | 175 | 176 | ## Nginx usage example 177 | 178 | Let's assume that you have authorization application on the port 3333 and file 179 | server on unix socket `/var/run/file-storage.sock`. Nginx configuration example: 180 | 181 | ```nginx 182 | server { 183 | listen 80; 184 | server_name file-storage.your.domain; 185 | 186 | access_log /usr/local/var/log/nginx/file-storage/access.log; 187 | error_log /usr/local/var/log/nginx/file-storage/error.log notice; 188 | 189 | location /auth { 190 | internal; 191 | 192 | proxy_pass "http://localhost:3333"; 193 | proxy_method GET; 194 | proxy_set_header Host $host; 195 | proxy_pass_request_body off; 196 | proxy_set_header Content-Length ""; 197 | 198 | # Set original request values uri and method to make 199 | # authorization though. 200 | proxy_set_header X-Original-Uri $request_uri; 201 | proxy_set_header X-Original-Method $request_method; 202 | } 203 | 204 | location / { 205 | auth_request /auth; 206 | 207 | # Use unix socket to avoid direct access from network 208 | proxy_pass "http://unix:/var/run/file-storage.sock"; 209 | } 210 | } 211 | ``` 212 | 213 | ## License 214 | 215 | MIT. 216 | 217 | 218 | ## Credits 219 | 220 | Inspired by [Pavo](https://github.com/kavkaz/pavo) and [Hermitage](https://github.com/LiveTyping/hermitage-skeleton). 221 | -------------------------------------------------------------------------------- /src/filestore.js: -------------------------------------------------------------------------------- 1 | const {defineConst} = require('../lib/utils.js'); 2 | 3 | class FileStore { 4 | constructor({dataStore, blobStore}) { 5 | if (! dataStore) { 6 | throw new Error('Datastore requried'); 7 | } 8 | 9 | if (! dataStore) { 10 | throw new Error('Blobstore requried'); 11 | } 12 | 13 | defineConst(this, 'dataStore', dataStore); 14 | defineConst(this, 'blobStore', blobStore); 15 | } 16 | 17 | put(id, meta, content) { 18 | if (typeof meta !== 'object') { 19 | throw new Error('Meta should be an object'); 20 | } 21 | 22 | return this.blobStore.put(content) 23 | .then((md5) => Object.assign({}, meta, {md5})) 24 | .then((meta) => this.dataStore.put(id, meta).then(() => meta)); 25 | } 26 | 27 | has(id) { 28 | return this.dataStore.has(id); 29 | } 30 | 31 | get(id) { 32 | return this.getMeta(id) 33 | .then((meta) => this.getBlob(meta.md5)); 34 | } 35 | 36 | delete(id) { 37 | return this.getMeta(id) 38 | .then((meta) => 39 | this.dataStore.delete(id) 40 | .then(() => this.dataStore.countBlobRefs(meta.md5)) 41 | .then((count) => { 42 | if (count > 0) { 43 | return; 44 | } 45 | 46 | return this.blobStore.delete(meta.md5); 47 | }) 48 | ); 49 | } 50 | 51 | setDeleted(id) { 52 | return this.dataStore.setDeleted(id); 53 | return this.getMeta(id) 54 | .then((meta) => 55 | this.dataStore.setDeleted(id) 56 | .then(() => this.dataStore.countBlobRefs(meta.md5)) 57 | .then((count) => { 58 | if (count > 0) { 59 | return; 60 | } 61 | 62 | return this.blobStore.delete(meta.md5); 63 | }) 64 | ); 65 | } 66 | 67 | setAccessDate(id, date) { 68 | return this.dataStore.setAccessDate(id, date); 69 | } 70 | 71 | getStream(id) { 72 | return this.getMeta(id) 73 | .then((meta) => this.getBlobStream(meta.md5) 74 | .then((stream) => [meta, stream])); 75 | } 76 | 77 | getMeta(id) { 78 | if (typeof id === 'object') { 79 | return Promise.resolve(id); 80 | } 81 | else { 82 | return this.dataStore.get(id); 83 | } 84 | } 85 | 86 | listMeta(skip, limit) { 87 | return this.dataStore.find(skip, limit); 88 | } 89 | 90 | countMeta(query, params = {}) { 91 | return this.dataStore.count(query, params); 92 | } 93 | 94 | getBlob(id) { 95 | return this.globStorage.get(id); 96 | } 97 | 98 | getBlobStream(id) { 99 | return this.blobStore.getStream(id); 100 | } 101 | 102 | listUpdated(date) { 103 | return this.dataStore.listUpdated(date); 104 | } 105 | 106 | countUpdated(date) { 107 | return this.dataStore.countUpdated(date); 108 | } 109 | } 110 | 111 | module.exports = FileStore; 112 | -------------------------------------------------------------------------------- /src/fs-blob-storage.js: -------------------------------------------------------------------------------- 1 | const path = require('path'); 2 | const fs = require('fs'); 3 | const crypto = require('crypto'); 4 | const mkdirp = require('mkdirp'); 5 | const pify = require('pify'); 6 | const mkdirpP = pify(mkdirp); 7 | const writeFileP = pify(fs.writeFile); 8 | const readFileP = pify(fs.readFile); 9 | const unlinkP = pify(fs.unlink); 10 | const renameP = pify(fs.rename); 11 | const statP = pify(fs.stat); 12 | const {Readable} = require('stream'); 13 | const uuid = require('node-uuid'); 14 | 15 | class FsBlobStorage { 16 | constructor({dir = process.cwd(), tmp = '/tmp', depth = 3} = {}) { 17 | defineConst(this, 'dir', dir); 18 | defineConst(this, 'depth', depth); 19 | this._tmp = tmp; 20 | } 21 | 22 | get tmp() { 23 | return this._tmp; 24 | } 25 | 26 | /** 27 | * Put binary data into storage. 28 | * 29 | * @param {Buffer|Stream} content Data to store. 30 | * @return {Promise} Promise resolves with md5 of the content. 31 | * @return {Promise} 32 | */ 33 | put(content) { 34 | if (content instanceof Readable) { 35 | return this.putStream(content); 36 | } 37 | 38 | var md5 = crypto.createHash('md5').update(content).digest('hex'); 39 | var dir = this.getDirpath(md5); 40 | var filepath = this.getFilepath(md5); 41 | 42 | return existsP(filepath) 43 | .then((exists) => { 44 | if (exists) { 45 | return md5; 46 | } 47 | else { 48 | return mkdirpP(dir) 49 | .then(() => writeFileP(filepath, content)) 50 | .then(() => md5); 51 | } 52 | }); 53 | } 54 | /** 55 | * Get item value as Buffer. 56 | * 57 | * @param {String} id Item id. 58 | * @return {Promise} Returns item content with promise. 59 | */ 60 | get(id) { 61 | var filepath = this.getFilepath(id); 62 | 63 | return readFileP(filepath); 64 | } 65 | /** 66 | * Put file as a stream. 67 | * @param {Stream.Readable} readStream Readable stream. 68 | * @return {Promise} Promise resolves with md5 of the content. 69 | */ 70 | putStream(readStream) { 71 | var tmpFile = this.tmpFilename(); 72 | 73 | return existsP(tmpFile) 74 | .then(exists => { 75 | if (exists) { 76 | throw new Error('File already exists'); 77 | } 78 | 79 | const hash = crypto.createHash('md5'); 80 | 81 | return new Promise((resolve, reject) => { 82 | var writeStream = fs.createWriteStream(tmpFile); 83 | readStream.pipe(writeStream); 84 | readStream.on('data', (chunk) => hash.update(chunk)); 85 | writeStream.on('finish', () => { 86 | resolve(hash.digest('hex')); 87 | }); 88 | writeStream.on('error', reject); 89 | }) 90 | .then((md5) => { 91 | var filepath = this.getFilepath(md5); 92 | 93 | return existsP(filepath) 94 | .then((exists) => { 95 | if (exists) { 96 | return unlinkP(tmpFile); 97 | } 98 | else { 99 | return mkdirpP(path.dirname(filepath)) 100 | .then(() => renameP(tmpFile, filepath)); 101 | } 102 | }) 103 | .then(() => md5); 104 | }); 105 | }); 106 | } 107 | /** 108 | * Get file read stream. 109 | * 110 | * @param {String} id Item id. 111 | * @return {Promise} Promise returning stram. 112 | */ 113 | getStream (id) { 114 | var filepath = this.getFilepath(id); 115 | 116 | return existsP(filepath) 117 | .then((exists) => { 118 | if (! exists) { 119 | throw new Error(`Item "${id}" not found`); 120 | } 121 | 122 | return fs.createReadStream(filepath); 123 | }); 124 | } 125 | /** 126 | * Remove item from storage. 127 | * 128 | * @param {String} id Item id. 129 | * @return {Promise} 130 | */ 131 | delete(id) { 132 | return unlinkP( 133 | this.getFilepath(id) 134 | ); 135 | } 136 | /** 137 | * Check if item exists in blob storage 138 | * @param {String} id Item id. 139 | * @return {Promise} Return promise returning boolean status. 140 | */ 141 | has(id) { 142 | return existsP( 143 | this.getFilepath(id) 144 | ); 145 | } 146 | 147 | 148 | // UTILS --------------------------------------------------------------- 149 | 150 | 151 | getDirpath(id) { 152 | var parts = []; 153 | var depth = this.depth; 154 | 155 | for (let i = 0; i < depth; i++) { 156 | parts.push(id.slice(i * 2, i * 2 + 2)); 157 | } 158 | 159 | return path.join(this.dir, ...parts); 160 | } 161 | 162 | getFilepath(id) { 163 | return path.join( 164 | this.getDirpath(id), id 165 | ); 166 | } 167 | 168 | getStat(id) { 169 | return statP( 170 | this.getFilepath(id) 171 | ); 172 | } 173 | 174 | tmpFilename() { 175 | return path.join(this._tmp, uuid()); 176 | } 177 | } 178 | 179 | function existsP(filepath) { 180 | return new Promise((resolve) => fs.exists(filepath, resolve)); 181 | } 182 | 183 | function defineConst(target, name, value) { 184 | Object.defineProperty(target, name, { 185 | value, 186 | enumerable: true, 187 | configurable: false, 188 | }); 189 | } 190 | 191 | module.exports = FsBlobStorage; 192 | -------------------------------------------------------------------------------- /src/index.js: -------------------------------------------------------------------------------- 1 | exports.FileStore = require('./filestore.js'); 2 | exports.FsBlobStorage = require('./fs-blob-storage.js'); 3 | exports.NedbDataStorage = require('./nedb-data-storage.js'); 4 | exports.MongodbDataStorage = require('./mongodb-data-storage.js'); 5 | exports.middleware = require('./middleware.js'); 6 | -------------------------------------------------------------------------------- /src/middleware.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | const _ = require('lodash'); 4 | const url = require('url'); 5 | const {promiseSeries} = require('../lib/utils.js'); 6 | const zlib = require('zlib'); 7 | 8 | module.exports = function(router, filestore, logger, debug) { 9 | const VERBOSE = !! logger; 10 | const DEBUG = !! debug; 11 | 12 | // Parse url query if it's not presented in request object. 13 | router.use((req, res, next) => { 14 | if (req.query) { 15 | next(); 16 | return; 17 | } 18 | 19 | req._url = req.url; 20 | req.parsedUrl = url.parse(req.url, true); 21 | req.query = req.parsedUrl.query; 22 | req.url = req.parsedUrl.pathname; 23 | 24 | next(); 25 | }); 26 | 27 | // Get file 28 | router.get('/files/:id', (req, res, next) => { 29 | const id = req.params.id; 30 | 31 | filestore.has(id) 32 | .then((status) => { 33 | if (! status) { 34 | next(); 35 | return; 36 | } 37 | 38 | 39 | return filestore.getStream(id) 40 | .then(([meta, stream]) => { 41 | if (meta.isDeleted) { 42 | res.writeHead(403, 'Deleted'); 43 | res.end(); 44 | return; 45 | } 46 | 47 | res.setHeader('content-type', meta.contentType); 48 | res.setHeader('content-length', meta.contentLength); 49 | res.setHeader('content-md5', meta.md5); 50 | 51 | if (meta.tags.length) { 52 | res.setHeader('x-tags', meta.tags.join(', ')); 53 | } 54 | 55 | if (req.query.download) { 56 | res.setHeader( 57 | 'content-disposition', 58 | `attachment; filename="${meta.name || id}"` 59 | ); 60 | } 61 | 62 | VERBOSE && logger.log('Sent', id); 63 | stream.pipe(res); 64 | 65 | filestore.setAccessDate(id, new Date()) 66 | .catch((error) => DEBUG && console.error(error)); 67 | }); 68 | }) 69 | .catch(next); 70 | }); 71 | 72 | // Get file status 73 | router.head('/files/:id', (req, res, next) => { 74 | const id = req.params.id; 75 | 76 | filestore.has(id) 77 | .then((status) => { 78 | if (! status) { 79 | next(); 80 | return; 81 | } 82 | 83 | 84 | return filestore.getMeta(id) 85 | .then((meta) => { 86 | if (meta.isDeleted) { 87 | res.writeHead(413, 'Deleted'); 88 | res.end(); 89 | return; 90 | } 91 | 92 | res.setHeader('content-type', meta.contentType); 93 | res.setHeader('content-length', meta.contentLength); 94 | res.setHeader('content-md5', meta.md5); 95 | 96 | if (meta.tags.length) { 97 | res.setHeader('x-tags', meta.tags.join(', ')); 98 | } 99 | res.end(); 100 | 101 | VERBOSE && logger.log('Check', id); 102 | }); 103 | }) 104 | .catch(next); 105 | }); 106 | 107 | // Put file 108 | router.post('/files/:id', (req, res, next) => { 109 | const id = req.params.id; 110 | 111 | filestore.has(id) 112 | .then((status) => { 113 | if (status) { 114 | // Item exists... 115 | res.statusCode = 409; 116 | res.statusText = 'File already exists'; 117 | res.end('File exists'); 118 | return; 119 | } 120 | 121 | var contentType = req.headers['content-type']; 122 | var contentLength = req.headers['content-length']; 123 | var filename = req.headers['content-disposition']; 124 | var tags = req.headers['x-tags']; 125 | 126 | if (filename) { 127 | let match = filename.match(/^attachment;\s+filename=(.+)/); 128 | if (match) { 129 | filename = match[1]; 130 | if (filename.charAt(0) === '"') { 131 | filename = filename.slice(1, -1); 132 | } 133 | 134 | if (! filename.length) { 135 | filename = undefined; 136 | } 137 | } 138 | else { 139 | filename = undefined; 140 | } 141 | } 142 | 143 | if (tags) { 144 | tags = tags.split(/\s*,\s*/); 145 | } 146 | 147 | var meta = { 148 | name: filename || '', 149 | contentType, 150 | contentLength, 151 | tags, 152 | }; 153 | 154 | return filestore.put(id, meta, req) 155 | .then(() => { 156 | VERBOSE && logger.log('Added', id); 157 | res.end('OK'); 158 | }); 159 | }) 160 | .catch(next); 161 | }); 162 | 163 | // Delete file 164 | router.delete('/files/:id', (req, res, next) => { 165 | const id = req.params.id; 166 | 167 | filestore.has(id) 168 | .then((exists) => { 169 | if (! exists) { 170 | next(); 171 | return; 172 | } 173 | 174 | return storage.getMeta(id) 175 | .then((meta) => { 176 | if (meta.isDeleted) { 177 | res.statusCode = 410; 178 | res.statusText = 'File deleted'; 179 | res.end('Deleted'); 180 | return; 181 | } 182 | 183 | return storage.setDeleted(id) 184 | .then(() => { 185 | VERBOSE && logger.log('Deleted', id); 186 | res.end('OK'); 187 | }); 188 | }); 189 | }) 190 | .catch(next); 191 | }); 192 | 193 | // Info routes 194 | 195 | router.get('/storage/updates', (req, res, next) => { 196 | var date = req.query.after || 0; 197 | 198 | filestore.listUpdated(date) 199 | .then((updates) => { 200 | var result = JSON.stringify(updates.map( 201 | (item) => _.pick(item, [ 202 | '_id', 203 | 'isDeleted', 204 | 'updateDate', 205 | 'createDate', 206 | 'accessDate', 207 | 'contentType', 208 | 'contentLength', 209 | 'name', 210 | ]) 211 | )); 212 | 213 | res.setHeader('content-type', 'application/json'); 214 | res.setHeader('content-length', result.length); 215 | res.end(result); 216 | }) 217 | .catch(next); 218 | }); 219 | 220 | router.get('/storage/updates/count', (req, res, next) => { 221 | var date = req.query.after || 0; 222 | 223 | filestore.countUpdated(date) 224 | .then((count) => { 225 | var result = JSON.stringify(count); 226 | 227 | res.setHeader('content-type', 'application/json'); 228 | res.setHeader('content-length', result.length); 229 | res.end(result); 230 | }) 231 | .catch(next); 232 | }); 233 | 234 | router.get('/storage/dump', (req, res, next) => { 235 | filestore.countMeta() 236 | .then((count) => { 237 | if (! count) { 238 | res.end('[]'); 239 | return; 240 | } 241 | 242 | var skip = 0; 243 | var limit = 1000; 244 | var output = res; 245 | 246 | res.setHeader('content-type', 'application/json'); 247 | 248 | // If accept gzipped. 249 | if ('accept-encoding' in req.headers) { 250 | let accept = req.headers['accept-encoding']; 251 | if (accept.includes('gzip')) { 252 | res.setHeader('content-encoding', 'gzip'); 253 | let gzip = zlib.createGzip(); 254 | gzip.pipe(res); 255 | output = gzip; 256 | } 257 | } 258 | 259 | // Start sending an array 260 | output.write('[\n'); 261 | 262 | return promiseSeries( 263 | () => skip < count, 264 | () => filestore.listMeta(skip, limit) 265 | .then((items) => { 266 | skip += Math.min(limit, items.length); 267 | 268 | // Convert items to JSON strings 269 | var result = items.map((item) => 270 | JSON.stringify(_.pick(item, [ 271 | '_id', 272 | 'isDeleted', 273 | 'updateDate', 274 | 'createDate', 275 | 'accessDate', 276 | 'contentType', 277 | 'contentLength', 278 | 'name', 279 | ])) 280 | ).join(',\n'); 281 | 282 | // Append final comma if not a last chunk 283 | if (skip < count) { 284 | result += ','; 285 | } 286 | 287 | // Write gzip data 288 | output.write(result + '\n'); 289 | }) 290 | ) 291 | .then(() => { 292 | output.write(']'); 293 | output.end(); 294 | }); 295 | }) 296 | .catch(next); 297 | }); 298 | 299 | return router; 300 | }; 301 | -------------------------------------------------------------------------------- /src/mongodb-data-storage.js: -------------------------------------------------------------------------------- 1 | const {defineConst} = require('../lib/utils.js'); 2 | const _ = require('lodash'); 3 | 4 | function bind(db, collection, method) { 5 | return function(...args) { 6 | return db.collection(collection)[method](...args); 7 | }; 8 | } 9 | 10 | class MongodbDataStorage { 11 | constructor({db, collection} = {}) { 12 | if (! db) { 13 | throw new Error('Database required'); 14 | } 15 | 16 | defineConst(this, 'db', db); 17 | // Promisify db methods 18 | defineConst(this, '_insert', bind(db, collection, 'insert')); 19 | defineConst(this, '_update', bind(db, collection, 'update')); 20 | defineConst(this, '_find', bind(db, collection, 'find')); 21 | defineConst(this, '_findOne', bind(db, collection, 'findOne')); 22 | defineConst(this, '_remove', bind(db, collection, 'remove')); 23 | defineConst(this, '_count', bind(db, collection, 'count')); 24 | } 25 | 26 | /** 27 | * Set item with id 28 | * 29 | * @param {String} id Item id. 30 | * @param {Object} data Item data. 31 | * @return {Promise} Promise resolves with inserted item. 32 | */ 33 | put(id, data) { 34 | // TODO Validate data object 35 | // Append default fields 36 | let item = _.merge({}, _.pick(data, [ 37 | '_id', 38 | 'name', 39 | 'md5', 40 | 'contentType', 41 | 'contentLength', 42 | 'isDeleted', 43 | 'createDate', 44 | 'updateDate', 45 | 'accessDate', 46 | 'tags', 47 | ]), { 48 | _id: id, 49 | isDeleted: false, 50 | createDate: new Date(), 51 | updateDate: new Date(), 52 | accessDate: null, 53 | tags: [], 54 | }); 55 | 56 | return this._insert(item).then(() => item); 57 | } 58 | 59 | /** 60 | * Get item by id. 61 | * 62 | * @param {String} id Item id. 63 | * @return {Promise} Promise resolves with document or null 64 | */ 65 | get(id) { 66 | return this._findOne({_id: id}); 67 | } 68 | 69 | /** 70 | * Check if item exists in storage 71 | * @param {String} id Item id 72 | * @returns {Promise} Promise resolves with boolean status. 73 | */ 74 | has(id) { 75 | return this._count({ 76 | _id: id 77 | }) 78 | .then((count) => count > 0); 79 | } 80 | 81 | /** 82 | * Delete item from sotrage 83 | * @param {String} id Item id 84 | * @return {Promise} 85 | */ 86 | delete(id) { 87 | return this._remove({_id: id}); 88 | } 89 | 90 | /** 91 | * Find items with query. 92 | * @param {object} query Search query. 93 | * @param {object} options Query options: select, slip, sort, limit, etc. 94 | * @return {Promise} Promise resolves with list of items. 95 | */ 96 | find(query, {select, sort, skip, limit}) { 97 | return new Promise((resolve, reject) => { 98 | var cursor = this.db.find(query); 99 | 100 | if (skip) { 101 | cursor.skip(skip); 102 | } 103 | 104 | if (limit) { 105 | cursor.limit(limit); 106 | } 107 | 108 | if (sort) { 109 | cursor.sort({ 110 | updateDate: -1, 111 | }); 112 | } 113 | 114 | if (select) { 115 | cursor.projection(select); 116 | } 117 | 118 | cursor.exec((err, docs) => { 119 | if (err) { 120 | reject(err); 121 | } 122 | else { 123 | resolve(docs); 124 | } 125 | }); 126 | }); 127 | } 128 | 129 | /** 130 | * Count items with query. 131 | * @param {object} query Search query. 132 | * @return {Promise} Promise resolves with number of items. 133 | */ 134 | count(query) { 135 | return this._count(query); 136 | } 137 | 138 | /** 139 | * Mark item as updated 140 | * @param {String} id Item id 141 | * @return {Promise} 142 | */ 143 | setUpdated(id) { 144 | return this._update({ 145 | _id: id, 146 | }, { 147 | $set: { 148 | updateDate: new Date(), 149 | }, 150 | }); 151 | } 152 | 153 | /** 154 | * Mark item as updated. 155 | * @param {String} id Item id. 156 | * @param {Date|String|Number} date Access date. 157 | * @return {Promise} 158 | */ 159 | setAccessDate(id, date) { 160 | return this._update({ 161 | _id: id, 162 | }, { 163 | $set: { 164 | accessDate: this._getDate(date), 165 | } 166 | }); 167 | } 168 | 169 | /** 170 | * Mark item as deleted 171 | * @param {String} id Item id 172 | * @return {Promise} 173 | */ 174 | setDeleted(id) { 175 | return this._update({ 176 | _id: id, 177 | }, { 178 | $set: { 179 | isDeleted: true, 180 | updateDate: new Date(), 181 | }, 182 | }); 183 | } 184 | 185 | /** 186 | * List all items. 187 | * 188 | * @param {Date|Number} date Minimum update date 189 | * @return {Promise} Promise resolvs with array of updated documents. 190 | */ 191 | listAll(skip, limit) { 192 | return this.find({}, {skip, limit}); 193 | } 194 | 195 | /** 196 | * List items updated after specified Database. 197 | * 198 | * @param {Date|Number} date Minimum update date 199 | * @return {Promise} Promise resolvs with array of updated documents. 200 | */ 201 | listUpdated(date, skip = 0, limit = 0) { 202 | return this.find({ 203 | updateDate: {$gte: this._getDate(date)}, 204 | }, { 205 | sort: { 206 | updateDate: -1, 207 | }, 208 | skip, 209 | limit 210 | }); 211 | } 212 | 213 | /** 214 | * Check if there is file updates in the store. 215 | * 216 | * @param {Date|Number} date Minimum update date 217 | * @return {Promise} Promise resolves with true if there is one. 218 | */ 219 | hasUpdates(date) { 220 | return this.countUpdates(date) 221 | .then((count) => count > 0); 222 | } 223 | 224 | /** 225 | * Count updated documents number. 226 | * 227 | * @param {Date|Number} date Minimum update date 228 | * @return {Promise} Promise resolves with number of documents. 229 | */ 230 | countUpdates(date) { 231 | return this.count({ 232 | updateDate: {$gte: this._getDate(date)}, 233 | }); 234 | } 235 | 236 | countBlobRefs(md5) { 237 | return this.count({ 238 | md5, 239 | isDeleted: false, 240 | }); 241 | } 242 | 243 | // COMMON METHODS ---------------------------------------------------------- 244 | 245 | 246 | // UTILS ------------------------------------------------------------------- 247 | 248 | 249 | _getDate(date) { 250 | if (! (date instanceof Date)) { 251 | return new Date(date); 252 | } 253 | else { 254 | return date; 255 | } 256 | } 257 | } 258 | 259 | module.exports = MongodbDataStorage; 260 | -------------------------------------------------------------------------------- /src/nedb-data-storage.js: -------------------------------------------------------------------------------- 1 | const pify = require('pify'); 2 | const {defineConst} = require('../lib/utils.js'); 3 | const _ = require('lodash'); 4 | 5 | class NedbDataStorage { 6 | constructor({db} = {}) { 7 | if (! db) { 8 | throw new Error('Database required'); 9 | } 10 | 11 | defineConst(this, 'db', db); 12 | // Promisify db methods 13 | defineConst(this, '_insert', pify(db.insert.bind(db))); 14 | defineConst(this, '_update', pify(db.update.bind(db))); 15 | defineConst(this, '_find', pify(db.find.bind(db))); 16 | defineConst(this, '_findOne', pify(db.findOne.bind(db))); 17 | defineConst(this, '_remove', pify(db.remove.bind(db))); 18 | defineConst(this, '_count', pify(db.count.bind(db))); 19 | } 20 | 21 | /** 22 | * Set item with id 23 | * 24 | * @param {String} id Item id. 25 | * @param {Object} data Item data. 26 | * @return {Promise} Promise resolves with inserted item. 27 | */ 28 | put(id, data) { 29 | // TODO Validate data object 30 | // Append default fields 31 | return this._insert(_.merge({}, _.pick(data, [ 32 | '_id', 33 | 'name', 34 | 'md5', 35 | 'contentType', 36 | 'contentLength', 37 | 'isDeleted', 38 | 'createDate', 39 | 'updateDate', 40 | 'accessDate', 41 | 'tags', 42 | ]), { 43 | _id: id, 44 | isDeleted: false, 45 | createDate: new Date(), 46 | updateDate: new Date(), 47 | accessDate: null, 48 | tags: [], 49 | })); 50 | } 51 | 52 | /** 53 | * Get item by id. 54 | * 55 | * @param {String} id Item id. 56 | * @return {Promise} Promise resolves with document or null 57 | */ 58 | get(id) { 59 | return this._findOne({_id: id}); 60 | } 61 | 62 | /** 63 | * Check if item exists in storage 64 | * @param {String} id Item id 65 | * @returns {Promise} Promise resolves with boolean status. 66 | */ 67 | has(id) { 68 | return this._count({ 69 | _id: id 70 | }) 71 | .then((count) => count > 0); 72 | } 73 | 74 | /** 75 | * Delete item from sotrage 76 | * @param {String} id Item id 77 | * @return {Promise} 78 | */ 79 | delete(id) { 80 | return this._remove({_id: id}); 81 | } 82 | 83 | /** 84 | * Find items with query. 85 | * @param {object} query Search query. 86 | * @param {object} options Query options: select, slip, sort, limit, etc. 87 | * @return {Promise} Promise resolves with list of items. 88 | */ 89 | find(query, {select, sort, skip, limit}) { 90 | return new Promise((resolve, reject) => { 91 | var cursor = this.db.find(query); 92 | 93 | if (skip) { 94 | cursor.skip(skip); 95 | } 96 | 97 | if (limit) { 98 | cursor.limit(limit); 99 | } 100 | 101 | if (sort) { 102 | cursor.sort({ 103 | updateDate: -1, 104 | }); 105 | } 106 | 107 | if (select) { 108 | cursor.projection(select); 109 | } 110 | 111 | cursor.exec((err, docs) => { 112 | if (err) { 113 | reject(err); 114 | } 115 | else { 116 | resolve(docs); 117 | } 118 | }); 119 | }); 120 | } 121 | 122 | /** 123 | * Count items with query. 124 | * @param {object} query Search query. 125 | * @return {Promise} Promise resolves with number of items. 126 | */ 127 | count(query) { 128 | return this._count(query); 129 | } 130 | 131 | /** 132 | * Mark item as updated 133 | * @param {String} id Item id 134 | * @return {Promise} 135 | */ 136 | setUpdated(id) { 137 | return this._update({ 138 | _id: id, 139 | }, { 140 | $set: { 141 | updateDate: new Date(), 142 | }, 143 | }); 144 | } 145 | 146 | /** 147 | * Mark item as updated. 148 | * @param {String} id Item id. 149 | * @param {Date|String|Number} date Access date. 150 | * @return {Promise} 151 | */ 152 | setAccessDate(id, date) { 153 | return this._update({ 154 | _id: id, 155 | }, { 156 | $set: { 157 | accessDate: this._getDate(date), 158 | } 159 | }); 160 | } 161 | 162 | /** 163 | * Mark item as deleted 164 | * @param {String} id Item id 165 | * @return {Promise} 166 | */ 167 | setDeleted(id) { 168 | return this._update({ 169 | _id: id, 170 | }, { 171 | $set: { 172 | isDeleted: true, 173 | updateDate: new Date(), 174 | }, 175 | }); 176 | } 177 | 178 | /** 179 | * List all items. 180 | * 181 | * @param {Date|Number} date Minimum update date 182 | * @return {Promise} Promise resolvs with array of updated documents. 183 | */ 184 | listAll(skip, limit) { 185 | return this.find({}, {skip, limit}); 186 | } 187 | 188 | /** 189 | * List items updated after specified Database. 190 | * 191 | * @param {Date|Number} date Minimum update date 192 | * @return {Promise} Promise resolvs with array of updated documents. 193 | */ 194 | listUpdated(date, skip = 0, limit = 0) { 195 | return this.find({ 196 | updateDate: {$gte: this._getDate(date)}, 197 | }, { 198 | sort: { 199 | updateDate: -1, 200 | }, 201 | skip, 202 | limit 203 | }); 204 | } 205 | 206 | /** 207 | * Check if there is file updates in the store. 208 | * 209 | * @param {Date|Number} date Minimum update date 210 | * @return {Promise} Promise resolves with true if there is one. 211 | */ 212 | hasUpdates(date) { 213 | return this.countUpdates(date) 214 | .then((count) => count > 0); 215 | } 216 | 217 | /** 218 | * Count updated documents number. 219 | * 220 | * @param {Date|Number} date Minimum update date 221 | * @return {Promise} Promise resolves with number of documents. 222 | */ 223 | countUpdates(date) { 224 | return this.count({ 225 | updateDate: {$gte: this._getDate(date)}, 226 | }); 227 | } 228 | 229 | countBlobRefs(md5) { 230 | return this.count({ 231 | md5, 232 | isDeleted: false, 233 | }); 234 | } 235 | 236 | // COMMON METHODS ---------------------------------------------------------- 237 | 238 | 239 | // UTILS ------------------------------------------------------------------- 240 | 241 | 242 | _getDate(date) { 243 | if (! (date instanceof Date)) { 244 | return new Date(date); 245 | } 246 | else { 247 | return date; 248 | } 249 | } 250 | } 251 | 252 | module.exports = NedbDataStorage; 253 | -------------------------------------------------------------------------------- /test/fs-blob-storage.spec.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | const {FsBlobStorage} = require('..'); 4 | const assert = require('assert'); 5 | const fs = require('fs'); 6 | const uuid = require('node-uuid'); 7 | const {Readable} = require('stream'); 8 | 9 | describe('FsBlobStorage', () => { 10 | describe('#constructor', () => { 11 | it('Should define default values', () => { 12 | var store = new FsBlobStorage(); 13 | 14 | assert.equal(typeof store.depth, 'number', 'Depth is a number'); 15 | assert.equal(typeof store.dir, 'string', 'Directory is a string'); 16 | }); 17 | }); 18 | 19 | describe('Storage methods', () => { 20 | var storage; 21 | before(() => { 22 | const dir = fs.mkdtempSync('/tmp/node-test-'); 23 | 24 | storage = new FsBlobStorage({ 25 | dir, 26 | depth: 1, 27 | }); 28 | }); 29 | 30 | it('It should put and get string', () => { 31 | var content = 'Hello'; 32 | 33 | return storage.put(content) 34 | .then((id) => storage.get(id)) 35 | .then((result) => { 36 | assert.ok(result instanceof Buffer, 'Result is a buffer'); 37 | assert.equal(result, content, 'Data from storage is the same as written'); 38 | }); 39 | }); 40 | 41 | it('It should put stream and get buffer', () => { 42 | var content = new Buffer('Hello', 'utf-8'); 43 | 44 | return storage.put(content) 45 | .then((id) => storage.get(id)) 46 | .then((result) => { 47 | assert.ok(result instanceof Buffer, 'Result is a buffer'); 48 | assert.equal(Buffer.compare(result, content), 0, 'Data from storage is the same as written'); 49 | }); 50 | }); 51 | 52 | it('It should put and get stream', () => { 53 | var content = new Buffer('Hello', 'utf-8'); 54 | 55 | class Stream extends Readable { 56 | constructor(opt) { 57 | super(opt); 58 | this.isRead = false; 59 | } 60 | 61 | _read() { 62 | if (this.isRead) { 63 | this.push(null); 64 | } 65 | else { 66 | this.isRead = true; 67 | this.push(content); 68 | } 69 | } 70 | } 71 | 72 | var stream = new Stream; 73 | 74 | return storage.put(stream) 75 | .then((id) => storage.get(id)) 76 | .then((result) => { 77 | assert.ok(result instanceof Buffer, 'Result is a buffer'); 78 | assert.equal(Buffer.compare(result, content), 0, 'Data from storage is the same as written'); 79 | }); 80 | }); 81 | 82 | it('It should get stream', () => { 83 | var content = new Buffer('Hello', 'utf-8'); 84 | 85 | return storage.put(content) 86 | .then((id) => storage.getStream(id)) 87 | .then((stream) => { 88 | assert.ok(stream instanceof Readable, 'Stream returned'); 89 | 90 | return new Promise((resolve, reject) => { 91 | var result = ''; 92 | stream.on('data', (chunk) => { 93 | result += chunk; 94 | }); 95 | 96 | stream.on('end', () => resolve(result)); 97 | stream.on('error', reject); 98 | }); 99 | }) 100 | .then((result) => { 101 | assert.equal(result, content, 'Data from storage is the same as written'); 102 | }); 103 | }); 104 | 105 | it('#has should return `true` when item exists', () => { 106 | var content = new Buffer('Hello', 'utf-8'); 107 | 108 | return storage.put(content) 109 | .then((id) => storage.has(id)) 110 | .then((result) => { 111 | assert.ok(result, 'Item exists'); 112 | }); 113 | }); 114 | 115 | it('#has should return `false` when item removed', () => { 116 | var content = new Buffer('Hello', 'utf-8'); 117 | var id; 118 | return storage.put(content) 119 | .then((id_) => (id = id_)) 120 | .then(() => storage.delete(id)) 121 | .then(() => storage.has(id)) 122 | .then((result) => { 123 | assert.ok(! result, 'Item not exists'); 124 | }); 125 | }); 126 | 127 | it('Should write item on disk', () => { 128 | var content = new Buffer('Hello', 'utf-8'); 129 | 130 | return storage.put(content) 131 | .then((id) => { 132 | var filepath = storage.getFilepath(id); 133 | 134 | assert.ok(fs.existsSync(filepath), 'File exists'); 135 | }); 136 | }); 137 | 138 | it('Should remove item from disk', () => { 139 | var content = new Buffer('Hello', 'utf-8'); 140 | var id; 141 | return storage.put(content) 142 | .then((id_) => (id = id_)) 143 | .then(() => storage.delete(id)) 144 | .then(() => { 145 | var filepath = storage.getFilepath(id); 146 | 147 | assert.ok(! fs.existsSync(filepath), 'File not exists'); 148 | }); 149 | }); 150 | }); 151 | 152 | 153 | describe('Utils', () => { 154 | var storage; 155 | 156 | before(() => { 157 | storage = new FsBlobStorage({ 158 | dir: '/tmp/blob', 159 | depth: 2, 160 | }); 161 | }); 162 | 163 | it('#getDirpath', () => { 164 | var dirname = storage.getDirpath('40b2da21-ac35-48af-a08a-1957cf3eb284'); 165 | 166 | assert.equal(dirname, '/tmp/blob/40/b2'); 167 | }); 168 | 169 | it('#getFilepath', () => { 170 | var filename = storage.getFilepath('40b2da21-ac35-48af-a08a-1957cf3eb284'); 171 | 172 | assert.equal(filename, '/tmp/blob/40/b2/40b2da21-ac35-48af-a08a-1957cf3eb284'); 173 | }); 174 | }); 175 | }); 176 | -------------------------------------------------------------------------------- /test/mongodb-data-storage.spec.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | const {MongoClient} = require('mongodb'); 4 | const {MongodbDataStorage} = require('..'); 5 | const assert = require('assert'); 6 | const uuid = require('node-uuid'); 7 | 8 | describe('MongodbDataStorage', () => { 9 | var storage; 10 | 11 | it('Should instantiate ', () => { 12 | return MongoClient.connect('mongodb://localhost:27017/test').then((db) => { 13 | storage = new MongodbDataStorage({ 14 | db, 15 | collection: 'files', 16 | }); 17 | }); 18 | }); 19 | 20 | it('#put should return valid item', () => { 21 | var id = uuid(); 22 | 23 | return storage.put(id, { 24 | contentType: 'text/plain', 25 | contentLength: 1, 26 | tags: ['test'], 27 | }) 28 | .then((result) => { 29 | assert.equal(typeof result, 'object', 'result is object'); 30 | [ 31 | '_id', 32 | 'contentLength', 33 | 'contentType', 34 | 'createDate', 35 | 'updateDate', 36 | 'accessDate', 37 | 'isDeleted', 38 | 'tags', 39 | ] 40 | .forEach((prop) => { 41 | assert.ok(result.hasOwnProperty(prop), `result has \`${prop}\``); 42 | }); 43 | assert.deepEqual(result.tags, ['test'], 'Tags returned'); 44 | }); 45 | }); 46 | 47 | it('#get should return item', () => { 48 | var id = uuid(); 49 | 50 | return storage.put(id, { 51 | contentType: 'text/plain', 52 | contentLength: 1, 53 | tags: ['test'], 54 | }) 55 | .then(() => storage.get(id)) 56 | .then((result) => { 57 | assert.equal(typeof result, 'object', 'result is object'); 58 | [ 59 | '_id', 60 | 'contentLength', 61 | 'contentType', 62 | 'createDate', 63 | 'updateDate', 64 | 'accessDate', 65 | 'isDeleted', 66 | 'tags', 67 | ] 68 | .forEach((prop) => { 69 | assert.ok(result.hasOwnProperty(prop), `result has \`${prop}\``); 70 | }); 71 | assert.deepEqual(result.tags, ['test'], 'Tags returned'); 72 | }); 73 | }); 74 | 75 | it('#has should return `true` when exists', () => { 76 | var id = uuid(); 77 | 78 | return storage.put(id, { 79 | contentType: 'text/plain', 80 | contentLength: 1, 81 | }) 82 | .then(() => storage.has(id)) 83 | .then((result) => { 84 | assert.ok(result, 'Item exists'); 85 | }); 86 | }); 87 | 88 | it('#has should return `false` when item deleted', () => { 89 | var id = uuid(); 90 | 91 | return storage.put(id, { 92 | contentType: 'text/plain', 93 | contentLength: 1, 94 | }) 95 | .then(() => storage.delete(id)) 96 | .then(() => storage.has(id)) 97 | .then((result) => { 98 | assert.ok(! result, 'Item not exists'); 99 | }); 100 | }); 101 | 102 | it('#setDeleted should set isDeleted `true`', () => { 103 | var id = uuid(); 104 | 105 | return storage.put(id, { 106 | contentType: 'text/plain', 107 | contentLength: 1, 108 | }) 109 | .then(() => storage.setDeleted(id)) 110 | .then(() => storage.get(id)) 111 | .then((result) => { 112 | assert.ok(result.isDeleted, 'isDeleted is `true`'); 113 | }); 114 | }); 115 | 116 | it('#countUpdates should return updated items count', () => { 117 | var id = uuid(); 118 | var date; 119 | 120 | return storage.put(id, { 121 | contentType: 'text/plain', 122 | contentLength: 1, 123 | }) 124 | .then(() =>new Promise( 125 | (resolve) => setTimeout(() => { 126 | date = new Date(); 127 | resolve(); 128 | }, 10)) 129 | ) 130 | // .then(() => storage.listUpdated(date).then(updates => console.log('updates', updates, date))) 131 | .then(() => storage.countUpdates(date)) 132 | .then((count) => assert.equal(count, 0, 'no updates in db')) 133 | .then(() => storage.setDeleted(id)) 134 | .then(() => storage.countUpdates(date)) 135 | .then((count) => assert.equal(count, 1, '1 document updated')) 136 | ; 137 | }); 138 | }); 139 | -------------------------------------------------------------------------------- /test/nedb-data-storage.spec.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | const Datastore = require('nedb'); 4 | const {NedbDataStorage} = require('..'); 5 | const assert = require('assert'); 6 | const uuid = require('node-uuid'); 7 | 8 | describe('NedbDataStorage', () => { 9 | var storage; 10 | 11 | it('Should instantiate ', () => { 12 | storage = new NedbDataStorage({ 13 | db: new Datastore(), 14 | }); 15 | }); 16 | 17 | it('#put should return valid item', () => { 18 | var id = uuid(); 19 | 20 | return storage.put(id, { 21 | contentType: 'text/plain', 22 | contentLength: 1, 23 | tags: ['test'], 24 | }) 25 | .then((result) => { 26 | assert.equal(typeof result, 'object', 'result is object'); 27 | [ 28 | '_id', 29 | 'contentLength', 30 | 'contentType', 31 | 'createDate', 32 | 'updateDate', 33 | 'accessDate', 34 | 'isDeleted', 35 | 'tags', 36 | ] 37 | .forEach((prop) => { 38 | assert.ok(result.hasOwnProperty(prop), `result has \`${prop}\``); 39 | }); 40 | assert.deepEqual(result.tags, ['test'], 'Tags returned'); 41 | }); 42 | }); 43 | 44 | it('#get should return item', () => { 45 | var id = uuid(); 46 | 47 | return storage.put(id, { 48 | contentType: 'text/plain', 49 | contentLength: 1, 50 | tags: ['test'], 51 | }) 52 | .then(() => storage.get(id)) 53 | .then((result) => { 54 | assert.equal(typeof result, 'object', 'result is object'); 55 | [ 56 | '_id', 57 | 'contentLength', 58 | 'contentType', 59 | 'createDate', 60 | 'updateDate', 61 | 'accessDate', 62 | 'isDeleted', 63 | 'tags', 64 | ] 65 | .forEach((prop) => { 66 | assert.ok(result.hasOwnProperty(prop), `result has \`${prop}\``); 67 | }); 68 | assert.deepEqual(result.tags, ['test'], 'Tags returned'); 69 | }); 70 | }); 71 | 72 | it('#has should return `true` when exists', () => { 73 | var id = uuid(); 74 | 75 | return storage.put(id, { 76 | contentType: 'text/plain', 77 | contentLength: 1, 78 | }) 79 | .then(() => storage.has(id)) 80 | .then((result) => { 81 | assert.ok(result, 'Item exists'); 82 | }); 83 | }); 84 | 85 | it('#has should return `false` when item deleted', () => { 86 | var id = uuid(); 87 | 88 | return storage.put(id, { 89 | contentType: 'text/plain', 90 | contentLength: 1, 91 | }) 92 | .then(() => storage.delete(id)) 93 | .then(() => storage.has(id)) 94 | .then((result) => { 95 | assert.ok(! result, 'Item not exists'); 96 | }); 97 | }); 98 | 99 | it('#setDeleted should set isDeleted `true`', () => { 100 | var id = uuid(); 101 | 102 | return storage.put(id, { 103 | contentType: 'text/plain', 104 | contentLength: 1, 105 | }) 106 | .then(() => storage.setDeleted(id)) 107 | .then(() => storage.get(id)) 108 | .then((result) => { 109 | assert.ok(result.isDeleted, 'isDeleted is `true`'); 110 | }); 111 | }); 112 | 113 | it('#countUpdates should return updated items count', () => { 114 | var id = uuid(); 115 | var date; 116 | 117 | return storage.put(id, { 118 | contentType: 'text/plain', 119 | contentLength: 1, 120 | }) 121 | .then(() =>new Promise( 122 | (resolve) => setTimeout(() => { 123 | date = new Date(); 124 | resolve(); 125 | }, 10)) 126 | ) 127 | // .then(() => storage.listUpdated(date).then(updates => console.log('updates', updates, date))) 128 | .then(() => storage.countUpdates(date)) 129 | .then((count) => assert.equal(count, 0, 'no updates in db')) 130 | .then(() => storage.setDeleted(id)) 131 | .then(() => storage.countUpdates(date)) 132 | .then((count) => assert.equal(count, 1, '1 document updated')) 133 | ; 134 | }); 135 | }); 136 | -------------------------------------------------------------------------------- /test/server.spec.js: -------------------------------------------------------------------------------- 1 | const fs = require('fs'); 2 | const connect = require('connect'); 3 | const hall = require('hall'); 4 | const {middleware, FileStore, NedbDataStorage, FsBlobStorage} = require('..'); 5 | const assert = require('assert'); 6 | const nedb = require('nedb'); 7 | const fetch = require('node-fetch'); 8 | const uuid = require('node-uuid'); 9 | const qs = require('querystring'); 10 | const DEBUG = process.env.DEBUG === '1'; 11 | 12 | describe('HTTP server', () => { 13 | var server, storage, dir, port; 14 | 15 | before(() => { 16 | dir = fs.mkdtempSync('/tmp/node-test-'); 17 | storage = new FileStore({ 18 | dataStore: new NedbDataStorage({ 19 | db: new nedb({ 20 | filename: dir + '/files.db', 21 | autoload: true, 22 | }), 23 | }), 24 | blobStore: new FsBlobStorage({dir}), 25 | }); 26 | 27 | var router = hall(); 28 | 29 | server = connect() 30 | .use(middleware(router, storage)) 31 | .use((err, req, res, next) => { 32 | if (! err) { 33 | res.statusCode = 404; 34 | res.statusText = 'Nothing found'; 35 | res.end(); 36 | } 37 | else { 38 | res.statusCode = 500; 39 | res.statusText = 'Internal error'; 40 | res.end(err.message); 41 | DEBUG && console.log(err); 42 | } 43 | }) 44 | .listen(); 45 | 46 | port = server.address().port; 47 | }); 48 | 49 | it('Should post, get and head file', () => { 50 | const id = uuid(); 51 | const data = new Buffer('Hello'); 52 | const url = `http://localhost:${port}/files/${id}`; 53 | 54 | return fetch(url, { 55 | method: 'POST', 56 | headers: { 57 | 'content-type': 'text/plain', 58 | 'content-length': data.length, 59 | 'content-disposition': 'attachment; filename=test', 60 | 'x-tags': 'test, file', 61 | }, 62 | body: data, 63 | }) 64 | .then((res) => { 65 | assert.equal(res.status, 200, 'Status is 200'); 66 | }) 67 | .then(() => fetch(url, {method: 'HEAD'})) 68 | .then((res) => { 69 | assert.equal(res.status, 200, 'Status is 200'); 70 | var filepath = storage.blobStore.getFilepath(res.headers.get('content-md5')); 71 | assert.ok(fs.existsSync(filepath), 'File created'); 72 | }) 73 | .then(() => fetch(url) 74 | .then((res) => { 75 | assert.equal(res.status, 200, 'Status is 200'); 76 | assert.equal(res.headers.get('content-type'), 'text/plain', 'Content type header'); 77 | assert.equal(res.headers.get('content-length'), data.length, 'Content length header'); 78 | return res.text(); 79 | }) 80 | .then((result) => { 81 | assert.equal(result, data, 'result is `Hello`'); 82 | }) 83 | .then(() => fetch( 84 | `http://localhost:${port}/storage/dump` 85 | )) 86 | .then((res) => res.json()) 87 | .then((items) => { 88 | var item = items[0]; 89 | 90 | assert.ok(item._id === id, 'Item exists'); 91 | assert.ok(item.accessDate !== null, 'Access date is set'); 92 | }) 93 | ); 94 | }); 95 | 96 | it('Should return updates list and count', () => { 97 | const id = uuid(); 98 | const data = new Buffer('Hello'); 99 | const url = `http://localhost:${port}/files/${id}`; 100 | const date = new Date(); 101 | 102 | return fetch(url, { 103 | method: 'POST', 104 | headers: { 105 | 'content-type': 'text/plain', 106 | 'content-length': data.length, 107 | 'content-disposition': 'attachment; filename=test', 108 | }, 109 | body: data, 110 | }) 111 | .then((res) => assert.equal(res.status, 200, 'Status is 200')) 112 | .then(() => fetch( 113 | `http://localhost:${port}/storage/updates?` + qs.stringify({after: date.toISOString()}) 114 | )) 115 | .then((res) => { 116 | assert.equal(res.status, 200, 'Status is 200'); 117 | assert.equal(res.headers.get('content-type'), 'application/json', 'Content type is JSON'); 118 | 119 | return res.json(); 120 | }) 121 | .then((result) => { 122 | assert.equal(result.length, 1, '1 file updated'); 123 | assert.equal(result[0].name, 'test', 'Filename is `test`'); 124 | }) 125 | ; 126 | }); 127 | }); 128 | -------------------------------------------------------------------------------- /test/test.spec.js: -------------------------------------------------------------------------------- 1 | const assert = require('assert'); 2 | const fs = require('fs'); 3 | const {middleware, FileStore, NedbDataStorage, FsBlobStorage} = require('..'); 4 | const nedb = require('nedb'); 5 | const uuid = require('node-uuid'); 6 | const DEBUG = process.env.DEBUG === '1'; 7 | 8 | describe('Filestore', () => { 9 | var storage, dir; 10 | 11 | before(() => { 12 | dir = fs.mkdtempSync('/tmp/node-test-'); 13 | storage = new FileStore({ 14 | dataStore: new NedbDataStorage({ 15 | db: new nedb({ 16 | filename: dir + '/files.db', 17 | autoload: true, 18 | }), 19 | }), 20 | blobStore: new FsBlobStorage({dir}), 21 | }); 22 | 23 | }); 24 | 25 | it('Should add file', () => { 26 | var id = uuid(); 27 | var content = new Buffer('Hello'); 28 | var meta = { 29 | name: 'test.txt', 30 | contentLength: content.length, 31 | contentType: 'text/plain', 32 | }; 33 | 34 | return storage.put(id, meta, content) 35 | .then(({md5}) => { 36 | let filepath = storage.blobStore.getFilepath(md5); 37 | assert.ok(fs.existsSync(filepath), 'File exists'); 38 | }); 39 | }); 40 | 41 | it('Should delete file', () => { 42 | var id = uuid(); 43 | var content = new Buffer(uuid); 44 | var meta = { 45 | name: 'delete.txt', 46 | contentLength: content.length, 47 | contentType: 'text/plain', 48 | }; 49 | 50 | return storage.put(id, meta, content) 51 | .then(({md5}) => storage.delete(id).then(() => md5)) 52 | .then((md5) => { 53 | let filepath = storage.blobStore.getFilepath(md5); 54 | assert.ok(! fs.existsSync(filepath), 'File not exists'); 55 | }); 56 | }); 57 | }); 58 | -------------------------------------------------------------------------------- /todo.md: -------------------------------------------------------------------------------- 1 | # TODOLIST 2 | 3 | ## v0.3 4 | 5 | * Complete delete. 6 | * Sharding. 7 | 8 | ## v0.4 9 | 10 | * Storage size limitation. 11 | -------------------------------------------------------------------------------- /var/systemd/file-store.service: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=File Store Application 3 | 4 | [Service] 5 | Restart=always 6 | Type=simple 7 | ExecStart=/usr/bin/env node file-store /var/data/file-store/ --port=/var/run/file-store.sock 8 | 9 | [Install] 10 | WantedBy=default.target 11 | --------------------------------------------------------------------------------