├── .dockerignore ├── .gitattributes ├── .gitignore ├── .vscode ├── launch.json ├── settings.json └── tasks.json ├── Dockerfile ├── README.md ├── app.js ├── build.sh ├── config ├── config.json ├── express.config.js ├── logger.config.js └── mongoose.config.js ├── models └── files.model.js ├── package.json └── routings └── v1 ├── download └── documents-downloader.js ├── index.js └── upload └── documents-uploader.js /.dockerignore: -------------------------------------------------------------------------------- 1 | **/.classpath 2 | **/.dockerignore 3 | **/.env 4 | **/.git 5 | **/.gitignore 6 | **/.project 7 | **/.settings 8 | **/.toolstarget 9 | **/.vs 10 | **/.vscode 11 | **/*.*proj.user 12 | **/*.dbmdl 13 | **/*.jfm 14 | **/azds.yaml 15 | **/charts 16 | **/docker-compose* 17 | **/compose* 18 | **/Dockerfile* 19 | **/node_modules 20 | **/npm-debug.log 21 | **/obj 22 | **/secrets.dev.yaml 23 | **/values.dev.yaml 24 | README.md 25 | -------------------------------------------------------------------------------- /.gitattributes: -------------------------------------------------------------------------------- 1 | # Auto detect text files and perform LF normalization 2 | * text=auto 3 | 4 | # Custom for Visual Studio 5 | *.cs diff=csharp 6 | 7 | # Standard to msysgit 8 | *.doc diff=astextplain 9 | *.DOC diff=astextplain 10 | *.docx diff=astextplain 11 | *.DOCX diff=astextplain 12 | *.dot diff=astextplain 13 | *.DOT diff=astextplain 14 | *.pdf diff=astextplain 15 | *.PDF diff=astextplain 16 | *.rtf diff=astextplain 17 | *.RTF diff=astextplain 18 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules/ 2 | package-lock.json 3 | -------------------------------------------------------------------------------- /.vscode/launch.json: -------------------------------------------------------------------------------- 1 | { 2 | "configurations": [ 3 | { 4 | "name": "Docker Node.js Launch", 5 | "type": "docker", 6 | "request": "launch", 7 | "preLaunchTask": "docker-run: debug", 8 | "platform": "node" 9 | } 10 | ] 11 | } -------------------------------------------------------------------------------- /.vscode/settings.json: -------------------------------------------------------------------------------- 1 | { 2 | "workbench.colorCustomizations": { 3 | "titleBar.activeForeground": "#15202b", 4 | "titleBar.inactiveForeground": "#15202b99", 5 | "titleBar.activeBackground": "#ffc600", 6 | "titleBar.inactiveBackground": "#ffc60099", 7 | "activityBar.activeBackground": "#ffd133", 8 | "activityBar.activeBorder": "#009f7b", 9 | "activityBar.background": "#ffd133", 10 | "activityBar.foreground": "#15202b", 11 | "activityBar.inactiveForeground": "#15202b99", 12 | "activityBarBadge.background": "#009f7b", 13 | "activityBarBadge.foreground": "#e7e7e7", 14 | "statusBar.background": "#ffc600", 15 | "statusBar.foreground": "#15202b", 16 | "statusBarItem.hoverBackground": "#cc9e00" 17 | }, 18 | "peacock.color": "#FFC600" 19 | } -------------------------------------------------------------------------------- /.vscode/tasks.json: -------------------------------------------------------------------------------- 1 | { 2 | "version": "2.0.0", 3 | "tasks": [ 4 | { 5 | "type": "docker-build", 6 | "label": "docker-build", 7 | "platform": "node", 8 | "dockerBuild": { 9 | "dockerfile": "${workspaceFolder}/Dockerfile", 10 | "context": "${workspaceFolder}", 11 | "pull": true 12 | } 13 | }, 14 | { 15 | "type": "docker-run", 16 | "label": "docker-run: release", 17 | "dependsOn": [ 18 | "docker-build" 19 | ], 20 | "platform": "node" 21 | }, 22 | { 23 | "type": "docker-run", 24 | "label": "docker-run: debug", 25 | "dependsOn": [ 26 | "docker-build" 27 | ], 28 | "dockerRun": { 29 | "env": { 30 | "DEBUG": "*", 31 | "NODE_ENV": "development" 32 | } 33 | }, 34 | "node": { 35 | "enableDebugging": true 36 | } 37 | } 38 | ] 39 | } -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | FROM ubuntu:20.04 2 | ENV DEBIAN_FRONTEND noninteractive 3 | 4 | # Install dependencies 5 | RUN apt-get update && apt-get install -y --no-install-recommends apt-utils 6 | RUN apt-get update && apt-get install -y software-properties-common 7 | RUN add-apt-repository universe 8 | RUN apt-get update && apt-get install -y curl wget 9 | 10 | RUN curl -sL https://deb.nodesource.com/setup_14.x| bash 11 | RUN apt-get install nodejs -y 12 | RUN apt-get install build-essential -y 13 | 14 | # Work Directory 15 | RUN mkdir -p /usr/src/file_uploader 16 | WORKDIR /usr/src/file_uploader 17 | COPY ["package.json", "./"] 18 | 19 | #Envirenment Variables 20 | ENV NODE_ENV staging 21 | 22 | #Metadata 23 | LABEL version="1.5.0-d" 24 | LABEL description="Typical file uploader to MongoDB GridFS." 25 | LABEL maintainer "houssem.yahiaoui.ai@gmail.com" 26 | 27 | # Install Backend Depedencies 28 | RUN cd /usr/src/file_uploader && npm i 29 | 30 | # Copy and Install Packages 31 | COPY . . 32 | 33 | EXPOSE 3001 34 | 35 | ADD build.sh /usr/src/file_uploader/ 36 | RUN ["chmod", "+x", "/usr/src/file_uploader/build.sh"] 37 | ENTRYPOINT ["/usr/src/file_uploader/build.sh"] -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # MongoDB/GridFS File upload with NodeJS. 2 | [![Open Source Love](https://badges.frapsoft.com/os/v1/open-source.svg?v=102)](https://github.com/ellerbrock/open-source-badge/) 3 | [![Open Source Love](https://badges.frapsoft.com/os/mit/mit.svg?v=102)](https://github.com/ellerbrock/open-source-badge/) 4 | [![code style: prettier](https://img.shields.io/badge/code_style-prettier-ff69b4.svg?style=flat-square)](https://github.com/prettier/prettier) 5 | 6 | **PS**: This project have a **JS/Es\*** Version and **Typescript** version that i will ry to keep in sync in term of integration, so for sake of clarity the **DEV**(this branch) will have the **JS/ES\*** Version and [**ts-build**](https://github.com/houssem-yahiaoui/fileupload-nodejs/tree/ts-build) Branch will have the **Typescript** Version. 7 | 8 | 9 | As mouthfull as it sounds, but functional it will be, file upload is needed in any application 10 | there for this demo shows how we can upload files directly to our MongoDB using it's GridFS System. 11 | 12 | ## NOTICE: ## 13 | This project will be improved in near future to use the new supported version of file management 14 | in MongoDB, which is the GridFSBucket, this final support will be ported soon. 15 | 16 | ## Dependencies Management : 17 | 18 | In order to use this repository, please feel free to clone directly to your working directory and simply enter this command to your Terminal/CMD : 19 | 20 | ``` 21 | # yarn install 22 | ``` 23 | or, in case your don't dispose yarn, please use the following command: 24 | ``` 25 | # npm install 26 | ``` 27 | 28 | ## Launching the app : 29 | 30 | In case you want to run the app, simply use the following command : 31 | 32 | ``` 33 | # npm start 34 | ``` 35 | 36 | ## **Updates**: 37 | ### **File Upload** : 38 | I've changed the **upload** endpoints so no in order to upload your documents you will just call for the endpoint bellow : 39 | 40 | ``` 41 | /v1/bucket/upload 42 | ``` 43 | 44 | ### **Checking Uplaoded files** : 45 | As an added endpoint, now you can view all your uploaded files using this endpoint: 46 | 47 | ``` 48 | /v1/home 49 | ``` 50 | 51 | This endpoint will give you all uploaded docs within you Database in an array of object that will have the following: 52 | ``` 53 | [{ 54 | file_name: , 55 | file_id: , 56 | file_link: 57 | }] 58 | ``` 59 | 60 | 1. **file_name** : The uploaded file name. 61 | 2. **file_id** : An auto generated _if upon file upload over GridFS. 62 | 3. **file_link** : An auto generated download link of your file for easy download. 63 | 64 | ### **File Download** : 65 | I've changed the **download** endpoints so no in order to download your documents you will just call for the endpoint bellow : 66 | 67 | ``` 68 | /v1/bucket/download?document_id= 69 | ``` 70 | 71 | ### **Change Log** : 72 | 1 - Depedencies upgrade. 73 | 2 - Logger updates to match winston v3.3 74 | 75 | 76 | Finally, for even more details please check this Youtube tutorial I've created that explains just that. 77 | 78 | Link : https://youtu.be/pXHOF4GWuZQ 79 | 80 | ## Collaborators : 81 | - Houssem Yahiaoui (@houssem-yahiaoui). 82 | 83 | **Happy Coding =D** 84 | -------------------------------------------------------------------------------- /app.js: -------------------------------------------------------------------------------- 1 | const app = require('express')(); 2 | const config = require('./config/config'); 3 | const logger = require('./config/logger.config'); 4 | 5 | // Express conf ! 6 | require('./config/express.config')(app); 7 | 8 | // Mongoose Conf ! 9 | require('./config/mongoose.config')(config, logger); 10 | 11 | app.listen(config.port, () => { 12 | logger.info(`[*] 🚀 Listening on port : ${config.port} ..`); 13 | }); 14 | -------------------------------------------------------------------------------- /build.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | echo "[*] Welcome to Fileuploader Build Script." 3 | # here you can handle any configuraion/needs certification management before app startup. 4 | echo "[*] Starting Fileuploader .." 5 | node app.js 6 | -------------------------------------------------------------------------------- /config/config.json: -------------------------------------------------------------------------------- 1 | { 2 | "port": 3000, 3 | "db" : "mongodb://localhost:27017/testgridfs", 4 | "debug": true 5 | } 6 | -------------------------------------------------------------------------------- /config/express.config.js: -------------------------------------------------------------------------------- 1 | // Logic 2 | const logger = require('morgan'); 3 | const busboyBodyParser = require('busboy-body-parser'); 4 | 5 | module.exports = app => { 6 | app.use(logger('dev')); 7 | app.use((req, res, next) => { 8 | res.header('Access-Control-Allow-Origin', '*'); 9 | res.header('Access-Control-Allow-Methods', 'GET,PUT,POST,DELETE'); 10 | res.header('Access-Control-Allow-Headers', 'Content-Type, Authorization'); 11 | next(); 12 | }); 13 | app.use(busboyBodyParser({ limit: '50mb' })); 14 | 15 | //[*] V1 Routes Configuration. 16 | let viRoutes = require('../routings/v1/'); 17 | app.use('/v1', viRoutes); 18 | }; 19 | -------------------------------------------------------------------------------- /config/logger.config.js: -------------------------------------------------------------------------------- 1 | const { createLogger, format, transports } = require('winston'); 2 | 3 | const { 4 | combine, timestamp, printf, colorize, 5 | } = format; 6 | 7 | // TODO : add support for request ids. 8 | 9 | // a custom format that outputs request id 10 | const applicationLoggerFormat = printf(({ level, message, timestamp }) => { 11 | const possibleENV = ['production', 'staging']; 12 | const timestampShowcase = process.env.SHOW_LOGGER_TIME_SPAN || !possibleENV.includes(process.env.NODE_ENV) ? timestamp : ''; 13 | return `🔔 ${timestampShowcase} ${level} : ${message}`; 14 | }); 15 | 16 | const logger = createLogger({ 17 | format: process.env.NODE_ENV == "staging" || process.env.NODE_ENV == "production" ? combine(timestamp(), applicationLoggerFormat): combine(timestamp(), colorize(), applicationLoggerFormat), 18 | transports: [new transports.Console()], 19 | }); 20 | 21 | module.exports = logger; 22 | -------------------------------------------------------------------------------- /config/mongoose.config.js: -------------------------------------------------------------------------------- 1 | // Logic 2 | const mongoose = require('mongoose'); 3 | 4 | module.exports = (config, logger) => { 5 | // using new syntax for mongoose library 6 | // check the mongoose doc for more info 7 | mongoose.Promise = global.Promise; 8 | mongoose.set('useCreateIndex', true); 9 | mongoose.set('debug', config.debug); // for dev needs, help you understand executed db quiries. 10 | const options = { 11 | useNewUrlParser: true, 12 | keepAlive: 30000, 13 | useFindAndModify: false, 14 | useUnifiedTopology: true 15 | } 16 | mongoose.connect(config.db, options); 17 | 18 | // CONNECTION EVENTS 19 | mongoose.connection.on('open', () => { 20 | logger.info(`[*] Establishing Database Connection.`); 21 | }); 22 | mongoose.connection.on('connected', () => { 23 | logger.info(`[*] Database Connected.`); 24 | }); 25 | mongoose.connection.on('error', (err) => { 26 | logger.error(`[*] Database connection error: ${err}`); 27 | mongoose.connect(config.db, options); 28 | }); 29 | mongoose.connection.on('disconnected', () => { 30 | logger.warn('[*] Database Disconnected.'); 31 | mongoose.connect(config.db, options); 32 | }); 33 | }; 34 | -------------------------------------------------------------------------------- /models/files.model.js: -------------------------------------------------------------------------------- 1 | const mongoose = require('mongoose'); 2 | 3 | const fileSchema = new mongoose.Schema({ 4 | doc_id: { 5 | type: String 6 | }, 7 | length : { 8 | type: Number 9 | }, 10 | name: { 11 | type: String 12 | }, 13 | type: { 14 | type: String 15 | } 16 | }); 17 | 18 | module.exports = mongoose.model('File', fileSchema); 19 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "fileuploader-node-mongodbdemo", 3 | "version": "1.5.0-d", 4 | "description": "NodeJS file uploader using mongoDB gridFS system", 5 | "main": "app.js", 6 | "repository": { 7 | "type": "git", 8 | "url": "git+https://github.com/houssem-yahiaoui/fileupload-nodejs.git" 9 | }, 10 | "scripts": { 11 | "start": "node app.js", 12 | "format": "prettier-standard '**/*.js'" 13 | }, 14 | "lint-staged": { 15 | "linters": { 16 | "**/*.js": [ 17 | "prettier-standard", 18 | "git add" 19 | ] 20 | } 21 | }, 22 | "husky": { 23 | "hooks": { 24 | "pre-commit": "lint-staged" 25 | } 26 | }, 27 | "author": "Houssam Yahiaoui ", 28 | "license": "MIT", 29 | "dependencies": { 30 | "busboy-body-parser": "^0.3.2", 31 | "express": "^4.17.1", 32 | "file-type": "^16.3.0", 33 | "gridfs-stream": "^1.1.1", 34 | "mongoose": "^5.7.5", 35 | "morgan": "^1.10.0", 36 | "winston": "^3.3.3" 37 | }, 38 | "devDependencies": { 39 | "husky": "^6.0.0", 40 | "lint-staged": "^10.5.4", 41 | "prettier-standard": "^16.4.1" 42 | } 43 | } 44 | -------------------------------------------------------------------------------- /routings/v1/download/documents-downloader.js: -------------------------------------------------------------------------------- 1 | // Config 2 | const logger = require('../../../config/logger.config'); 3 | 4 | // Logic 5 | const mongoose = require("mongoose"); 6 | const fs = require('fs'); 7 | const Grid = require("gridfs-stream"); 8 | const fileType = require('file-type'); 9 | 10 | //models 11 | const Files = require('../../../models/files.model'); 12 | 13 | module.exports = router => { 14 | const conn = mongoose.connection; 15 | Grid.mongo = mongoose.mongo; 16 | let gfs; 17 | 18 | conn.once("open", () => { 19 | gfs = Grid(conn.db); 20 | 21 | router.get('/home', (req, res) => { 22 | Files.find() 23 | .exec() 24 | .then(files => { 25 | let uploadedFiles = files.map(file => ({ 26 | file_name: file.name, 27 | file_type: file.type, 28 | file_link: `http://${req.headers.host}/v1/bucket/download?document_id=${file.doc_id}` 29 | })); 30 | res.json({ 31 | success: true, 32 | uploadedFiles 33 | }) 34 | }) 35 | .catch(err => { 36 | // TODO : improve logging with slack msgs. 37 | logger.error(`[*] Error, while getting all uploaded file, with error: ${err}`); 38 | res.status(400).send({ 39 | message: `Error, while getting all uploaded file, with error: ${err}` 40 | }); 41 | }); 42 | }); 43 | 44 | router.get('/bucket/download', (req, res) => { 45 | let { 46 | document_id 47 | } = req.query; 48 | gfs.findOne({ 49 | _id: document_id 50 | }, (err, file) => { 51 | if (!file) { 52 | return res.status(404).send({ 53 | message: 'File was not found' 54 | }); 55 | } 56 | let data = []; 57 | let readstream = gfs.createReadStream({ 58 | filename: file.filename 59 | }); 60 | readstream.on('data', (chunk) => { 61 | data.push(chunk); 62 | }); 63 | readstream.on('end', () => { 64 | data = Buffer.concat(data); 65 | let type = fileType(data); 66 | res.writeHead(200, { 67 | 'Content-Type': type.mime, 68 | 'Content-disposition': 'attachment; filename=' + file.filename + '.' + type.ext, 69 | 'Content-Length': file.length 70 | }); 71 | res.end(data); 72 | }); 73 | readstream.on('error', (err) => { 74 | // TODO : improve logging with slack msgs. 75 | logger.error(`[*] Error, while downloading a file, with error: ${err}`); 76 | res.status(400).send({ 77 | message: `Error, while downloading a file, with error: ${err}` 78 | }); 79 | }); 80 | }); 81 | }); 82 | }); 83 | } -------------------------------------------------------------------------------- /routings/v1/index.js: -------------------------------------------------------------------------------- 1 | const router = require('express').Router(); 2 | 3 | // Requiring the document-uploader file here from index file. 4 | require('./upload/documents-uploader')(router); 5 | require('./download/documents-downloader')(router); 6 | 7 | module.exports = router; 8 | -------------------------------------------------------------------------------- /routings/v1/upload/documents-uploader.js: -------------------------------------------------------------------------------- 1 | // Config 2 | const logger = require('../../../config/logger.config'); 3 | 4 | // Logic 5 | const mongoose = require("mongoose"); 6 | const Grid = require('gridfs-stream'); 7 | 8 | //models 9 | const Files = require('../../../models/files.model'); 10 | 11 | module.exports = router => { 12 | const conn = mongoose.connection; 13 | Grid.mongo = mongoose.mongo; 14 | let gfs; 15 | 16 | conn.once("open", () => { 17 | gfs = Grid(conn.db); 18 | 19 | router.post('/bucket/upload', (req, res) => { 20 | let { 21 | file 22 | } = req.files; 23 | let writeStream = gfs.createWriteStream({ 24 | filename: `${file.name}`, 25 | mode: 'w', 26 | content_type: file.mimetype 27 | }); 28 | writeStream.on('close', function (uploadedFile) { 29 | Files.create({ 30 | doc_id: uploadedFile._id, 31 | length: uploadedFile.length, 32 | name: uploadedFile.filename, 33 | type: uploadedFile.contentType 34 | }) 35 | .then(_ => res.json({ 36 | success: true, 37 | message: "File was saved with success" 38 | })) 39 | .catch(err => { 40 | console.log(err); 41 | // TODO : improve logging with slack messages 42 | logger.error(`[*] Error, while uploading new files, with error: ${err}`); 43 | res.status(500).json({ 44 | message: `[*] Error while uploading new files, with error: ${err}` 45 | }) 46 | }) 47 | }); 48 | writeStream.write(file.data); 49 | writeStream.end(); 50 | }); 51 | }); 52 | } --------------------------------------------------------------------------------