├── assets ├── .keep └── images │ ├── cb_logo.png │ ├── upload.svg │ └── syncing.svg ├── logs └── .keep ├── uploads └── .keep ├── DEMO.gif ├── scripts ├── configuration.sh └── validate-coverage.sh ├── controllers ├── index.js └── root.js ├── .gitignore ├── envConfigLoader.js ├── sentry.js ├── routes ├── mapper.js ├── dynamicRoutes.js └── index.js ├── ecosystem.config.js ├── app.js ├── README.md ├── package.json ├── .circleci └── config.yml ├── services └── csvProcessor.js ├── logger.js └── views └── pages └── home.ejs /assets/.keep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /logs/.keep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /uploads/.keep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /DEMO.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Codebrahma/csv-to-graphql/HEAD/DEMO.gif -------------------------------------------------------------------------------- /scripts/configuration.sh: -------------------------------------------------------------------------------- 1 | cp ./../shared/config.json ./config/config.json 2 | cp ./../shared/*.env ./config/ -------------------------------------------------------------------------------- /assets/images/cb_logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Codebrahma/csv-to-graphql/HEAD/assets/images/cb_logo.png -------------------------------------------------------------------------------- /controllers/index.js: -------------------------------------------------------------------------------- 1 | const RootController = require('./root'); 2 | 3 | module.exports = { 4 | RootController 5 | }; 6 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | credentials.json 2 | token.json 3 | config/*.env 4 | node_modules/ 5 | logs/*.log 6 | .nyc_output/ 7 | coverage/ 8 | uploads/* 9 | !uploads/.keep -------------------------------------------------------------------------------- /envConfigLoader.js: -------------------------------------------------------------------------------- 1 | const path = require('path'); 2 | 3 | const loader = (envName) => { 4 | const configPath = path.join(__dirname, 'config', `${envName || 'development'}.env`) 5 | require('dotenv').config({ path: configPath }); 6 | } 7 | 8 | module.exports = loader; 9 | -------------------------------------------------------------------------------- /sentry.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Sentry initialization 3 | */ 4 | const Sentry = require('@sentry/node'); 5 | module.exports = () => { 6 | if (process.env.NODE_ENV === 'test') { return; } 7 | Sentry.init({ 8 | environment: (process.env.NODE_ENV || 'development'), 9 | dsn: process.env.SENTRY_DSN 10 | }); 11 | } 12 | -------------------------------------------------------------------------------- /routes/mapper.js: -------------------------------------------------------------------------------- 1 | const capitalize = require('lodash/capitalize'); 2 | const controllers = require('./../controllers'); 3 | 4 | const mapper = (to) => { 5 | let [ controllerKey = 'root', actionName = 'index' ] = (to || "").split("#"); 6 | const controllerName = `${capitalize(controllerKey)}Controller`; 7 | return controllers[controllerName][actionName]; 8 | } 9 | 10 | module.exports = mapper; 11 | -------------------------------------------------------------------------------- /scripts/validate-coverage.sh: -------------------------------------------------------------------------------- 1 | COVERAGE_THRESHOLD=90 2 | 3 | node_modules/.bin/nyc \ 4 | --reporter html \ 5 | --reporter text-summary \ 6 | --reporter lcov \ 7 | npm run specs 8 | 9 | percentage=`printf "%.0f" $(node_modules/.bin/coverage-percentage ./coverage/lcov.info --lcov)` 10 | echo "Test Coverage: $percentage%" 11 | 12 | if [ "$percentage" -lt "$COVERAGE_THRESHOLD" ]; then 13 | echo "Code coverage is lesser than $COVERAGE_THRESHOLD%. Please add tests for your code changes." 14 | exit 1 15 | fi 16 | -------------------------------------------------------------------------------- /routes/dynamicRoutes.js: -------------------------------------------------------------------------------- 1 | const multer = require('multer'); 2 | const upload = multer({ dest: 'uploads/' }); 3 | 4 | const dynamicRoutes = [ 5 | { 6 | path: '/', 7 | to: 'root#index', 8 | via: ['get'], 9 | }, 10 | { 11 | path: '/upload', 12 | to: 'root#upload', 13 | via: ['post'], 14 | middlewares: [upload.single('csv')], 15 | }, 16 | { 17 | path: '/csv-to-graphql', 18 | to: 'root#csvToGraphql', 19 | via: ['get', 'post'] 20 | } 21 | ]; 22 | 23 | module.exports = dynamicRoutes; 24 | -------------------------------------------------------------------------------- /routes/index.js: -------------------------------------------------------------------------------- 1 | const { getRequestLogger, getResponseLogger } = require('./../logger'); 2 | const mapper = require('./mapper'); 3 | const dynamicRoutes = require('./dynamicRoutes'); 4 | 5 | module.exports = (app) => { 6 | // Adding logger middlewares 7 | app.use(getRequestLogger()); 8 | app.use(getResponseLogger()); 9 | 10 | dynamicRoutes.forEach((dynamicRoute) => { 11 | const { path, via: httpMethods, to, middlewares: routeMiddlewares = [] } = dynamicRoute; 12 | httpMethods.forEach((httpMethod) => { 13 | const method = httpMethod.toLowerCase(); 14 | app[method](path, ...routeMiddlewares, mapper(to)); 15 | }); 16 | }); 17 | } 18 | -------------------------------------------------------------------------------- /ecosystem.config.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | apps: [{ 3 | name: '[APP NAME]', 4 | script: './app.js' 5 | }], 6 | deploy: { 7 | staging: { 8 | user: 'ubuntu', 9 | host: '[PRODUCTION HOST NAME]', 10 | key: '[PATH TO SSH PUBLIC KEY]', 11 | ref: '[BRANCH TO DEPLOY]', 12 | repo: '[GIT REPO]', 13 | path: '[DEPLOYMENT PATH]', 14 | 'post-deploy': 'sh scripts/configuration.sh && npm install && pm2 startOrRestart ecosystem.config.js', 15 | env: { 16 | "NODE_ENV": "staging" 17 | } 18 | }, 19 | production: { 20 | user: 'ubuntu', 21 | host: '[PRODUCTION HOST NAME]', 22 | key: '[PATH TO SSH PUBLIC KEY]', 23 | ref: '[BRANCH TO DEPLOY]', 24 | repo: '[GIT REPO]', 25 | path: '[DEPLOYMENT PATH]', 26 | 'post-deploy': 'sh scripts/configuration.sh && npm install && pm2 startOrRestart ecosystem.config.js', 27 | "env" : { 28 | "NODE_ENV": "production" 29 | } 30 | } 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /controllers/root.js: -------------------------------------------------------------------------------- 1 | const fs = require('fs'); 2 | const isFileExists = require('util').promisify(fs.exists); 3 | const graphqlHTTP = require('express-graphql'); 4 | 5 | const CSVProcessor = require('./../services/csvProcessor'); 6 | 7 | class RootController { 8 | static index(_req, res) { 9 | return res.render('pages/home'); 10 | } 11 | 12 | static upload(req, res) { 13 | if (!req.file || !req.file.path) { 14 | return res.redirect('/?notice=CSV File is required to proceed'); 15 | } 16 | 17 | res.redirect(`/csv-to-graphql?path=${req.file.path}`); 18 | } 19 | 20 | static async csvToGraphql(req, res) { 21 | const { 22 | query: { path = '' }, 23 | } = req; 24 | 25 | if (!path) { 26 | return res.redirect('/?notice=CSV File is required to proceed'); 27 | } 28 | 29 | if (!(await isFileExists(path))) { 30 | return res.redirect('/?notice=CSV File not found. Perhaps it could be expired or deleted.'); 31 | } 32 | 33 | const schema = await CSVProcessor.process(path); 34 | graphqlHTTP({ 35 | schema, 36 | graphiql: true, 37 | })(req, res); 38 | } 39 | } 40 | 41 | module.exports = RootController; 42 | -------------------------------------------------------------------------------- /app.js: -------------------------------------------------------------------------------- 1 | require('./envConfigLoader')(process.env.NODE_ENV); // Load env based config 2 | 3 | const express = require('express'); 4 | const bodyParser = require('body-parser'); 5 | const cookieParser = require('cookie-parser'); 6 | 7 | const app = express(); 8 | const http = require('http'); 9 | const initializeSentry = require('./sentry'); 10 | 11 | const initializeApp = () => { 12 | initializeSentry(); // Sentry config initialization 13 | 14 | /** 15 | * Logger creation and configuration 16 | */ 17 | const logger = require('./logger').makeLogger('APP'); 18 | logger.info(`Loading env... ${process.env.NODE_ENV}`); 19 | 20 | app.set('view engine', 'ejs'); 21 | app.use(function(_req, res, next) { 22 | res.header("Access-Control-Allow-Origin", "*"); 23 | res.header("Access-Control-Allow-Headers", "Origin, X-Requested-With, Content-Type, Accept, jwt"); 24 | next(); 25 | }); 26 | 27 | app.use(bodyParser.json()); 28 | app.use(cookieParser()); 29 | app.use(express.static(`${__dirname}/assets`)); 30 | require('./routes')(app); 31 | 32 | const httpServer = http.createServer(app); 33 | httpServer.listen(process.env.PORT || 3000, () => logger.info('http 3000')); 34 | return httpServer; 35 | } 36 | 37 | const mainApp = initializeApp(); 38 | module.exports = mainApp; 39 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # CSV to GraphQL 2 | 3 | [DEMO](https://csv-to-graphql.herokuapp.com) 4 | 5 | ![CSV to GraphQL demo](./DEMO.gif) 6 | 7 | ## Dev environment setup 8 | 9 | ### Setup Prerequisites 10 | Install the below softwares 11 | 12 | | S.No | Software | Version | Reference | 13 | | :--------- | :-------- | :------- | :-------- | 14 | | 1 | node | >= 9.10.0 | [Node.js installation with nvm](https://blog.pm2.io/install-node-js-with-nvm/) | 15 | | 2 | npm | >= 6.4.1 | [npm installation](https://www.npmjs.com/get-npm) | 16 | 17 | ### Setup Instructions 18 | 1. Clone the project 19 | 20 | ```shell 21 | git clone git@github.com:Codebrahma/csv-to-graphql.git 22 | ``` 23 | 2. Switch your current working directory to the root directory of the project 24 | 25 | ```shell 26 | cd csv-to-graphql 27 | ``` 28 | 3. Install the dependencies required by the app and development environment 29 | 30 | ```shell 31 | npm install 32 | ``` 33 | 4. Start the server 34 | 35 | ```shell 36 | npm start 37 | ``` 38 | 39 | ## Developer notes 40 | 41 | ### To Contribute 42 | 1. Create a new feature branch from `master` branch 43 | 2. Make your contribution or changes 44 | 3. Push your branch to github.com 45 | 4. Raise PR to `master` from your feature branch. 46 | 5. Assign `ready-for-review` label to your PR once you are done. -------------------------------------------------------------------------------- /assets/images/upload.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | upload 5 | Created with Sketch. 6 | 7 | 8 | 9 | 10 | 11 | 12 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "nodejs-backend-boilerplate", 3 | "version": "0.0.1", 4 | "description": "[My product description]", 5 | "main": "index.js", 6 | "scripts": { 7 | "start": "nodemon app.js", 8 | "dev": "nodemon app.js", 9 | "specs": "node_modules/.bin/mocha --exit", 10 | "test": "sh scripts/validate-coverage.sh" 11 | }, 12 | "author": "Anand Narayan Sivaprakasam", 13 | "license": "ISC", 14 | "dependencies": { 15 | "@sentry/node": "^4.1.1", 16 | "body-parser": "^1.18.3", 17 | "cookie-parser": "^1.4.3", 18 | "csv-parser": "^2.3.1", 19 | "dotenv": "^6.0.0", 20 | "ejs": "^2.7.1", 21 | "express": "^4.16.3", 22 | "express-certbot-endpoint": "^1.0.0", 23 | "express-graphql": "^0.9.0", 24 | "express-winston": "^3.0.0", 25 | "graphql": "^14.5.8", 26 | "lodash": "^4.17.21", 27 | "migrate": "^1.6.1", 28 | "multer": "^1.3.1", 29 | "opn": "^5.3.0", 30 | "request": "^2.88.0", 31 | "request-promise": "^4.2.2", 32 | "winston": "^3.1.0", 33 | "winston-papertrail": "^1.0.5" 34 | }, 35 | "devDependencies": { 36 | "chai": "^4.2.0", 37 | "chai-http": "^4.2.0", 38 | "coverage-percentage": "0.0.2", 39 | "factory-girl": "^5.0.2", 40 | "mocha": "^5.2.0", 41 | "moment": "^2.22.2", 42 | "nock": "^10.0.1", 43 | "nock-vcr-recorder": "^0.1.5", 44 | "nodemon": "^1.18.4", 45 | "nyc": "^13.1.0", 46 | "sequelize-cli": "^5.4.0", 47 | "sinon": "^7.0.0" 48 | }, 49 | "engines": { 50 | "node": "9.10.0" 51 | } 52 | } 53 | -------------------------------------------------------------------------------- /assets/images/syncing.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | Shape 5 | Created with Sketch. 6 | 7 | 8 | 9 | 10 | 11 | 12 | -------------------------------------------------------------------------------- /.circleci/config.yml: -------------------------------------------------------------------------------- 1 | # Javascript Node CircleCI 2.0 configuration file 2 | # 3 | # Check https://circleci.com/docs/2.0/language-javascript/ for more details 4 | # 5 | version: 2 6 | jobs: 7 | build: 8 | docker: 9 | # specify the version you desire here 10 | - image: circleci/node:9.10.0 11 | - image: circleci/postgres:9.6-alpine 12 | environment: 13 | POSTGRES_USER: ubuntu 14 | POSTGRES_DB: test 15 | POSTGRES_PASSWORD: "" 16 | 17 | # Specify service dependencies here if necessary 18 | # CircleCI maintains a library of pre-built images 19 | # documented at https://circleci.com/docs/2.0/circleci-images/ 20 | # - image: circleci/mongo:3.4.4 21 | 22 | working_directory: ~/repo 23 | 24 | steps: 25 | - checkout 26 | 27 | # Download and cache dependencies 28 | - restore_cache: 29 | keys: 30 | - v1-dependencies-{{ checksum "package.json" }} 31 | # fallback to using the latest cache if no exact match is found 32 | - v1-dependencies- 33 | 34 | - run: sudo apt install -y postgresql-client || true 35 | - run: sudo npm install sequelize-cli -g 36 | - run: sudo npm install sequelize -g 37 | - run: sudo npm install pg -g 38 | - run: sudo npm install codecov -g 39 | - run: cp config/config.json.example config/config.json 40 | - run: cp config/sample.env.example config/test.env 41 | - run: 42 | name: DB Creation 43 | command: NODE_ENV=test sequelize db:create 44 | background: true 45 | 46 | - run: npm install 47 | 48 | - save_cache: 49 | paths: 50 | - node_modules 51 | key: v1-dependencies-{{ checksum "package.json" }} 52 | 53 | - run: NODE_ENV=test sequelize db:migrate 54 | - run: echo $CIRCLE_BRANCH 55 | - run: npm run test 56 | - run: codecov 57 | -------------------------------------------------------------------------------- /services/csvProcessor.js: -------------------------------------------------------------------------------- 1 | const csv = require('csv-parser'); 2 | const fs = require('fs'); 3 | const _ = require('lodash'); 4 | const { 5 | GraphQLSchema, 6 | GraphQLObjectType, 7 | GraphQLString, 8 | GraphQLList, 9 | GraphQLInt, 10 | } = require('graphql'); 11 | 12 | class CSVProcessor { 13 | static async parse(file) { 14 | return new Promise(resolve => { 15 | const results = []; 16 | fs.createReadStream(file) 17 | .pipe( 18 | csv({ 19 | mapHeaders: ({ header }) => _.snakeCase(header.trim()), 20 | mapValues: ({ value }) => 21 | value ? unescape(new String(value).trim()) : null, 22 | }), 23 | ) 24 | .on('data', data => results.push(data)) 25 | .on('end', () => resolve(results)); 26 | }); 27 | } 28 | 29 | static getType(headers) { 30 | return new GraphQLObjectType({ 31 | name: 'csv', 32 | description: '...', 33 | 34 | fields: () => 35 | headers.reduce( 36 | (fields, header) => ({ 37 | ...fields, 38 | [header]: { 39 | type: GraphQLString, 40 | resolve: row => row[header], 41 | }, 42 | }), 43 | {}, 44 | ), 45 | }); 46 | } 47 | 48 | static generateGraphQLSchema(rows, headers) { 49 | return new GraphQLSchema({ 50 | query: new GraphQLObjectType({ 51 | name: 'csv_to_graphql', 52 | description: '...', 53 | 54 | fields: () => ({ 55 | csv: { 56 | type: new GraphQLList(CSVProcessor.getType(headers)), 57 | args: { 58 | limit: { type: GraphQLInt }, 59 | ...headers.reduce( 60 | (args, header) => ({ 61 | ...args, 62 | [header]: { type: GraphQLString }, 63 | }), 64 | {}, 65 | ), 66 | }, 67 | resolve: (_root, args) => { 68 | const sortables = _.pick(args, headers); 69 | const orderedRows = _.orderBy( 70 | rows, 71 | _.keys(sortables), 72 | _.chain(sortables) 73 | .values() 74 | .map(e => e.toLowerCase()) 75 | .value(), 76 | ); 77 | 78 | return typeof args.limit === 'undefined' 79 | ? orderedRows 80 | : orderedRows.slice(0, args.limit); 81 | }, 82 | }, 83 | }), 84 | }), 85 | }); 86 | } 87 | 88 | static async process(file) { 89 | const rows = await CSVProcessor.parse(file); 90 | const headers = _.keys(rows[0]); 91 | 92 | return CSVProcessor.generateGraphQLSchema(rows, headers); 93 | } 94 | } 95 | 96 | module.exports = CSVProcessor; 97 | -------------------------------------------------------------------------------- /logger.js: -------------------------------------------------------------------------------- 1 | const { createLogger, format, transports } = require('winston'); 2 | const expressWinston = require('express-winston'); 3 | //const { Papertrail } = require('winston-papertrail'); 4 | const { combine, timestamp, label, printf } = format; 5 | const env = process.env.NODE_ENV || 'development'; 6 | 7 | const formatMessageWithTimestampAndLabel = printf( 8 | process.env.PAPERTRAIL_HOST ? 9 | (info) => (`<<<<[${info.level}] - [${info.label}]: ${info.message}>>>>`) 10 | : (info) => (`[${info.level}] - [${info.label}]: ${info.message}`) 11 | ); 12 | 13 | const logTransports = () => { 14 | switch(env) { 15 | case 'development': 16 | case 'staging': 17 | return [ new transports.Console() ]; 18 | case 'test': 19 | return [ new transports.File({ filename: `./logs/${process.env.NODE_ENV}.log` }) ]; 20 | default: 21 | return [ new transports.Console() ]; 22 | // return [ 23 | // new transports.Papertrail({ 24 | // host: process.env.PAPERTRAIL_HOST, 25 | // port: process.env.PAPERTRAIL_PORT, 26 | // inlineMeta: true, 27 | // logFormat: (_, message) => { 28 | // const messagesToDisplay = message.match(/<<<<(.*)>>>>/) 29 | // return (messagesToDisplay && messagesToDisplay[1]) 30 | // }, 31 | // }) 32 | // ] 33 | } 34 | } 35 | 36 | const formatter = (tagName) => combine( 37 | label({ label: tagName }), 38 | timestamp(), 39 | formatMessageWithTimestampAndLabel 40 | ); 41 | 42 | class Logger { 43 | static getInstance() { 44 | if (Logger.instance) { return Logger.instance; } 45 | Logger.instance = new Logger(); 46 | return Logger.instance; 47 | } 48 | 49 | constructor() { 50 | this.handlers = {}; 51 | this.makeLogger = this.makeLogger.bind(this); 52 | this.controllerLogger = this.controllerLogger.bind(this); 53 | this.getRequestLogger = this.getRequestLogger.bind(this); 54 | this.getResponseLogger = this.getResponseLogger.bind(this); 55 | } 56 | 57 | makeLogger(tagName) { 58 | if (this.handlers[tagName]) { return this.handlers[tagName]; } 59 | this.handlers[tagName] = createLogger({ 60 | transports: logTransports(), 61 | format: formatter(tagName) 62 | }); 63 | return this.handlers[tagName]; 64 | } 65 | 66 | controllerLogger(tagName, message) { 67 | if (this.handlers[tagName]) { return this.handlers[tagName] } 68 | this.handlers[tagName] = expressWinston.logger({ 69 | winstonInstance: this.makeLogger(tagName), 70 | meta: true, 71 | msg: message 72 | }); 73 | return this.handlers[tagName]; 74 | } 75 | 76 | getRequestLogger() { 77 | const requestLoggerInstance = this.makeLogger('REQUEST'); 78 | return (req, _res, next) => { 79 | requestLoggerInstance.info(`Started ${req.method}: ${req.url}`); 80 | return next(); 81 | } 82 | } 83 | 84 | getResponseLogger() { 85 | const message = "Processed {{req.method}}: {{req.url}} in {{res.responseTime}}ms with {{res.statusCode}}"; 86 | return this.controllerLogger('RESPONSE', message); 87 | } 88 | } 89 | 90 | Logger.instance = null; 91 | 92 | module.exports = Logger.getInstance(); 93 | -------------------------------------------------------------------------------- /views/pages/home.ejs: -------------------------------------------------------------------------------- 1 | 2 | 3 | 8 | 9 | 13 | 14 | 216 | 217 | 218 | 219 |
220 |
221 | 222 | 223 | 224 |
225 |
226 |
Drop CSV file to upload
227 |
228 |
229 | 233 | 234 | 235 |
236 |
237 | 241 | 242 |
243 |
244 |
245 | 246 | 247 | 314 | 315 | --------------------------------------------------------------------------------