├── .eslintrc.json ├── .gitignore ├── .nvmrc ├── API_DOCUMENTATION.apib ├── README.md ├── config.js ├── db-schema.png ├── index.js ├── models ├── contribution │ ├── contribution.js │ ├── contribution.spec.js │ └── index.js ├── db │ ├── config.js │ ├── index.js │ ├── migrations │ │ ├── 1-create-user.js │ │ ├── 2-create-repository.js │ │ ├── 3-create-contribution.js │ │ └── 4-add-indexes.js │ └── utils.js ├── github │ ├── api.js │ ├── api.spec.js │ └── index.js ├── redis │ ├── config.js │ └── index.js ├── repository │ ├── index.js │ ├── repository.js │ └── repository.spec.js ├── test.setup.js └── user │ ├── index.js │ ├── user.js │ └── user.spec.js ├── package-lock.json ├── package.json ├── scripts ├── migrate-db.js └── trigger.js ├── web ├── config.js ├── index.js ├── middleware │ ├── index.js │ ├── queryParser.js │ ├── requestLogger.js │ └── validator.js ├── router │ ├── contribution │ │ ├── getById.js │ │ ├── getById.spec.js │ │ ├── getByName.js │ │ ├── getByName.spec.js │ │ └── index.js │ ├── healthz │ │ ├── get.js │ │ ├── get.spec.js │ │ └── index.js │ ├── index.js │ ├── repository │ │ ├── getById.js │ │ ├── getById.spec.js │ │ ├── getByName.js │ │ ├── getByName.spec.js │ │ └── index.js │ ├── router.js │ └── trigger │ │ ├── index.js │ │ ├── post.js │ │ └── post.spec.js ├── server.js ├── server.spec.js └── test.setup.js └── worker ├── config.js ├── handlers ├── contributions.js ├── contributions.spec.js ├── index.js ├── repository.js ├── repository.spec.js ├── trigger.js └── trigger.spec.js ├── index.js ├── router ├── healthz │ ├── get.js │ ├── get.spec.js │ └── index.js ├── index.js └── router.js ├── server.js ├── test.setup.js └── worker.js /.eslintrc.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": [ 3 | "airbnb-base" 4 | ], 5 | "env": { 6 | "es6": true, 7 | "node": true, 8 | "mocha": true 9 | }, 10 | "parserOptions": { 11 | "sourceType": "script" 12 | }, 13 | "rules": { 14 | "semi": ["error", "never"], 15 | "comma-dangle": "off", 16 | "no-use-before-define": "off", 17 | "arrow-parens": ["error", "always"], 18 | "max-len": ["error", 120], 19 | "import/no-extraneous-dependencies": ["error", { "devDependencies": true }] 20 | } 21 | } 22 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Logs 2 | logs 3 | *.log 4 | 5 | # Dependency directory 6 | # https://www.npmjs.org/doc/misc/npm-faq.html#should-i-check-my-node_modules-folder-into-git 7 | node_modules 8 | 9 | # Misc 10 | .env 11 | -------------------------------------------------------------------------------- /.nvmrc: -------------------------------------------------------------------------------- 1 | 8 2 | -------------------------------------------------------------------------------- /API_DOCUMENTATION.apib: -------------------------------------------------------------------------------- 1 | FORMAT: 1A 2 | 3 | # Polls 4 | 5 | Polls is a simple API allowing consumers to view polls and vote in them. 6 | 7 | ## Questions Collection [/questions] 8 | 9 | ### List All Questions [GET] 10 | 11 | + Response 200 (application/json) 12 | 13 | { 14 | "question": "Favourite programming language?", 15 | "choices": [ 16 | { 17 | "choice": "Swift", 18 | "votes": 2048 19 | }, { 20 | "choice": "Python", 21 | "votes": 1024 22 | } 23 | ] 24 | } 25 | 26 | ### Create a New Question [POST] 27 | You may create your own question using this action. 28 | This action takes a JSON payload as part of the request. 29 | Response then return specific header and body. 30 | 31 | + Request (application/json) 32 | 33 | { 34 | "question": "Favourite programming language?", 35 | "choices": [ 36 | "Swift", 37 | "Python", 38 | "Objective-C", 39 | "Ruby" 40 | ] 41 | } 42 | 43 | + Response 201 (application/json) 44 | 45 | + Headers 46 | 47 | Location: /questions/1 48 | 49 | + Body 50 | 51 | { 52 | "question": "Favourite programming language?", 53 | "published_at": "2014-11-11T08:40:51.620Z", 54 | "url": "/questions/1", 55 | "choices": [ 56 | { 57 | "choice": "Swift", 58 | "url": "/questions/1/choices/1", 59 | "votes": 0 60 | }, { 61 | "choice": "Python", 62 | "url": "/questions/1/choices/2", 63 | "votes": 0 64 | }, { 65 | "choice": "Objective-C", 66 | "url": "/questions/1/choices/3", 67 | "votes": 0 68 | }, { 69 | "choice": "Ruby", 70 | "url": "/questions/1/choices/4", 71 | "votes": 0 72 | } 73 | ] 74 | } 75 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # RisingStack bootcamp 2 | 3 | 4 | 5 | - [RisingStack bootcamp](#risingstack-bootcamp) 6 | - [General](#general) 7 | - [Installation](#installation) 8 | - [Install Node 8 and the latest `npm`.](#install-node-8-and-the-latest-npm) 9 | - [Install PostgreSQL on your system](#install-postgresql-on-your-system) 10 | - [Install Redis on your system](#install-redis-on-your-system) 11 | - [Install project dependencies](#install-project-dependencies) 12 | - [Set up your development environment](#set-up-your-development-environment) 13 | - [Steps](#steps) 14 | - [1. Create a simple web application and make the test pass](#1-create-a-simple-web-application-and-make-the-test-pass) 15 | - [2. Create a model for the Github API](#2-create-a-model-for-the-github-api) 16 | - [3. Implement the database models](#3-implement-the-database-models) 17 | - [4. Implement helper functions for the database models](#4-implement-helper-functions-for-the-database-models) 18 | - [5. Create a worker process](#5-create-a-worker-process) 19 | - [6. Implement a REST API](#6-implement-a-rest-api) 20 | - [7. Prepare your service for production](#7-prepare-your-service-for-production) 21 | 22 | 23 | 24 | ## General 25 | 26 | - Always handle errors (at the right place, sometimes it's better to let it propagate) 27 | - Write unit tests for everything 28 | - Use [`winston`](https://github.com/winstonjs/winston) for logging 29 | - Use [`joi`](https://github.com/hapijs/joi) for schema validation (you can validate anything from environment variables to the body of an HTTP request) 30 | - Leverage [`lodash`](https://lodash.com/docs) and [`lodash/fp`](https://github.com/lodash/lodash/wiki/FP-Guide) functions 31 | - Avoid callbacks, use [`util.promisify`](https://nodejs.org/api/util.html#util_util_promisify_original) to convert them 32 | - Read the [You don't know JS](https://github.com/getify/You-Dont-Know-JS) books 33 | - Use the [latest and greatest ES features](http://node.green/) 34 | - Follow the [clean coding](https://blog.risingstack.com/javascript-clean-coding-best-practices-node-js-at-scale/) guidelines 35 | - Enjoy! :) 36 | 37 | ## Installation 38 | 39 | ### Install Node 8 and the latest `npm`. 40 | 41 | For this, use [nvm](https://github.com/creationix/nvm), the Node version manager. 42 | 43 | ```sh 44 | $ nvm install 8 45 | # optional: set it as default 46 | $ nvm alias default 8 47 | # install latest npm 48 | $ npm install -g npm 49 | ``` 50 | 51 | ### Install PostgreSQL on your system 52 | 53 | Preferably with [Homebrew](https://brew.sh/). 54 | 55 | ```sh 56 | $ brew install postgresql 57 | # create a table 58 | $ createdb risingstack_bootcamp 59 | ``` 60 | 61 | You should also install a visual tool for PostgreSQL, pick one: 62 | - [pgweb](https://github.com/sosedoff/pgweb) (`$ brew cask install pgweb`) 63 | - [postico](https://eggerapps.at/postico/) 64 | 65 | ### Install Redis on your system 66 | 67 | Preferably with [Homebrew](https://brew.sh/). 68 | 69 | ```sh 70 | $ brew install redis 71 | ``` 72 | 73 | ### Install project dependencies 74 | 75 | You only need to install them once, necessary packages are included for all of the steps. 76 | 77 | ```sh 78 | $ npm install 79 | ``` 80 | 81 | ### Set up your development environment 82 | 83 | *If you already have a favorite editor or IDE, you can skip this step.* 84 | 85 | 1. Download [Visual Studio Code](https://code.visualstudio.com/) 86 | 2. Install the following extensions: 87 | - [ESLint](https://marketplace.visualstudio.com/items?itemName=dbaeumer.vscode-eslint) 88 | - [npm](https://marketplace.visualstudio.com/items?itemName=eg2.vscode-npm-script) 89 | - [npm-intellisense](https://marketplace.visualstudio.com/items?itemName=christian-kohler.npm-intellisense) 90 | - [Flat UI Theme (optional)](https://marketplace.visualstudio.com/items?itemName=danibram.theme-flatui) 91 | 3. Read the [Node.js tutorial](https://code.visualstudio.com/docs/nodejs/nodejs-tutorial) 92 | 93 | ## Steps 94 | 95 | ### 1. Create a simple web application and make the test pass 96 | 97 | Tasks: 98 | - [ ] Create a `GET` endpoint `/hello` returning `Hello Node.js!` in the response body, use the middleware of the `koa-router` package 99 | - [ ] Use the `PORT` environment variable to set the port, **make it required** 100 | - [ ] Make the tests pass (`npm run test-web`) 101 | - [ ] Run the application (eg. `PORT=3000 npm start` and try if it breaks when `PORT` is not provided) 102 | 103 | Readings: 104 | - [12 factor - Config](https://12factor.net/config) 105 | - [Node `process`](https://nodejs.org/api/process.html) 106 | - [Koa](http://koajs.com/) web framework 107 | - [Koa router](https://github.com/alexmingoia/koa-router/tree/master) 108 | - [Mocha](https://mochajs.org/) test framework 109 | - [Chai](http://chaijs.com/api/bdd/) assertion library 110 | 111 | ### 2. Create a model for the Github API 112 | 113 | In this step you will implement two functions, wrappers for the GitHub API. You will use them to get information from GitHub later. 114 | 115 | Tasks: 116 | - [ ] `searchRepositories(query)`: should search for repositories given certain programming languages and/or keywords 117 | - The `query` function parameter is an `Object` of key-value pairs of the request query parameters (eg. `{ q: 'language:javascript' }`, defaults to `{}`) 118 | - It returns a `Promise` of the HTTP response without modification 119 | - [ ] `getContributors(repository, query)`: get contributors list with additions, deletions, and commit counts (statistics) 120 | - `repository` function parameter is a String of the repository full name, including the owner (eg. `RisingStack/cache`) 121 | - The `query` function parameter is an `Object` of key-value pairs of the request query parameters (defaults to `{}`) 122 | - It returns a `Promise` of the HTTP response without modification 123 | - [ ] Write unit tests for each function, use `nock` to intercept HTTP calls to the GitHub API endpoints 124 | 125 | Readings: 126 | - [Github API v3](https://developer.github.com/v3) 127 | - [`request`](https://www.npmjs.com/package/request-promise) & [`request-promise-native`](https://www.npmjs.com/package/request-promise-native) packages 128 | - [`nock`](https://github.com/node-nock/nock) for mocking endpoints 129 | 130 | Extra: 131 | - Use the [Github API v4 - GraphQL API](https://developer.github.com/v4) instead 132 | 133 | ### 3. Implement the database models 134 | 135 | In this step you will create the database tables, where the data will be stored, using migrations. 136 | 137 | Your model should look like this: 138 | 139 | ![DB schema](db-schema.png) 140 | 141 | It consists of 3 tables: `user`, `repository`, `contribution`. Rows in the `repository` table have foreign keys to a record in the `user` table, `owner`. The `contribution` table is managing many-to-many relationship between the `user` and `repository` tables with foreign keys. 142 | 143 | Tasks: 144 | - [ ] Edit the config and specify the `migrations` field in the knex initialization `Object`, for example: 145 | ```js 146 | { 147 | client: 'pg', 148 | connection: '...', 149 | migrations: { 150 | directory: path.join(__dirname, './migrations') 151 | } 152 | } 153 | ``` 154 | - [ ] Create one migration file per table (eg. `1-create-user.js`, `2-create-repository.js`, `3-create-contribution.js`) with the following skeleton: 155 | - `up` method has the logic for the migration, `down` is for reverting it 156 | - The migrations are executed in transactions 157 | - The files are executed in alphabetical order 158 | ```js 159 | 'use strict' 160 | 161 | const tableName = '...' 162 | 163 | function up(knex) { 164 | return knex.schema.createTable(tableName, (table) => { 165 | // your code goes here 166 | }) 167 | } 168 | 169 | function down(knex) { 170 | return knex.schema.dropTableIfExists(tableName) 171 | } 172 | 173 | module.exports = { 174 | up, 175 | down 176 | } 177 | ``` 178 | - [ ] Add a `migrate-db` script to the scripts in `package.json`, edit `scripts/migrate-db.js` to add the migration call. Finally, run your migration script to create the tables: 179 | ```sh 180 | $ npm run migrate-db -- --local 181 | ``` 182 | 183 | Readings: 184 | - [`knex`](http://knexjs.org/) SQL query builder 185 | - [`knex` migrations API](http://knexjs.org/#Migrations-API) 186 | - [npm scripts](https://docs.npmjs.com/misc/scripts) 187 | 188 | ### 4. Implement helper functions for the database models 189 | 190 | In this step you will implement and test helper functions for inserting, changing and reading data from the database. 191 | 192 | Tasks: 193 | - [ ] Implement the user model: 194 | - `User.insert({ id, login, avatar_url, html_url, type })` 195 | - validate the parameters 196 | - `User.read({ id, login })` 197 | - validate the parameters 198 | - one is required: `id` or `login` 199 | - [ ] Implement the repository model: 200 | - `Repository.insert({ id, owner, full_name, description, html_url, language, stargazers_count })` 201 | - Validate the parameters 202 | - `description` and `language` can be empty `String`s 203 | - `Repository.read({ id, full_name })` 204 | - Validate the parameters 205 | - One is required: `id` or `full_name` 206 | - Return the owner as well as an object (join tables and reorganize fields) 207 | - [ ] Implement the contribution model: 208 | - `Contribution.insert({ repository, user, line_count })` 209 | - Validate the parameters 210 | - `Contribution.insertOrReplace({ repository, user, line_count })` 211 | - Validate the parameters 212 | - Use a [raw query](http://knexjs.org/#Raw-Queries) and the [`ON CONFLICT`](https://www.postgresql.org/docs/9.6/static/sql-insert.html) SQL expression 213 | - `Contribution.read({ user: { id, login }, repository: { id, full_name } })` 214 | - Validate the parameters 215 | - The function parameter should be an Object, it should contain either a user, either a repository field or both of them. 216 | 217 | If only the user is provided, then all the contributions of that user will be resolved. 218 | If only the repository is provided, than all the users who contributed to that repository will be resolved. 219 | If both are provided, then it will match the contribution of a particular user to a particular repo. 220 | 221 | - The functions resolves to an Array of contributions (when both a user and a repository identifier is passed, it will only have 1 element) 222 | - Return the repository and user as well as an object 223 | (*This requirement is just for the sake of making up a problem, when you actually need this function, you will most likely have the user or the repository Object in a whole*) 224 | ```js 225 | { 226 | line_count: 10, 227 | user: { id: 1, login: 'coconut', ... }, 228 | repository: { id: 1, full_name: 'risingstack/repo', ... } 229 | } 230 | ``` 231 | - Use a **single** SQL query 232 | - When you join the tables, there will be conflicting column names (`id`, `html_url`). Use the `as` keyword when selecting columns (eg. `repository.id as repository_id`) to avoid this 233 | 234 | Notes: 235 | - `user` is a reserved keyword in PG, use double quotes where you reference the table in a raw query 236 | - You can get the columns of a table by querying `information_schema.columns`, which can be useful to select fields dinamically when joining tables, eg.: 237 | ```sql 238 | SELECT column_name FROM information_schema.columns WHERE table_name='contribution'; 239 | ``` 240 | 241 | ### 5. Create a worker process 242 | 243 | In this step you will implement another process of the application, the worker. We will trigger a request to collect the contributions for repositories based on some query. The trigger will send messages to another channel, the handler for this channel is reponsible to fetch the repositories. The third channel is used to fetch and save the contributions. 244 | 245 | **Make a drawing of the message flow, it will help you a lot!** 246 | 247 | Tasks: 248 | - [ ] Start Redis locally 249 | - [ ] Implement the contributions handler: 250 | - The responsibility of the contributions handler is to fetch the contributions of a repository from the GitHub API and to save the contributors and their line counts to the database 251 | - Validate the `message`, it has two fields: `date` and `repository` with `id` and `full_name` fields 252 | - Get the contributions from the GitHub API (use your models created in step 2) 253 | - Count all the lines currently in the repository per users (use `lodash` and `Array` functions) 254 | - Save the users to the database, don't fail if the user already exists (use your models created in step 3) 255 | - Save the contributions to the database, insert or replace (use your models created in step 3) 256 | - [ ] Implement the repository handler: 257 | - Validate the `message`, it has three fields: `date`, `query` and `page` 258 | - Get the repositories from the GitHub API (use your models created in step 2) with the `q`, `page` and `per_page` (set to 100) query parameters. 259 | - Modify the response to a format which is close to the database models (try to use [`lodash/fp`](https://github.com/lodash/lodash/wiki/FP-Guide)) 260 | - Save the owner to the database, don't fail if the user already exists (use your models created in step 3) 261 | - Save the repository to the database, don't fail if the repository already exists (use your models created in step 3) 262 | - Publish a message to the `contributions` channel with the same `date` 263 | - [ ] Implement the trigger handler: 264 | - The responsibility of the trigger handler is to send 10 messages to the `repository` collect channel implemented above. 10, because GitHub only gives access to the first 1000 (10 * page size of 100) search results 265 | - Validate the `message`, it has two fields: `date` and `query` 266 | - [ ] We would like to make our first search and data collection from GitHub. 267 | - For this, create a trigger.js file in the scripts folder. It should be a simple run once Node script which will publish a message to the `trigger` channel with the query passed in as an environment variable (`TRIGGER_QUERY`), then exit. It should have the same `--local`, `-L` flag, but for setting the `REDIS_URI`, as the migrate-db script. 268 | - Add a `trigger` field to the scripts in `package.json` that calls your `trigger.js` script. 269 | 270 | Readings: 271 | - [12 factor - Processes](https://12factor.net/processes) 272 | - [12 factor - Concurrency](https://12factor.net/concurrency) 273 | - [Redis pub/sub](https://redis.io/topics/pubsub) 274 | - [`ioredis`](https://github.com/luin/ioredis) 275 | 276 | ### 6. Implement a REST API 277 | 278 | In this step you will add a few routes to the existing web application to trigger a data crawl and to expose the collected data. 279 | 280 | Tasks: 281 | - [ ] The database requirements changed in the meantime, create a new migration (call it `4-add-indexes.js`), add indexes to `user.login` and `repository.full_name` (use `knex.schema.alterTable`) 282 | - [ ] Implement the `POST /api/v1/trigger` route, the body contains an object with a string `query` field, you will use this query to send a message to the corresponding Redis channel. Return `201` when it was successful 283 | - [ ] Implement the `GET /api/v1/repository/:id` and `GET /api/v1/repository/:owner/:name` endpoints 284 | - [ ] Implement the `GET /api/v1/repository/:id/contributions` and `GET /api/v1/repository/:owner/:name/contributions` endpoints 285 | - [ ] Create a middleware (`requestLogger({ level = 'silly' })`) and add it to your server, that logs out: 286 | - The method and original url of the request 287 | - Request headers (except `authorization` and `cookie`) and body 288 | - The request duration in `ms` 289 | - Response headers (except `authorization` and `cookie`) and body 290 | - Response status code (based on it: log level should be `error` when server error, `warn` when client error) 291 | - [ ] Document your API using [Apiary](https://apiary.io/)'s Blueprint format (edit the `API_DOCUMENTATION.apib`). 292 | 293 | Notes: 294 | - Make use of [koa-compose](https://github.com/koajs/compose) and the validator middleware 295 | ```js 296 | compose([ 297 | middleware.validator({ 298 | params: paramsSchema, 299 | query: querySchema, 300 | body: bodySchema 301 | }), 302 | // additional middleware 303 | ]) 304 | ``` 305 | 306 | Readings: 307 | - [Pragmatic RESTful API](http://www.vinaysahni.com/best-practices-for-a-pragmatic-restful-api) 308 | - [Koa middleware & cascade](http://koajs.com/) 309 | - [API Blueprint tutorial](https://help.apiary.io/api_101/api_blueprint_tutorial/) 310 | 311 | ### 7. Prepare your service for production 312 | 313 | In this step you will add some features, which are required to have your application running in production environment. 314 | 315 | Tasks: 316 | - [ ] Listen on the `SIGTERM` signal in `web/index.js`. 317 | - Create a function called `gracefulShutdown` 318 | - Use koa's `.callback()` function to create a `http` server (look for `http.createServer`) and convert `server.close` with `util.promisify` 319 | - Close the server and destroy the database and redis connections (use the `destroy` function to the redis model, which calls `disconnect` on both redis clients and returns a `Promise`) 320 | - Log out and exit the process with code `1` if something fails 321 | - Exit the process with code `0` if everything is closed succesfully 322 | - [ ] Implement the same for the worker process 323 | - [ ] Add a health check endpoint for the web server 324 | - Add a `healthCheck` function for the database model, use the `PG_HEALTH_CHECK_TIMEOUT` environment variable to set the query timeout (set default to `2000` ms) 325 | - Add a `healthCheck` function to the redis model 326 | - Implement the `GET /healthz` endpoint, return `200` with JSON body `{ "status": "ok" }`when everything is healthy, `500` if any of the database or redis connections are not healthy and `503` if the process got `SIGTERM` signal 327 | - [ ] Create a http server and add a similar health check endpoint for the worker process 328 | 329 | Readings: 330 | - [Signal events](https://nodejs.org/api/process.html#process_signal_events) 331 | - [Graceful shutdown](https://blog.risingstack.com/graceful-shutdown-node-js-kubernetes/) 332 | - [Health checks](http://microservices.io/patterns/observability/health-check-api.html) 333 | -------------------------------------------------------------------------------- /config.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | const joi = require('joi') 4 | 5 | const envVarsSchema = joi.object({ 6 | NODE_ENV: joi.string() 7 | .allow(['development', 'production', 'test']) 8 | .default('production'), 9 | PROCESS_TYPE: joi.string() 10 | .allow(['web', 'worker']) 11 | .required(), 12 | LOGGER_LEVEL: joi.string() 13 | .allow(['test', 'error', 'warn', 'info', 'verbose', 'debug', 'silly']) 14 | .when('NODE_ENV', { 15 | is: 'development', 16 | then: joi.default('silly') 17 | }) 18 | .when('NODE_ENV', { 19 | is: 'production', 20 | then: joi.default('info') 21 | }) 22 | .when('NODE_ENV', { 23 | is: 'test', 24 | then: joi.default('warn') 25 | }) 26 | }) 27 | .unknown().required() 28 | 29 | const envVars = joi.attempt(process.env, envVarsSchema) 30 | 31 | const config = { 32 | env: envVars.NODE_ENV, 33 | process: { 34 | type: envVars.PROCESS_TYPE 35 | }, 36 | logger: { 37 | level: envVars.LOGGER_LEVEL 38 | } 39 | } 40 | 41 | module.exports = config 42 | -------------------------------------------------------------------------------- /db-schema.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/RisingStack/risingstack-bootcamp/754f643172322c1dee039f040be5995a03d62988/db-schema.png -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | /* eslint-disable global-require */ 2 | 3 | 'use strict' 4 | 5 | // load .env in local development 6 | if (process.env.NODE_ENV === 'development') { 7 | require('dotenv').config({ silent: true }) 8 | } 9 | 10 | const logger = require('winston') 11 | const semver = require('semver') 12 | const pkg = require('./package.json') 13 | const config = require('./config') 14 | 15 | // validate Node version requirement 16 | const runtime = { 17 | expected: semver.validRange(pkg.engines.node), 18 | actual: semver.valid(process.version) 19 | } 20 | const valid = semver.satisfies(runtime.actual, runtime.expected) 21 | if (!valid) { 22 | throw new Error( 23 | `Expected Node.js version ${runtime.expected}, but found v${runtime.actual}. Please update or change your runtime!` 24 | ) 25 | } 26 | 27 | // configure logger 28 | logger.default.transports.console.colorize = true 29 | logger.default.transports.console.timestamp = true 30 | logger.default.transports.console.prettyPrint = config.env === 'development' 31 | logger.level = config.logger.level 32 | 33 | // start process 34 | logger.info(`Starting ${config.process.type} process`, { pid: process.pid }) 35 | 36 | if (config.process.type === 'web') { 37 | require('./web') 38 | } else if (config.process.type === 'worker') { 39 | require('./worker') 40 | } else { 41 | throw new Error(`${config.process.type} is an unsupported process type.`) 42 | } 43 | -------------------------------------------------------------------------------- /models/contribution/contribution.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | const fp = require('lodash/fp') 4 | const joi = require('joi') 5 | const db = require('../db') 6 | const { getColumns, addPrefixAliasToColumns, getColumnsByTableNamePrefix } = require('../db/utils') 7 | const User = require('../user') 8 | const Repository = require('../repository') 9 | 10 | const tableName = 'contribution' 11 | 12 | const insertSchema = joi.object({ 13 | user: joi.number().integer().required(), 14 | repository: joi.number().integer().required(), 15 | line_count: joi.number().integer().default(0) 16 | }).required() 17 | 18 | async function insert(params) { 19 | const contribution = joi.attempt(params, insertSchema) 20 | 21 | return db(tableName) 22 | .insert(contribution) 23 | .returning('*') 24 | .then(fp.first) 25 | } 26 | 27 | const readSchema = joi.object({ 28 | user: joi.object({ 29 | id: joi.number().integer(), 30 | login: joi.string() 31 | }).xor('id', 'login'), 32 | repository: joi.object({ 33 | id: joi.number().integer(), 34 | full_name: joi.string() 35 | }).xor('id', 'full_name') 36 | }) 37 | .or('user', 'repository') 38 | .required() 39 | 40 | async function read(params) { 41 | const { repository = {}, user = {} } = joi.attempt(params, readSchema) 42 | 43 | // at least one field is required in the condition 44 | // filter out undefined values 45 | const condition = fp.omitBy(fp.isUndefined, { 46 | [`${User.tableName}.id`]: user.id, 47 | [`${User.tableName}.login`]: user.login, 48 | [`${Repository.tableName}.id`]: repository.id, 49 | [`${Repository.tableName}.full_name`]: repository.full_name 50 | }) 51 | 52 | // get the columns of the user and repository tables 53 | // add alias prefixes to avoid column name collisions 54 | const [userColumns, repositoryColumns] = await Promise.all([ 55 | getColumns(User.tableName) 56 | .then(addPrefixAliasToColumns(User.tableName)), 57 | getColumns(Repository.tableName) 58 | .then(addPrefixAliasToColumns(Repository.tableName)) 59 | ]) 60 | const selection = [`${tableName}.*`, ...userColumns, ...repositoryColumns] 61 | 62 | const contributions = await db(tableName) 63 | .where(condition) 64 | .leftJoin(User.tableName, `${tableName}.user`, `${User.tableName}.id`) 65 | .leftJoin(Repository.tableName, `${tableName}.repository`, `${Repository.tableName}.id`) 66 | .select(selection) 67 | 68 | // format contributions 69 | const omitUserAndRepositoryColumns = fp.compose([ 70 | fp.omitBy((value, column) => fp.startsWith(`${User.tableName}_`, column)), 71 | fp.omitBy((value, column) => fp.startsWith(`${Repository.tableName}_`, column)) 72 | ]) 73 | 74 | return fp.map((contribution) => Object.assign(omitUserAndRepositoryColumns(contribution), { 75 | user: getColumnsByTableNamePrefix(User.tableName, contribution), 76 | repository: getColumnsByTableNamePrefix(Repository.tableName, contribution) 77 | }), contributions) 78 | } 79 | 80 | async function insertOrReplace(params) { 81 | const contribution = joi.attempt(params, insertSchema) 82 | 83 | const query = ` 84 | INSERT INTO :tableName: ("user", repository, line_count) 85 | VALUES (:user, :repository, :line_count) 86 | ON CONFLICT ("user", repository) DO UPDATE SET line_count = :line_count 87 | RETURNING *; 88 | ` 89 | 90 | return db.raw(query, Object.assign({ tableName }, contribution)) 91 | .then(fp.first) 92 | } 93 | 94 | module.exports = { 95 | tableName, 96 | insert, 97 | insertOrReplace, 98 | read 99 | } 100 | -------------------------------------------------------------------------------- /models/contribution/contribution.spec.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | const { expect } = require('chai') 4 | const _ = require('lodash') 5 | const db = require('../db') 6 | const Repository = require('../repository') 7 | const User = require('../user') 8 | const Contribution = require('./contribution') 9 | 10 | describe('Contribution', () => { 11 | let repositoryId 12 | let userId 13 | let contributionToInsert 14 | let userToInsert 15 | let repositoryToInsert 16 | 17 | beforeEach(async () => { 18 | repositoryId = _.random(1000) 19 | userId = _.random(1000) 20 | contributionToInsert = { 21 | repository: repositoryId, 22 | user: userId, 23 | line_count: _.random(1000) 24 | } 25 | 26 | userToInsert = { 27 | id: userId, 28 | login: 'developer', 29 | avatar_url: 'https://developer.com/avatar.png', 30 | html_url: 'https://github.com/developer', 31 | type: 'User' 32 | } 33 | 34 | repositoryToInsert = { 35 | id: repositoryId, 36 | owner: userId, 37 | full_name: '@risingstack/foo', 38 | description: 'Very foo package, using bar technologies', 39 | html_url: 'https://github.com/risingstack/foo', 40 | language: 'Baz', 41 | stargazers_count: 123 42 | } 43 | 44 | await db(User.tableName) 45 | .insert(userToInsert) 46 | 47 | await db(Repository.tableName) 48 | .insert(repositoryToInsert) 49 | }) 50 | 51 | afterEach(async () => { 52 | await db(Repository.tableName) 53 | .where({ id: repositoryId }) 54 | .delete() 55 | 56 | await db(User.tableName) 57 | .where({ id: userId }) 58 | .delete() 59 | }) 60 | 61 | describe('.insert', () => { 62 | it('should insert a new contribution', async () => { 63 | const contributionReturned = await Contribution.insert(contributionToInsert) 64 | const contributionInDB = await db(Contribution.tableName) 65 | .where({ repository: repositoryId, user: userId }) 66 | .first() 67 | 68 | expect(contributionToInsert).to.eql(contributionInDB) 69 | expect(contributionReturned).to.eql(contributionInDB) 70 | }) 71 | 72 | it('should validate the input params', async () => { 73 | delete contributionToInsert.user 74 | try { 75 | await Contribution.insert(contributionToInsert) 76 | } catch (err) { 77 | expect(err.name).to.be.eql('ValidationError') 78 | return 79 | } 80 | 81 | throw new Error('Did not validate') 82 | }) 83 | }) 84 | 85 | describe('.insertOrReplace', () => { 86 | it('should insert the contribution if it does not exist', async () => { 87 | await Contribution.insertOrReplace(contributionToInsert) 88 | const contributionInDB = await db(Contribution.tableName) 89 | .where({ repository: repositoryId, user: userId }) 90 | .first() 91 | 92 | expect(contributionToInsert).to.eql(contributionInDB) 93 | }) 94 | 95 | it('should replace the line_count of an existing contribution', async () => { 96 | await db(Contribution.tableName) 97 | .insert(contributionToInsert) 98 | 99 | contributionToInsert.line_count = _.random(1000) 100 | await Contribution.insertOrReplace(contributionToInsert) 101 | const contributionInDB = await db(Contribution.tableName) 102 | .where({ repository: repositoryId, user: userId }) 103 | .first() 104 | 105 | expect({ 106 | repository: repositoryId, 107 | user: userId, 108 | line_count: contributionToInsert.line_count 109 | }).to.eql(contributionInDB) 110 | }) 111 | }) 112 | 113 | describe('.read', () => { 114 | it('should return a contribution with the repository and user', async () => { 115 | await db(Contribution.tableName) 116 | .insert(contributionToInsert) 117 | 118 | const expected = [{ 119 | line_count: contributionToInsert.line_count, 120 | user: userToInsert, 121 | repository: repositoryToInsert 122 | }] 123 | 124 | let result = await Contribution.read({ 125 | repository: { 126 | id: contributionToInsert.repository 127 | } 128 | }) 129 | expect(result).to.eql(expected) 130 | 131 | result = await Contribution.read({ 132 | repository: { 133 | full_name: repositoryToInsert.full_name 134 | } 135 | }) 136 | expect(result).to.eql(expected) 137 | }) 138 | }) 139 | }) 140 | -------------------------------------------------------------------------------- /models/contribution/index.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | module.exports = require('./contribution') 4 | -------------------------------------------------------------------------------- /models/db/config.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | const path = require('path') 4 | const joi = require('joi') 5 | const { parse } = require('pg-connection-string') 6 | 7 | const envVarsSchema = joi.object({ 8 | PG_URI: joi.string().uri({ scheme: 'postgres' }).required(), 9 | PG_SSL_CA: joi.string(), 10 | PG_SSL_KEY: joi.string(), 11 | PG_SSL_CERT: joi.string(), 12 | PG_SSL_ALLOW_UNAUTHORIZED: joi.boolean().truthy('true').falsy('false').default(true), 13 | PG_POOL_MIN: joi.number().integer().default(1), 14 | PG_POOL_MAX: joi.number().integer().default(20), 15 | PG_HEALTH_CHECK_TIMEOUT: joi.number().integer().default(2000) 16 | }).unknown() 17 | .required() 18 | 19 | const envVars = joi.attempt(process.env, envVarsSchema) 20 | 21 | const config = { 22 | client: 'pg', 23 | connection: Object.assign( 24 | parse(envVars.PG_URI), 25 | envVars.PG_SSL_CA || envVars.PG_SSL_KEY || envVars.PG_SSL_CERT 26 | ? { 27 | ssl: { 28 | ca: envVars.PG_SSL_CA, 29 | key: envVars.PG_SSL_KEY, 30 | cert: envVars.PG_SSL_CERT, 31 | rejectUnauthorized: !envVars.PG_SSL_ALLOW_UNAUTHORIZED 32 | } 33 | } 34 | : {} 35 | ), 36 | pool: { 37 | min: envVars.PG_POOL_MIN, 38 | max: envVars.PG_POOL_MAX 39 | }, 40 | migrations: { 41 | directory: path.join(__dirname, './migrations') 42 | }, 43 | healthCheck: { 44 | timeout: envVars.PG_HEALTH_CHECK_TIMEOUT 45 | } 46 | } 47 | 48 | module.exports = config 49 | -------------------------------------------------------------------------------- /models/db/index.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | const config = require('./config') 4 | const knex = require('knex') 5 | 6 | const db = knex(config) 7 | 8 | function healthCheck() { 9 | return db.select(1).timeout(config.healthCheck.timeout) 10 | } 11 | 12 | module.exports = Object.assign(db, { 13 | healthCheck 14 | }) 15 | -------------------------------------------------------------------------------- /models/db/migrations/1-create-user.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | const tableName = 'user' 4 | 5 | function up(knex) { 6 | return knex.schema.createTable(tableName, (table) => { 7 | table.integer('id').unsigned().primary() 8 | table.string('login').notNullable() 9 | table.string('avatar_url').notNullable() 10 | table.string('html_url').notNullable() 11 | table.string('type').notNullable() 12 | }) 13 | } 14 | 15 | function down(knex) { 16 | return knex.schema.dropTableIfExists(tableName) 17 | } 18 | 19 | module.exports = { 20 | up, 21 | down 22 | } 23 | -------------------------------------------------------------------------------- /models/db/migrations/2-create-repository.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | const tableName = 'repository' 4 | 5 | function up(knex) { 6 | return knex.schema.createTable(tableName, (table) => { 7 | table.integer('id').unsigned().primary() 8 | table.integer('owner').notNullable() 9 | table.foreign('owner').references('user.id').onDelete('CASCADE') 10 | table.string('full_name').notNullable() 11 | table.string('description').notNullable() 12 | table.string('html_url').notNullable() 13 | table.string('language').notNullable() 14 | table.integer('stargazers_count').unsigned().notNullable() 15 | }) 16 | } 17 | 18 | function down(knex) { 19 | return knex.schema.dropTableIfExists(tableName) 20 | } 21 | 22 | module.exports = { 23 | up, 24 | down 25 | } 26 | -------------------------------------------------------------------------------- /models/db/migrations/3-create-contribution.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | const tableName = 'contribution' 4 | 5 | function up(knex) { 6 | return knex.schema.createTable(tableName, (table) => { 7 | table.integer('user') 8 | table.foreign('user').references('user.id').onDelete('CASCADE') 9 | table.integer('repository') 10 | table.foreign('repository').references('repository.id').onDelete('CASCADE') 11 | table.integer('line_count').unsigned().notNullable() 12 | table.primary(['user', 'repository']) 13 | }) 14 | } 15 | 16 | function down(knex) { 17 | return knex.schema.dropTableIfExists(tableName) 18 | } 19 | 20 | module.exports = { 21 | up, 22 | down 23 | } 24 | -------------------------------------------------------------------------------- /models/db/migrations/4-add-indexes.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | const repositoryTableName = 'repository' 4 | const userTableName = 'user' 5 | 6 | function up(knex) { 7 | return Promise.all([ 8 | knex.schema.alterTable(repositoryTableName, (table) => { 9 | table.index(['full_name']) 10 | }), 11 | knex.schema.alterTable(userTableName, (table) => { 12 | table.index(['login']) 13 | }) 14 | ]) 15 | } 16 | 17 | function down(knex) { 18 | return Promise.all([ 19 | knex.schema.alterTable(repositoryTableName, (table) => { 20 | table.dropIndex(['full_name']) 21 | }), 22 | knex.schema.alterTable(userTableName, (table) => { 23 | table.dropIndex(['login']) 24 | }) 25 | ]) 26 | } 27 | 28 | module.exports = { 29 | up, 30 | down 31 | } 32 | -------------------------------------------------------------------------------- /models/db/utils.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | const fp = require('lodash/fp') 4 | const db = require('./') 5 | 6 | function getColumns(tableName) { 7 | return db('information_schema.columns') 8 | .where({ table_name: tableName }) 9 | .select('column_name') 10 | .options({ rowMode: 'array' }) 11 | .then(fp.map(fp.first)) 12 | } 13 | 14 | function addPrefixAliasToColumns(tableName, columns) { 15 | const fn = fp.map((column) => `${tableName}.${column} as ${tableName}_${column}`) 16 | 17 | if (columns) { 18 | return fn(columns) 19 | } 20 | 21 | return fn 22 | } 23 | 24 | function getColumnsByTableNamePrefix(tableName, columns) { 25 | const fn = fp.compose([ 26 | fp.mapKeys(fp.replace(`${tableName}_`, '')), 27 | fp.pickBy((value, key) => fp.startsWith(`${tableName}_`, key)) 28 | ]) 29 | 30 | if (columns) { 31 | return fn(columns) 32 | } 33 | 34 | return fn 35 | } 36 | 37 | module.exports = { 38 | getColumns, 39 | addPrefixAliasToColumns, 40 | getColumnsByTableNamePrefix 41 | } 42 | -------------------------------------------------------------------------------- /models/github/api.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | const request = require('request-promise-native') 4 | 5 | const API_URL = 'https://api.github.com' 6 | const USER_AGENT = 'RisingStack-Bootcamp' 7 | 8 | function searchRepositories(query = {}) { 9 | return request({ 10 | method: 'GET', 11 | uri: `${API_URL}/search/repositories`, 12 | headers: { 13 | Accept: 'application/vnd.github.v3+json', 14 | 'User-Agent': USER_AGENT 15 | }, 16 | qs: query, 17 | json: true 18 | }) 19 | } 20 | 21 | function getContributors(repository, query = {}) { 22 | return request({ 23 | method: 'GET', 24 | uri: `${API_URL}/repos/${repository}/stats/contributors`, 25 | headers: { 26 | Accept: 'application/vnd.github.v3+json', 27 | 'User-Agent': USER_AGENT 28 | }, 29 | qs: query, 30 | json: true 31 | }) 32 | } 33 | 34 | module.exports = { 35 | searchRepositories, 36 | getContributors 37 | } 38 | -------------------------------------------------------------------------------- /models/github/api.spec.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | const nock = require('nock') 4 | const { expect } = require('chai') 5 | const api = require('./api') 6 | 7 | describe('GitHub API', () => { 8 | it('should search repositories', async () => { 9 | const githubAPI = nock('https://api.github.com', { 10 | reqheaders: { 11 | accept: 'application/vnd.github.v3+json', 12 | 'user-agent': 'RisingStack-Bootcamp' 13 | } 14 | }) 15 | .get('/search/repositories') 16 | .query({ q: 'language:javascript' }) 17 | .reply(200, { items: [] }) 18 | 19 | const result = await api.searchRepositories({ q: 'language:javascript' }) 20 | expect(githubAPI.isDone()).to.eql(true) 21 | expect(result).to.eql({ items: [] }) 22 | }) 23 | 24 | it('should get contributors', async () => { 25 | const repository = 'owner/repository' 26 | const githubAPI = nock('https://api.github.com', { 27 | reqheaders: { 28 | accept: 'application/vnd.github.v3+json', 29 | 'user-agent': 'RisingStack-Bootcamp' 30 | } 31 | }) 32 | .get(`/repos/${repository}/stats/contributors`) 33 | .reply(200, [{ author: {}, weeks: [] }]) 34 | 35 | const result = await api.getContributors(repository) 36 | expect(githubAPI.isDone()).to.eql(true) 37 | expect(result).to.eql([{ author: {}, weeks: [] }]) 38 | }) 39 | }) 40 | -------------------------------------------------------------------------------- /models/github/index.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | const api = require('./api') 4 | 5 | module.exports = { 6 | api 7 | } 8 | -------------------------------------------------------------------------------- /models/redis/config.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | const joi = require('joi') 4 | 5 | const envVarsSchema = joi.object({ 6 | REDIS_URI: joi.string().uri({ scheme: 'redis' }).required() 7 | }).unknown() 8 | .required() 9 | 10 | const envVars = joi.attempt(process.env, envVarsSchema) 11 | 12 | const config = { 13 | uri: envVars.REDIS_URI 14 | } 15 | 16 | module.exports = config 17 | -------------------------------------------------------------------------------- /models/redis/index.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | const Redis = require('ioredis') 4 | const config = require('./config') 5 | 6 | const CHANNELS = { 7 | collect: { 8 | trigger: { 9 | v1: 'bootcamp.collect.trigger.v1' 10 | }, 11 | repository: { 12 | v1: 'bootcamp.collect.repository.v1' 13 | }, 14 | contributions: { 15 | v1: 'bootcamp.collect.contributions.v1' 16 | } 17 | } 18 | } 19 | 20 | const publisher = new Redis(config.uri, { lazyConnect: true, dropBufferSupport: true }) 21 | const subscriber = new Redis(config.uri, { lazyConnect: true, dropBufferSupport: true }) 22 | 23 | function publishObject(channel, message) { 24 | return publisher.publish(channel, JSON.stringify(message)) 25 | } 26 | 27 | async function destroy() { 28 | subscriber.disconnect() 29 | publisher.disconnect() 30 | } 31 | 32 | async function healthCheck() { 33 | try { 34 | await Promise.all([ 35 | // check first if not connected yet (lazy connect) 36 | subscriber.status === 'wait' ? Promise.resolve() : subscriber.ping(), 37 | publisher.status === 'wait' ? Promise.resolve() : publisher.ping() 38 | ]) 39 | } catch (err) { 40 | const error = new Error('One or more client status are not healthy') 41 | error.status = { 42 | subscriber: subscriber.status, 43 | publisher: publisher.status 44 | } 45 | 46 | throw error 47 | } 48 | } 49 | 50 | module.exports = Object.assign(subscriber, { 51 | subscriber, 52 | publisher, 53 | publishObject, 54 | destroy, 55 | healthCheck, 56 | CHANNELS 57 | }) 58 | -------------------------------------------------------------------------------- /models/repository/index.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | module.exports = require('./repository') 4 | -------------------------------------------------------------------------------- /models/repository/repository.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | const fp = require('lodash/fp') 4 | const joi = require('joi') 5 | const db = require('../db') 6 | const { getColumns, addPrefixAliasToColumns, getColumnsByTableNamePrefix } = require('../db/utils') 7 | const User = require('../user') 8 | 9 | const tableName = 'repository' 10 | 11 | const insertSchema = joi.object({ 12 | id: joi.number().integer().required(), 13 | owner: joi.number().integer().required(), 14 | full_name: joi.string().required(), 15 | description: joi.string().allow('').required(), 16 | html_url: joi.string().uri().required(), 17 | language: joi.string().allow('').required(), 18 | stargazers_count: joi.number().integer().required() 19 | }).required() 20 | 21 | async function insert(params) { 22 | const repository = joi.attempt(params, insertSchema) 23 | 24 | return db(tableName) 25 | .insert(repository) 26 | .returning('*') 27 | .then(fp.first) 28 | } 29 | 30 | const readSchema = joi.object({ 31 | id: joi.number().integer(), 32 | full_name: joi.string() 33 | }) 34 | .xor('id', 'full_name') 35 | .required() 36 | 37 | async function read(params) { 38 | const repository = joi.attempt(params, readSchema) 39 | 40 | // at least one field is required in the condition 41 | // filter out undefined values 42 | const condition = fp.omitBy(fp.isUndefined, { 43 | [`${tableName}.id`]: repository.id, 44 | [`${tableName}.full_name`]: repository.full_name 45 | }) 46 | 47 | // get the columns of the user table 48 | // add alias prefixes to avoid column name collisions 49 | const userColumns = await getColumns(User.tableName) 50 | .then(addPrefixAliasToColumns(User.tableName)) 51 | const selection = [`${tableName}.*`, ...userColumns] 52 | 53 | const repo = await db(tableName) 54 | .where(condition) 55 | .leftJoin(User.tableName, `${tableName}.owner`, `${User.tableName}.id`) 56 | .select(selection) 57 | .first() 58 | 59 | if (!repo) { 60 | return undefined 61 | } 62 | 63 | 64 | const omitUserColumns = fp.omitBy((value, column) => fp.startsWith(`${User.tableName}_`, column)) 65 | return Object.assign(omitUserColumns(repo), { 66 | owner: getColumnsByTableNamePrefix(User.tableName, repo), 67 | }) 68 | } 69 | 70 | module.exports = { 71 | tableName, 72 | insert, 73 | read 74 | } 75 | -------------------------------------------------------------------------------- /models/repository/repository.spec.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | const { expect } = require('chai') 4 | const _ = require('lodash') 5 | const db = require('../db') 6 | const User = require('../user') 7 | const Repository = require('./repository') 8 | 9 | describe('Repository', () => { 10 | let id 11 | let userId 12 | let repositoryToInsert 13 | let userToInsert 14 | 15 | beforeEach(async () => { 16 | id = _.random(1000) 17 | userId = _.random(1000) 18 | 19 | repositoryToInsert = { 20 | id, 21 | owner: userId, 22 | full_name: '@risingstack/foo', 23 | description: 'Very foo package, using bar technologies', 24 | html_url: 'https://github.com/risingstack/foo', 25 | language: 'Baz', 26 | stargazers_count: _.random(1000) 27 | } 28 | 29 | userToInsert = { 30 | id: userId, 31 | login: 'developer', 32 | avatar_url: 'https://developer.com/avatar.png', 33 | html_url: 'https://github.com/developer', 34 | type: 'User' 35 | } 36 | 37 | await db(User.tableName) 38 | .insert(userToInsert) 39 | }) 40 | 41 | afterEach(async () => { 42 | await db(Repository.tableName) 43 | .where({ id }) 44 | .delete() 45 | 46 | await db(User.tableName) 47 | .where({ id: userId }) 48 | .delete() 49 | }) 50 | 51 | describe('.insert', () => { 52 | it('should insert a new repository', async () => { 53 | const repositoryReturned = await Repository.insert(repositoryToInsert) 54 | const repositoryInDB = await db(Repository.tableName) 55 | .where({ id }) 56 | .first() 57 | 58 | expect(repositoryInDB).to.eql(repositoryToInsert) 59 | expect(repositoryReturned).to.eql(repositoryToInsert) 60 | }) 61 | 62 | it('should validate the input params', async () => { 63 | delete repositoryToInsert.full_name 64 | try { 65 | await Repository.insert(repositoryToInsert) 66 | } catch (err) { 67 | expect(err.name).to.be.eql('ValidationError') 68 | return 69 | } 70 | 71 | throw new Error('Did not validate') 72 | }) 73 | }) 74 | 75 | describe('.read', () => { 76 | it('should return a repository', async () => { 77 | await db(Repository.tableName) 78 | .insert(repositoryToInsert) 79 | 80 | const result = await Repository.read({ id: repositoryToInsert.id }) 81 | 82 | expect(result).to.eql(Object.assign(repositoryToInsert, { 83 | owner: userToInsert 84 | })) 85 | }) 86 | 87 | it('should return undefined if the repository is not in the db', async () => { 88 | const result = await Repository.read({ id: repositoryToInsert.id }) 89 | 90 | expect(result).to.eql(undefined) 91 | }) 92 | }) 93 | }) 94 | -------------------------------------------------------------------------------- /models/test.setup.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | const sinon = require('sinon') 4 | const winston = require('winston') 5 | 6 | // postgres 7 | const user = process.env.PG_USER || process.env.USER || 'root' 8 | const pw = process.env.PG_PASSWORD || '' 9 | const db = process.env.PG_DATABASE || 'risingstack_bootcamp' 10 | process.env.PG_URI = `postgres://${user}:${pw}@localhost:5432/${db}` 11 | 12 | // logger 13 | winston.setLevels({ test: 0, error: 1, warn: 2, info: 3, verbose: 4, debug: 5, silly: 6 }) 14 | winston.addColors({ 15 | test: 'cyan', error: 'red', warn: 'yellow', info: 'cyan', verbose: 'cyan', debug: 'blue', silly: 'magenta' 16 | }) 17 | winston.remove(winston.transports.Console) 18 | winston.add(winston.transports.Console, { 19 | level: process.env.LOGGER_LEVEL || 'test', colorize: true, prettyPrint: true 20 | }) 21 | 22 | beforeEach(function beforeEach() { 23 | this.sandbox = sinon.sandbox.create() 24 | }) 25 | 26 | afterEach(function afterEach() { 27 | this.sandbox.restore() 28 | }) 29 | -------------------------------------------------------------------------------- /models/user/index.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | module.exports = require('./user') 4 | -------------------------------------------------------------------------------- /models/user/user.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | const fp = require('lodash/fp') 4 | const joi = require('joi') 5 | const db = require('../db') 6 | 7 | const tableName = 'user' 8 | 9 | const insertSchema = joi.object({ 10 | id: joi.number().integer().required(), 11 | login: joi.string().required(), 12 | avatar_url: joi.string().uri().required(), 13 | html_url: joi.string().uri().required(), 14 | type: joi.string().required() 15 | }).required() 16 | 17 | async function insert(params) { 18 | const user = joi.attempt(params, insertSchema) 19 | 20 | return db(tableName) 21 | .insert(user) 22 | .returning('*') 23 | .then(fp.first) 24 | } 25 | 26 | const readSchema = joi.object({ 27 | id: joi.number().integer(), 28 | login: joi.string() 29 | }) 30 | .xor('id', 'login') 31 | .required() 32 | 33 | async function read(params) { 34 | const selection = joi.attempt(params, readSchema) 35 | 36 | return db(tableName) 37 | .where(selection) 38 | .select() 39 | .first() 40 | } 41 | 42 | module.exports = { 43 | tableName, 44 | insert, 45 | read 46 | } 47 | -------------------------------------------------------------------------------- /models/user/user.spec.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | const { expect } = require('chai') 4 | const _ = require('lodash') 5 | const db = require('../db') 6 | const User = require('./user') 7 | 8 | describe('User', () => { 9 | let id 10 | let userToInsert 11 | 12 | beforeEach(async () => { 13 | id = _.random(1000) 14 | 15 | userToInsert = { 16 | id, 17 | login: 'developer', 18 | avatar_url: 'https://developer.com/avatar.png', 19 | html_url: 'https://github.com/developer', 20 | type: 'User' 21 | } 22 | }) 23 | 24 | afterEach(async () => { 25 | await db(User.tableName) 26 | .where({ id }) 27 | .delete() 28 | }) 29 | 30 | describe('.insert', () => { 31 | it('should insert a new user', async () => { 32 | const userReturned = await User.insert(userToInsert) 33 | const userInDB = await db(User.tableName) 34 | .where({ id }) 35 | .first() 36 | 37 | expect(userInDB).to.eql(userToInsert) 38 | expect(userReturned).to.eql(userToInsert) 39 | }) 40 | 41 | it('should validate the input params', async () => { 42 | delete userToInsert.login 43 | 44 | try { 45 | await User.insert(userToInsert) 46 | } catch (err) { 47 | expect(err.name).to.be.eql('ValidationError') 48 | return 49 | } 50 | 51 | throw new Error('Did not validate') 52 | }) 53 | }) 54 | 55 | describe('.read', () => { 56 | it('should return a user', async () => { 57 | await db(User.tableName) 58 | .insert(userToInsert) 59 | 60 | const result = await User.read({ id: userToInsert.id }) 61 | 62 | expect(result).to.eql(userToInsert) 63 | }) 64 | 65 | it('should return undefined if the user is not in the db', async () => { 66 | const result = await User.read({ id: userToInsert.id }) 67 | 68 | expect(result).to.eql(undefined) 69 | }) 70 | }) 71 | }) 72 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "risingstack-bootcamp", 3 | "version": "1.0.0", 4 | "description": "", 5 | "main": "index.js", 6 | "scripts": { 7 | "start": "node .", 8 | "dev": "NODE_ENV=development nodemon .", 9 | "test-web": "NODE_ENV=test PROCESS_TYPE=web mocha 'web/test.setup.js' 'web/**/*.spec.js'", 10 | "test-worker": "NODE_ENV=test PROCESS_TYPE=worker mocha 'worker/test.setup.js' 'worker/**/*.spec.js'", 11 | "test-models": "NODE_ENV=test mocha 'models/test.setup.js' 'models/**/*.spec.js'", 12 | "test-only": "npm run test-web && npm run test-worker && npm run test-models", 13 | "lint": "eslint .", 14 | "test": "npm run lint && npm run test-only", 15 | "migrate-db": "node ./scripts/migrate-db", 16 | "trigger": "node ./scripts/trigger" 17 | }, 18 | "repository": "git+https://github.com/RisingStack/risingstack-bootcamp.git", 19 | "author": "Andras Toth ", 20 | "license": "ISC", 21 | "bugs": { 22 | "url": "https://github.com/RisingStack/risingstack-bootcamp/issues" 23 | }, 24 | "engines": { 25 | "node": ">=8.1.2" 26 | }, 27 | "homepage": "https://github.com/RisingStack/risingstack-bootcamp#readme", 28 | "dependencies": { 29 | "ioredis": "3.1.1", 30 | "joi": "10.6.0", 31 | "knex": "0.13.0", 32 | "koa": "2.3.0", 33 | "koa-bodyparser": "4.2.0", 34 | "koa-compose": "4.0.0", 35 | "koa-router": "7.2.1", 36 | "lodash": "4.17.4", 37 | "pg": "6.4.0", 38 | "pg-connection-string": "0.1.3", 39 | "qs": "6.5.0", 40 | "request": "2.81.0", 41 | "request-promise-native": "1.0.4", 42 | "semver": "5.3.0", 43 | "sinon": "2.3.6", 44 | "sinon-chai": "2.11.0", 45 | "winston": "2.3.1" 46 | }, 47 | "devDependencies": { 48 | "chai": "4.0.2", 49 | "dotenv": "4.0.0", 50 | "eslint": "4.0.0", 51 | "eslint-config-airbnb-base": "11.2.0", 52 | "eslint-plugin-import": "2.3.0", 53 | "mocha": "3.4.2", 54 | "nock": "9.0.13", 55 | "nodemon": "1.11.0", 56 | "super-request": "1.2.0" 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /scripts/migrate-db.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | const args = process.argv.slice(2) 4 | if (args.includes('--local') || args.includes('-L')) { 5 | const user = process.env.PG_USER || process.env.USER || 'root' 6 | const pw = process.env.PG_PASSWORD || '' 7 | const db = process.env.PG_DATABASE || 'risingstack_bootcamp' 8 | process.env.PG_URI = `postgres://${user}:${pw}@localhost:5432/${db}` 9 | } 10 | 11 | const knex = require('../models/db') 12 | 13 | knex.migrate.latest() 14 | .then(() => { 15 | // eslint-disable-next-line no-console 16 | console.log('Database synced successfully!') 17 | process.exit(0) 18 | }) 19 | .catch((err) => { 20 | // eslint-disable-next-line no-console 21 | console.error(err) 22 | process.exit(1) 23 | }) 24 | -------------------------------------------------------------------------------- /scripts/trigger.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | const joi = require('joi') 4 | 5 | const envVarsSchema = joi.object({ 6 | TRIGGER_QUERY: joi.string().required(), 7 | }).unknown() 8 | .required() 9 | 10 | const envVars = joi.attempt(process.env, envVarsSchema) 11 | 12 | const args = process.argv.slice(2) 13 | if (args.includes('--local') || args.includes('-L')) { 14 | process.env.REDIS_URI = 'redis://localhost' 15 | } 16 | 17 | const redis = require('../models/redis') 18 | 19 | const { CHANNELS } = redis 20 | 21 | redis.publishObject(CHANNELS.collect.trigger.v1, { 22 | query: envVars.TRIGGER_QUERY, 23 | date: new Date().toISOString() 24 | }) 25 | .then(() => { 26 | // eslint-disable-next-line no-console 27 | console.log('Trigger published successfully!') 28 | process.exit(0) 29 | }) 30 | .catch((err) => { 31 | // eslint-disable-next-line no-console 32 | console.error(err) 33 | process.exit(1) 34 | }) 35 | -------------------------------------------------------------------------------- /web/config.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | const joi = require('joi') 4 | 5 | const envVarsSchema = joi.object({ 6 | PORT: joi.number().integer().min(0).max(65535) 7 | .required() 8 | }).unknown() 9 | .required() 10 | 11 | const envVars = joi.attempt(process.env, envVarsSchema) 12 | 13 | const config = { 14 | port: envVars.PORT 15 | } 16 | 17 | module.exports = config 18 | -------------------------------------------------------------------------------- /web/index.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | const { promisify } = require('util') 4 | const http = require('http') 5 | const logger = require('winston') 6 | const db = require('../models/db') 7 | const redis = require('../models/redis') 8 | const config = require('./config') 9 | const app = require('./server') 10 | 11 | const server = http.createServer(app.callback()) 12 | const serverListen = promisify(server.listen).bind(server) 13 | const serverClose = promisify(server.close).bind(server) 14 | 15 | serverListen(config.port) 16 | .then(() => logger.info(`Server is listening on port ${config.port}`)) 17 | .catch((err) => { 18 | logger.error('Error happened during server start', err) 19 | process.exit(1) 20 | }) 21 | 22 | process.on('SIGTERM', gracefulShutdown) 23 | 24 | let shuttingDown = false 25 | async function gracefulShutdown() { 26 | logger.info('Got kill signal, starting graceful shutdown') 27 | 28 | if (shuttingDown) { 29 | return 30 | } 31 | 32 | shuttingDown = true 33 | try { 34 | await serverClose() 35 | await db.destroy() 36 | await redis.destroy() 37 | } catch (err) { 38 | logger.error('Error happened during graceful shutdown', err) 39 | process.exit(1) 40 | } 41 | 42 | logger.info('Graceful shutdown finished') 43 | process.exit() 44 | } 45 | -------------------------------------------------------------------------------- /web/middleware/index.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | const queryParser = require('./queryParser') 4 | const requestLogger = require('./requestLogger') 5 | const validator = require('./validator') 6 | 7 | module.exports = { 8 | queryParser, 9 | requestLogger, 10 | validator 11 | } 12 | -------------------------------------------------------------------------------- /web/middleware/queryParser.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | const qs = require('qs') 4 | 5 | function parseQueryFactory(options) { 6 | return async function parseQuery(ctx, next) { 7 | ctx.query = qs.parse(ctx.querystring, options) 8 | ctx.request.query = ctx.query 9 | await next() 10 | } 11 | } 12 | 13 | module.exports = parseQueryFactory 14 | -------------------------------------------------------------------------------- /web/middleware/requestLogger.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | const logger = require('winston') 4 | const _ = require('lodash') 5 | 6 | function createRequestLogger(options = {}) { 7 | let { level = 'silly' } = options 8 | 9 | return async function requestLogger(ctx, next) { 10 | const start = Date.now() 11 | 12 | const { method, originalUrl, headers: requestHeaders, body: requestBody } = ctx.request 13 | try { 14 | await next() 15 | } catch (err) { 16 | logger.error(`${method}: ${originalUrl}`, { error: err.message }) 17 | throw err 18 | } 19 | 20 | if (ctx.status >= 500) { 21 | level = 'error' 22 | } else if (ctx.status >= 400) { 23 | level = 'warn' 24 | } 25 | 26 | const ms = new Date() - start 27 | 28 | const { status, headers: responseHeaders, body: responseBody = '' } = ctx.response 29 | const logContext = { 30 | method, 31 | originalUrl, 32 | duration: `${ms}ms`, 33 | request: _.omitBy({ 34 | headers: _.omit(requestHeaders, ['authorization', 'cookie']), 35 | body: requestBody 36 | }, _.isNil), 37 | response: _.omitBy({ 38 | status, 39 | headers: _.omit(responseHeaders, ['authorization', 'cookie']), 40 | body: responseBody 41 | }, _.isNil) 42 | } 43 | 44 | logger.log(level, `${method}: ${originalUrl}`, logContext) 45 | } 46 | } 47 | 48 | module.exports = createRequestLogger 49 | -------------------------------------------------------------------------------- /web/middleware/validator.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | const fp = require('lodash/fp') 4 | const joi = require('joi') 5 | 6 | function validatorFactory(schemas) { 7 | return async function validatorMiddleware(ctx, next) { 8 | try { 9 | ['params', 'query', 'body'] 10 | .forEach((partToValidate) => { 11 | const toValidate = ctx.request[partToValidate] || ctx[partToValidate] 12 | if (schemas[partToValidate]) { 13 | const validatedObject = joi.attempt(toValidate, schemas[partToValidate].label(partToValidate)) 14 | 15 | Object.assign(toValidate, validatedObject) 16 | } 17 | }) 18 | 19 | if (schemas.body && ctx.request.body) { 20 | ctx.request.body = joi.attempt(ctx.request.body, schemas.body.label('body')) 21 | } 22 | } catch (err) { 23 | if (!err.isJoi) { 24 | throw err 25 | } 26 | 27 | const errors = fp.compose([ 28 | fp.mapValues(fp.map('message')), 29 | fp.groupBy('context.key') 30 | ])(err.details) 31 | 32 | ctx.status = 400 33 | ctx.body = { errors } 34 | 35 | return 36 | } 37 | 38 | await next() 39 | } 40 | } 41 | 42 | module.exports = validatorFactory 43 | -------------------------------------------------------------------------------- /web/router/contribution/getById.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | const joi = require('joi') 4 | const compose = require('koa-compose') 5 | const Contribution = require('../../../models/contribution') 6 | const middleware = require('../../middleware') 7 | 8 | const paramsSchema = joi.object({ 9 | id: joi.number().integer().required() 10 | }) 11 | .required() 12 | 13 | async function getById(ctx) { 14 | const result = await Contribution.read({ repository: ctx.params }) 15 | if (!result.length) { 16 | ctx.status = 404 17 | return 18 | } 19 | 20 | ctx.body = result 21 | } 22 | 23 | module.exports = compose([ 24 | middleware.validator({ 25 | params: paramsSchema 26 | }), 27 | getById 28 | ]) 29 | -------------------------------------------------------------------------------- /web/router/contribution/getById.spec.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | const _ = require('lodash') 4 | const request = require('super-request') 5 | const { expect } = require('chai') 6 | const Contribution = require('../../../models/contribution') 7 | const server = require('../../server') 8 | 9 | const url = '/api/v1/repository/:id/contributions' 10 | describe(`GET ${url}`, () => { 11 | it('should response with 200 when the repository exists with contributions', async function () { 12 | const id = _.random(1000) 13 | const lineCount = _.random(1000) 14 | this.sandbox.stub(Contribution, 'read').resolves([{ line_count: lineCount }]) 15 | 16 | const { body } = await request(server.listen()) 17 | .get(url.replace(':id', id)) 18 | .expect(200) 19 | .json(true) 20 | .end() 21 | 22 | expect(body).to.eql([{ line_count: lineCount }]) 23 | expect(Contribution.read).to.have.been.calledWith({ repository: { id } }) 24 | }) 25 | 26 | it('should response with 404 when the repository does not exist', async function () { 27 | const id = _.random(1000) 28 | this.sandbox.stub(Contribution, 'read').resolves([]) 29 | 30 | await request(server.listen()) 31 | .get(url.replace(':id', id)) 32 | .expect(404) 33 | .json(true) 34 | .end() 35 | 36 | expect(Contribution.read).to.have.been.calledWith({ repository: { id } }) 37 | }) 38 | }) 39 | -------------------------------------------------------------------------------- /web/router/contribution/getByName.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | const joi = require('joi') 4 | const compose = require('koa-compose') 5 | const Contribution = require('../../../models/contribution') 6 | const middleware = require('../../middleware') 7 | 8 | const paramsSchema = joi.object({ 9 | owner: joi.string().required(), 10 | name: joi.string().required() 11 | }) 12 | .required() 13 | 14 | async function getByName(ctx) { 15 | const { owner, name } = ctx.params 16 | const fullName = `${owner}/${name}` 17 | const result = await Contribution.read({ repository: { full_name: fullName } }) 18 | if (!result.length) { 19 | ctx.status = 404 20 | return 21 | } 22 | 23 | ctx.body = result 24 | } 25 | 26 | module.exports = compose([ 27 | middleware.validator({ 28 | params: paramsSchema 29 | }), 30 | getByName 31 | ]) 32 | -------------------------------------------------------------------------------- /web/router/contribution/getByName.spec.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | const _ = require('lodash') 4 | const request = require('super-request') 5 | const { expect } = require('chai') 6 | const Contribution = require('../../../models/contribution') 7 | const server = require('../../server') 8 | 9 | const url = '/api/v1/repository/:owner/:name/contributions' 10 | describe(`GET ${url}`, () => { 11 | it('should response with 200 when the repository exists with contributions', async function () { 12 | const owner = 'RisingStack' 13 | const name = 'foo' 14 | const fullName = `${owner}/${name}` 15 | const lineCount = _.random(1000) 16 | this.sandbox.stub(Contribution, 'read').resolves([{ line_count: lineCount }]) 17 | 18 | const { body } = await request(server.listen()) 19 | .get(url.replace(':owner', owner).replace(':name', name)) 20 | .expect(200) 21 | .json(true) 22 | .end() 23 | 24 | expect(body).to.eql([{ line_count: lineCount }]) 25 | expect(Contribution.read).to.have.been.calledWith({ repository: { full_name: fullName } }) 26 | }) 27 | 28 | it('should response with 404 when the repository does not exist', async function () { 29 | const owner = 'RisingStack' 30 | const name = 'foo' 31 | const fullName = `${owner}/${name}` 32 | this.sandbox.stub(Contribution, 'read').resolves([]) 33 | 34 | await request(server.listen()) 35 | .get(url.replace(':owner', owner).replace(':name', name)) 36 | .expect(404) 37 | .json(true) 38 | .end() 39 | 40 | expect(Contribution.read).to.have.been.calledWith({ repository: { full_name: fullName } }) 41 | }) 42 | }) 43 | -------------------------------------------------------------------------------- /web/router/contribution/index.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | const getById = require('./getById') 4 | const getByName = require('./getByName') 5 | 6 | module.exports = { 7 | getById, 8 | getByName 9 | } 10 | -------------------------------------------------------------------------------- /web/router/healthz/get.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | const logger = require('winston') 4 | const db = require('../../../models/db') 5 | const redis = require('../../../models/redis') 6 | 7 | const state = { 8 | shutdown: false 9 | } 10 | 11 | process.on('SIGTERM', () => { 12 | state.shutdown = true 13 | }) 14 | 15 | async function get(ctx) { 16 | if (state.shutdown) { 17 | ctx.throw(503, 'Service is shutting down') 18 | } 19 | 20 | try { 21 | await db.healthCheck() 22 | } catch (err) { 23 | const message = 'Database health check failed' 24 | logger.error(message, err) 25 | ctx.throw(500, message) 26 | } 27 | 28 | try { 29 | await redis.healthCheck() 30 | } catch (err) { 31 | const message = 'Redis health check failed' 32 | logger.error(message, err) 33 | ctx.throw(500, message) 34 | } 35 | 36 | ctx.body = { 37 | status: 'ok' 38 | } 39 | } 40 | 41 | module.exports = get 42 | -------------------------------------------------------------------------------- /web/router/healthz/get.spec.js: -------------------------------------------------------------------------------- 1 | /* eslint-disable no-unused-expressions */ 2 | 3 | 'use strict' 4 | 5 | const { expect } = require('chai') 6 | const request = require('super-request') 7 | const db = require('../../../models/db') 8 | const redis = require('../../../models/redis') 9 | const server = require('../../server') 10 | 11 | const url = '/healthz' 12 | describe(`GET ${url}`, () => { 13 | it('should be healthy', async function () { 14 | this.sandbox.stub(db, 'healthCheck').resolves() 15 | this.sandbox.stub(redis, 'healthCheck').resolves() 16 | 17 | const { body } = await request(server.listen()) 18 | .get(url) 19 | .json(true) 20 | .expect(200) 21 | .end() 22 | 23 | expect(body).to.be.eql({ 24 | status: 'ok' 25 | }) 26 | 27 | expect(db.healthCheck).to.have.been.called 28 | expect(redis.healthCheck).to.have.been.called 29 | }) 30 | 31 | it('should return 500 if the db is not healthy', async function () { 32 | this.sandbox.stub(db, 'healthCheck').rejects(new Error()) 33 | this.sandbox.stub(redis, 'healthCheck').resolves() 34 | 35 | await request(server.listen()) 36 | .get(url) 37 | .expect(500) 38 | .end() 39 | 40 | expect(db.healthCheck).to.have.been.called 41 | }) 42 | 43 | it('should return 500 if the redis is not healthy', async function () { 44 | this.sandbox.stub(db, 'healthCheck').resolves() 45 | this.sandbox.stub(redis, 'healthCheck').rejects(new Error()) 46 | 47 | await request(server.listen()) 48 | .get(url) 49 | .expect(500) 50 | .end() 51 | 52 | expect(redis.healthCheck).to.have.been.called 53 | }) 54 | 55 | it('should return 503 if the process got SIGTERM', async function () { 56 | this.sandbox.stub(db, 'healthCheck').resolves() 57 | this.sandbox.stub(redis, 'healthCheck').resolves() 58 | 59 | process.emit('SIGTERM') 60 | 61 | await request(server.listen()) 62 | .get(url) 63 | .expect(503) 64 | .end() 65 | 66 | expect(db.healthCheck).not.to.have.been.called 67 | expect(redis.healthCheck).not.to.have.been.called 68 | }) 69 | }) 70 | -------------------------------------------------------------------------------- /web/router/healthz/index.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | const get = require('./get') 4 | 5 | module.exports = { 6 | get 7 | } 8 | -------------------------------------------------------------------------------- /web/router/index.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | module.exports = require('./router') 4 | -------------------------------------------------------------------------------- /web/router/repository/getById.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | const joi = require('joi') 4 | const compose = require('koa-compose') 5 | const Repository = require('../../../models/repository') 6 | const middleware = require('../../middleware') 7 | 8 | const paramsSchema = joi.object({ 9 | id: joi.number().integer().required() 10 | }) 11 | .required() 12 | 13 | async function get(ctx) { 14 | const result = await Repository.read(ctx.params) 15 | if (!result) { 16 | ctx.status = 404 17 | return 18 | } 19 | 20 | ctx.body = result 21 | } 22 | 23 | module.exports = compose([ 24 | middleware.validator({ 25 | params: paramsSchema 26 | }), 27 | get 28 | ]) 29 | -------------------------------------------------------------------------------- /web/router/repository/getById.spec.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | const _ = require('lodash') 4 | const request = require('super-request') 5 | const { expect } = require('chai') 6 | const Repository = require('../../../models/repository') 7 | const server = require('../../server') 8 | 9 | const url = '/api/v1/repository/:id' 10 | describe(`GET ${url}`, () => { 11 | it('should response with 200 when the repository exists', async function () { 12 | const id = _.random(1000) 13 | this.sandbox.stub(Repository, 'read').resolves({ id }) 14 | 15 | const { body } = await request(server.listen()) 16 | .get(url.replace(':id', id)) 17 | .expect(200) 18 | .json(true) 19 | .end() 20 | 21 | expect(body).to.eql({ id }) 22 | expect(Repository.read).to.have.been.calledWith({ id }) 23 | }) 24 | 25 | it('should response with 404 when the repository does not exist', async function () { 26 | const id = _.random(1000) 27 | this.sandbox.stub(Repository, 'read').resolves(undefined) 28 | 29 | await request(server.listen()) 30 | .get(url.replace(':id', id)) 31 | .expect(404) 32 | .json(true) 33 | .end() 34 | 35 | expect(Repository.read).to.have.been.calledWith({ id }) 36 | }) 37 | }) 38 | -------------------------------------------------------------------------------- /web/router/repository/getByName.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | const joi = require('joi') 4 | const compose = require('koa-compose') 5 | const Repository = require('../../../models/repository') 6 | const middleware = require('../../middleware') 7 | 8 | const paramsSchema = joi.object({ 9 | owner: joi.string().required(), 10 | name: joi.string().required() 11 | }) 12 | .required() 13 | 14 | async function get(ctx) { 15 | const { owner, name } = ctx.params 16 | const fullName = `${owner}/${name}` 17 | const result = await Repository.read({ full_name: fullName }) 18 | if (!result) { 19 | ctx.status = 404 20 | return 21 | } 22 | 23 | ctx.body = result 24 | } 25 | 26 | module.exports = compose([ 27 | middleware.validator({ 28 | params: paramsSchema 29 | }), 30 | get 31 | ]) 32 | -------------------------------------------------------------------------------- /web/router/repository/getByName.spec.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | const _ = require('lodash') 4 | const request = require('super-request') 5 | const { expect } = require('chai') 6 | const Repository = require('../../../models/repository') 7 | const server = require('../../server') 8 | 9 | const url = '/api/v1/repository/:owner/:name' 10 | describe(`GET ${url}`, () => { 11 | it('should response with 200 when the repository exists', async function () { 12 | const id = _.random(1000) 13 | const owner = 'RisingStack' 14 | const name = 'foo' 15 | const fullName = `${owner}/${name}` 16 | this.sandbox.stub(Repository, 'read').resolves({ id, full_name: fullName }) 17 | 18 | const { body } = await request(server.listen()) 19 | .get(url.replace(':owner', owner).replace(':name', name)) 20 | .expect(200) 21 | .json(true) 22 | .end() 23 | 24 | expect(body).to.eql({ id, full_name: fullName }) 25 | expect(Repository.read).to.have.been.calledWith({ full_name: fullName }) 26 | }) 27 | 28 | it('should response with 404 when the repository does not exist', async function () { 29 | const owner = 'RisingStack' 30 | const name = 'foo' 31 | const fullName = `${owner}/${name}` 32 | this.sandbox.stub(Repository, 'read').resolves(undefined) 33 | 34 | await request(server.listen()) 35 | .get(url.replace(':owner', owner).replace(':name', name)) 36 | .expect(404) 37 | .json(true) 38 | .end() 39 | 40 | expect(Repository.read).to.have.been.calledWith({ full_name: fullName }) 41 | }) 42 | }) 43 | -------------------------------------------------------------------------------- /web/router/repository/index.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | const getById = require('./getById') 4 | const getByName = require('./getByName') 5 | 6 | module.exports = { 7 | getById, 8 | getByName 9 | } 10 | -------------------------------------------------------------------------------- /web/router/router.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | const Router = require('koa-router') 4 | const bodyParser = require('koa-bodyparser') 5 | const middleware = require('../middleware') 6 | const healthz = require('./healthz') 7 | const trigger = require('./trigger') 8 | const repository = require('./repository') 9 | const contribution = require('./contribution') 10 | 11 | const router = new Router() 12 | 13 | router 14 | .use(bodyParser()) 15 | .use(middleware.queryParser({ allowDots: true })) 16 | 17 | router.get('/hello', (ctx) => { 18 | ctx.body = 'Hello Node.js!' 19 | }) 20 | 21 | router.get('/healthz', healthz.get) 22 | 23 | router.post('/api/v1/trigger', trigger.post) 24 | router.get('/api/v1/repository/:id', repository.getById) 25 | router.get('/api/v1/repository/:id/contributions', contribution.getById) 26 | router.get('/api/v1/repository/:owner/:name', repository.getByName) 27 | router.get('/api/v1/repository/:owner/:name/contributions', contribution.getByName) 28 | 29 | module.exports = router 30 | -------------------------------------------------------------------------------- /web/router/trigger/index.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | const post = require('./post') 4 | 5 | module.exports = { 6 | post 7 | } 8 | -------------------------------------------------------------------------------- /web/router/trigger/post.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | const joi = require('joi') 4 | const compose = require('koa-compose') 5 | const redis = require('../../../models/redis') 6 | const middleware = require('../../middleware') 7 | 8 | const { CHANNELS } = redis 9 | 10 | const bodySchema = joi.object({ 11 | query: joi.string().required() 12 | }).required() 13 | 14 | async function post(ctx) { 15 | const { query } = ctx.request.body 16 | 17 | await redis.publishObject(CHANNELS.collect.trigger.v1, { 18 | query, 19 | date: new Date().toISOString() 20 | }) 21 | 22 | ctx.status = 201 23 | } 24 | 25 | module.exports = compose([ 26 | middleware.validator({ 27 | body: bodySchema 28 | }), 29 | post 30 | ]) 31 | -------------------------------------------------------------------------------- /web/router/trigger/post.spec.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | const request = require('super-request') 4 | const { expect } = require('chai') 5 | const redis = require('../../../models/redis') 6 | const server = require('../../server') 7 | 8 | const { CHANNELS } = redis 9 | 10 | const url = '/api/v1/trigger' 11 | describe(`POST ${url}`, () => { 12 | let now 13 | 14 | beforeEach(function () { 15 | now = Date.now() 16 | this.sandbox.useFakeTimers(now) 17 | }) 18 | 19 | it('should validate the body', async () => { 20 | const { body } = await request(server.listen()) 21 | .post(url) 22 | .expect(400) 23 | .json(true) 24 | .end() 25 | 26 | expect(body).to.eql({ errors: { query: ['"query" is required'] } }) 27 | }) 28 | 29 | it('should response with 201 and send to the trigger queue', async function () { 30 | this.sandbox.stub(redis, 'publishObject').resolves() 31 | 32 | const query = 'language:javascript' 33 | await request(server.listen()) 34 | .post(url) 35 | .body({ 36 | query 37 | }) 38 | .expect(201) 39 | .json(true) 40 | .end() 41 | 42 | expect(redis.publishObject).to.have.been.calledWith(CHANNELS.collect.trigger.v1, { 43 | query, 44 | date: new Date(now).toISOString() 45 | }) 46 | }) 47 | }) 48 | -------------------------------------------------------------------------------- /web/server.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | const Koa = require('koa') 4 | const logger = require('winston') 5 | const middleware = require('./middleware') 6 | const router = require('./router') 7 | 8 | const app = new Koa() 9 | 10 | app 11 | .use(middleware.requestLogger()) 12 | .use(router.routes()) 13 | .use(router.allowedMethods()) 14 | 15 | app.on('error', (err) => { 16 | logger.error('Server error', { error: err.message }) 17 | }) 18 | 19 | module.exports = app 20 | -------------------------------------------------------------------------------- /web/server.spec.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | const request = require('super-request') 4 | const { expect } = require('chai') 5 | const server = require('./server') 6 | 7 | describe('GET /hello', () => { 8 | it('should response with `Hello Node.js!`', async () => { 9 | const { body } = await request(server.listen()) 10 | .get('/hello') 11 | .expect(200) 12 | .end() 13 | 14 | expect(body).to.eql('Hello Node.js!') 15 | }) 16 | }) 17 | -------------------------------------------------------------------------------- /web/test.setup.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | const sinon = require('sinon') 4 | const chai = require('chai') 5 | const sinonChai = require('sinon-chai') 6 | const winston = require('winston') 7 | 8 | chai.use(sinonChai) 9 | 10 | // postgres 11 | const user = process.env.PG_USER || process.env.USER || 'root' 12 | const pw = process.env.PG_PASSWORD || '' 13 | const db = process.env.PG_DATABASE || 'risingstack_bootcamp' 14 | process.env.PG_URI = `postgres://${user}:${pw}@localhost:5432/${db}` 15 | 16 | // amqp 17 | process.env.REDIS_URI = 'redis://localhost' 18 | 19 | // logger 20 | winston.setLevels({ test: 0, error: 1, warn: 2, info: 3, verbose: 4, debug: 5, silly: 6 }) 21 | winston.addColors({ 22 | test: 'cyan', error: 'red', warn: 'yellow', info: 'cyan', verbose: 'cyan', debug: 'blue', silly: 'magenta' 23 | }) 24 | winston.remove(winston.transports.Console) 25 | winston.add(winston.transports.Console, { 26 | level: process.env.LOGGER_LEVEL || 'test', colorize: true, prettyPrint: true 27 | }) 28 | 29 | beforeEach(function beforeEach() { 30 | this.sandbox = sinon.sandbox.create() 31 | }) 32 | 33 | afterEach(function afterEach() { 34 | this.sandbox.restore() 35 | }) 36 | -------------------------------------------------------------------------------- /worker/config.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | const joi = require('joi') 4 | 5 | const envVarsSchema = joi.object({ 6 | PORT: joi.number().integer().min(0).max(65535) 7 | .required() 8 | }).unknown() 9 | .required() 10 | 11 | const envVars = joi.attempt(process.env, envVarsSchema) 12 | 13 | const config = { 14 | port: envVars.PORT 15 | } 16 | 17 | module.exports = config 18 | -------------------------------------------------------------------------------- /worker/handlers/contributions.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | const _ = require('lodash') 4 | const joi = require('joi') 5 | const logger = require('winston') 6 | const GitHub = require('../../models/github') 7 | const User = require('../../models/user') 8 | const Contribution = require('../../models/contribution') 9 | 10 | const schema = joi.object({ 11 | date: joi.date().raw().required(), 12 | repository: joi.object({ 13 | id: joi.number().integer().required(), 14 | full_name: joi.string().required() 15 | }).unknown().required() 16 | }).required() 17 | 18 | async function onContributions(message) { 19 | logger.debug('contributions: received', message) 20 | 21 | // validate data 22 | let data 23 | try { 24 | data = joi.attempt(message, schema) 25 | } catch (err) { 26 | logger.error('contributions: invalid message', { 27 | data: message, 28 | error: err.message 29 | }) 30 | 31 | return 32 | } 33 | 34 | const { repository } = data 35 | 36 | let contributors = await GitHub.api.getContributors(repository.full_name) 37 | contributors = contributors.map(({ author, weeks }) => ({ 38 | user: _.pick(author, ['id', 'login', 'avatar_url', 'html_url', 'type']), 39 | line_count: weeks.reduce((lines, { a: additions, d: deletions }) => lines + (additions - deletions), 0) 40 | })) 41 | 42 | await Promise.all(contributors.map(({ user, line_count }) => 43 | User.insert(user) 44 | .catch((err) => logger.warn('contributions: User insert error', err)) 45 | .then(() => Contribution.insertOrReplace({ 46 | line_count, 47 | repository: repository.id, 48 | user: user.id 49 | })) 50 | )) 51 | 52 | logger.debug('contributions: finished', message) 53 | } 54 | 55 | module.exports = onContributions 56 | -------------------------------------------------------------------------------- /worker/handlers/contributions.spec.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | const _ = require('lodash') 4 | const { expect } = require('chai') 5 | const worker = require('../worker') 6 | const redis = require('../../models/redis') 7 | const GitHub = require('../../models/github') 8 | const User = require('../../models/user') 9 | const Contribution = require('../../models/contribution') 10 | const handlers = require('./') 11 | 12 | const { CHANNELS } = redis 13 | 14 | describe('Worker "contributions" channel', () => { 15 | it(`should handle messages on the ${CHANNELS.collect.contributions.v1} channel`, async function () { 16 | const date = new Date().toISOString() 17 | const repository = { 18 | id: _.random(1000), 19 | full_name: 'project/name' 20 | } 21 | 22 | const done = new Promise((resolve, reject) => { 23 | this.sandbox.stub(handlers, 'contributions').callsFake(async (params) => { 24 | await worker.halt() 25 | 26 | try { 27 | expect(params).to.be.eql({ 28 | date, 29 | repository 30 | }) 31 | } catch (err) { 32 | reject(err) 33 | return 34 | } 35 | 36 | resolve() 37 | }) 38 | }) 39 | 40 | await worker.init() 41 | 42 | await redis.publishObject(CHANNELS.collect.contributions.v1, { 43 | date, 44 | repository 45 | }) 46 | 47 | return done 48 | }) 49 | 50 | it('should fetch the contributions from GitHub & save them to the database', 51 | async function () { 52 | const date = new Date().toISOString() 53 | const repository = { 54 | id: _.random(1000), 55 | full_name: 'project/name' 56 | } 57 | 58 | const author = { 59 | id: _.random(1000), 60 | login: 'user' 61 | } 62 | 63 | this.sandbox.stub(GitHub.api, 'getContributors').resolves([{ 64 | author, 65 | weeks: [{ 66 | a: 100, 67 | d: 10 68 | }, { 69 | a: 30, 70 | d: 50 71 | }] 72 | }]) 73 | this.sandbox.stub(User, 'insert').resolves() 74 | this.sandbox.stub(Contribution, 'insertOrReplace').resolves() 75 | 76 | await handlers.contributions({ 77 | date, 78 | repository 79 | }) 80 | 81 | expect(GitHub.api.getContributors).to.have.been.calledWith(repository.full_name) 82 | expect(User.insert).to.have.been.calledWith(author) 83 | expect(Contribution.insertOrReplace).to.have.been.calledWith({ 84 | line_count: 70, 85 | repository: repository.id, 86 | user: author.id 87 | }) 88 | }) 89 | }) 90 | -------------------------------------------------------------------------------- /worker/handlers/index.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | const trigger = require('./trigger') 4 | const repository = require('./repository') 5 | const contributions = require('./contributions') 6 | 7 | module.exports = { 8 | trigger, 9 | repository, 10 | contributions 11 | } 12 | -------------------------------------------------------------------------------- /worker/handlers/repository.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | const fp = require('lodash/fp') 4 | const joi = require('joi') 5 | const logger = require('winston') 6 | const redis = require('../../models/redis') 7 | const GitHub = require('../../models/github') 8 | const User = require('../../models/user') 9 | const Repository = require('../../models/repository') 10 | 11 | const { CHANNELS } = redis 12 | 13 | const schema = joi.object({ 14 | date: joi.date().raw().required(), 15 | query: joi.string().required(), 16 | page: joi.number().integer().required() 17 | }).required() 18 | 19 | async function onRepository(message) { 20 | logger.debug('repository: received', message) 21 | 22 | // validate data 23 | let data 24 | try { 25 | data = joi.attempt(message, schema) 26 | } catch (err) { 27 | logger.error('repository: invalid message', { 28 | data: message, 29 | error: err.message 30 | }) 31 | 32 | return 33 | } 34 | 35 | const { date, query, page } = data 36 | 37 | let { items } = await GitHub.api.searchRepositories({ q: query, page, per_page: 100 }) 38 | items = items.map((item) => ({ 39 | owner: fp.pick(['id', 'login', 'avatar_url', 'html_url', 'type'], item.owner), 40 | repository: fp.compose([ 41 | fp.assign({ owner: item.owner.id }), 42 | fp.defaults({ description: '', language: '' }), 43 | fp.omitBy(fp.isNil), 44 | fp.pick(['id', 'full_name', 'description', 'html_url', 'language', 'stargazers_count']) 45 | ])(item) 46 | })) 47 | 48 | await Promise.all(items.map(({ owner, repository }) => 49 | User.insert(owner) 50 | .catch((err) => logger.warn('repository: User insert error', err)) 51 | .then(() => Repository.insert(repository)) 52 | .catch((err) => logger.warn('repository: Repository insert error', err)) 53 | .then(() => redis.publishObject(CHANNELS.collect.contributions.v1, { 54 | date, 55 | repository 56 | })) 57 | )) 58 | 59 | logger.debug('repository: finished', message) 60 | } 61 | 62 | module.exports = onRepository 63 | -------------------------------------------------------------------------------- /worker/handlers/repository.spec.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | const _ = require('lodash') 4 | const { expect } = require('chai') 5 | const worker = require('../worker') 6 | const redis = require('../../models/redis') 7 | const GitHub = require('../../models/github') 8 | const User = require('../../models/user') 9 | const Repository = require('../../models/repository') 10 | const handlers = require('./') 11 | 12 | const { CHANNELS } = redis 13 | 14 | describe('Worker "repository" channel', () => { 15 | it(`should handle messages on the ${CHANNELS.collect.repository.v1} channel`, async function () { 16 | const date = new Date().toISOString() 17 | const query = 'language:javascript' 18 | const page = _.random(10) 19 | 20 | const done = new Promise((resolve, reject) => { 21 | this.sandbox.stub(handlers, 'repository').callsFake(async (params) => { 22 | await worker.halt() 23 | 24 | try { 25 | expect(params).to.be.eql({ 26 | date, 27 | query, 28 | page 29 | }) 30 | } catch (err) { 31 | reject(err) 32 | return 33 | } 34 | 35 | resolve() 36 | }) 37 | }) 38 | 39 | await worker.init() 40 | 41 | await redis.publishObject(CHANNELS.collect.repository.v1, { 42 | date, 43 | query, 44 | page 45 | }) 46 | 47 | return done 48 | }) 49 | 50 | it(`should fetch repositories from GitHub & send the messages to the ${CHANNELS.collect.contributions.v1} channel`, 51 | async function () { 52 | const owner = { 53 | id: _.random(1000), 54 | login: 'project-owner', 55 | avatar_url: 'https://github.com/project-owner.png', 56 | html_url: 'https://github.com/project-owner.png', 57 | type: 'User' 58 | } 59 | 60 | const repository = { 61 | id: _.random(1000), 62 | full_name: 'project/name', 63 | description: 'Very project', 64 | html_url: 'https://github.com/project/name', 65 | language: 'JavaScript', 66 | stargazers_count: _.random(1000) 67 | } 68 | 69 | this.sandbox.stub(GitHub.api, 'searchRepositories').resolves({ items: [Object.assign({ owner }, repository)] }) 70 | this.sandbox.stub(User, 'insert').resolves() 71 | this.sandbox.stub(Repository, 'insert').resolves() 72 | this.sandbox.stub(redis, 'publishObject').resolves() 73 | 74 | const date = new Date().toISOString() 75 | const query = 'language:javascript' 76 | const page = 0 77 | 78 | await handlers.repository({ 79 | date, 80 | query, 81 | page 82 | }) 83 | 84 | expect(GitHub.api.searchRepositories).to.have.been.calledWith({ q: query, page, per_page: 100 }) 85 | expect(User.insert).to.have.been.calledWith(owner) 86 | expect(Repository.insert).to.have.been.calledWith(Object.assign({ owner: owner.id }, repository)) 87 | expect(redis.publishObject).to.have.been.calledWith(CHANNELS.collect.contributions.v1, { 88 | date, 89 | repository: Object.assign({ owner: owner.id }, repository) 90 | }) 91 | }) 92 | }) 93 | -------------------------------------------------------------------------------- /worker/handlers/trigger.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | const _ = require('lodash') 4 | const joi = require('joi') 5 | const logger = require('winston') 6 | const redis = require('../../models/redis') 7 | 8 | const { CHANNELS } = redis 9 | 10 | const schema = joi.object({ 11 | date: joi.date().raw().required(), 12 | query: joi.string().required() 13 | }).required() 14 | 15 | async function onTrigger(message) { 16 | logger.debug('trigger: received', message) 17 | 18 | // validate data 19 | let data 20 | try { 21 | data = joi.attempt(message, schema) 22 | } catch (err) { 23 | logger.error('trigger: invalid message', { 24 | data: message, 25 | error: err.message 26 | }) 27 | 28 | return 29 | } 30 | 31 | const { date, query } = data 32 | // only the first 1000 search results are available 33 | await Promise.all(_.range(10).map((page) => 34 | redis.publishObject(CHANNELS.collect.repository.v1, { 35 | date, 36 | page, 37 | query 38 | }) 39 | )) 40 | 41 | logger.debug('trigger: finished', message) 42 | } 43 | 44 | module.exports = onTrigger 45 | -------------------------------------------------------------------------------- /worker/handlers/trigger.spec.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | const { expect } = require('chai') 4 | const _ = require('lodash') 5 | const redis = require('../../models/redis') 6 | const worker = require('../worker') 7 | const handlers = require('./') 8 | 9 | const { CHANNELS } = redis 10 | 11 | describe('Worker "trigger" channel', () => { 12 | it(`should handle messages on the ${CHANNELS.collect.trigger.v1} channel`, async function () { 13 | const date = new Date().toISOString() 14 | const query = 'language:javascript' 15 | 16 | const done = new Promise((resolve, reject) => { 17 | this.sandbox.stub(handlers, 'trigger').callsFake(async (params) => { 18 | await worker.halt() 19 | 20 | try { 21 | expect(params).to.be.eql({ 22 | date, 23 | query 24 | }) 25 | } catch (err) { 26 | reject(err) 27 | return 28 | } 29 | 30 | resolve() 31 | }) 32 | }) 33 | 34 | await worker.init() 35 | 36 | await redis.publishObject(CHANNELS.collect.trigger.v1, { 37 | date, 38 | query 39 | }) 40 | 41 | return done 42 | }) 43 | 44 | it(`should send the messages to the ${CHANNELS.collect.repository.v1} channel`, async function () { 45 | this.sandbox.stub(redis, 'publishObject').resolves() 46 | 47 | const date = new Date().toISOString() 48 | const query = 'language:javascript' 49 | await handlers.trigger({ 50 | date, 51 | query 52 | }) 53 | 54 | expect(redis.publishObject).to.have.callCount(10) 55 | _.range(10).forEach((page) => { 56 | expect(redis.publishObject).to.have.been.calledWith(CHANNELS.collect.repository.v1, { 57 | date, 58 | query, 59 | page 60 | }) 61 | }) 62 | }) 63 | }) 64 | -------------------------------------------------------------------------------- /worker/index.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | const { promisify } = require('util') 4 | const http = require('http') 5 | const logger = require('winston') 6 | const db = require('../models/db') 7 | const redis = require('../models/redis') 8 | const worker = require('./worker') 9 | const config = require('./config') 10 | const app = require('./server') 11 | 12 | const server = http.createServer(app.callback()) 13 | const serverListen = promisify(server.listen).bind(server) 14 | const serverClose = promisify(server.close).bind(server) 15 | 16 | Promise.all([worker.init(), serverListen(config.port)]) 17 | .then(() => { 18 | logger.info('Worker is running') 19 | logger.info(`Server is listening on port ${config.port}`) 20 | }) 21 | .catch((err) => { 22 | logger.error('Error happened during worker initialization', err) 23 | process.exit(1) 24 | }) 25 | 26 | process.on('SIGTERM', gracefulShutdown) 27 | 28 | let shuttingDown = false 29 | async function gracefulShutdown() { 30 | logger.info('Got kill signal, starting graceful shutdown') 31 | 32 | if (shuttingDown) { 33 | return 34 | } 35 | 36 | shuttingDown = true 37 | try { 38 | await redis.destroy() 39 | await db.destroy() 40 | await serverClose() 41 | } catch (err) { 42 | logger.error('Error happened during graceful shutdown', err) 43 | process.exit(1) 44 | } 45 | 46 | logger.info('Graceful shutdown finished') 47 | process.exit() 48 | } 49 | -------------------------------------------------------------------------------- /worker/router/healthz/get.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | const logger = require('winston') 4 | const db = require('../../../models/db') 5 | const redis = require('../../../models/redis') 6 | 7 | const state = { 8 | shutdown: false 9 | } 10 | 11 | process.on('SIGTERM', () => { 12 | state.shutdown = true 13 | }) 14 | 15 | async function get(ctx) { 16 | if (state.shutdown) { 17 | ctx.throw(503, 'Service is shutting down') 18 | } 19 | 20 | try { 21 | await db.healthCheck() 22 | } catch (err) { 23 | const message = 'Database health check failed' 24 | logger.error(message, err) 25 | ctx.throw(500, message) 26 | } 27 | 28 | try { 29 | await redis.healthCheck() 30 | } catch (err) { 31 | const message = 'Redis health check failed' 32 | logger.error(message, err) 33 | ctx.throw(500, message) 34 | } 35 | 36 | ctx.body = { 37 | status: 'ok' 38 | } 39 | } 40 | 41 | module.exports = get 42 | -------------------------------------------------------------------------------- /worker/router/healthz/get.spec.js: -------------------------------------------------------------------------------- 1 | /* eslint-disable no-unused-expressions */ 2 | 3 | 'use strict' 4 | 5 | const { expect } = require('chai') 6 | const request = require('super-request') 7 | const db = require('../../../models/db') 8 | const redis = require('../../../models/redis') 9 | const server = require('../../server') 10 | 11 | const url = '/healthz' 12 | describe(`GET ${url}`, () => { 13 | it('should be healthy', async function () { 14 | this.sandbox.stub(db, 'healthCheck').resolves() 15 | this.sandbox.stub(redis, 'healthCheck').resolves() 16 | 17 | const { body } = await request(server.listen()) 18 | .get(url) 19 | .json(true) 20 | .expect(200) 21 | .end() 22 | 23 | expect(body).to.be.eql({ 24 | status: 'ok' 25 | }) 26 | 27 | expect(db.healthCheck).to.have.been.called 28 | expect(redis.healthCheck).to.have.been.called 29 | }) 30 | 31 | it('should return 500 if the db is not healthy', async function () { 32 | this.sandbox.stub(db, 'healthCheck').rejects(new Error()) 33 | this.sandbox.stub(redis, 'healthCheck').resolves() 34 | 35 | await request(server.listen()) 36 | .get(url) 37 | .expect(500) 38 | .end() 39 | 40 | expect(db.healthCheck).to.have.been.called 41 | }) 42 | 43 | it('should return 500 if the redis is not healthy', async function () { 44 | this.sandbox.stub(db, 'healthCheck').resolves() 45 | this.sandbox.stub(redis, 'healthCheck').rejects(new Error()) 46 | 47 | await request(server.listen()) 48 | .get(url) 49 | .expect(500) 50 | .end() 51 | 52 | expect(redis.healthCheck).to.have.been.called 53 | }) 54 | 55 | it('should return 503 if the process got SIGTERM', async function () { 56 | this.sandbox.stub(db, 'healthCheck').resolves() 57 | this.sandbox.stub(redis, 'healthCheck').resolves() 58 | 59 | process.emit('SIGTERM') 60 | 61 | await request(server.listen()) 62 | .get(url) 63 | .expect(503) 64 | .end() 65 | 66 | expect(db.healthCheck).not.to.have.been.called 67 | expect(redis.healthCheck).not.to.have.been.called 68 | }) 69 | }) 70 | -------------------------------------------------------------------------------- /worker/router/healthz/index.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | const get = require('./get') 4 | 5 | module.exports = { 6 | get 7 | } 8 | -------------------------------------------------------------------------------- /worker/router/index.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | module.exports = require('./router') 4 | -------------------------------------------------------------------------------- /worker/router/router.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | const Router = require('koa-router') 4 | const healthz = require('./healthz') 5 | 6 | const router = new Router() 7 | 8 | router.get('/healthz', healthz.get) 9 | 10 | module.exports = router 11 | -------------------------------------------------------------------------------- /worker/server.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | const Koa = require('koa') 4 | const logger = require('winston') 5 | const router = require('./router') 6 | 7 | const app = new Koa() 8 | 9 | app.use(router.routes()) 10 | 11 | app.on('error', (err) => { 12 | logger.error('Server error', { error: err.message }) 13 | }) 14 | 15 | module.exports = app 16 | -------------------------------------------------------------------------------- /worker/test.setup.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | const sinon = require('sinon') 4 | const chai = require('chai') 5 | const sinonChai = require('sinon-chai') 6 | const winston = require('winston') 7 | 8 | chai.use(sinonChai) 9 | 10 | // postgres 11 | const user = process.env.PG_USER || process.env.USER || 'root' 12 | const pw = process.env.PG_PASSWORD || '' 13 | const db = process.env.PG_DATABASE || 'risingstack_bootcamp' 14 | process.env.PG_URI = `postgres://${user}:${pw}@localhost:5432/${db}` 15 | 16 | // amqp 17 | process.env.REDIS_URI = 'redis://localhost' 18 | 19 | // logger 20 | winston.setLevels({ test: 0, error: 1, warn: 2, info: 3, verbose: 4, debug: 5, silly: 6 }) 21 | winston.addColors({ 22 | test: 'cyan', error: 'red', warn: 'yellow', info: 'cyan', verbose: 'cyan', debug: 'blue', silly: 'magenta' 23 | }) 24 | winston.remove(winston.transports.Console) 25 | winston.add(winston.transports.Console, { 26 | level: process.env.LOGGER_LEVEL || 'test', colorize: true, prettyPrint: true 27 | }) 28 | 29 | beforeEach(function beforeEach() { 30 | this.sandbox = sinon.sandbox.create() 31 | }) 32 | 33 | afterEach(function afterEach() { 34 | this.sandbox.restore() 35 | }) 36 | -------------------------------------------------------------------------------- /worker/worker.js: -------------------------------------------------------------------------------- 1 | 'use strict' 2 | 3 | const logger = require('winston') 4 | const redis = require('../models/redis') 5 | const handlers = require('./handlers') 6 | 7 | const { subscriber, publisher, CHANNELS } = redis 8 | 9 | async function init() { 10 | await Promise.all([ 11 | subscriber.connect(), 12 | publisher.connect() 13 | ]) 14 | 15 | await subscriber.subscribe( 16 | CHANNELS.collect.trigger.v1, 17 | CHANNELS.collect.repository.v1, 18 | CHANNELS.collect.contributions.v1 19 | ) 20 | 21 | await subscriber.on('message', (channel, message) => { 22 | let messageObject 23 | try { 24 | messageObject = JSON.parse(message) 25 | } catch (err) { 26 | logger.warn('Invalid message, failed to parse', { 27 | message, 28 | error: err.message 29 | }) 30 | return 31 | } 32 | 33 | switch (channel) { 34 | case CHANNELS.collect.trigger.v1: 35 | handlers.trigger(messageObject) 36 | .catch(logError) 37 | break 38 | case CHANNELS.collect.repository.v1: 39 | handlers.repository(messageObject) 40 | .catch(logError) 41 | break 42 | case CHANNELS.collect.contributions.v1: 43 | handlers.contributions(messageObject) 44 | .catch(logError) 45 | break 46 | default: 47 | logger.warn(`Redis message is not handled on channel '${channel}'`, message) 48 | } 49 | 50 | function logError(err) { 51 | logger.debug('Message handling error', { 52 | message, 53 | error: err.message 54 | }) 55 | } 56 | }) 57 | 58 | logger.info('Channels are initialized') 59 | } 60 | 61 | async function halt() { 62 | await redis.destroy() 63 | 64 | logger.info('Channels are canceled') 65 | } 66 | 67 | module.exports = { 68 | init, 69 | halt 70 | } 71 | --------------------------------------------------------------------------------