├── .gitignore ├── LICENSE ├── README.md ├── github-usernames.csv ├── package-lock.json ├── package.json └── src ├── main.js └── transform-username-to-github-repos.js /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules/ 2 | .idea/ 3 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2019 Tommy May 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # dev.to article - How to Process Epic Amounts of Data in NodeJS 2 | -------------------------------------------------------------------------------- /github-usernames.csv: -------------------------------------------------------------------------------- 1 | "itmayziii" 2 | "dhershman1" 3 | "HetaRZinzuvadia" 4 | "joeswislocki" 5 | "justinvoelkel" 6 | "mandarm2593" 7 | "mfrost503" 8 | "wmontgomery" 9 | "kentcdodds" 10 | "gaearon" 11 | "btholt" 12 | "paulirish" 13 | "ryanflorence" 14 | -------------------------------------------------------------------------------- /package-lock.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "dev-to-nodejs-epic-data", 3 | "version": "0.0.1", 4 | "lockfileVersion": 1, 5 | "requires": true, 6 | "dependencies": { 7 | "axios": { 8 | "version": "0.18.0", 9 | "resolved": "https://registry.npmjs.org/axios/-/axios-0.18.0.tgz", 10 | "integrity": "sha1-MtU+SFHv3AoRmTts0AB4nXDAUQI=", 11 | "requires": { 12 | "follow-redirects": "^1.3.0", 13 | "is-buffer": "^1.1.5" 14 | } 15 | }, 16 | "csv-parse": { 17 | "version": "4.3.1", 18 | "resolved": "https://registry.npmjs.org/csv-parse/-/csv-parse-4.3.1.tgz", 19 | "integrity": "sha512-1V98UTtfefu8yKdYIGX1LFhfE2yMllveq2uCBay5y4ybfTzvW6I4M6r8Yc2YnKJdJBUig5ksEMh/bLqKg4vEMQ==" 20 | }, 21 | "debug": { 22 | "version": "3.1.0", 23 | "resolved": "https://registry.npmjs.org/debug/-/debug-3.1.0.tgz", 24 | "integrity": "sha512-OX8XqP7/1a9cqkxYw2yXss15f26NKWBpDXQd0/uK/KPqdQhxbPa994hnzjcE2VqQpDslf55723cKPUOGSmMY3g==", 25 | "requires": { 26 | "ms": "2.0.0" 27 | } 28 | }, 29 | "follow-redirects": { 30 | "version": "1.6.1", 31 | "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.6.1.tgz", 32 | "integrity": "sha512-t2JCjbzxQpWvbhts3l6SH1DKzSrx8a+SsaVf4h6bG4kOXUuPYS/kg2Lr4gQSb7eemaHqJkOThF1BGyjlUkO1GQ==", 33 | "requires": { 34 | "debug": "=3.1.0" 35 | } 36 | }, 37 | "is-buffer": { 38 | "version": "1.1.6", 39 | "resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-1.1.6.tgz", 40 | "integrity": "sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w==" 41 | }, 42 | "ms": { 43 | "version": "2.0.0", 44 | "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", 45 | "integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g=" 46 | } 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "dev-to-nodejs-epic-data", 3 | "version": "0.0.1", 4 | "description": "Dev.to article on handling big data with NodeJS", 5 | "main": "src/main.js", 6 | "scripts": { 7 | "start": "node src/main.js" 8 | }, 9 | "author": "", 10 | "license": "MIT", 11 | "dependencies": { 12 | "axios": "^0.18.0", 13 | "csv-parse": "^4.3.1" 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /src/main.js: -------------------------------------------------------------------------------- 1 | const fs = require('fs') 2 | const path = require('path') 3 | const CsvParser = require('csv-parse') 4 | const TransformUsernameToGithubRepos = require('./transform-username-to-github-repos') 5 | const stream = require('stream') 6 | 7 | const readGithubUsernamesStream = fs.createReadStream(path.resolve(__dirname, '../github-usernames.csv')) 8 | const csvParser = new CsvParser({ columns: false }) 9 | const transformUsernameToGithubRepos = new TransformUsernameToGithubRepos() 10 | const writeStream = fs.createWriteStream(path.resolve(__dirname, '../github-user-repositories.txt')) 11 | 12 | stream.pipeline( 13 | readGithubUsernamesStream, 14 | csvParser, 15 | transformUsernameToGithubRepos, 16 | writeStream, 17 | (error) => { 18 | if (error) { 19 | console.error('error ', error) 20 | return process.exit(1) 21 | } 22 | 23 | process.exit() 24 | } 25 | ) 26 | -------------------------------------------------------------------------------- /src/transform-username-to-github-repos.js: -------------------------------------------------------------------------------- 1 | const axios = require('axios') 2 | const stream = require('stream') 3 | 4 | module.exports = class TransformUsernameToGithubRepos extends stream.Transform { 5 | constructor (options = {}) { 6 | super({ ...options, objectMode: true }) 7 | this.requests = [] 8 | } 9 | 10 | _transform (chunk, encoding, callback) { 11 | const username = chunk[0] 12 | const githubRequest = this.getGithubRepositoriesForUser(username) 13 | this.requests.push(this.prepareGithubRequest(username, githubRequest)) 14 | if (this.requests.length < 5) { 15 | return callback() 16 | } 17 | 18 | this.processRequests(callback) 19 | } 20 | 21 | _flush (callback) { 22 | this.processRequests(callback) 23 | } 24 | 25 | getGithubRepositoriesForUser (username) { 26 | return axios.get(`https://api.github.com/users/${username}/repos`, { 27 | headers: { 28 | Authorization: `Token ${process.env.GITHUB_ACCESS_TOKEN}` 29 | } 30 | }) 31 | } 32 | 33 | prepareGithubRequest (username, githubRequest) { 34 | return githubRequest 35 | .then((response) => { 36 | let repositories = [] 37 | if (response.data) { 38 | repositories = response.data.map((repository) => repository.name) 39 | } 40 | 41 | return { 42 | username, 43 | repositories 44 | } 45 | }) 46 | } 47 | 48 | processRequests (callback) { 49 | return Promise.all(this.requests) 50 | .then((responses) => { 51 | this.requests = [] 52 | 53 | this.push(responses.reduce((accumulator, currentValue) => { 54 | return accumulator + JSON.stringify(currentValue) 55 | }, '')) 56 | callback() 57 | }) 58 | .catch(callback) 59 | } 60 | } 61 | --------------------------------------------------------------------------------