├── .babelrc ├── .editorconfig ├── .eslintignore ├── .eslintrc.js ├── .gitignore ├── .travis.yml ├── HISTORY.md ├── LICENSE.md ├── Readme.md ├── package.json ├── server.js ├── src ├── client.js ├── codemod-repo.js ├── console.js ├── db.js ├── get-repos-for-user.js ├── index.js ├── process-all-users.js ├── process-user.js └── read-client.js └── test └── index.js /.babelrc: -------------------------------------------------------------------------------- 1 | { 2 | "presets": ["forbeslindesay"] 3 | } 4 | -------------------------------------------------------------------------------- /.editorconfig: -------------------------------------------------------------------------------- 1 | root = true 2 | 3 | [*] 4 | indent_style = space 5 | charset = utf-8 6 | end_of_line = lf 7 | indent_size = 2 8 | insert_final_newline = true 9 | trim_trailing_whitespace = true 10 | -------------------------------------------------------------------------------- /.eslintignore: -------------------------------------------------------------------------------- 1 | lib 2 | node_modules 3 | server.js 4 | -------------------------------------------------------------------------------- /.eslintrc.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | extends: "forbeslindesay", 3 | rules: { 4 | 'no-unused-vars': [0], 5 | }, 6 | }; 7 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Logs 2 | logs 3 | *.log 4 | 5 | # Runtime data 6 | pids 7 | *.pid 8 | *.seed 9 | 10 | # Coverage directory used by tools like istanbul 11 | coverage 12 | 13 | # Compiled binary addons (http://nodejs.org/api/addons.html) 14 | build/Release 15 | 16 | # Dependency directory 17 | node_modules 18 | 19 | # Users Environment Variables 20 | .lock-wscript 21 | 22 | # Babel build output 23 | /lib 24 | 25 | # Config files 26 | environment.toml 27 | 28 | # Temporrary file for repos being code-modded 29 | /temp 30 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: node_js 2 | 3 | sudo: false 4 | 5 | node_js: 6 | - "6.5.0" 7 | 8 | deploy: 9 | provider: script 10 | script: npm run deploy 11 | on: 12 | branch: master 13 | node_js: 6.5.0 14 | -------------------------------------------------------------------------------- /HISTORY.md: -------------------------------------------------------------------------------- 1 | # Changelog 2 | 3 | ## v0.0.1: 2016-xx-xx 4 | 5 | - Initial release 6 | -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | # The MIT License (MIT) 2 | 3 | Copyright (c) 2016 [Forbes Lindesay](https://github.com/ForbesLindesay) 4 | 5 | > Permission is hereby granted, free of charge, to any person obtaining a copy 6 | > of this software and associated documentation files (the "Software"), to deal 7 | > in the Software without restriction, including without limitation the rights 8 | > to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | > copies of the Software, and to permit persons to whom the Software is 10 | > furnished to do so, subject to the following conditions: 11 | > 12 | > The above copyright notice and this permission notice shall be included in 13 | > all copies or substantial portions of the Software. 14 | > 15 | > THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | > IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | > FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | > AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | > LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | > OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | > SOFTWARE. 22 | -------------------------------------------------------------------------------- /Readme.md: -------------------------------------------------------------------------------- 1 | # unpkg-bot 2 | 3 | Bot to convert npmcdn urls to unpkg. 4 | 5 | Log in to https://unpkg-bot.herokuapp.com/ to: 6 | 7 | 1. get pull requests for any of your repos that need updating 8 | 2. lend your access tokens/rate limit towards updating all the other github users 9 | 10 | [![Build Status](https://img.shields.io/travis/ForbesLindesay/unpkg-bot/master.svg)](https://travis-ci.org/ForbesLindesay/unpkg-bot) 11 | [![Dependency Status](https://img.shields.io/david/ForbesLindesay/unpkg-bot/master.svg)](http://david-dm.org/ForbesLindesay/unpkg-bot) 12 | [![NPM version](https://img.shields.io/npm/v/unpkg-bot.svg)](https://www.npmjs.org/package/unpkg-bot) 13 | 14 | ## Installation 15 | 16 | ``` 17 | npm install unpkg-bot --save 18 | ``` 19 | 20 | ## Usage 21 | 22 | ```js 23 | var unpkgBot = require('unpkg-bot'); 24 | 25 | // ... 26 | ``` 27 | 28 | ## License 29 | 30 | MIT 31 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "unpkg-bot", 3 | "private": true, 4 | "version": "0.0.0", 5 | "main": "lib/index.js", 6 | "description": "Bot to convert npmcdn urls to unpkg", 7 | "keywords": [], 8 | "files": [ 9 | "lib/" 10 | ], 11 | "dependencies": { 12 | "babel-runtime": "^6.3.19", 13 | "cookie-session": "^2.0.0-alpha.1", 14 | "deck": "0.0.4", 15 | "express": "^4.14.0", 16 | "gethub": "^2.0.2", 17 | "github-basic": "^6.0.0", 18 | "lsr": "^1.0.0", 19 | "ms": "^0.7.1", 20 | "passport": "^0.3.2", 21 | "passport-github2": "^0.1.10", 22 | "prepare-response": "^1.1.3", 23 | "promise": "^7.1.1", 24 | "rimraf": "^2.5.4", 25 | "then-mongo": "^2.3.2", 26 | "then-request": "^2.2.0", 27 | "throat": "^3.0.0" 28 | }, 29 | "devDependencies": { 30 | "babel-cli": "*", 31 | "babel-preset-forbeslindesay": "*", 32 | "babel-register": "^6.14.0", 33 | "babelify": "^7.3.0", 34 | "browserify": "^13.1.0", 35 | "browserify-middleware": "^7.0.0", 36 | "envify": "^3.4.1", 37 | "eslint": "*", 38 | "eslint-config-forbeslindesay": "*", 39 | "react": "^15.3.1", 40 | "react-dom": "^15.3.1", 41 | "testit": "*", 42 | "uglify-js": "^2.7.3" 43 | }, 44 | "scripts": { 45 | "deploy": "npm install && npm run build && npm prune --prod && npm i heroku-release && heroku-release --app unpkg-bot", 46 | "build": "NODE_ENV=production babel src --out-dir lib && NODE_ENV=production browserify --global-transform envify lib/client.js | uglifyjs --compress --mangle > lib/bundle.js", 47 | "lint": "eslint src", 48 | "test": "babel-node test/index.js && npm run lint" 49 | }, 50 | "repository": { 51 | "type": "git", 52 | "url": "https://github.com/ForbesLindesay/unpkg-bot.git" 53 | }, 54 | "author": { 55 | "name": "Forbes Lindesay", 56 | "url": "http://github.com/ForbesLindesay" 57 | }, 58 | "license": "MIT", 59 | "engines": { 60 | "node": "6.5.0" 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /server.js: -------------------------------------------------------------------------------- 1 | if (process.env.NODE_ENV === 'production') { 2 | require('./lib'); 3 | } else { 4 | require('babel-register'); 5 | require('./src'); 6 | } 7 | -------------------------------------------------------------------------------- /src/client.js: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | import ReactDOM from 'react-dom'; 3 | import request from 'then-request'; 4 | import ms from 'ms'; 5 | 6 | const colors = [ 7 | '#3d9df2', 8 | '#e673cf', 9 | '#8800cc', 10 | '#005fb3', 11 | '#a69d7c', 12 | '#397358', 13 | '#ffbffb', 14 | '#ffd580', 15 | '#00e2f2', 16 | '#9173e6', 17 | '#a099cc', 18 | '#5995b3', 19 | '#994d6b', 20 | '#2d2080', 21 | '#736039', 22 | '#0c0059', 23 | '#00401a', 24 | '#1a3320', 25 | '#f240ff', 26 | '#ff8800', 27 | '#00f2c2', 28 | '#e59173', 29 | '#3347cc', 30 | '#18b300', 31 | '#269954', 32 | '#205380', 33 | '#733d00', 34 | '#161f59', 35 | '#364010', 36 | '#332b1a', 37 | '#0000ff', 38 | '#ffc8bf', 39 | '#79f299', 40 | '#0000d9', 41 | '#99cca7', 42 | '#b29559', 43 | '#8c004b', 44 | '#7f7700', 45 | '#730000', 46 | '#305900', 47 | '#402200', 48 | '#331a1a', 49 | '#bfe1ff', 50 | '#ff0000', 51 | '#baf279', 52 | '#a3d5d9', 53 | '#cc8533', 54 | '#b38686', 55 | '#59468c', 56 | '#7f4840', 57 | '#66001b', 58 | '#134d49', 59 | '#401100', 60 | '#40bfff', 61 | '#ff8080', 62 | '#def2b6', 63 | '#d94c36', 64 | '#cc5200', 65 | '#a67c98', 66 | '#23858c', 67 | '#6d1d73', 68 | '#005266', 69 | '#4d3e39', 70 | '#330014', 71 | '#73ff40', 72 | '#f2b6c6', 73 | '#f2d6b6', 74 | '#cc3347', 75 | '#0000bf', 76 | '#29a68d', 77 | '#8c6246', 78 | '#565a73', 79 | '#57664d', 80 | '#400033', 81 | '#331a31', 82 | '#ccff00', 83 | '#f279aa', 84 | '#f20000', 85 | '#cc0088', 86 | '#b2bf00', 87 | '#95a653', 88 | '#8c7769', 89 | '#566973', 90 | '#592d39', 91 | '#002240', 92 | '#070033', 93 | ]; 94 | let colorIndex = 0; 95 | function nextColor() { 96 | if (colorIndex < colors.length) { 97 | return colors[colorIndex++]; 98 | } else { 99 | colorIndex = 0; 100 | return nextColor(); 101 | } 102 | } 103 | let scrolledDown = false; 104 | window.addEventListener('scroll', () => { 105 | scrolledDown = document.body.scrollTop > document.getElementById('log-heading').offsetTop; 106 | }, false); 107 | 108 | const colorsByMessage = {}; 109 | const App = React.createClass({ 110 | _maxLogIndex: -1, 111 | getInitialState() { 112 | return {loading: true, maxUserIDProcessed: 0, rateLimits: [], log: []}; 113 | }, 114 | componentDidMount() { 115 | this._poll(); 116 | }, 117 | _poll() { 118 | request('get', '/ajax').getBody('utf8').then(JSON.parse).done(status => { 119 | this.setState({ 120 | loading: false, 121 | maxUserIDProcessed: status.maxUserIDProcessed, 122 | rateLimits: status.rateLimits, 123 | log: ( 124 | scrolledDown 125 | ? this.state.log 126 | : status.log.filter(logEntry => { 127 | if (logEntry.index > this._maxLogIndex) { 128 | this._maxLogIndex = logEntry.index; 129 | return true; 130 | } else { 131 | return false; 132 | } 133 | }).slice().reverse().concat(this.state.log) 134 | ), 135 | }); 136 | setTimeout(this._poll, 3000); 137 | }, this._poll); 138 | }, 139 | render() { 140 | if (this.state.loading) { 141 | return
Loading...
; 142 | } 143 | const now = Date.now() / 1000; 144 | return ( 145 |
146 |

Code Mod Status

147 |

148 | Max User ID Processed: {this.state.maxUserIDProcessed} 149 |

150 |

Rate Limits Remaining

151 |
152 | { 153 | this.state.rateLimits.map((rateLimit, i) => { 154 | const reset = Math.floor(rateLimit.reset - now); 155 | return ( 156 |
157 |
162 |
167 |
168 |

{reset <= 0 ? 'now' : ms(reset * 1000)}

169 |

{rateLimit.remaining}

170 |
171 |
172 | ); 173 | }) 174 | } 175 |
176 |

Log

177 | 
178 |           {this.state.log.map(log => {
179 |             if (log.level === 'error') {
180 |               return (
181 |                 

182 |                   
{log.date} {log.context}

183 |                   
{log.stack}

184 |                 

185 |               );
186 |             }
187 |             if (log.level === 'warn') {
188 |               return (
189 |                 

190 |                   
{log.date} {log.message}

191 |                 

192 |               );
193 |             }
194 |             if (log.level === 'log') {
195 |               const color = colorsByMessage[log.name] || (colorsByMessage[log.name] = nextColor());
196 |               return (
197 |                 

198 |                   
{log.date} {log.name} {log.message}

199 |                 

200 |               );
201 |             }
202 |             return 
{log.date} 
;
203 |           })}
204 |
205 |
206 | ); 207 | }, 208 | }); 209 | 210 | ReactDOM.render(, document.getElementById('container')); 211 | -------------------------------------------------------------------------------- /src/codemod-repo.js: -------------------------------------------------------------------------------- 1 | import fs from 'fs'; 2 | import Promise from 'promise'; 3 | import github from 'github-basic'; 4 | import gethub from 'gethub'; 5 | import rimraf from 'rimraf'; 6 | import lsr from 'lsr'; 7 | import throat from 'throat'; 8 | import {log, error} from './console'; 9 | import {pushError} from './db'; 10 | 11 | const client = github({version: 3, auth: process.env.GITHUB_BOT_TOKEN}); 12 | 13 | const readFile = Promise.denodeify(fs.readFile); 14 | const rm = Promise.denodeify(rimraf); 15 | const directory = __dirname + '/../temp'; 16 | 17 | // TODO: client.exists doesn't work because http-basic does not work with head requests that also support gzip 18 | client.exists = function (owner, repo) { 19 | return this.get('/repos/:owner/:repo', {owner, repo}).then( 20 | () => true, 21 | () => false, 22 | ); 23 | }; 24 | 25 | if (process.env.NODE_ENV !== 'production') { 26 | console.log('===== DRY RUN ====='); 27 | console.log(''); 28 | console.log('To actually run code-mod, set NODE_ENV=production'); 29 | console.log(''); 30 | client.exists = () => Promise.resolve(false); 31 | [ 32 | 'fork', 33 | 'branch', 34 | 'commit', 35 | 'pull', 36 | ].forEach(method => { 37 | client[method] = () => Promise.resolve(null); 38 | }); 39 | } 40 | 41 | const blackList = [ 42 | 'qdot/gecko-hg', 43 | 'angular/code.angularjs.org', 44 | 'Belxjander/tenfourfox', 45 | 'tijuca/icedove', 46 | 'jsdelivr/jsdelivr', 47 | 'andrit/uiUXWebsite', 48 | 'WhoAmID/angular-whoamid', 49 | 'rubber-duckies/the-frank-tank', 50 | ]; 51 | function codemodRepo(fullName) { 52 | if (blackList.includes(fullName)) { 53 | return Promise.resolve(null); 54 | } 55 | const [owner, name] = fullName.split('/'); 56 | 57 | return client.exists('npmcdn-to-unpkg-bot', name).then(exists => { 58 | if (exists) { 59 | return; 60 | } 61 | log('Code Modding', fullName); 62 | return rm(directory).then(() => { 63 | log('Downloading', fullName); 64 | return gethub(owner, name, 'master', directory); 65 | }).then(() => { 66 | log('Fetched', fullName); 67 | return lsr(directory); 68 | }).then(entries => { 69 | log('Processing Files', fullName); 70 | return Promise.all(entries.map(entry => { 71 | if (entry.isFile()) { 72 | return readFile(entry.fullPath, 'utf8').then(content => { 73 | const newContent = content.replace(/\bnpmcdn\b/g, 'unpkg'); 74 | if (newContent !== content) { 75 | return { 76 | path: entry.path.substr(2), // remove the `./` that entry paths start with 77 | content: newContent, 78 | }; 79 | } 80 | }, err => { 81 | if (err.code === 'ENOENT') { 82 | return; 83 | } 84 | throw err; 85 | }); 86 | } 87 | })); 88 | }).then(updates => { 89 | updates = updates.filter(Boolean); 90 | if (updates.length === 0) { 91 | log('No Changes Made', fullName); 92 | return; 93 | } 94 | log('Forking', fullName); 95 | return client.fork(owner, name).then(() => { 96 | return new Promise(resolve => setTimeout(resolve, 10000)); 97 | }).then(() => { 98 | log('Branching', fullName); 99 | return client.branch('npmcdn-to-unpkg-bot', name, 'master', 'npmcdn-to-unpkg'); 100 | }).then(() => { 101 | log('Committing', fullName); 102 | return client.commit('npmcdn-to-unpkg-bot', name, { 103 | branch: 'npmcdn-to-unpkg', 104 | message: 'Replace npmcdn.com with unpkg.com', 105 | updates, 106 | }); 107 | }).then(() => { 108 | log('Submitting Pull Request', fullName); 109 | return client.pull( 110 | {user: 'npmcdn-to-unpkg-bot', repo: name, branch: 'npmcdn-to-unpkg'}, 111 | {user: owner, repo: name}, 112 | { 113 | title: 'Replace npmcdn.com with unpkg.com', 114 | body: ( 115 | 'To avoid potential naming conflicts with npm, npmcdn.com is being renamed to unpkg.com. This is an ' + 116 | 'automated pull request to update your project to use the new domain.' 117 | ), 118 | }, 119 | ); 120 | }).then(() => { 121 | log('Codemod Complete', fullName); 122 | }); 123 | }); 124 | }).then(null, err => { 125 | error('Error processing ' + fullName, err.stack); 126 | return pushError(owner, name, err.message || err); 127 | }); 128 | } 129 | 130 | // only allow codemodding one repo at a time 131 | export default throat(Promise)(1, codemodRepo); 132 | -------------------------------------------------------------------------------- /src/console.js: -------------------------------------------------------------------------------- 1 | let index = 0; 2 | const logEntries = []; 3 | for (let i = 0; i < 500; i++) { 4 | logEntries.push(null); 5 | } 6 | 7 | export function getLog() { 8 | return logEntries; 9 | } 10 | export function log(name, message) { 11 | logEntries.shift(); 12 | logEntries.push({ 13 | index: index++, 14 | date: (new Date()).toISOString(), 15 | level: 'log', 16 | name, 17 | message, 18 | }); 19 | } 20 | export function warn(message) { 21 | logEntries.shift(); 22 | logEntries.push({ 23 | index: index++, 24 | date: (new Date()).toISOString(), 25 | level: 'warn', 26 | message, 27 | }); 28 | } 29 | export function error(context, stack = '') { 30 | if ( 31 | /The listed users and repositories cannot be searched/.test(stack) 32 | ) { 33 | stack = 'The listed user cannot be searched'; 34 | } 35 | logEntries.shift(); 36 | logEntries.push({ 37 | index: index++, 38 | date: (new Date()).toISOString(), 39 | level: 'error', 40 | context, 41 | stack, 42 | }); 43 | console.error(context + '\n' + stack); 44 | } 45 | -------------------------------------------------------------------------------- /src/db.js: -------------------------------------------------------------------------------- 1 | import mongo from 'then-mongo'; 2 | import Promise from 'promise'; 3 | 4 | const db = mongo(process.env.MONGO_DB, ['users', 'maxUserIDProcessed', 'errors']); 5 | 6 | export function saveUser(username, accessToken) { 7 | return db.users.update({_id: username}, {_id: username, username, accessToken}, {upsert: true}); 8 | } 9 | export function getUsers() { 10 | return db.users.find(); 11 | } 12 | 13 | export function getMaxUserIDProcessed() { 14 | return db.maxUserIDProcessed.findOne({_id: 'unpkg-bot'}).then(o => (o ? o.value : undefined)); 15 | } 16 | export function setMaxUserIDProcessed(value) { 17 | if (process.env.NODE_ENV === 'production') { 18 | return db.maxUserIDProcessed.update({_id: 'unpkg-bot'}, {_id: 'unpkg-bot', value}, {upsert: true}); 19 | } else { 20 | return Promise.resolve(null); 21 | } 22 | } 23 | export function pushError(username, repo, message) { 24 | return db.errors.insert({username, repo, message}); 25 | } 26 | -------------------------------------------------------------------------------- /src/get-repos-for-user.js: -------------------------------------------------------------------------------- 1 | import Promise from 'promise'; 2 | import {get} from './read-client'; 3 | 4 | export default function getReposForUser(username) { 5 | return new Promise((resolve, reject) => { 6 | const repos = []; 7 | processPage(get('/search/code', {q: 'npmcdn user:' + username})); 8 | function processPage(page) { 9 | page.done(p => { 10 | p.items.forEach(item => { 11 | const repo = item.repository.full_name; 12 | if (!repos.includes(repo)) { 13 | repos.push(repo); 14 | } 15 | }); 16 | if (p.urlNext) { 17 | processPage(get(p.urlNext)); 18 | } else { 19 | resolve(repos); 20 | } 21 | }, reject); 22 | } 23 | }); 24 | } 25 | -------------------------------------------------------------------------------- /src/index.js: -------------------------------------------------------------------------------- 1 | import {readFileSync} from 'fs'; 2 | import {createHash} from 'crypto'; 3 | import passport from 'passport'; 4 | import {Strategy as GitHubStrategy} from 'passport-github2'; 5 | import cookieSession from 'cookie-session'; 6 | import express from 'express'; 7 | import prepareResponse from 'prepare-response'; 8 | import {getLog} from './console'; 9 | import {saveUser, getUsers, getMaxUserIDProcessed} from './db'; 10 | import {addToken, getRateLimitStatus} from './read-client'; 11 | import processUser from './process-user'; 12 | import './process-all-users'; 13 | 14 | const app = express(); 15 | 16 | getUsers().done(users => { 17 | users.forEach(user => { 18 | addToken(user.accessToken); 19 | }); 20 | }); 21 | 22 | app.use(cookieSession({ 23 | keys: [process.env.SESSION_KEY], 24 | // session expires after 1 hour 25 | maxAge: 60 * 60 * 1000, 26 | // session is not accessible from JavaScript 27 | httpOnly: true, 28 | })); 29 | passport.serializeUser((user, done) => { 30 | done(null, user); 31 | }); 32 | 33 | passport.deserializeUser((user, done) => { 34 | done(null, user); 35 | }); 36 | 37 | passport.use(new GitHubStrategy( 38 | { 39 | clientID: process.env.GITHUB_CLIENT_ID, 40 | clientSecret: process.env.GITHUB_CLIENT_SECRET, 41 | callbackURL: process.env.GITHUB_CALLBACK || 'http://localhost:3000/auth/github/callback', 42 | }, 43 | (accessToken, refreshToken, profile, done) => { 44 | addToken(accessToken); 45 | processUser(profile.username).done(); 46 | saveUser(profile.username, accessToken).done( 47 | () => done(null, {username: profile.username, accessToken}), 48 | done, 49 | ); 50 | } 51 | )); 52 | app.use(passport.initialize()); 53 | app.use(passport.session()); 54 | 55 | app.get('/auth/github', passport.authenticate('github', {scope: []})); 56 | 57 | app.get('/auth/github/callback', passport.authenticate('github'), (req, res) => { 58 | res.redirect('/'); 59 | }); 60 | app.get('/auth/logout', passport.authenticate('github'), (req, res) => { 61 | res.logout(); 62 | res.sendStatus(200); 63 | }); 64 | 65 | app.get('/ajax', (req, res, next) => { 66 | getMaxUserIDProcessed().done(maxUserIDProcessed => { 67 | res.json({maxUserIDProcessed, log: getLog().filter(Boolean), rateLimits: getRateLimitStatus()}); 68 | }, next); 69 | }); 70 | let hash = 'development'; 71 | if (process.env.NODE_ENV !== 'production') { 72 | app.get('/client/' + hash + '.js', require('browserify-middleware')(__dirname + '/client.js', { 73 | transform: [require('babelify')], 74 | })); 75 | } else { 76 | const src = readFileSync(__dirname + '/bundle.js'); 77 | const response = prepareResponse(src, { 78 | 'content-type': 'js', 79 | 'cache-control': '1 year', 80 | }); 81 | hash = createHash('md5').update(src).digest("hex"); 82 | app.get('/client/' + hash + '.js', (req, res, next) => { 83 | response.send(req, res, next); 84 | }); 85 | } 86 | 87 | app.get('/', (req, res, next) => { 88 | if (!req.isAuthenticated()) { 89 | return res.redirect('/auth/github'); 90 | } 91 | res.send( 92 | ` 93 | 94 | Code Mod 95 | 96 | 125 |
126 | 127 | ` 128 | ); 129 | }); 130 | 131 | app.listen(process.env.PORT || 3000); 132 | -------------------------------------------------------------------------------- /src/process-all-users.js: -------------------------------------------------------------------------------- 1 | import throat from 'throat'; 2 | import Promise from 'promise'; 3 | import {getMaxUserIDProcessed, setMaxUserIDProcessed, pushError} from './db'; 4 | import {get} from './read-client'; 5 | import processUser from './process-user'; 6 | import {error} from './console'; 7 | 8 | function getPage(since) { 9 | get('/users', {since}).done(users => { 10 | const maxID = users.reduce((id, user) => { 11 | return Math.max(id, user.id); 12 | }, since || -1); 13 | Promise.all(users.map(throat(Promise)(10, user => processUser(user.login)))).then(() => { 14 | return setMaxUserIDProcessed(maxID); 15 | }).done(() => getPage(maxID)); 16 | }, err => { 17 | error('Error getting users', err.stack); 18 | pushError(null, null, err.message || err).done( 19 | () => setTimeout(() => getPage(since), 5000), 20 | ); 21 | }); 22 | } 23 | getMaxUserIDProcessed().done(getPage); 24 | -------------------------------------------------------------------------------- /src/process-user.js: -------------------------------------------------------------------------------- 1 | import getReposForUser from './get-repos-for-user'; 2 | import codemodRepo from './codemod-repo'; 3 | import {pushError} from './db'; 4 | import {log, error} from './console'; 5 | 6 | // codemod repos for a given user one at a time 7 | export default function processUser(username) { 8 | return getReposForUser(username).then(repos => { 9 | log('Processing', username); 10 | return new Promise((resolve, reject) => { 11 | function next(i) { 12 | if (i >= repos.length) { 13 | return resolve(); 14 | } 15 | codemodRepo(repos[i]).done(() => next(i + 1), reject); 16 | } 17 | next(0); 18 | }); 19 | }).then(null, err => { 20 | error('Error processing ' + username, err.stack); 21 | return pushError(username, null, err.message || err); 22 | }); 23 | } 24 | -------------------------------------------------------------------------------- /src/read-client.js: -------------------------------------------------------------------------------- 1 | import github from 'github-basic'; 2 | import Promise from 'promise'; 3 | import deck from 'deck'; 4 | import {warn} from './console'; 5 | 6 | const tokens = []; 7 | const clients = []; 8 | 9 | export function addToken(token) { 10 | if (!tokens.includes(token)) { 11 | tokens.push(token); 12 | clients.push(github({version: 3, auth: token})); 13 | } 14 | } 15 | export function getRateLimitStatus() { 16 | return clients.map(client => client.rateLimit); 17 | } 18 | 19 | export function get(...args) { 20 | return new Promise((resolve, reject) => { 21 | function retry() { 22 | if (clients.length === 0) { 23 | warn('No clients available, waiting for a client to be added'); 24 | setTimeout(retry, 5000); 25 | return; 26 | } 27 | const clientWeights = {}; 28 | for (let i = 0; i < clients.length; i++) { 29 | clientWeights[i] = clients[i].rateLimit.remaining || 1; 30 | } 31 | const clientToUse = clients[deck.pick(clientWeights)]; 32 | clientToUse.get(...args).done(resolve, err => { 33 | if (err.statusCode === 401) { 34 | console.error('====================='); 35 | console.error(err.stack); 36 | console.dir(clientToUse); 37 | console.error('====================='); 38 | warn('Client token seems to have expired!'); 39 | const index = clients.indexOf(clientToUse); 40 | if (index !== -1) { 41 | clients.splice(index, 1); 42 | } 43 | setTimeout(retry, 1000); 44 | } else if (err.statusCode === 403) { 45 | warn('Rate limit exceeded, waiting 1 second then trying again'); 46 | setTimeout(retry, 1000); 47 | } else { 48 | reject(err); 49 | } 50 | }); 51 | } 52 | retry(); 53 | }); 54 | } 55 | -------------------------------------------------------------------------------- /test/index.js: -------------------------------------------------------------------------------- 1 | console.error('Oops, no tests :('); 2 | --------------------------------------------------------------------------------