├── app ├── blacklist.json ├── routes.js └── updater.js ├── jsconfig.json ├── README.md ├── .editorconfig ├── .gitignore ├── bin ├── server.js └── cli.js ├── package.json ├── lib └── plugin-cache │ ├── update.js │ ├── npm-keyword-search.js │ ├── npm-download-counts.js │ ├── npm-package-info.js │ ├── merge-info.js │ ├── github-stats.js │ └── index.js └── LICENSE /app/blacklist.json: -------------------------------------------------------------------------------- 1 | [ 2 | 3 | ] -------------------------------------------------------------------------------- /jsconfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "compilerOptions": { 3 | "target": "ES6", 4 | "module": "commonjs" 5 | } 6 | } -------------------------------------------------------------------------------- /app/routes.js: -------------------------------------------------------------------------------- 1 | const PluginCache = require('../lib/plugin-cache'); 2 | 3 | let pluginCache = new PluginCache('yeoman-generator'); 4 | 5 | module.exports = function (server) { 6 | 7 | }; 8 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Pixi.js Build Tool 2 | 3 | Directory listing: 4 | 5 | ``` 6 | app/ # Server files 7 | lib/ # Common library files 8 | bin/ # Executables, cli and server 9 | ``` 10 | 11 | Required Env Vars: 12 | 13 | - `GITHUB_CLIENT_ID` 14 | - `GITHUB_CLIENT_SECRET` 15 | -------------------------------------------------------------------------------- /.editorconfig: -------------------------------------------------------------------------------- 1 | # This file is for unifying the coding style for different editors and IDEs. 2 | # More information at http://EditorConfig.org 3 | root = true 4 | 5 | [*] 6 | end_of_line = lf 7 | insert_final_newline = true 8 | trim_trailing_whitespace = true 9 | indent_style = space 10 | indent_size = 4 11 | 12 | [*.md] 13 | trim_trailing_whitespace = false 14 | 15 | [{*.json,.*rc,.travis.yml}] 16 | indent_size = 2 17 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # sublime text files 2 | *.sublime* 3 | *.*~*.TMP 4 | 5 | # temp files 6 | .DS_Store 7 | Thumbs.db 8 | Desktop.ini 9 | npm-debug.log 10 | 11 | # project files 12 | .project 13 | 14 | # vim swap files 15 | *.sw* 16 | 17 | # emacs temp files 18 | *~ 19 | \#*# 20 | 21 | # project ignores 22 | !.gitkeep 23 | *__temp 24 | node_modules 25 | docs/ 26 | examples_old/ 27 | 28 | # jetBrains IDE ignores 29 | .idea 30 | 31 | # Application ignores 32 | .env 33 | .cache 34 | -------------------------------------------------------------------------------- /bin/server.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | const cp = require('child_process'); 4 | const path = require('path'); 5 | const restify = require('restify'); 6 | const env = require('node-env-file'); 7 | const routes = require('./app/routes'); 8 | 9 | env(path.join(__dirname, '..', '.env')); 10 | 11 | // setup and run server 12 | let server = restify.createServer({ name: 'pixi-build-tool' }); 13 | 14 | require('../app/routes')(server, cache); 15 | 16 | server.listen(process.env.PORT || 8085); 17 | 18 | // run the updater 19 | cp.fork('../app/updater'); 20 | -------------------------------------------------------------------------------- /bin/cli.js: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | 3 | 'use strict'; 4 | 5 | // Set higher maximum http sockets 6 | require('http').globalAgent.maxSockets = 100; 7 | require('https').globalAgent.maxSockets = 100; 8 | 9 | const path = require('path'); 10 | const env = require('node-env-file'); 11 | const pkg = require('../package.json'); 12 | const PluginCache = require('../lib/plugin-cache'); 13 | 14 | env(path.join(__dirname, '..', '.env')); 15 | 16 | // Create cache and update 17 | let cache = new PluginCache(pkg.toolConfig.keyword); 18 | 19 | cache.update(); 20 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@pixi/build-tool", 3 | "version": "1.0.0", 4 | "description": "Build tool for pixi.js", 5 | "bin": { 6 | "pbt": "cli.js" 7 | }, 8 | "author": "Chad Engler ", 9 | "license": "MIT", 10 | "scripts": { 11 | "start": "node ./bin/server.js", 12 | "update": "node ./bin/cli.js" 13 | }, 14 | "dependencies": { 15 | "async": "^1.5.0", 16 | "aws-sdk": "^2.2.19", 17 | "github-url-to-object": "^2.1.0", 18 | "mkdirp": "^0.5.1", 19 | "node-env-file": "^0.1.8", 20 | "npm-keyword": "^4.2.0", 21 | "package-json": "^2.2.1", 22 | "request": "^2.67.0", 23 | "restify": "^4.0.3" 24 | }, 25 | "engines": { 26 | "node": "^4.2.2" 27 | }, 28 | "toolConfig": { 29 | "keyword": "pixi-plugin" 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /lib/plugin-cache/update.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | const async = require('async'); 4 | const npmKeywordSearch = require('./npm-keyword-search'); 5 | const npmPackageInfo = require('./npm-package-info'); 6 | const npmDownloadCounts = require('./npm-download-counts'); 7 | const ghStats = require('./github-stats'); 8 | const log = process.env.LOGGER || console; 9 | 10 | module.exports = function (keyword, limit, blacklist, cb) { 11 | log.info('Update: keyword: %s, GH limit: %s', keyword, limit); 12 | 13 | async.waterfall([ 14 | function (_done) { 15 | npmKeywordSearch(keyword, blacklist, _done); 16 | }, 17 | npmPackageInfo, 18 | npmDownloadCounts, 19 | function (list, _done) { 20 | ghStats(list, limit, _done); 21 | } 22 | ], function (err, results) { 23 | if (err) { 24 | log.error('Could not update the list: ', err); 25 | return cb(err); 26 | } 27 | 28 | cb(null, results); 29 | }); 30 | }; -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2016, Chad Engler 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in 13 | all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 21 | THE SOFTWARE. 22 | -------------------------------------------------------------------------------- /lib/plugin-cache/npm-keyword-search.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | const async = require('async'); 4 | const npmKeyword = require('npm-keyword'); 5 | const log = process.env.LOGGER || console; 6 | 7 | module.exports = function (keyword, blacklist, cb) { 8 | log.info('Starting keyword search: %s', keyword); 9 | let start = Date.now(); 10 | 11 | npmKeyword.names(keyword) 12 | .then(function (packages) { 13 | var found = packages.length; 14 | packages = packages.filter(function (pkg) { 15 | return blacklist.indexOf(pkg) === -1; 16 | }); 17 | 18 | log.info('Found %s packages in %sms (%s ignored due to blacklist)', 19 | found, (Date.now() - start), (found - packages.length)); 20 | 21 | if (!packages.length) { 22 | var err = new Error('No packages found.'); 23 | return cb(err); 24 | } 25 | 26 | cb(null, packages); 27 | }) 28 | .catch(function (err) { 29 | log.error('Unable to search for keywords ', err); 30 | cb(err); 31 | }) 32 | }; 33 | -------------------------------------------------------------------------------- /lib/plugin-cache/npm-download-counts.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | const async = require('async'); 4 | const request = require('request'); 5 | const log = process.env.LOGGER || console; 6 | 7 | module.exports = function (list, cb) { 8 | log.info('Getting downloads for %s packages', list.length); 9 | 10 | let count = 0; 11 | let url = 'https://api.npmjs.org/downloads/point/last-month/'; 12 | 13 | async.each(list, function (plugin, _done) { 14 | request({ url: url + encodeURIComponent(plugin.name), json: true }, function (err, res) { 15 | if (!err && res.statusCode === 200) { 16 | plugin.downloads = res.body.downloads || 0; 17 | count++; 18 | } 19 | 20 | _done(null, plugin); 21 | }); 22 | }, function (err) { 23 | if (err) { 24 | log.error('Could not get download counts ', err); 25 | return cb(err); 26 | } 27 | 28 | let message = 'Fetched download stats for %s packages'; 29 | let skipped = list.length - count; 30 | 31 | if (skipped) { 32 | message += '. Skipped %s packages'; 33 | log.info(message, count, skipped); 34 | } 35 | else { 36 | log.info(message, count); 37 | } 38 | 39 | cb(null, list); 40 | }); 41 | }; 42 | -------------------------------------------------------------------------------- /lib/plugin-cache/npm-package-info.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | const async = require('async'); 4 | const packageJson = require('package-json'); 5 | const log = process.env.LOGGER || console; 6 | 7 | module.exports = function (list, cb) { 8 | log.info('Fetching package info for %s packages', list.length); 9 | let start = Date.now(); 10 | 11 | async.map(list, function (plugin, _done) { 12 | packageJson(plugin) 13 | .then(function (pkg) { 14 | _done(null, { 15 | name: pkg.name, 16 | author: pkg.author, 17 | description: pkg.description, 18 | version: pkg['dist-tags'] && pkg['dist-tags'].latest, 19 | repo: pkg.repository && pkg.repository.type === 'git' ? pkg.repository.url : false, 20 | website: pkg.homepage || false, 21 | updated: pkg.time.modified || pkg.time.created || '' 22 | }); 23 | }) 24 | .catch(function (err) { 25 | log.error('Unable to fetch package info for %s ', plugin, err); 26 | _done(); 27 | }); 28 | }, function (err, packages) { 29 | if (err) { 30 | log.error('Could not fetch package info ', err); 31 | return cb(err); 32 | } 33 | 34 | packages.filter(function (pkg) { 35 | return !!pkg; 36 | }); 37 | 38 | log.info('Fetched info for %s packages in %sms', packages.length, (Date.now() - start)); 39 | 40 | cb(null, packages); 41 | }); 42 | }; 43 | -------------------------------------------------------------------------------- /lib/plugin-cache/merge-info.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | function cleanupDescription(str) { 4 | if (!str) { 5 | return str; 6 | } 7 | 8 | str = str.trim() 9 | .replace(/:\w+:/, '') // remove GitHub emojis 10 | .replace(/ ?plugin for (?:pixi(?:\.?js)?) ?/i, '') 11 | .replace(/(?:a )?(?:pixi(?:\.js)?) (?:plugin (?:for|to|that|which)?)?/i, '') 12 | .replace(/(?:pixi(?:\.js)?) plugin$/i, '') 13 | .replace(/ ?application ?/i, 'app') 14 | .trim() 15 | .replace(/\.$/, ''); 16 | 17 | str = str.charAt(0).toUpperCase() + str.slice(1); 18 | 19 | return str; 20 | } 21 | 22 | module.exports = function (npm, gh) { 23 | gh = gh || {}; 24 | 25 | var description; 26 | if (npm.description && gh.description) { 27 | if (npm.description.length > gh.description.length) { 28 | description = npm.description; 29 | } 30 | else { 31 | description = gh.description; 32 | } 33 | } 34 | else { 35 | description = npm.description || gh.description; 36 | } 37 | 38 | var ownerWebsite = npm.author && npm.author.url; 39 | ownerWebsite = ownerWebsite || gh.owner && gh.owner.html_url; 40 | 41 | return { 42 | name: npm.name.replace(/^pixi-/, ''), 43 | description: cleanupDescription(description || ''), 44 | stars: gh.stargazers_count || 0, 45 | downloads: npm.downloads, 46 | site: npm.website || gh.html_url || '', 47 | owner: { 48 | name: npm.author && npm.author.name || gh.owner && gh.owner.login || '', 49 | site: ownerWebsite || '' 50 | }, 51 | updated: npm.updated 52 | }; 53 | }; -------------------------------------------------------------------------------- /app/updater.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | const fs = require('fs'); 4 | const path = require('path'); 5 | const AWS = require('aws-sdk'); 6 | const async = require('async'); 7 | const env = require('node-env-file'); 8 | const pkg = require('../package.json'); 9 | const blacklist = require('./blacklist.json'); 10 | const PluginCache = require('../lib/plugin-cache'); 11 | const log = process.env.LOGGER || console; 12 | 13 | env(path.join(__dirname, '..', '.env')); 14 | 15 | let cache = new PluginCache(pkg.toolConfig.keyword, 0, blacklist); 16 | 17 | beginUpdate(); 18 | 19 | function beginUpdate() { 20 | log.info('Update starting.'); 21 | 22 | async.waterfall([ 23 | updateCache, 24 | uploadCache 25 | ], function () { 26 | log.info('Update complete.'); 27 | }); 28 | 29 | setTimeout(beginUpdate, 3610000); // every hour + 10 seconds 30 | } 31 | 32 | function updateCache(cb) { 33 | cache.update(cb); 34 | } 35 | 36 | function uploadCache(cb) { 37 | let key = 'list-cache.json'; 38 | let s3obj = new AWS.S3({ params: { Bucket: 'pixi-build-tool', Key: key } }); 39 | 40 | s3obj.upload({ 41 | ACL: 'public-read', 42 | CacheControl: 'max-age=3600', 43 | ContentType: 'application/json', 44 | Body: cache.createReadStream(key) 45 | }) 46 | .on('httpUploadProgress', function (evt) { 47 | log.info('Uploading list-cache:', evt); 48 | }) 49 | .send(function (err, data) { 50 | if (err) { 51 | log.error('Error uploading list cache:', err); 52 | } 53 | else { 54 | log.info('List cache uploaded.\n CDN URL: %s\n S3 URL: %s\n ETag: %s', 55 | process.env.AWS_CDN_URL + key, data.Location, data.ETag); 56 | } 57 | }); 58 | } 59 | -------------------------------------------------------------------------------- /lib/plugin-cache/github-stats.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | const request = require('request'); 4 | const async = require('async'); 5 | const gh = require('github-url-to-object'); 6 | const merge = require('./merge-info'); 7 | 8 | const log = process.env.LOGGER || console; 9 | 10 | module.exports = function (list, limit, cb) { 11 | limit = limit || process.env.GITHUB_API_LIMIT || list.length; 12 | limit = list.length < limit ? list.length : limit; 13 | log.info('Fetching GH stats for %s packages', limit); 14 | 15 | let count = 0; 16 | let limitMessage = false; 17 | let start = Date.now(); 18 | 19 | async.each(list, function (plugin, _done) { 20 | if (plugin.repo) { 21 | var url = gh(plugin.repo); 22 | 23 | if (url && url.user && url.repo) { 24 | plugin.repo = url.user + '/' + url.repo; 25 | } 26 | else { 27 | plugin.repo = false; 28 | } 29 | } 30 | 31 | if (limit && limit === count && !limitMessage) { 32 | log.info('Limit (%s) reached for GH API calls', limit); 33 | limitMessage = true; 34 | } 35 | 36 | if (!plugin.repo || (limit && limit < count)) { 37 | _done(null, merge(plugin)); 38 | } 39 | else { 40 | request({ 41 | url: 'https://api.github.com/repos/' + plugin.repo, 42 | json: true, 43 | auth: { 44 | username: process.env.GITHUB_CLIENT_ID, 45 | password: process.env.GITHUB_CLIENT_SECRET 46 | }, 47 | headers: { 48 | 'accept': 'application/vnd.github.v3+json', 49 | 'user-agent': 'https://github.com/pixijs/build-tool' 50 | } 51 | }, function (err, res) { 52 | if (err || res.statusCode !== 200) { 53 | return _done(null, merge(plugin)); 54 | } 55 | 56 | count++; 57 | _done(null, merge(plugin, res.body)); 58 | }); 59 | } 60 | }, function (err) { 61 | if (err) { 62 | log.error('Could not fetch GH stats ', err); 63 | return cb(err); 64 | } 65 | 66 | let message = 'Fetched GH stats for %s packages in %sms'; 67 | let skipped = list.length - count; 68 | let time = Date.now() - start; 69 | 70 | if (skipped) { 71 | message += '. Skipped %s packages'; 72 | log.info(message, count, time, skipped); 73 | } 74 | else { 75 | log.info(message, count, time); 76 | } 77 | 78 | cb(null, list); 79 | }); 80 | }; 81 | -------------------------------------------------------------------------------- /lib/plugin-cache/index.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | const crypto = require('crypto'); 4 | const fs = require('fs'); 5 | const mkdirp = require('mkdirp'); 6 | const os = require('os'); 7 | const path = require('path'); 8 | const updateList = require('./update'); 9 | const log = process.env.LOGGER || console; 10 | 11 | class PluginCache { 12 | constructor(keyword, limit, blacklist) { 13 | if (typeof keyword !== 'string') { 14 | log.error('Keyword is required'); 15 | return; 16 | } 17 | 18 | this._listHash = null; 19 | this._cachePath = null; 20 | 21 | this.keyword = keyword; 22 | this.limit = limit || Infinity; 23 | this.blacklist = blacklist || []; 24 | 25 | this.cachePath = path.join(__dirname, '..', '..', '.cache'); 26 | } 27 | 28 | get listHash() { 29 | return this._listHash; 30 | } 31 | 32 | get cachePath() { 33 | return this._cachePath; 34 | } 35 | 36 | set cachePath(value) { 37 | this._cachePath = path.normalize(value); 38 | this.checkCache(); 39 | } 40 | 41 | checkCache() { 42 | try { 43 | mkdirp.sync(this._cachePath); 44 | } 45 | catch (e) { 46 | log.error('Unable to create cache path: %s', this._cachePath, e); 47 | return false; 48 | } 49 | 50 | return checkCacheFile(this._cachePath, 'list-cache.json'); 51 | } 52 | 53 | createReadStream(fname) { 54 | return fs.createReadStream(path.join(this._cachePath, fname)); 55 | } 56 | 57 | update(cb) { 58 | updateList(this.keyword, this.limit, this.blacklist, (err, data) => { 59 | if (!data || data.length) { 60 | return; 61 | } 62 | 63 | let json = JSON.stringify(data, null, 4); 64 | let fpath = path.join(this._cachePath, 'list-cache.json'); 65 | this._listHash = createHash(json); 66 | 67 | fs.writeFile(fpath, json, (err) => { 68 | if (err) { 69 | log.error('Failed to write to cache file at %s', fpath, err); 70 | return cb(err); 71 | } 72 | log.info('Updated %s plugins', data.length); 73 | cb(); 74 | }); 75 | }); 76 | } 77 | } 78 | 79 | module.exports = PluginCache; 80 | 81 | function createHash(str) { 82 | var shasum = crypto.createHash('sha1'); 83 | shasum.update(str); 84 | return shasum.digest('hex'); 85 | } 86 | 87 | function checkCacheFile(cachePath, fname) { 88 | let cache = path.join(cachePath, fname); 89 | 90 | try { 91 | fs.accessSync(cache, fs.R_OK | fs.W_OK); 92 | log.info('Cache file exists and is accessible at %s', cache); 93 | } 94 | catch (e) { 95 | if (e.code === 'ENOENT') { 96 | log.info('Cache file %s doesn\'t exist, creating a new one.', fname); 97 | try { 98 | fs.writeFileSync(cache, '[]'); 99 | log.info('Cache file created at %s', cache); 100 | } 101 | catch (e) { 102 | log.error('Unable to create cache file %s', cache, e); 103 | return false; 104 | } 105 | } 106 | else { 107 | log.error('Cache file %s exists but is inaccessible.', fname); 108 | return false; 109 | } 110 | } 111 | 112 | return true; 113 | } --------------------------------------------------------------------------------