├── .gitignore ├── exec-git.js ├── github.js ├── package.json └── test.js /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | package-lock.json 3 | -------------------------------------------------------------------------------- /exec-git.js: -------------------------------------------------------------------------------- 1 | const exec = require('child_process').exec; 2 | const os = require('os'); 3 | 4 | class Pool { 5 | constructor (count) { 6 | this.count = count; 7 | this.queue = []; 8 | this.promises = new Array(count); 9 | } 10 | } 11 | 12 | /* Run the function immediately. */ 13 | function run (pool, idx, executionFunction) { 14 | var p = Promise.resolve() 15 | .then(executionFunction) 16 | .then(() => { 17 | delete pool.promises[idx]; 18 | var next = pool.queue.pop(); 19 | if (next) 20 | pool.execute(next); 21 | }); 22 | pool.promises[idx] = p; 23 | return p; 24 | } 25 | 26 | /* Defer function to run once all running and queued functions have run. */ 27 | function enqueue (pool, executeFunction) { 28 | return new Promise(resolve => { 29 | pool.queue.push(() => { 30 | return Promise.resolve() 31 | .then(executeFunction) 32 | .then(resolve); 33 | }); 34 | }); 35 | } 36 | 37 | /* Take a function to execute within pool, and return promise delivering the functions 38 | * result immediately once it is run. */ 39 | Pool.prototype.execute = function (executionFunction) { 40 | var idx = -1; 41 | for (var i = 0; i < this.count; i++) 42 | if (!this.promises[i]) 43 | idx = i; 44 | if (idx !== -1) 45 | return run(this, idx, executionFunction); 46 | else 47 | return enqueue(this, executionFunction); 48 | }; 49 | 50 | if (process.platform === 'win32') { 51 | var gitPool = new Pool(Math.min(os.cpus().length, 2)); 52 | module.exports = function (command, execOpt) { 53 | return new Promise((topResolve, topReject) => { 54 | return gitPool.execute(function() { 55 | return new Promise(resolve => { 56 | exec('git ' + command, execOpt, (err, stdout, stderr) => { 57 | if (err) 58 | topReject(stderr || err); 59 | else 60 | topResolve(stdout); 61 | resolve(); 62 | }); 63 | }); 64 | }); 65 | }); 66 | }; 67 | } 68 | else { 69 | module.exports = (command, execOpt) => new Promise((resolve, reject) => { 70 | exec('git ' + command, execOpt, (err, stdout, stderr) => { 71 | if (err) 72 | reject(stderr || err); 73 | else 74 | resolve(stdout); 75 | }); 76 | }); 77 | } 78 | -------------------------------------------------------------------------------- /github.js: -------------------------------------------------------------------------------- 1 | const { Semver, SemverRange } = require('sver'); 2 | const execGit = require('./exec-git'); 3 | const { URL } = require('url'); 4 | 5 | const githubApiAcceptHeader = 'application/vnd.github.v3+json'; 6 | const githubApiRawAcceptHeader = 'application/vnd.github.v3.raw'; 7 | 8 | const commitRegEx = /^[a-f0-9]{6,}$/; 9 | const wildcardRange = new SemverRange('*'); 10 | 11 | const githubApiAuth = process.env.JSPM_GITHUB_AUTH_TOKEN ? { 12 | username: 'envtoken', 13 | password: process.env.JSPM_GITHUB_AUTH_TOKEN 14 | } : null; 15 | 16 | module.exports = class GithubEndpoint { 17 | constructor (util, config) { 18 | this.userInput = config.userInput; 19 | this.util = util; 20 | 21 | this.timeout = config.timeout; 22 | this.strictSSL = config.strictSSL; 23 | this.instanceId = Math.round(Math.random() * 10**10); 24 | 25 | if (config.auth) { 26 | this._auth = readAuth(config.auth); 27 | if (!this._auth) 28 | this.util.log.warn(`${this.util.bold(`registries.github.auth`)} global github registry auth token is not a valid token format.`); 29 | } 30 | else { 31 | this._auth = undefined; 32 | } 33 | 34 | this.credentialsAttempts = 0; 35 | 36 | if (config.host && config.host !== 'github.com') { 37 | // github enterprise support 38 | this.githubUrl = 'https://' + (config.host[config.host.length - 1] === '/' ? config.host.substr(0, config.host.length - 1) : config.host); 39 | this.githubApiUrl = `https://${this.githubApiHost}/api/v3`; 40 | } 41 | else { 42 | this.githubUrl = 'https://github.com'; 43 | this.githubApiUrl = 'https://api.github.com'; 44 | } 45 | 46 | this.execOpt = { 47 | timeout: this.timeout, 48 | killSignal: 'SIGKILL', 49 | maxBuffer: 100 * 1024 * 1024, 50 | env: Object.assign({ 51 | GIT_TERMINAL_PROMPT: '0', 52 | GIT_SSL_NO_VERIFY: this.strictSSL ? '0' : '1' 53 | }, process.env) 54 | }; 55 | 56 | this.gettingCredentials = false; 57 | this.rateLimited = false; 58 | this.freshLookups = {}; 59 | 60 | // by default, "dependencies" are taken to be from npm registry 61 | // unless there is an explicit "registry" property 62 | this.dependencyRegistry = 'npm'; 63 | } 64 | 65 | dispose () { 66 | } 67 | 68 | /* 69 | * Registry config 70 | */ 71 | async configure () { 72 | this.gettingCredentials = true; 73 | await this.ensureAuth(this.util.getCredentials(this.githubUrl), true); 74 | this.gettingCredentials = false; 75 | this.util.log.ok('GitHub authentication updated.'); 76 | } 77 | 78 | async auth (url, _method, credentials, unauthorizedHeaders) { 79 | if (unauthorizedHeaders || this._auth) { 80 | const origin = url.origin; 81 | if (origin === this.githubUrl || origin === this.githubApiUrl) { 82 | // unauthorized -> fresh auth token 83 | if (unauthorizedHeaders) 84 | await this.ensureAuth(credentials, true); 85 | // update old jspm auth format to an automatically generated token, so we always use tokens 86 | // (can be deprecated eventually) 87 | else if (this._auth && !isGithubApiToken(this._auth.password) && !this.gettingCredentials) 88 | await this.ensureAuth(credentials); 89 | credentials.basicAuth = githubApiAuth || this._auth; 90 | return true; 91 | } 92 | } 93 | } 94 | 95 | async ensureAuth (credentials, invalid) { 96 | if (invalid || !this._auth) { 97 | if (!this.userInput) 98 | return; 99 | 100 | const username = await this.util.input('Enter your GitHub username', this._auth && this._auth.username !== 'Token' ? this._auth.username : '', { 101 | edit: true, 102 | info: `jspm can generate an authentication token to install packages from GitHub with the best performance and for private repo support. Leave blank to remove jspm credentials.` 103 | }); 104 | 105 | if (!username) { 106 | this.util.globalConfig.set('registries.github.auth', undefined); 107 | return; 108 | } 109 | else { 110 | const password = await this.util.input('Enter your GitHub password or access token', { 111 | info: `Your password is not saved locally and is only used to generate a token with the permission for repo access ${this.util.bold('repo')} to be saved into the jspm global configuration. Alternatively, you can generate an access token manually from ${this.util.bold(`${this.githubUrl}/settings/tokens`)}.`, 112 | silent: true, 113 | validate (input) { 114 | if (!input) 115 | return 'Please enter a valid GitHub password or token.'; 116 | } 117 | }); 118 | if (isGithubApiToken(password)) { 119 | this.util.globalConfig.set('registries.github.auth', password); 120 | return; 121 | } 122 | 123 | credentials.basicAuth = { username, password }; 124 | } 125 | } 126 | 127 | const getAPIToken = async (otp) => { 128 | // get an API token if using basic auth 129 | const res = await this.util.fetch(`${this.githubApiUrl}/authorizations`, { 130 | method: 'POST', 131 | headers: { 132 | accept: githubApiAcceptHeader, 133 | 'X-GitHub-OTP': otp 134 | }, 135 | body: JSON.stringify({ 136 | scopes: ['repo'], 137 | note: 'jspm token ' + Math.round(Math.random() * 10**10) 138 | }), 139 | timeout: this.timeout, 140 | credentials, 141 | reauthorize: false 142 | }); 143 | switch (res.status) { 144 | case 201: 145 | const response = await res.json(); 146 | this.util.globalConfig.set('registries.github.auth', response.token); 147 | this._auth = credentials.basicAuth = { 148 | username: 'Token', 149 | password: response.token 150 | }; 151 | this.util.log.ok('GitHub token generated successfully from basic auth credentials.'); 152 | break; 153 | case 401: 154 | if (!this.userInput) 155 | return; 156 | if (++this.credentialsAttempts === 3) 157 | throw new Error(`Unable to setup GitHub credentials.`); 158 | const otpHeader = res.headers.get('x-github-otp'); 159 | if (otpHeader && otpHeader.startsWith('required')) { 160 | const otp = await this.util.input('Please enter your GitHub 2FA token', { 161 | validate (input) { 162 | if (!input || input.length !== 6 || !input.match(/^[0-9]{6}$/)) 163 | return 'Please enter a valid GitHub 6 digit 2FA Token.'; 164 | } 165 | }); 166 | return getAPIToken(otp); 167 | } 168 | this.util.log.warn('GitHub username and password combination is invalid. Please enter your details again.'); 169 | return await this.ensureAuth(credentials, true); 170 | break; 171 | default: 172 | throw new Error(`Bad GitHub API response code ${res.status}: ${res.statusText}`); 173 | } 174 | }; 175 | return getAPIToken(); 176 | } 177 | 178 | /* 179 | * Resolved object has the shape: 180 | * { source?, dependencies?, peerDependencies?, optionalDependencies?, deprecated?, override? } 181 | */ 182 | async lookup (packageName, versionRange, lookup) { 183 | if (lookup.redirect && this.freshLookups[packageName]) 184 | return false; 185 | 186 | // first check if we have a redirect 187 | try { 188 | var res = await this.util.fetch(`${this.githubUrl}/${packageName[0] === '@' ? packageName.substr(1) : packageName}`, { 189 | headers: { 190 | 'User-Agent': 'jspm' 191 | }, 192 | redirect: 'manual', 193 | timeout: this.timeout 194 | }); 195 | } 196 | catch (err) { 197 | err.retriable = true; 198 | throw err; 199 | } 200 | 201 | switch (res.status) { 202 | case 301: 203 | lookup.redirect = `github:${res.headers.get('location').split('/').splice(3).join('/')}`; 204 | return true; 205 | 206 | // it might be a private repo, so wait for the lookup to fail as well 207 | case 200: 208 | case 404: 209 | case 302: 210 | break 211 | 212 | case 401: 213 | var e = new Error(`Invalid GitHub authentication details. Run ${this.util.bold(`jspm registry config github`)} to configure.`); 214 | e.hideStack = true; 215 | throw e; 216 | 217 | default: 218 | throw new Error(`Invalid status code ${res.status}: ${res.statusText}`); 219 | } 220 | 221 | // cache lookups per package for process execution duration 222 | if (this.freshLookups[packageName]) 223 | return false; 224 | 225 | // could filter to range in this lookup, but testing of eg `git ls-remote https://github.com/twbs/bootstrap.git refs/tags/v4.* resf/tags/v.*` 226 | // didn't reveal any significant improvement 227 | let url = this.githubUrl; 228 | let credentials = await this.util.getCredentials(this.githubUrl); 229 | if (credentials.basicAuth) { 230 | let urlObj = new URL(url); 231 | ({ username: urlObj.username, password: urlObj.password } = credentials.basicAuth); 232 | url = urlObj.href; 233 | // href includes trailing `/` 234 | url = url.substr(0, url.length - 1); 235 | } 236 | 237 | try { 238 | var stdout = await execGit(`ls-remote ${url}/${packageName[0] === '@' ? packageName.substr(1) : packageName}.git refs/tags/* refs/heads/*`, this.execOpt); 239 | } 240 | catch (err) { 241 | const str = err.toString(); 242 | // not found 243 | if (str.indexOf('not found') !== -1) 244 | return; 245 | // invalid credentials 246 | if (str.indexOf('Invalid username or password') !== -1 || str.indexOf('fatal: could not read Username') !== -1) { 247 | let e = new Error(`git authentication failed resolving GitHub package ${this.util.highlight(packageName)}. 248 | Make sure that git is locally configured with permissions to ${this.githubUrl} or run ${this.util.bold(`jspm registry config github`)}.`, err); 249 | e.hideStack = true; 250 | throw e; 251 | } 252 | throw err; 253 | } 254 | 255 | let refs = stdout.split('\n'); 256 | for (let ref of refs) { 257 | if (!ref) 258 | continue; 259 | 260 | let hash = ref.substr(0, ref.indexOf('\t')); 261 | let refName = ref.substr(hash.length + 1); 262 | let version; 263 | 264 | if (refName.substr(0, 11) === 'refs/heads/') { 265 | version = refName.substr(11); 266 | } 267 | else if (refName.substr(0, 10) === 'refs/tags/') { 268 | if (refName.substr(refName.length - 3, 3) === '^{}') 269 | version = refName.substr(10, refName.length - 13); 270 | else 271 | version = refName.substr(10); 272 | 273 | if (version.substr(0, 1) === 'v' && Semver.isValid(version.substr(1))) 274 | version = version.substr(1); 275 | } 276 | 277 | const encoded = this.util.encodeVersion(version); 278 | const existingVersion = lookup.versions[encoded]; 279 | if (!existingVersion) 280 | lookup.versions[encoded] = { resolved: undefined, meta: { expected: hash, resolved: undefined } }; 281 | else 282 | existingVersion.meta.expected = hash; 283 | } 284 | return true; 285 | } 286 | 287 | async resolve (packageName, version, lookup) { 288 | let changed = false; 289 | let versionEntry; 290 | 291 | // first ensure we have the right ref hash 292 | // an exact commit is immutable 293 | if (commitRegEx.test(version)) { 294 | versionEntry = lookup.versions[version] = { resolved: undefined, meta: { expected: version, resolved: undefined } }; 295 | } 296 | else { 297 | versionEntry = lookup.versions[version]; 298 | // we get refs through the full remote-ls lookup 299 | if (!(packageName in this.freshLookups)) { 300 | await this.lookup(packageName, wildcardRange, lookup); 301 | changed = true; 302 | versionEntry = lookup.versions[version]; 303 | if (!versionEntry) 304 | return changed; 305 | } 306 | } 307 | 308 | // next we fetch the package.json file for that ref hash, to get the dependency information 309 | // to populate into the resolved object 310 | if (!versionEntry.resolved || versionEntry.meta.resolved !== versionEntry.meta.expected) { 311 | changed = true; 312 | const hash = versionEntry.meta.expected; 313 | 314 | const resolved = versionEntry.resolved = { 315 | source: `${this.githubUrl}/${packageName[0] === '@' ? packageName.substr(1) : packageName}/archive/${hash}.tar.gz`, 316 | override: undefined 317 | }; 318 | 319 | // if this fails, we just get no preloading 320 | if (!this.rateLimited) { 321 | const res = await this.util.fetch(`${this.githubApiUrl}/repos/${packageName[0] === '@' ? packageName.substr(1) : packageName}/contents/package.json?ref=${hash}`, { 322 | headers: { 323 | 'User-Agent': 'jspm', 324 | accept: githubApiRawAcceptHeader 325 | }, 326 | timeout: this.timeout 327 | }); 328 | switch (res.status) { 329 | case 404: 330 | // repo can not have a package.json 331 | break; 332 | case 200: 333 | const pjson = await res.json(); 334 | resolved.override = { 335 | dependencies: pjson.dependencies, 336 | peerDependencies: pjson.peerDependencies, 337 | optionalDepdnencies: pjson.optionalDependencies 338 | } 339 | break; 340 | case 401: 341 | apiWarn(this.util, `Invalid GitHub API credentials`); 342 | break; 343 | case 403: 344 | apiWarn(this.util, `GitHub API rate limit reached`); 345 | this.rateLimited = true; 346 | break; 347 | case 406: 348 | apiWarn(this.util, `GitHub API token doesn't have the right access permissions`); 349 | break; 350 | default: 351 | apiWarn(this.util, `Invalid GitHub API response code ${res.status}`); 352 | } 353 | function apiWarn (util, msg) { 354 | util.log.warn(`${msg} attempting to preload dependencies for ${packageName}.`); 355 | }; 356 | } 357 | 358 | versionEntry.meta.resolved = hash; 359 | } 360 | 361 | return changed; 362 | } 363 | }; 364 | 365 | function readAuth (auth) { 366 | // no auth 367 | if (!auth) 368 | return; 369 | // auth is an object 370 | if (typeof auth === 'object' && typeof auth.username === 'string' && typeof auth.password === 'string') 371 | return auth; 372 | else if (typeof auth !== 'string') 373 | return; 374 | // jspm 2 auth form - just a token 375 | if (isGithubApiToken(auth)) { 376 | return { username: 'Token', password: auth }; 377 | } 378 | // jspm 0.16/0.17 auth form backwards compat 379 | // (base64(encodeURI(username):encodeURI(password))) 380 | try { 381 | let auth = new Buffer(auth, 'base64').toString('utf8').split(':'); 382 | if (auth.length !== 2) 383 | return; 384 | let username = decodeURIComponent(auth[0]); 385 | let password = decodeURIComponent(auth[1]); 386 | return { username, password }; 387 | } 388 | // invalid auth 389 | catch (e) { 390 | return; 391 | } 392 | } 393 | 394 | 395 | function isGithubApiToken (str) { 396 | if (str && str.length === 40 && str.match(/^[a-f0-9]+$/)) 397 | return true; 398 | else 399 | return false; 400 | } -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "@jspm/github", 3 | "version": "1.0.4", 4 | "description": "jspm GitHub endpoint", 5 | "main": "github.js", 6 | "author": "Guy Bedford", 7 | "license": "MIT", 8 | "repository": { 9 | "type": "git", 10 | "url": "https://github.com/jspm/github.git" 11 | }, 12 | "homepage": "https://github.com/jspm/github", 13 | "dependencies": { 14 | "sver": "^1.6.0" 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /test.js: -------------------------------------------------------------------------------- 1 | var github = require('./github'); 2 | 3 | github = new github({ 4 | baseDir: '.', 5 | log: true, 6 | tmpDir: '.', 7 | username: '', 8 | password: '' 9 | }); 10 | 11 | github.lookup('angular/bower-angular') 12 | .then(function(versions) { 13 | console.log(versions); 14 | return github.download('angular/bower-angular', 'v1.2.12', 'e8a1df5f060bf7e6631554648e0abde150aedbe4', {}, 'test-repo'); 15 | }) 16 | .then(function() { 17 | console.log('done'); 18 | }) 19 | .catch(function(err) { 20 | console.log(err); 21 | }); 22 | --------------------------------------------------------------------------------