├── .npmrc ├── .npmignore ├── .gitignore ├── .eslintrc.json ├── .babelrc ├── publish.sh ├── src ├── progress.js └── index.js ├── webpack.config.js ├── package.json ├── CHANGELOG.md └── README.md /.npmrc: -------------------------------------------------------------------------------- 1 | package-lock=false 2 | -------------------------------------------------------------------------------- /.npmignore: -------------------------------------------------------------------------------- 1 | .eslintrc.json 2 | src/ 3 | webpack.config.json 4 | .babelrc -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .idea/* 2 | node_modules/* 3 | .DS_Store 4 | package-lock.json 5 | dist/ 6 | -------------------------------------------------------------------------------- /.eslintrc.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "airbnb-base", 3 | "rules": { 4 | "no-console": "off", 5 | "max-len": "off", 6 | "no-plusplus": "off", 7 | "no-shadow": "off" 8 | } 9 | } -------------------------------------------------------------------------------- /.babelrc: -------------------------------------------------------------------------------- 1 | { 2 | "presets": [ 3 | [ 4 | "@babel/preset-env", 5 | { 6 | "targets": { 7 | "node": "6.0.0" 8 | }, 9 | "useBuiltIns": "usage", 10 | "corejs": 3 11 | } 12 | ] 13 | ] 14 | } -------------------------------------------------------------------------------- /publish.sh: -------------------------------------------------------------------------------- 1 | #! /bin/bash 2 | 3 | 4 | # Step1: add !/usr/bin/env node to index.js 5 | pushd dist 1>/dev/null 6 | 7 | sed -i '/#! *\/usr\/bin\/env *node/d' index.js 8 | 9 | touch tempfile.$$ 10 | echo '#!/usr/bin/env node' >> tempfile.$$ 11 | cat index.js >> tempfile.$$ 12 | mv tempfile.$$ index.js 13 | 14 | # Step2: run npm publish 15 | npm publish 16 | 17 | pushd 1>/dev/null -------------------------------------------------------------------------------- /src/progress.js: -------------------------------------------------------------------------------- 1 | const progress = require('@gyumeijie/cli-progress'); 2 | const colors = require('colors'); 3 | 4 | // Customizable progress bar 5 | const bar = new progress.Bar({ 6 | format: `fetcher |${colors.cyan('{bar}')}| {percentage}% || {value}/{total} File(s) || {status} || authenticated: {doesUseAuth}`, 7 | barCompleteChar: '\u2588', 8 | barIncompleteChar: '\u2591', 9 | hideCursor: true, 10 | }); 11 | 12 | module.exports = { 13 | bar, 14 | }; 15 | -------------------------------------------------------------------------------- /webpack.config.js: -------------------------------------------------------------------------------- 1 | const path = require('path'); 2 | const CopyWebpackPlugin = require('copy-webpack-plugin'); 3 | const WriteToFilePlugin = require('write-to-file-webpack'); 4 | const config = require('./package.json'); 5 | 6 | module.exports = { 7 | mode: 'production', 8 | entry: './src/index.js', 9 | output: { 10 | filename: 'index.js', 11 | path: path.resolve(__dirname, 'dist'), 12 | }, 13 | module: { 14 | rules: [ 15 | { test: /\.js$/, exclude: /node_modules/, loader: 'babel-loader' }, 16 | ], 17 | }, 18 | target: 'node', 19 | externals: [ 20 | 'electron', 21 | ], 22 | plugins: [ 23 | new CopyWebpackPlugin({ 24 | patterns: [ 25 | { from: 'CHANGELOG.md', to: path.resolve(__dirname, 'dist', 'CHANGELOG.md') }, 26 | { from: 'README.md', to: path.resolve(__dirname, 'dist', 'README.md') }, 27 | ] 28 | }), 29 | new WriteToFilePlugin({ 30 | filename: path.resolve(__dirname, 'dist/package.json'), 31 | data() { 32 | // We publish the bundled file, so we don't need the following option in pacakage.json 33 | return JSON.stringify({ 34 | ...config, 35 | dependencies: undefined, 36 | devDependencies: undefined, 37 | scripts: undefined, 38 | config: undefined, 39 | }); 40 | }, 41 | }), 42 | ], 43 | }; 44 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "github-files-fetcher", 3 | "version": "1.6.0", 4 | "description": "download files from github repository", 5 | "keywords": [ 6 | "github", 7 | "repo", 8 | "repository", 9 | "download", 10 | "fetch", 11 | "fetcher", 12 | "files" 13 | ], 14 | "main": "index.js", 15 | "scripts": { 16 | "test": "echo \"Error: no test specified\" && exit 1", 17 | "version": "conventional-changelog -p angular -i CHANGELOG.md -s -r 0 && git add CHANGELOG.md", 18 | "build": "webpack", 19 | "publish": "./publish.sh" 20 | }, 21 | "bin": { 22 | "fetcher": "index.js", 23 | "github-files-fetcher": "index.js" 24 | }, 25 | "author": "Gyumeijie", 26 | "license": "MIT", 27 | "repository": { 28 | "type": "git", 29 | "url": "https://github.com/Gyumeijie/github-files-fetcher.git" 30 | }, 31 | "bugs": { 32 | "url": "https://github.com/Gyumeijie/github-files-fetcher/issues" 33 | }, 34 | "homepage": "https://github.com/Gyumeijie/github-files-fetcher", 35 | "dependencies": { 36 | "@gyumeijie/cli-progress": "^1.0.2", 37 | "args-parser": "^1.3.0", 38 | "axios": "^0.21.4", 39 | "colors": "^1.4.0", 40 | "core-js": "^3.17.3", 41 | "is-online": "^9.0.1", 42 | "shelljs": "^0.8.4" 43 | }, 44 | "devDependencies": { 45 | "@babel/core": "^7.15.5", 46 | "@babel/preset-env": "^7.15.6", 47 | "babel-loader": "^8.2.2", 48 | "copy-webpack-plugin": "^9.0.1", 49 | "cz-conventional-changelog": "^3.3.0", 50 | "eslint": "^7.32.0", 51 | "eslint-config-airbnb": "^18.2.1", 52 | "eslint-plugin-import": "^2.24.2", 53 | "eslint-plugin-jsx-a11y": "^6.4.1", 54 | "eslint-plugin-react": "^7.25.2", 55 | "eslint-plugin-react-hooks": "^4.2.0", 56 | "webpack": "^5.53.0", 57 | "webpack-cli": "^4.8.0", 58 | "write-to-file-webpack": "^1.0.6" 59 | }, 60 | "config": { 61 | "commitizen": { 62 | "path": "./node_modules/cz-conventional-changelog" 63 | } 64 | } 65 | } 66 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | 2 | # 1.2.0 (2018-09-18) 3 | 4 | 5 | ### Bug Fixes 6 | 7 | * [#1](https://github.com/Gyumeijie/github-files-fetcher/issues/1) ([d347841](https://github.com/Gyumeijie/github-files-fetcher/commit/d347841)) 8 | * [#2](https://github.com/Gyumeijie/github-files-fetcher/issues/2) ([5b00b60](https://github.com/Gyumeijie/github-files-fetcher/commit/5b00b60)) 9 | * add code for client-side errors ([e181f0d](https://github.com/Gyumeijie/github-files-fetcher/commit/e181f0d)) 10 | * command line output ([ff482e6](https://github.com/Gyumeijie/github-files-fetcher/commit/ff482e6)) 11 | * default output directory ([58f690e](https://github.com/Gyumeijie/github-files-fetcher/commit/58f690e)) 12 | * offset by one error ([d4455d6](https://github.com/Gyumeijie/github-files-fetcher/commit/d4455d6)) 13 | * the type of clientside response status code ([d934a9c](https://github.com/Gyumeijie/github-files-fetcher/commit/d934a9c)) 14 | * unknown commandline options ([f326bc3](https://github.com/Gyumeijie/github-files-fetcher/commit/f326bc3)) 15 | 16 | 17 | ### Features 18 | 19 | * add --alwaysUseAuth option ([f8ea220](https://github.com/Gyumeijie/github-files-fetcher/commit/f8ea220)) 20 | * add --file option ([01e6a77](https://github.com/Gyumeijie/github-files-fetcher/commit/01e6a77)) 21 | * add --help option ([a64a3b0](https://github.com/Gyumeijie/github-files-fetcher/commit/a64a3b0)) 22 | * add auth status in progroess bar ([e60b4f8](https://github.com/Gyumeijie/github-files-fetcher/commit/e60b4f8)) 23 | * add event listener for CTRL+C ([2b2441d](https://github.com/Gyumeijie/github-files-fetcher/commit/2b2441d)) 24 | * add progress bar ([d983874](https://github.com/Gyumeijie/github-files-fetcher/commit/d983874)) 25 | * add support for alwaysUseAuth in config file ([f3eda1b](https://github.com/Gyumeijie/github-files-fetcher/commit/f3eda1b)) 26 | * finish basic features. ([d7aace5](https://github.com/Gyumeijie/github-files-fetcher/commit/d7aace5)) 27 | * finish parsing the repo url ([6ea892c](https://github.com/Gyumeijie/github-files-fetcher/commit/6ea892c)) 28 | * support command line authentication ([16126b3](https://github.com/Gyumeijie/github-files-fetcher/commit/16126b3)) 29 | * support commondline args ([5a8cb3e](https://github.com/Gyumeijie/github-files-fetcher/commit/5a8cb3e)) 30 | 31 | 32 | ### Performance Improvements 33 | 34 | * support request with or without auth ([25b98c6](https://github.com/Gyumeijie/github-files-fetcher/commit/25b98c6)) 35 | 36 | 37 | 38 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Introduction [![License: MIT](https://img.shields.io/badge/License-MIT-blue.svg)](https://opensource.org/licenses/MIT) 2 | 3 | The `github-files-fetcher` is designed for downloading parts of a github repository. This is very useful if you have a low bandwidth network or only need a particular file or subdirectory from a large repository. If you want to download a whole repository, prefer `git clone`. 4 | 5 | # Installation 6 | 7 | Run `npm install -g github-files-fetcher` 8 | 9 | ## Basic usage 10 | 11 | ```sh 12 | fetcher --url=resource_url --out=output_directory 13 | ``` 14 | 15 | For example: 16 | ```sh 17 | fetcher --url="https://github.com/Gyumeijie/github-files-fetcher/blob/master/CHANGELOG.md" --out=/tmp 18 | ``` 19 | ![](https://github.com/Gyumeijie/assets/blob/master/github-files-fetcher/fetcher-result.png) 20 | 21 | ## Authentication 22 | 23 | The default unauthorized API access rate is **60** times per hour, which is usually enough. 24 | You can surpass this with authentication, using one of the following three ways: 25 | 26 | 1. The --auth commandline option 27 | 28 | This option takes the form of `--auth=username:password`, where the password can be either the login password for your github account or the personal access token which can be generated in https://github.com/settings/tokens. 29 | 30 | 2. Default configuration file 31 | 32 | The default configuration file is `~/.download_github`, and the config file is a **json file**. 33 | 34 | 3. Designate via --file commandline option 35 | 36 | For example, you can use `~/config.json` as configuration file. 37 | ```sh 38 | # download a directory 39 | fetcher --file="~/config.json" --url="https://github.com/reduxjs/redux/tree/master/examples/async" --out="~/" 40 | 41 | # download a single file 42 | fetcher --file="~/config.json" --url="https://github.com/Gyumeijie/github-files-fetcher/blob/master/index.js" --out="~/" 43 | ``` 44 | 45 | This is a **template** for the configuration file: 46 | ```json 47 | { 48 | "auth": { 49 | "username" : "your_github_name", 50 | "password" : "password_or_api_access_token" 51 | }, 52 | "alwaysUseAuth" : true, 53 | "timeout" : 5000 54 | } 55 | ``` 56 | 57 | ### Behavior 58 | 59 | When the default unauthorized API access rate exceeded, `github-files-fetcher` will automatically switch to use authentication if provided through one of the ways above. 60 | 61 | `github-files-fetcher` requests resources without authentication by default to improve performance. However, this incurs a delay once the default unauthorized API access rate exceeded. To avoid this problem you can specify the `--alwaysUseAuth` option so `github-files-fetcher` always uses authentication. 62 | 63 | # Environment 64 | `node >= 6` 65 | 66 | # Related works 67 | There are some other good tools that function similarly: 68 | - GitZip (Credits to Kino, Browser Extensions) 69 | - [Firefox Addon](https://addons.mozilla.org/en-US/firefox/addon/gitzip/) 70 | - [Chrome Extension](https://chrome.google.com/webstore/detail/gitzip-for-github/ffabmkklhbepgcgfonabamgnfafbdlkn) 71 | - [DownGit](https://minhaskamal.github.io/DownGit/#/home) (Credits to Minhas Kamal, Web Page) 72 | -------------------------------------------------------------------------------- /src/index.js: -------------------------------------------------------------------------------- 1 | const fs = require('fs'); 2 | const os = require('os'); 3 | const url = require('url'); 4 | const axios = require('axios'); 5 | const shell = require('shelljs'); 6 | const argsParser = require('args-parser'); 7 | const isOnline = require('is-online'); 8 | const progress = require('./progress'); 9 | 10 | const AUTHOR = 1; 11 | const REPOSITORY = 2; 12 | const BRANCH = 4; 13 | 14 | // Progress managment 15 | const progressBar = progress.bar; 16 | const fileStats = { 17 | downloaded: 0, 18 | currentTotal: 0, 19 | done: false, 20 | doesDownloadDirectory: false, 21 | }; 22 | 23 | // A utility function for expand `~` 24 | function tilde(pathString) { 25 | if (pathString[0] === '~') { 26 | return os.homedir() + pathString.substring(1); 27 | } 28 | return pathString; 29 | } 30 | 31 | // The default output directory is the current directory 32 | let outputDirectory = `${process.cwd()}/`; 33 | let localRootDirectory = ''; 34 | let currentDownloadingFile = ''; 35 | // Default authentication setting 36 | const authentication = {}; 37 | let authenticationSwitch = {}; 38 | let doesUseAuth = false; 39 | // Defalut configuration file 40 | let configFile = tilde('~/.download_github'); 41 | // `timeout` specifies the number of milliseconds to exit after the internet is detected disconnected. 42 | let timeout; 43 | let timer; 44 | // Root directory when clean up 45 | let rootDirectoryForCleanUp; 46 | 47 | function checkGithubRepoURLValidity(downloadUrl) { 48 | const { hostname, pathname } = url.parse(downloadUrl, true); 49 | 50 | if (hostname !== 'github.com') { 51 | throw new Error('Invalid domain: github.com is expected!'); 52 | } 53 | 54 | if (pathname.split('/').length < 3) { 55 | throw new Error('Invalid url: https://github.com/user/repository is expected'); 56 | } 57 | } 58 | 59 | function printHelpInformation() { 60 | console.log(` 61 | Usage: download [OPTION]... 62 | Example: download --url='https://github.com/user/repository' --out='~/output' 63 | 64 | Resource URL: 65 | --url=URL the url of resource to be downloaded 66 | 67 | Output: 68 | --out=output_directory the directory holds your download resource 69 | 70 | Authentication: 71 | --auth=username:password the password can be either you login password of github account or access token 72 | --alwaysUseAuth if set true, every request is authenticated and in this way we can have more API 73 | access rate 74 | 75 | Configuration file: 76 | --file=config_file the default configuration file is the '~/.download_github' 77 | 78 | Timeout: 79 | --timeout=number(ms) timeout specifies the number of milliseconds to exit after the internet is detected disconnected 80 | `); 81 | } 82 | 83 | // Find root directory for clean up when exit unexpectedly 84 | function findRootDirectoryForCleanUp(out) { 85 | const segments = out.split(/\/+/); 86 | let path = segments[0]; 87 | let rootDirectoryForCleanUp; 88 | 89 | for (let i = 1; i < segments.length; i++) { 90 | path = `${path}/${segments[i]}`; 91 | if (!fs.existsSync(path)) { 92 | rootDirectoryForCleanUp = path; 93 | break; 94 | } 95 | } 96 | 97 | return rootDirectoryForCleanUp; 98 | } 99 | 100 | const args = argsParser(process.argv); 101 | let doseJustPrintHelpInfo = args.help || (Object.keys(args).length === 0); 102 | try { 103 | (function tackleArgs() { 104 | if (doseJustPrintHelpInfo) { 105 | printHelpInformation(); 106 | return; 107 | } 108 | 109 | // The url is required and should be a valid github repository url 110 | if (!args.url) { 111 | throw new Error(' Bad option: a URL is needed!'); 112 | } else { 113 | checkGithubRepoURLValidity(args.url); 114 | } 115 | 116 | if (args.out) { 117 | outputDirectory = tilde(args.out); 118 | if (outputDirectory[args.out.length - 1] !== '/') { 119 | outputDirectory = `${outputDirectory}/`; 120 | } 121 | rootDirectoryForCleanUp = findRootDirectoryForCleanUp(outputDirectory); 122 | } 123 | 124 | if (args.auth) { 125 | const { auth } = args; 126 | 127 | const colonPos = auth.indexOf(':'); 128 | if (colonPos === -1 || colonPos === auth.length - 1) { 129 | throw new Error('Bad auth option: username:password is expected!'); 130 | } 131 | 132 | const [username, password] = auth.split(':'); 133 | authentication.auth = { 134 | username, 135 | password, 136 | }; 137 | 138 | if (args.alwaysUseAuth) { 139 | authenticationSwitch = authentication; 140 | doesUseAuth = true; 141 | } 142 | } 143 | 144 | if (args.timeout !== undefined) { 145 | timeout = parseInt(args.timeout, 10); 146 | // [eslint] Unexpected use of 'isNaN'. (no-restricted-globals) 147 | timeout = Number.isNaN(timeout) ? undefined : timeout; 148 | } 149 | 150 | if (args.file) { 151 | configFile = tilde(args.file); 152 | } 153 | }()); 154 | } catch (error) { 155 | console.log(error.message); 156 | printHelpInformation(); 157 | // No more action, just quit after printing help information 158 | doseJustPrintHelpInfo = true; 159 | } 160 | 161 | const parameters = { 162 | url: args.url, 163 | fileName: undefined, 164 | rootDirectory: undefined, 165 | }; 166 | 167 | // If there no `auth`, `alwaysUseAuth`, `timeout` provided in the commandline, then we need 168 | // to read the configuration file 169 | function doesNeedReadConfiguration() { 170 | if (!args.auth || !args.alwaysUseAuth || !args.timeout) return true; 171 | return false; 172 | } 173 | 174 | // Read the configuration file if exists 175 | // WARNING: options provided in commandline prioritize those in configuration file 176 | if (fs.existsSync(configFile) && doesNeedReadConfiguration()) { 177 | const data = fs.readFileSync(configFile, 'utf8'); 178 | const config = JSON.parse(data); 179 | 180 | if (!args.auth && config.auth) authentication.auth = config.auth; 181 | 182 | if (!args.alwaysUseAuth && config.alwaysUseAuth) { 183 | authenticationSwitch = authentication; 184 | doesUseAuth = true; 185 | } 186 | 187 | if (!args.timeout && config.timeout) { 188 | timeout = parseInt(config.timeout, 10); 189 | timeout = Number.isNaN(timeout) ? undefined : timeout; 190 | } 191 | } 192 | 193 | function preprocessURL(repoURL) { 194 | // We just simply fix issue#2(https://github.com/Gyumeijie/github-files-fetcher/issues/2) 195 | // not to guarantee the validity of the url of the repository 196 | const len = repoURL.length; 197 | if (repoURL[len - 1] === '/') { 198 | return repoURL.slice(0, len - 1); 199 | } 200 | 201 | return repoURL; 202 | } 203 | 204 | function parseInfo(repoInfo) { 205 | const repoURL = preprocessURL(repoInfo.url); 206 | const repoPath = url.parse(repoURL, true).pathname; 207 | const splitPath = repoPath.split('/'); 208 | const info = {}; 209 | 210 | info.author = splitPath[AUTHOR]; 211 | info.repository = splitPath[REPOSITORY]; 212 | info.branch = splitPath[BRANCH]; 213 | info.rootName = splitPath[splitPath.length - 1]; 214 | 215 | // Common parts of url for downloading 216 | info.urlPrefix = `https://api.github.com/repos/${info.author}/${info.repository}/contents/`; 217 | info.urlPostfix = `?ref=${info.branch}`; 218 | 219 | if (splitPath[BRANCH]) { 220 | info.resPath = repoPath.substring(repoPath.indexOf(splitPath[BRANCH]) + splitPath[BRANCH].length + 1); 221 | } 222 | 223 | if (!repoInfo.fileName || repoInfo.fileName === '') { 224 | info.downloadFileName = info.rootName; 225 | } else { 226 | info.downloadFileName = repoInfo.fileName; 227 | } 228 | 229 | if (repoInfo.rootDirectory === 'false') { 230 | info.rootDirectoryName = ''; 231 | } else if (!repoInfo.rootDirectory || repoInfo.rootDirectory === '' 232 | || repoInfo.rootDirectory === 'true') { 233 | info.rootDirectoryName = `${info.rootName}/`; 234 | } else { 235 | info.rootDirectoryName = `${parameters.rootDirectory}/`; 236 | } 237 | 238 | return info; 239 | } 240 | 241 | const basicOptions = { 242 | method: 'get', 243 | responseType: 'arrayBuffer', 244 | }; 245 | // Global variable 246 | let repoInfo = {}; 247 | 248 | function cleanUpOutputDirectory() { 249 | if (rootDirectoryForCleanUp !== undefined) { 250 | shell.rm('-rf', rootDirectoryForCleanUp); 251 | return; 252 | } 253 | 254 | if (fileStats.doesDownloadDirectory) { 255 | shell.rm('-rf', localRootDirectory); 256 | } else { 257 | // Mainly for remove .zip file, when download a whole repo 258 | shell.rm('-f', localRootDirectory + currentDownloadingFile); 259 | } 260 | } 261 | 262 | function processClientError(error, retryCallback) { 263 | // Due to the cli-process, all console output before `stop` will be cleared, 264 | // to avoid that we put `stop` before the console output 265 | progressBar.stop(); 266 | 267 | // WARNING: never trust the returned data 268 | if (error.response === undefined) { 269 | console.error(` 270 | No internet, try:\n 271 | - Checking the network cables, modem, and router 272 | - Reconnecting to Wi-Fi`); 273 | if (localRootDirectory !== '') cleanUpOutputDirectory(); 274 | // Must use exit here not return 275 | process.exit(); 276 | } 277 | 278 | if (error.response.status === 401) { 279 | // Unauthorized 280 | console.error('\nBad credentials, please check your username or password(or access token)!'); 281 | } else if (error.response.status === 403) { 282 | if (authentication.auth) { 283 | // If the default API access rate without authentication exceeds and the command line 284 | // authentication is provided, then we switch to use authentication 285 | console.warn('\nThe unauthorized API access rate exceeded, we are now retrying with authentication......\n'); 286 | authenticationSwitch = authentication; 287 | doesUseAuth = true; 288 | retryCallback(); 289 | } else { 290 | // API rate limit exceeded 291 | console.error('\nAPI rate limit exceeded, Authenticated requests get a higher rate limit.' 292 | + ' Check out the documentation for more details. https://developer.github.com/v3/#rate-limiting'); 293 | 294 | // If there already has part of files downloaded, then clean up the output directory 295 | if (localRootDirectory !== '') cleanUpOutputDirectory(); 296 | } 297 | } else { 298 | let errMsg = `\n${error.message}`; 299 | if (error.response.status === 404) { 300 | errMsg += ', please check the repo URL!'; 301 | } 302 | console.error(errMsg); 303 | } 304 | 305 | // Must use exit here not return 306 | process.exit(); 307 | } 308 | 309 | function extractFilenameAndDirectoryFrom(path) { 310 | const components = path.split('/'); 311 | const filename = components[components.length - 1]; 312 | const directory = path.substring(0, path.length - filename.length); 313 | 314 | return { 315 | filename, 316 | directory, 317 | }; 318 | } 319 | 320 | /* 321 | * @example 322 | * take fether --url='https://github.com/reduxjs/redux/tree/master/examples/async' for example: 323 | * all paths of files under the 'async' directory are prefixed with the so-called 'resPath', which 324 | * equals to 'example/async', and the 'rootDirectoryName' is 'async'. The 'resPath' could be very long, 325 | * and we don't need that deep path locally in fact. So we just remove the 'resPath' from the path of a file. 326 | */ 327 | function removeResPathFrom(path) { 328 | return path.substring(decodeURI(repoInfo.resPath).length + 1); 329 | } 330 | 331 | function constructLocalPathname(repoPath) { 332 | const partialPath = extractFilenameAndDirectoryFrom(removeResPathFrom(repoPath)); 333 | localRootDirectory = outputDirectory + repoInfo.rootDirectoryName; 334 | const localDirectory = localRootDirectory + partialPath.directory; 335 | 336 | return { 337 | filename: partialPath.filename, 338 | directory: localDirectory, 339 | }; 340 | } 341 | 342 | const pathCache = []; 343 | function downloadFile(url, pathname) { 344 | axios({ 345 | ...basicOptions, 346 | responseType: 'stream', 347 | url, 348 | ...authenticationSwitch, 349 | }).then((response) => { 350 | if (pathCache.indexOf(pathname.directory) === -1) { 351 | shell.mkdir('-p', pathname.directory); 352 | pathCache.push(pathname.directory); 353 | } 354 | 355 | const localPathname = pathname.directory + pathname.filename; 356 | response.data.pipe(fs.createWriteStream(localPathname)) 357 | .on('error', error => processClientError({ 358 | response: { }, 359 | message: error, 360 | }, null)) 361 | .on('close', () => { 362 | fileStats.downloaded++; 363 | // Avoid falsy 100% progress, it is a sheer trick of presentation, not the logic 364 | if (fileStats.downloaded < fileStats.currentTotal) { 365 | progressBar.update(fileStats.downloaded, { 366 | doesUseAuth, 367 | }); 368 | } 369 | 370 | if (fileStats.downloaded === fileStats.currentTotal && fileStats.done) { 371 | progressBar.update(fileStats.downloaded, { 372 | status: 'downloaded', 373 | }); 374 | progressBar.stop(); 375 | process.exit(); 376 | } 377 | }); 378 | }).catch((error) => { 379 | processClientError(error, downloadFile.bind(null, url, pathname)); 380 | }); 381 | } 382 | 383 | function iterateDirectory(dirPaths) { 384 | axios({ 385 | ...basicOptions, 386 | url: repoInfo.urlPrefix + dirPaths.pop() + repoInfo.urlPostfix, 387 | ...authenticationSwitch, 388 | }).then((response) => { 389 | const { data } = response; 390 | for (let i = 0; i < data.length; i++) { 391 | if (data[i].type === 'dir') { 392 | dirPaths.push(data[i].path); 393 | } else if (data[i].download_url) { 394 | const pathname = constructLocalPathname(data[i].path); 395 | downloadFile(data[i].download_url, pathname); 396 | 397 | fileStats.currentTotal++; 398 | progressBar.setTotal(fileStats.currentTotal); 399 | } else { 400 | console.log(data[i]); 401 | } 402 | } 403 | 404 | if (dirPaths.length !== 0) { 405 | iterateDirectory(dirPaths); 406 | } else { 407 | fileStats.done = true; 408 | } 409 | }).catch((error) => { 410 | processClientError(error, iterateDirectory.bind(null, dirPaths)); 411 | }); 412 | } 413 | 414 | function downloadDirectory() { 415 | const dirPaths = []; 416 | dirPaths.push(repoInfo.resPath); 417 | iterateDirectory(dirPaths); 418 | } 419 | 420 | function initializeDownload(paras) { 421 | repoInfo = parseInfo(paras); 422 | 423 | if (!repoInfo.resPath || repoInfo.resPath === '') { 424 | if (!repoInfo.branch || repoInfo.branch === '') { 425 | repoInfo.branch = 'master'; 426 | } 427 | 428 | // Download the whole repository as a zip file 429 | const repoURL = `https://github.com/${repoInfo.author}/${repoInfo.repository}/archive/${repoInfo.branch}.zip`; 430 | downloadFile(repoURL, { directory: outputDirectory, filename: `${repoInfo.repository}.zip` }); 431 | localRootDirectory = outputDirectory; 432 | currentDownloadingFile = `${repoInfo.repository}.zip`; 433 | fileStats.done = true; 434 | fileStats.currentTotal = 1; 435 | progressBar.setTotal(fileStats.currentTotal); 436 | } else { 437 | // Download part(s) of repository 438 | axios({ 439 | ...basicOptions, 440 | url: repoInfo.urlPrefix + repoInfo.resPath + repoInfo.urlPostfix, 441 | ...authenticationSwitch, 442 | }).then((response) => { 443 | if (response.data instanceof Array) { 444 | downloadDirectory(); 445 | fileStats.doesDownloadDirectory = true; 446 | } else { 447 | const partialPath = extractFilenameAndDirectoryFrom(decodeURI(repoInfo.resPath)); 448 | downloadFile(response.data.download_url, { ...partialPath, directory: outputDirectory }); 449 | localRootDirectory = outputDirectory; 450 | currentDownloadingFile = partialPath.filename; 451 | fileStats.done = true; 452 | fileStats.currentTotal = 1; 453 | progressBar.setTotal(fileStats.currentTotal); 454 | } 455 | }).catch((error) => { 456 | processClientError(error, initializeDownload.bind(null, paras)); 457 | }); 458 | } 459 | } 460 | 461 | // Enable to detect CTRL+C 462 | process.on('SIGINT', () => { 463 | if (localRootDirectory !== '') cleanUpOutputDirectory(); 464 | progressBar.stop(); // Recover cursor 465 | process.exit(); 466 | }); 467 | 468 | // Enable to detect internet connectivity 469 | function detectInternetConnectivity() { 470 | isOnline().then((online) => { 471 | if (!online && timeout !== undefined) { 472 | setTimeout(() => { 473 | clearInterval(timer); 474 | // Construct a falsy error object to indicate the disconnectivity 475 | processClientError({ response: undefined }, null); 476 | }, timeout); 477 | } 478 | }); 479 | } 480 | timer = setInterval(detectInternetConnectivity, 2000); 481 | 482 | if (!doseJustPrintHelpInfo) { 483 | // Initialize progress bar 484 | console.log(''); 485 | progressBar.start(fileStats.currentTotal, fileStats.downloaded, { 486 | status: 'downloading...', 487 | doesUseAuth, 488 | }); 489 | 490 | initializeDownload(parameters); 491 | } else { 492 | clearInterval(timer); 493 | } 494 | --------------------------------------------------------------------------------