├── .gitignore ├── README.md ├── bin └── start ├── docker └── Dockerfile ├── lib ├── .eslintrc ├── calculateSize.js ├── config.js ├── confirmLoading.js ├── eslint-config-drupal.json ├── loadFiles.js ├── loadIndex.js ├── loadReleasesInfo.js ├── logger.js └── start.js └── package.json /.gitignore: -------------------------------------------------------------------------------- 1 | /.idea 2 | /node_modules 3 | /drupal-codebase 4 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Drupal project loader 2 | 3 | Downloads the entire codebase for all Drupal projects. 4 | 5 | ## Installation 6 | ```bash 7 | npm install -g drupal-project-loader 8 | ``` 9 | 10 | ## Usage 11 | 12 | Options: 13 | 14 | Long | Short | Default value |Description 15 | :----:|:-----:|:-------:|----------- 16 | branch| b | 8.x | Drupal API branch (6.x, 7.x, 8.x, etc). 17 | type | t | module | Filter project by type (module, theme, distribution, etc). Pass "any" to load all project types. 18 | destination | d | ./drupal-codebase/[branch] | Path to which the projects will be copied. 19 | extract | e | true | Whether or not the files should be extracted (uncompressed). 20 | concurrency | c | 15 | Number of multiple requests to perform at a time. 21 | yes | y | false | Assume "yes" as answer to all prompts. 22 | log-level | l | warn | Loging level. 23 | version | v | false | Output the version number. 24 | timeout | m | 15000 | The number of milliseconds to wait for a server to send response headers. 25 | 26 | Example: 27 | ```bash 28 | drupal-project-loader --branch=7.x --type=theme --destination=/tmp/d7-themes --extract=0 --yes 29 | ``` 30 | ## Running in docker 31 | ```bash 32 | docker run --rm -v $(pwd):/data -it attr/drupal-project-loader 33 | ``` 34 | 35 | ## License 36 | GNU General Public License, version 2. 37 | -------------------------------------------------------------------------------- /bin/start: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | 3 | require('../lib/start.js'); 4 | -------------------------------------------------------------------------------- /docker/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM node:4 2 | 3 | RUN npm install -g drupal-project-loader 4 | 5 | RUN mkdir /data 6 | 7 | WORKDIR /data 8 | 9 | ENTRYPOINT ["drupal-project-loader"] 10 | -------------------------------------------------------------------------------- /lib/.eslintrc: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "./eslint-config-drupal.json", 3 | "env": { 4 | "node": true 5 | }, 6 | "rules": { 7 | "no-console": 0, 8 | "eqeqeq": 0, 9 | "no-process-exit": 0, 10 | "one-var": 0, 11 | "max-nested-callbacks": [1, 5] 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /lib/calculateSize.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | var fileSize = require('filesize'); 4 | var config = require('./config'); 5 | 6 | module.exports = function (releases) { 7 | var size = 0; 8 | releases.forEach(function (release) { 9 | size += parseInt(release.size); 10 | }); 11 | 12 | if (config.extract) { 13 | // Take into account an approximate compress rate. 14 | size = size * 3.8; 15 | } 16 | return fileSize(size); 17 | }; 18 | -------------------------------------------------------------------------------- /lib/config.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | require('colors'); 4 | var argv = require('minimist')(process.argv.slice(2)); 5 | 6 | var config = {}; 7 | 8 | // Storage service URL. 9 | config.storageUrl = 'https://updates.drupal.org'; 10 | 11 | // Branch name. 12 | config.branch = argv.branch || argv.b || '8.x'; 13 | if (!isNaN(config.branch)) { 14 | config.branch += '.x'; 15 | } 16 | if (!config.branch.match(/^(\d\.){1,2}x$/)) { 17 | console.log('%s is not valid branch name.'.red, config.branch.bold); 18 | process.exit(1); 19 | } 20 | 21 | // Project type. 22 | config.type = argv.type || argv.t || 'module'; 23 | var allowedProjectTypes = [ 24 | 'any', 25 | 'module', 26 | 'theme', 27 | 'distribution', 28 | 'core', 29 | 'drupalorg', 30 | 'theme_engine', 31 | 'translation' 32 | ]; 33 | if (config.type && allowedProjectTypes.indexOf(config.type) == -1) { 34 | console.log('%s is not valid project type.'.red, config.type.bold); 35 | process.exit(1); 36 | } 37 | 38 | // Destination. 39 | config.destination = argv.destination || argv.d || './drupal-codebase/' + config.branch; 40 | 41 | // Extract. 42 | config.extract = typeof argv.extract == 'undefined' && typeof argv.e == 'undefined' ? 43 | true : Boolean(argv.extract || argv.e); 44 | 45 | // Concurrency. 46 | config.concurrency = argv.concurrency || argv.c || 15; 47 | if (!config.concurrency.toString().match(/\d{1,3}/)) { 48 | console.log('%s is not valid concurrency value.'.red, config.concurrency.toString().bold); 49 | process.exit(1); 50 | } 51 | 52 | // Yes. 53 | config.yes = argv.yes || argv.y || false; 54 | 55 | // Log level. 56 | config.logLevel = argv['log-level'] || argv.l || 'warn'; 57 | 58 | // Timeout. 59 | config.timeout = argv.timeout || argv.m || 15000; 60 | if (!config.timeout.toString().match(/^[1-9][0-9]*$/)) { 61 | console.log('%s is not valid timout value.'.red, config.timeout.toString().bold); 62 | process.exit(1); 63 | } 64 | 65 | module.exports = config; 66 | -------------------------------------------------------------------------------- /lib/confirmLoading.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | var Table = require('cli-table'); 4 | var ReadLine = require('readline'); 5 | var util = require('util'); 6 | var calculateSize = require('./calculateSize'); 7 | var config = require('./config'); 8 | 9 | module.exports = function (releases, callback) { 10 | 11 | var table = new Table({ 12 | colAligns: ['left', 'middle'], 13 | style: { 14 | 'head': ['cyan'], 15 | 'padding-left': 0, 16 | 'padding-right': 0 17 | } 18 | }); 19 | 20 | table.push( 21 | {'API branch:': config.branch}, 22 | {'Project type:': config.type}, 23 | {'Destination directory:': config.destination}, 24 | {'Total projects:': releases.length}, 25 | {'Required disc space:': calculateSize(releases)} 26 | ); 27 | 28 | console.log(); 29 | console.log(table.toString()); 30 | 31 | var question = util.format('%s [%s]: ', 'Start loading?'.green, 'Y/n'.yellow); 32 | 33 | if (config.yes) { 34 | console.log(question + 'Y'); 35 | callback(null, releases); 36 | } 37 | else { 38 | var readLine = ReadLine.createInterface({ 39 | input: process.stdin, 40 | output: process.stdout 41 | }); 42 | readLine.question(util.format('%s [%s]: ', 'Start loading?'.green, 'Y/n'.yellow), function (answer) { 43 | if (answer == '' || answer.toLowerCase()[0] == 'y') { 44 | callback(null, releases); 45 | } 46 | readLine.close(); 47 | }); 48 | } 49 | 50 | }; 51 | -------------------------------------------------------------------------------- /lib/eslint-config-drupal.json: -------------------------------------------------------------------------------- 1 | { 2 | "extends": "eslint:recommended", 3 | "env": { 4 | "browser": true 5 | }, 6 | "globals": { 7 | "Drupal": true, 8 | "drupalSettings": true, 9 | "drupalTranslations": true, 10 | "domready": true, 11 | "jQuery": true, 12 | "_": true, 13 | "matchMedia": true, 14 | "Backbone": true, 15 | "Modernizr": true, 16 | "CKEDITOR": true 17 | }, 18 | "rules": { 19 | // Errors. 20 | "array-bracket-spacing": [2, "never"], 21 | "block-scoped-var": 2, 22 | "brace-style": [2, "stroustrup", {"allowSingleLine": true}], 23 | "comma-dangle": [2, "never"], 24 | "comma-spacing": 2, 25 | "comma-style": [2, "last"], 26 | "computed-property-spacing": [2, "never"], 27 | "curly": [2, "all"], 28 | "eol-last": 2, 29 | "eqeqeq": [2, "smart"], 30 | "guard-for-in": 2, 31 | "indent": [2, 2, {"SwitchCase": 1}], 32 | "key-spacing": [2, {"beforeColon": false, "afterColon": true}], 33 | "keyword-spacing": [2, {"before": true, "after": true}], 34 | "linebreak-style": [2, "unix"], 35 | "lines-around-comment": [2, {"beforeBlockComment": true, "afterBlockComment": false}], 36 | "new-parens": 2, 37 | "no-array-constructor": 2, 38 | "no-caller": 2, 39 | "no-catch-shadow": 2, 40 | "no-eval": 2, 41 | "no-extend-native": 2, 42 | "no-extra-bind": 2, 43 | "no-extra-parens": [2, "functions"], 44 | "no-implied-eval": 2, 45 | "no-iterator": 2, 46 | "no-label-var": 2, 47 | "no-labels": 2, 48 | "no-lone-blocks": 2, 49 | "no-loop-func": 2, 50 | "no-multi-spaces": 2, 51 | "no-multi-str": 2, 52 | "no-native-reassign": 2, 53 | "no-nested-ternary": 2, 54 | "no-new-func": 2, 55 | "no-new-object": 2, 56 | "no-new-wrappers": 2, 57 | "no-octal-escape": 2, 58 | "no-process-exit": 2, 59 | "no-proto": 2, 60 | "no-return-assign": 2, 61 | "no-script-url": 2, 62 | "no-sequences": 2, 63 | "no-shadow-restricted-names": 2, 64 | "no-spaced-func": 2, 65 | "no-trailing-spaces": 2, 66 | "no-undef-init": 2, 67 | "no-undefined": 2, 68 | "no-unused-expressions": 2, 69 | "no-unused-vars": [2, {"vars": "all", "args": "none"}], 70 | "no-with": 2, 71 | "object-curly-spacing": [2, "never"], 72 | "one-var": [2, "never"], 73 | "quote-props": [2, "consistent-as-needed"], 74 | "quotes": [2, "single", "avoid-escape"], 75 | "semi": [2, "always"], 76 | "semi-spacing": [2, {"before": false, "after": true}], 77 | "space-before-blocks": [2, "always"], 78 | "space-before-function-paren": [2, {"anonymous": "always", "named": "never"}], 79 | "space-in-parens": [2, "never"], 80 | "space-infix-ops": 2, 81 | "space-unary-ops": [2, { "words": true, "nonwords": false }], 82 | "spaced-comment": [2, "always"], 83 | "strict": 2, 84 | "yoda": [2, "never"], 85 | // Warnings. 86 | "max-nested-callbacks": [1, 3], 87 | "valid-jsdoc": [1, { 88 | "prefer": { 89 | "returns": "return", 90 | "property": "prop" 91 | }, 92 | "requireReturn": false 93 | }] 94 | } 95 | } 96 | -------------------------------------------------------------------------------- /lib/loadFiles.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | require('colors'); 4 | var config = require('./config'); 5 | var request = require('request'); 6 | var zlib = require('zlib'); 7 | var tar = require('tar'); 8 | var logger = require('./logger'); 9 | var async = require('async'); 10 | var ProgressBar = require('progress'); 11 | var mkdirp = require('mkdirp'); 12 | var fs = require('fs'); 13 | var util = require('util'); 14 | 15 | module.exports = function (releases, callback) { 16 | 17 | // Prepare directory. 18 | try { 19 | mkdirp.sync(config.destination); 20 | } 21 | catch (error) { 22 | logger.error(error.message); 23 | process.exit(1); 24 | } 25 | 26 | console.log('Loading files...'.yellow); 27 | 28 | var barLabel = util.format(':current%s:total %s:percent%s [:bar] :elapsed sec.', '/'.gray, '('.gray, ')'.gray); 29 | var bar = new ProgressBar(barLabel, { 30 | complete: '■'.blue, 31 | incomplete: '-'.gray, 32 | width: 50, 33 | total: releases.length 34 | }); 35 | 36 | // This will prevent freezing timer when download is slow. 37 | var interval = setInterval(function () { 38 | bar.tick(0); 39 | }, 100); 40 | 41 | var queue = async.queue(function (release, callback) { 42 | var downloadError; 43 | var result = request(release.downloadLink, {timeout: config.timeout}, function (error, response) { 44 | if (error || response.statusCode != 200) { 45 | downloadError = true; 46 | logger.error('Could not download file. %s', release.downloadLink.cyan); 47 | } 48 | bar.tick(); 49 | callback(); 50 | }); 51 | 52 | if (config.extract) { 53 | var gunzip = zlib.createGunzip(); 54 | gunzip.on('error', function () { 55 | if (!downloadError) { 56 | logger.error('Could not unzip file. %s', release.downloadLink.cyan); 57 | } 58 | }); 59 | 60 | var extractor = tar.Extract({path: config.destination}); 61 | extractor.on('error', function () { 62 | logger.error('Could not untar file. %s', release.downloadLink.cyan); 63 | }); 64 | 65 | result 66 | .pipe(gunzip) 67 | .pipe(extractor); 68 | 69 | } 70 | else { 71 | var fileName = release.downloadLink.split('/').slice(-1)[0]; 72 | var fileStream = fs.createWriteStream(config.destination + '/' + fileName); 73 | fileStream.on('error', function (error) { 74 | logger.warn(error); 75 | }); 76 | 77 | result.pipe(fileStream); 78 | } 79 | 80 | }, config.concurrency); 81 | 82 | releases.forEach(function (release) { 83 | queue.push(release); 84 | }); 85 | 86 | queue.drain = function () { 87 | clearInterval(interval); 88 | callback(); 89 | }; 90 | 91 | }; 92 | 93 | -------------------------------------------------------------------------------- /lib/loadIndex.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | require('colors'); 4 | var request = require('request'); 5 | var parseString = require('xml2js').parseString; 6 | var config = require('./config'); 7 | var url = config.storageUrl + '/release-history/project-list/all'; 8 | 9 | module.exports = function (callback) { 10 | 11 | console.log('Loading project index...'.yellow); 12 | 13 | request(url, function (error, response, body) { 14 | if (error) { 15 | return callback(error); 16 | } 17 | 18 | console.log('Processing project index...'.yellow); 19 | if (response.statusCode == 200) { 20 | 21 | var projects = []; 22 | parseString(body, function (error, result) { 23 | if (error) { 24 | return callback('Could not parse release-history file.'); 25 | } 26 | 27 | result.projects.project.forEach(function (project) { 28 | 29 | var typeIsValid = config.type == 'any' || 'project_' + config.type == project.type[0]; 30 | var versionIsValid = project.api_versions && project.api_versions[0].api_version.indexOf(config.branch) != -1; 31 | 32 | if (typeIsValid && versionIsValid) { 33 | projects.push(project.short_name[0]); 34 | } 35 | 36 | }); 37 | 38 | if (projects.length > 0) { 39 | callback(null, projects); 40 | } 41 | else { 42 | callback('No projects were found'); 43 | } 44 | }); 45 | 46 | } 47 | else { 48 | callback('Could not download project index.'); 49 | } 50 | }); 51 | 52 | }; 53 | -------------------------------------------------------------------------------- /lib/loadReleasesInfo.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | var request = require('request'); 4 | var parseString = require('xml2js').parseString; 5 | var async = require('async'); 6 | var logger = require('./logger'); 7 | var ProgressBar = require('progress'); 8 | var config = require('./config'); 9 | var util = require('util'); 10 | 11 | module.exports = function (projects, callback) { 12 | 13 | console.log('Loading releases history...'.yellow); 14 | 15 | var barLabel = util.format(':current%s:total %s:percent%s [:bar] :elapsed sec.', '/'.gray, '('.gray, ')'.gray); 16 | var bar = new ProgressBar(barLabel, { 17 | complete: '▶'.blue, 18 | incomplete: '-'.gray, 19 | width: 50, 20 | total: projects.length 21 | }); 22 | 23 | var info = []; 24 | 25 | var queue = async.queue(function (project, callback) { 26 | var url = config.storageUrl + '/release-history/' + project + '/' + config.branch; 27 | 28 | logger.debug(url.cyan); 29 | 30 | request(url, {timeout: config.timeout}, function (error, response, body) { 31 | if (error || response.statusCode != 200) { 32 | logger.error('Release history is not available. %s', url.cyan); 33 | callback(); 34 | return; 35 | } 36 | 37 | parseString(body, function (error, result) { 38 | 39 | if (error || typeof result.project == 'undefined') { 40 | logger.error('Release history is not available. %s', url.cyan); 41 | callback(); 42 | return; 43 | } 44 | 45 | var releaseInfo = {}; 46 | 47 | var majorVersion = 0, date = 0; 48 | result.project.releases[0].release.forEach(function (release) { 49 | 50 | // Some releases have no date. 51 | var releaseDate = release.date ? release.date[0] : 0; 52 | 53 | var isLatest = releaseDate >= date; 54 | var isCurrentMajor = release.version_major && release.version_major[0] >= majorVersion; 55 | if (isLatest && isCurrentMajor && release.download_link) { 56 | releaseInfo.downloadLink = release.download_link[0]; 57 | releaseInfo.size = release.filesize[0]; 58 | majorVersion = release.version_major[0]; 59 | date = releaseDate; 60 | } 61 | }); 62 | 63 | if (!releaseInfo.downloadLink) { 64 | logger.warn('Project %s has no downloadable releases. %s', project.bold, url.cyan); 65 | } 66 | else if (!releaseInfo.size) { 67 | logger.warn('Project %s has no release size. %s', project.bold, url.cyan); 68 | } 69 | else { 70 | info.push(releaseInfo); 71 | } 72 | 73 | }); 74 | 75 | bar.tick(); 76 | callback(); 77 | }); 78 | 79 | }, config.concurrency); 80 | 81 | projects.forEach(function (project) { 82 | queue.push(project); 83 | }); 84 | 85 | queue.drain = function () { 86 | if (info.length > 0) { 87 | callback(null, info); 88 | } 89 | else { 90 | callback('No releases were found.'); 91 | } 92 | }; 93 | 94 | }; 95 | -------------------------------------------------------------------------------- /lib/logger.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | var config = require('./config'); 4 | var winston = require('winston'); 5 | winston.emitErrs = true; 6 | 7 | var logger = new winston.Logger({ 8 | transports: [ 9 | new winston.transports.Memory({ 10 | level: config.logLevel, 11 | json: false, 12 | colorize: true 13 | }) 14 | ], 15 | exitOnError: false 16 | }); 17 | 18 | module.exports = logger; 19 | -------------------------------------------------------------------------------- /lib/start.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | require('colors'); 4 | var async = require('async'); 5 | var loadIndex = require('./loadIndex'); 6 | var logger = require('./logger'); 7 | var loadReleasesInfo = require('./loadReleasesInfo'); 8 | var confirmLoading = require('./confirmLoading'); 9 | var loadFiles = require('./loadFiles'); 10 | var argv = require('minimist')(process.argv.slice(2)); 11 | 12 | var stack = [ 13 | function (callback) { 14 | if (argv.v || argv.version) { 15 | console.log('v' + require('../package.json').version); 16 | } 17 | else { 18 | callback(); 19 | } 20 | }, 21 | loadIndex, 22 | loadReleasesInfo, 23 | confirmLoading, 24 | loadFiles 25 | ]; 26 | 27 | async.waterfall(stack, function (error) { 28 | if (error) { 29 | logger.error(error.message || error); 30 | } 31 | else { 32 | console.log('Done!'.green); 33 | } 34 | }); 35 | 36 | process.on('exit', function () { 37 | // Dump logs. 38 | var memoryTransport = logger.transports.memory; 39 | var messages = memoryTransport.errorOutput.concat(memoryTransport.writeOutput); 40 | if (messages.length > 0) { 41 | console.log('-- Log messages --'.gray); 42 | messages.forEach(function (message) { 43 | console.log(message); 44 | }); 45 | } 46 | }); 47 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "drupal-project-loader", 3 | "version": "1.1.1", 4 | "description": "Drupal project loader.", 5 | "repository": { 6 | "type": "git", 7 | "url": "https://github.com/Chi-teck/drupal-project-loader" 8 | }, 9 | "bin": "bin/start", 10 | "keywords": [ 11 | "drupal" 12 | ], 13 | "author": "Chi-teck", 14 | "dependencies": { 15 | "async": "^2.0.0-rc.4", 16 | "cli-table": "^0.3.1", 17 | "colors": "^1.1.2", 18 | "filesize": "^3.3.0", 19 | "minimist": "^1.2.0", 20 | "mkdirp": "^0.5.1", 21 | "progress": "^1.1.8", 22 | "request": "^2.72.0", 23 | "tar": "^2.2.1", 24 | "winston": "^2.2.0", 25 | "xml2js": "^0.4.16" 26 | }, 27 | "engines": { 28 | "node": ">= 0.10.0" 29 | }, 30 | "license": "GPL-2.0" 31 | } 32 | --------------------------------------------------------------------------------