├── .gitignore ├── .editorconfig ├── docs ├── getting-started.md ├── README.tmpl.md ├── examples.md └── options.md ├── LICENSE-MIT ├── package.json ├── Gruntfile.js ├── test └── fixtures │ └── issues.json ├── README.md └── tasks ├── github_api.js └── lib └── utils.js /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | npm-debug.log 3 | tmp 4 | TODO.md 5 | api-data 6 | .directory 7 | *.sublime-* 8 | -------------------------------------------------------------------------------- /.editorconfig: -------------------------------------------------------------------------------- 1 | # This file is for unifying the coding style for different editors and IDEs 2 | # editorconfig.org 3 | 4 | root = true 5 | 6 | [*] 7 | end_of_line = lf 8 | charset = UTF-8 9 | insert_final_newline = true 10 | trim_trailing_whitespace = true 11 | indent_style = space 12 | indent_size = 4 13 | -------------------------------------------------------------------------------- /docs/getting-started.md: -------------------------------------------------------------------------------- 1 | 2 | This plugin requires Grunt ~0.4.0 3 | 4 | If you haven't used Grunt before, be sure to check out the [Getting Started](http://gruntjs.com/getting-started) guide, as it explains how to create a Gruntfile as well as install and use Grunt plugins. Once you're familiar with that process, you may install this plugin with this command: 5 | 6 | npm install grunt-github-api --save-dev 7 | 8 | Once the plugin has been installed, it may be enabled inside your Gruntfile with this line of JavaScript: 9 | 10 | grunt.loadNpmTasks('grunt-github-api'); 11 | 12 | Run this task with the `grunt github` command. 13 | 14 | Task targets, files and options may be specified according to the grunt [Configuring tasks](http://gruntjs.com/configuring-tasks) guide. -------------------------------------------------------------------------------- /docs/README.tmpl.md: -------------------------------------------------------------------------------- 1 | # {%= name %} [![NPM version](https://badge.fury.io/js/{%= name %}.png)](http://badge.fury.io/js/{%= name %}) {% if (travis) { %} [![Build Status]({%= travis %}.png)]({%= travis %}){% } %} 2 | 3 | > {%= description %} 4 | 5 | Project authored and maintained by [github/{%= author.url %}]({%= author.url %}). 6 | 7 | ## Getting Started 8 | {%= _.doc("getting-started.md") %} 9 | 10 | ## Options 11 | {%= _.doc("options.md") %} 12 | 13 | ## Usage Examples 14 | {%= _.doc("examples.md") %} 15 | 16 | ## Contributing 17 | Please see the [Contributing to Assemble](http://assemble.io/contributing) guide for information on contributing to this project. 18 | 19 | ## Author 20 | 21 | + [github/{%= author.url %}]({%= author.url %}) 22 | 23 | {% if (changelog) { %} 24 | ## Release History 25 | {%= _.include("docs-changelog.md") %} {% } else { %}{% } %} 26 | 27 | ## License 28 | {%= _.copyright() %} 29 | {%= _.license() %} 30 | 31 | *** 32 | 33 | _This file was generated on {%= grunt.template.today() %}._ 34 | -------------------------------------------------------------------------------- /LICENSE-MIT: -------------------------------------------------------------------------------- 1 | Copyright (c) 2013 Jeffrey Herb 2 | 3 | Permission is hereby granted, free of charge, to any person 4 | obtaining a copy of this software and associated documentation 5 | files (the "Software"), to deal in the Software without 6 | restriction, including without limitation the rights to use, 7 | copy, modify, merge, publish, distribute, sublicense, and/or sell 8 | copies of the Software, and to permit persons to whom the 9 | Software is furnished to do so, subject to the following 10 | conditions: 11 | 12 | The above copyright notice and this permission notice shall be 13 | included in all copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 16 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES 17 | OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 18 | NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT 19 | HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, 20 | WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 21 | FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 22 | OTHER DEALINGS IN THE SOFTWARE. 23 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "grunt-github-api", 3 | "description": "Query Github's API and save the returned JSON files locally.", 4 | "version": "0.2.3", 5 | "homepage": "https://github.com/assemble/grunt-github-api", 6 | "author": { 7 | "name": "Jeffrey Herb", 8 | "email": "herb.jeff@gmail.com", 9 | "url": "https://github.com/jeffHerb" 10 | }, 11 | "repository": { 12 | "type": "git", 13 | "url": "git://github.com:assemble/grunt-github-api.git" 14 | }, 15 | "bugs": { 16 | "url": "https://github.com/assemble/grunt-github-api/issues" 17 | }, 18 | "licenses": [ 19 | { 20 | "type": "MIT", 21 | "url": "https://github.com/assemble/grunt-github-api/blob/master/LICENSE-MIT" 22 | } 23 | ], 24 | "main": "Gruntfile.js", 25 | "engines": { 26 | "node": ">= 0.8.0" 27 | }, 28 | "dependencies": { 29 | "crypto": "0.0.3", 30 | "mkdirp": "~0.3.5" 31 | }, 32 | "devDependencies": { 33 | "grunt": "~0.4.1", 34 | "grunt-readme": "~0.1.4", 35 | "grunt-contrib-jshint": "~0.6.4" 36 | }, 37 | "keywords": [ 38 | "gruntplugin", 39 | "assemble", 40 | "api", 41 | "json", 42 | "github", 43 | "github api", 44 | "git" 45 | ] 46 | } 47 | -------------------------------------------------------------------------------- /Gruntfile.js: -------------------------------------------------------------------------------- 1 | /*! 2 | * grunt-github-api 3 | * https://github.com/assemble/grunt-github-api 4 | * Authored by Jeffrey Herb 5 | * 6 | * Copyright (c) 2013 Jeffrey Herb, contributors 7 | * Licensed under the MIT license. 8 | */ 9 | 10 | module.exports = function(grunt) { 11 | 12 | // Project configuration. 13 | grunt.initConfig({ 14 | pkg: grunt.file.readJSON('package.json'), 15 | 16 | jshint: { 17 | options: { 18 | curly: true, 19 | eqeqeq: true, 20 | immed: true, 21 | latedef: true, 22 | newcap: true, 23 | noarg: true, 24 | sub: true, 25 | undef: true, 26 | boss: true, 27 | eqnull: true, 28 | node: true 29 | }, 30 | all: ['Gruntfile.js', 'tasks/**/*.js'] 31 | }, 32 | 33 | github: { 34 | exampleIssues: { 35 | options: { 36 | filters: { 37 | state: 'open' 38 | }, 39 | output: { 40 | format: { 41 | indent: 2 42 | } 43 | }, 44 | concat: true 45 | }, 46 | src: ['/repos/assemble/grunt-github-api-example/issues', '/repos/assemble/grunt-github-api/issues'], 47 | dest: 'combinded-issues.json' 48 | }, 49 | seperateIssues: { 50 | src: ['/repos/assemble/grunt-github-api-example/issues', '/repos/assemble/grunt-github-api/issues'], 51 | }, 52 | examplePkg: { 53 | options: { 54 | task: { 55 | type: 'file', 56 | } 57 | }, 58 | src: '/repos/assemble/grunt-github-api-example/contents/example.json', 59 | } 60 | } 61 | 62 | }); 63 | 64 | // Load tasks powered by npm plugins. 65 | grunt.loadNpmTasks('grunt-contrib-jshint'); 66 | grunt.loadNpmTasks('grunt-readme'); 67 | 68 | // Load this plugin's tasks. 69 | grunt.loadTasks('tasks'); 70 | 71 | // Default task(s). 72 | grunt.registerTask('default', ['jshint', 'github', 'readme']); 73 | 74 | }; 75 | -------------------------------------------------------------------------------- /docs/examples.md: -------------------------------------------------------------------------------- 1 | ## Targets 2 | 3 | * `src`: is the specified API query path. Source paths are everything that appears after `api.github.com`. Additional information about different query paths can be found in the [Github Developer Documentation](http://developer.github.com/). The `src` can be an `array` or a `string` value. 4 | * `dest`: (optional) - is the path and filename where the retured request should be saved. If nothing is given files will be saved into the same path as the task cache file and will be broken down into a folder structure that mimics its query path. (EXCEPTION: If you define multiple sources that are being concatenated together, you must define at least a filename). 5 | 6 | 7 | ## Configuration 8 | 9 | Options may be defined at either the task and/or target levels (_target-level options override task-level options_). 10 | 11 | ```js 12 | github: { 13 | // Concatentate returned JSON responses into a single file. 14 | combindedIssues: { 15 | options: { 16 | filters: { 17 | state: 'open' 18 | }, 19 | task: { 20 | concat: true 21 | } 22 | }, 23 | src: [ 24 | '/repos/assemble/grunt-github-api-example/issues', 25 | '/repos/assemble/grunt-github-api/issues' 26 | ], 27 | dest: 'combinded-issues.json' 28 | // File created will be save along the gruntfile. 29 | }, 30 | 31 | // Create two different files from two different repos. 32 | seperateIssues: { 33 | options: { 34 | // Access repo using credentials provided 35 | oAuth: { 36 | access_token: 'XXXXXXXXXXXXXXXXXX' 37 | } 38 | }, 39 | src: [ 40 | '/repos/assemble/grunt-github-api-example/issues', 41 | '/repos/assemble/grunt-github-api/issues' 42 | ] 43 | // Files created will be saved inside the api-data folder 44 | }, 45 | 46 | // Downloads a copy of the example.json file from GitHub. 47 | examplePkg: { 48 | options: { 49 | task: { 50 | type: 'file', 51 | } 52 | }, 53 | src: '/repos/assemble/grunt-github-api-example/contents/example.json' 54 | // File created will be saved inside the api-data folder 55 | } 56 | } 57 | ``` 58 | -------------------------------------------------------------------------------- /test/fixtures/issues.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "url": "https://api.github.com/repos/JeffHerb/buildsys/issues/1", 4 | "labels_url": "https://api.github.com/repos/JeffHerb/buildsys/issues/1/labels{/name}", 5 | "comments_url": "https://api.github.com/repos/JeffHerb/buildsys/issues/1/comments", 6 | "events_url": "https://api.github.com/repos/JeffHerb/buildsys/issues/1/events", 7 | "html_url": "https://github.com/JeffHerb/buildsys/issues/1", 8 | "id": 18075864, 9 | "number": 1, 10 | "title": "fake issue for project testing", 11 | "user": { 12 | "login": "JeffHerb", 13 | "id": 1133732, 14 | "avatar_url": "https://0.gravatar.com/avatar/c57c435c2b599aa93a9f27c786ca6f73?d=https%3A%2F%2Fidenticons.github.com%2F8da3e6ad52f643fa6f8fe376de6a9e00.png", 15 | "gravatar_id": "c57c435c2b599aa93a9f27c786ca6f73", 16 | "url": "https://api.github.com/users/JeffHerb", 17 | "html_url": "https://github.com/JeffHerb", 18 | "followers_url": "https://api.github.com/users/JeffHerb/followers", 19 | "following_url": "https://api.github.com/users/JeffHerb/following{/other_user}", 20 | "gists_url": "https://api.github.com/users/JeffHerb/gists{/gist_id}", 21 | "starred_url": "https://api.github.com/users/JeffHerb/starred{/owner}{/repo}", 22 | "subscriptions_url": "https://api.github.com/users/JeffHerb/subscriptions", 23 | "organizations_url": "https://api.github.com/users/JeffHerb/orgs", 24 | "repos_url": "https://api.github.com/users/JeffHerb/repos", 25 | "events_url": "https://api.github.com/users/JeffHerb/events{/privacy}", 26 | "received_events_url": "https://api.github.com/users/JeffHerb/received_events", 27 | "type": "User" 28 | }, 29 | "labels": [ 30 | { 31 | "url": "https://api.github.com/repos/JeffHerb/buildsys/labels/duplicate", 32 | "name": "duplicate", 33 | "color": "cccccc" 34 | }, 35 | { 36 | "url": "https://api.github.com/repos/JeffHerb/buildsys/labels/enhancement", 37 | "name": "enhancement", 38 | "color": "84b6eb" 39 | } 40 | ], 41 | "state": "open", 42 | "assignee": null, 43 | "milestone": null, 44 | "comments": 0, 45 | "created_at": "2013-08-14T20:46:26Z", 46 | "updated_at": "2013-08-14T20:46:26Z", 47 | "closed_at": null, 48 | "pull_request": { 49 | "html_url": null, 50 | "diff_url": null, 51 | "patch_url": null 52 | }, 53 | "body": "testing" 54 | } 55 | ] -------------------------------------------------------------------------------- /docs/options.md: -------------------------------------------------------------------------------- 1 | Options for this plugin are broken down into sub-option categories as defined below. Please note that all option sections by default are objects and contain simple key-value pairs unless otherwise noted. 2 | 3 | 4 | ## output 5 | > Used to idenify where cache and downloaded request data should be stored by default 6 | 7 | * **path**: _default: 'api-data'_ - special location where all data is collected when saved to disk, when no specific destination is definded under the targets `dest`. 8 | * **cache**: _default: '.cache.json'_ - plugin cache file name. Will be stored under under the output path defined above. 9 | 10 | ### format 11 | * **indent**: _default: 4_ - number of spaces each indent should take up. 12 | * **encoding**: _default: 'utf8'_ - file format all data written to disk should be in. 13 | 14 | Examples: 15 | 16 | ```js 17 | { 18 | options: { 19 | output: { 20 | path: 'my/api/data/', 21 | cache: 'my/api/cache/' 22 | format: { 23 | indent: 4, 24 | encoding: 'utf8' 25 | } 26 | } 27 | } 28 | } 29 | ``` 30 | 31 | 32 | ## connection 33 | > Connection headers used when connecting the GitHub. 34 | 35 | * **host**: _default: api.github.com_ - default GitHub API portal 36 | * **headers**: _default: {Object}_ used to define the nodejs `HTTPS` headers. 37 | - `User-Agent`: `node-http/0.10.1` 38 | - `Content-Type`: `application/json` 39 | 40 | Examples: 41 | 42 | ```js 43 | { 44 | options: { 45 | connection: { 46 | host: 'api.github.com', 47 | headers: { 48 | 'User-Agent': 'node-http/0.10.1', 49 | 'Content-Type': 'application/json' 50 | } 51 | } 52 | } 53 | } 54 | ``` 55 | 56 | ## type 57 | > Type of data the task will be downloading 58 | 59 | Indicates the type of request, may be set to either `data` or `file`. Default is `data`. 60 | 61 | ## cache 62 | > Control which files do or do not get tracked for changes. 63 | 64 | Specifies whether or not to cache API responses. Default is `true`. 65 | 66 | ## concat 67 | > Concatinate JSON data together before writting it to a file when property is set to `true`. Default is `false`. 68 | 69 | ## filters 70 | > Query search parameters 71 | 72 | Additional information about different filters can be found in the [Github Developer Documentation](http://developer.github.com/). 73 | 74 | 75 | ## oAuth 76 | > GitHub access credentials 77 | 78 | These credentials are required to preform many actions or continual usage of the plugin. In order to get access using oAuth the repo owner will need to create an access token via their [Application Settings](https://github.com/settings/applications) page. 79 | 80 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # grunt-github-api [![NPM version](https://badge.fury.io/js/grunt-github-api.png)](http://badge.fury.io/js/grunt-github-api) 2 | 3 | > Query Github's API and save the returned JSON files locally. 4 | 5 | Project authored and maintained by [github/https://github.com/jeffHerb](https://github.com/jeffHerb). 6 | 7 | ## Getting Started 8 | 9 | This plugin requires Grunt ~0.4.0 10 | 11 | If you haven't used Grunt before, be sure to check out the [Getting Started](http://gruntjs.com/getting-started) guide, as it explains how to create a Gruntfile as well as install and use Grunt plugins. Once you're familiar with that process, you may install this plugin with this command: 12 | 13 | npm install grunt-github-api --save-dev 14 | 15 | Once the plugin has been installed, it may be enabled inside your Gruntfile with this line of JavaScript: 16 | 17 | grunt.loadNpmTasks('grunt-github-api'); 18 | 19 | Run this task with the `grunt github` command. 20 | 21 | Task targets, files and options may be specified according to the grunt [Configuring tasks](http://gruntjs.com/configuring-tasks) guide. 22 | 23 | 24 | ## Options 25 | Options for this plugin are broken down into sub-option categories as defined below. Please note that all option sections by default are objects and contain simple key-value pairs unless otherwise noted. 26 | 27 | 28 | ### output 29 | > Used to idenify where cache and downloaded request data should be stored by default 30 | 31 | * **path**: _default: 'api-data'_ - special location where all data is collected when saved to disk, when no specific destination is definded under the targets `dest`. 32 | * **cache**: _default: '.cache.json'_ - plugin cache file name. Will be stored under under the output path defined above. 33 | 34 | #### format 35 | * **indent**: _default: 4_ - number of spaces each indent should take up. 36 | * **encoding**: _default: 'utf8'_ - file format all data written to disk should be in. 37 | 38 | Examples: 39 | 40 | ```js 41 | { 42 | options: { 43 | output: { 44 | path: 'my/api/data/', 45 | cache: 'my/api/cache/' 46 | format: { 47 | indent: 4, 48 | encoding: 'utf8' 49 | } 50 | } 51 | } 52 | } 53 | ``` 54 | 55 | 56 | ### connection 57 | > Connection headers used when connecting the GitHub. 58 | 59 | * **host**: _default: api.github.com_ - default GitHub API portal 60 | * **headers**: _default: {Object}_ used to define the nodejs `HTTPS` headers. 61 | - `User-Agent`: `node-http/0.10.1` 62 | - `Content-Type`: `application/json` 63 | 64 | Examples: 65 | 66 | ```js 67 | { 68 | options: { 69 | connection: { 70 | host: 'api.github.com', 71 | headers: { 72 | 'User-Agent': 'node-http/0.10.1', 73 | 'Content-Type': 'application/json' 74 | } 75 | } 76 | } 77 | } 78 | ``` 79 | 80 | ### type 81 | > Type of data the task will be downloading 82 | 83 | Indicates the type of request, may be set to either `data` or `file`. Default is `data`. 84 | 85 | ### cache 86 | > Control which files do or do not get tracked for changes. 87 | 88 | Specifies whether or not to cache API responses. Default is `true`. 89 | 90 | ### concat 91 | > Concatinate JSON data together before writting it to a file when property is set to `true`. Default is `false`. 92 | 93 | ### filters 94 | > Query search parameters 95 | 96 | Additional information about different filters can be found in the [Github Developer Documentation](http://developer.github.com/). 97 | 98 | 99 | ### oAuth 100 | > GitHub access credentials 101 | 102 | These credentials are required to preform many actions or continual usage of the plugin. In order to get access using oAuth the repo owner will need to create an access token via their [Application Settings](https://github.com/settings/applications) page. 103 | 104 | 105 | 106 | 107 | ## Usage Examples 108 | ### Targets 109 | 110 | * `src`: is the specified API query path. Source paths are everything that appears after `api.github.com`. Additional information about different query paths can be found in the [Github Developer Documentation](http://developer.github.com/). The `src` can be an `array` or a `string` value. 111 | * `dest`: (optional) - is the path and filename where the retured request should be saved. If nothing is given files will be saved into the same path as the task cache file and will be broken down into a folder structure that mimics its query path. (EXCEPTION: If you define multiple sources that are being concatenated together, you must define at least a filename). 112 | 113 | 114 | ### Configuration 115 | 116 | Options may be defined at either the task and/or target levels (_target-level options override task-level options_). 117 | 118 | ```js 119 | github: { 120 | // Concatentate returned JSON responses into a single file. 121 | combindedIssues: { 122 | options: { 123 | filters: { 124 | state: 'open' 125 | }, 126 | task: { 127 | concat: true 128 | } 129 | }, 130 | src: [ 131 | '/repos/assemble/grunt-github-api-example/issues', 132 | '/repos/assemble/grunt-github-api/issues' 133 | ], 134 | dest: 'combinded-issues.json' 135 | // File created will be save along the gruntfile. 136 | }, 137 | 138 | // Create two different files from two different repos. 139 | seperateIssues: { 140 | options: { 141 | // Access repo using credentials provided 142 | oAuth: { 143 | access_token: 'XXXXXXXXXXXXXXXXXX' 144 | } 145 | }, 146 | src: [ 147 | '/repos/assemble/grunt-github-api-example/issues', 148 | '/repos/assemble/grunt-github-api/issues' 149 | ] 150 | // Files created will be saved inside the api-data folder 151 | }, 152 | 153 | // Downloads a copy of the example.json file from GitHub. 154 | examplePkg: { 155 | options: { 156 | task: { 157 | type: 'file', 158 | } 159 | }, 160 | src: '/repos/assemble/grunt-github-api-example/contents/example.json' 161 | // File created will be saved inside the api-data folder 162 | } 163 | } 164 | ``` 165 | 166 | 167 | 168 | ## Contributing 169 | Please see the [Contributing to Assemble](http://assemble.io/contributing) guide for information on contributing to this project. 170 | 171 | ## Author 172 | 173 | + [github/https://github.com/jeffHerb](https://github.com/jeffHerb) 174 | 175 | 176 | 177 | ## License 178 | Copyright (c) 2013 Jeffrey Herb, contributors. 179 | Released under the MIT license 180 | 181 | *** 182 | 183 | _This file was generated on Tue Oct 29 2013 21:13:51._ 184 | -------------------------------------------------------------------------------- /tasks/github_api.js: -------------------------------------------------------------------------------- 1 | /*! 2 | * grunt-github-api 3 | * https://github.com/assemble/grunt-github-api 4 | * Authored by Jeffrey Herb 5 | * 6 | * Copyright (c) 2013 Jeffrey Herb, contributors 7 | * Licensed under the MIT license. 8 | */ 9 | 10 | 'use strict'; 11 | 12 | module.exports = function(grunt) { 13 | 14 | var github_api = require('./lib/utils'); 15 | 16 | // Setup the new multi task 17 | grunt.registerMultiTask('github', 'Simple Script to query the github API.', function() { 18 | 19 | var done = this.async(); 20 | var kindOf = grunt.util.kindOf; 21 | 22 | var getAPILimits = function(github_api, next) { 23 | 24 | // Check the rate limits only if the task is using a non oAuth access_token 25 | if (!github_api.options.oAuth) { 26 | 27 | github_api.request.add(github_api.options.connection, '/rate_limit', false, false); 28 | 29 | github_api.request.send(function(responseArray) { 30 | 31 | var requests = github_api.data.src; 32 | var rateOptions = github_api.options.rateLimit; 33 | var res = responseArray.shift(); 34 | var data = res[1]; 35 | var taskRequests = 0; 36 | var RLRemainingLimit = data[0].rate.remaining; 37 | var RLReset = data[0].rate.reset; 38 | 39 | // Determine how many request will be used in the current request. 40 | if (kindOf(requests) === 'array') { 41 | taskRequests = requests.length; 42 | } else { 43 | taskRequests = 1; 44 | } 45 | 46 | // Clean up reset date 47 | RLReset = grunt.template.date((RLReset * 1000), 'h:MM:ss TT'); 48 | 49 | // Check to see if the API is close or about to run out 50 | if (RLRemainingLimit < taskRequests) { 51 | 52 | console.log('You do not have enough public API request remaining to complete this task. Skipping this task.'); 53 | 54 | next(github_api, true); 55 | 56 | } else { 57 | 58 | if (RLRemainingLimit === 0) { 59 | 60 | } else { 61 | 62 | if (RLRemainingLimit <= rateOptions.warning) { 63 | 64 | console.log('You are about to hit your public API request limit. To avoid this issue it is suggested that you add an oAuth access token if possible. Public API limit reset at: ' + RLRemainingLimit); 65 | 66 | next(github_api); 67 | } else { 68 | // Nothing to worry about... This time! 69 | next(github_api); 70 | } 71 | } 72 | } 73 | }); 74 | 75 | } else { 76 | 77 | next(github_api); 78 | } 79 | 80 | }; 81 | 82 | var generateRequest = function(github_api, next) { 83 | 84 | function processPaths(src, dest, options, cb) { 85 | 86 | // Clean up the src 87 | src = github_api.path.cleaner(src, true, false); 88 | 89 | // Clean up the dest 90 | if (dest) { 91 | dest = github_api.path.cleaner(dest, false, false); 92 | } else { 93 | dest = github_api.path.cleaner(src, false, false); 94 | 95 | // Add output path if it exists. 96 | if (options.output.path) { 97 | dest = options.output.path + '/' + dest; 98 | } 99 | } 100 | 101 | // Create src parameters 102 | if (options.filters || options.oAuth) { 103 | src += "?" + github_api.path.parameters(options.filters, options.oAuth); 104 | } 105 | 106 | github_api.request.add(options.connection, src, dest, options); 107 | 108 | if (cb) { 109 | cb(); 110 | } 111 | 112 | } 113 | 114 | var data = github_api.data; 115 | var options = github_api.options; 116 | 117 | // Only execute if source exists 118 | if (data.src) { 119 | 120 | if (kindOf(data.src) === 'array') { 121 | 122 | (function multiSrc(dataSrc) { 123 | 124 | var curSrc = dataSrc.shift(); 125 | 126 | processPaths(curSrc, data.dest || false, options, function() { 127 | 128 | if (dataSrc.length === 0) { 129 | 130 | next(github_api); 131 | } else { 132 | 133 | multiSrc(dataSrc); 134 | } 135 | 136 | }); 137 | 138 | })(data.src); 139 | 140 | } else { 141 | 142 | processPaths(data.src, data.dest || false, options, function() { 143 | 144 | next(github_api); 145 | 146 | }); 147 | 148 | } 149 | 150 | } else { 151 | 152 | next(github_api); 153 | } 154 | 155 | }; 156 | 157 | var processRequest = function(github_api, next) { 158 | 159 | github_api.request.send(function(responseArray) { 160 | 161 | (function nextResponse(responseArray) { 162 | 163 | function leaveLoop() { 164 | 165 | if (responseArray.length === 0) { 166 | 167 | next(github_api); 168 | } else { 169 | 170 | nextResponse(responseArray); 171 | } 172 | 173 | } 174 | 175 | function checkCache(data, dest, name, type, cb) { 176 | 177 | var destPath = dest.split('.')[0]; 178 | var uniqueId = ''; 179 | 180 | if (type === 'file') { 181 | 182 | uniqueId = data[0].sha; 183 | 184 | } else { 185 | 186 | var jStr = JSON.stringify(data); 187 | 188 | // Deal with the strange changing gravatar url that breaks cache. 189 | jStr = jStr.replace(/https:\/\/\d\.gravatar/g, 'https://gravatar'); 190 | 191 | uniqueId = github_api.cache.generateId(jStr); 192 | 193 | } 194 | 195 | var cache = github_api.cache.get(name, destPath); 196 | 197 | if (cache) { 198 | 199 | if (uniqueId === cache.uniqueId) { 200 | 201 | grunt.log.writeln(destPath + ' is already up-to-date. (No data has been written)'); 202 | 203 | cb(false); 204 | 205 | } else { 206 | 207 | github_api.cache.set(name, destPath, type, uniqueId); 208 | 209 | cb(true); 210 | 211 | } 212 | 213 | } else { 214 | 215 | github_api.cache.set(name, destPath, type, uniqueId); 216 | 217 | cb(true); 218 | } 219 | 220 | } 221 | 222 | var res = responseArray.shift(), 223 | dest = res[0], 224 | data = res[1], 225 | options = res[2], 226 | format = options.output.format; 227 | 228 | if (data.length === 1) { 229 | 230 | if (options.cache) { 231 | 232 | checkCache(data, dest, options.name, options.type, function(results) { 233 | 234 | if (results) { 235 | github_api.write.add(data, dest, options.type, format); 236 | } 237 | 238 | leaveLoop(); 239 | 240 | }); 241 | 242 | } else { 243 | 244 | github_api.write.add(data, dest, options.type, format); 245 | 246 | leaveLoop(); 247 | 248 | } 249 | 250 | } else { 251 | 252 | (function collectData(data, type, collection, cb) { 253 | 254 | if (type === 'file') { 255 | 256 | console.log('Files can not merge at this time with this plugin'); 257 | 258 | cb(); 259 | 260 | } else { 261 | 262 | collection.push(data.shift()); 263 | 264 | if (data.length === 0) { 265 | 266 | if (options.cache) { 267 | 268 | checkCache(data, dest, options.name, type, function(results) { 269 | 270 | if (results) { 271 | 272 | collection = { 273 | data: collection 274 | }; 275 | 276 | 277 | github_api.write.add(collection, dest, type, format); 278 | 279 | } 280 | 281 | cb(); 282 | 283 | }); 284 | 285 | } else { 286 | 287 | collection = { 288 | data: collection 289 | }; 290 | 291 | github_api.write.add(collection, dest, type, format); 292 | 293 | cb(); 294 | 295 | } 296 | 297 | } else { 298 | 299 | collectData(data, type, collection, cb); 300 | 301 | } 302 | 303 | } 304 | 305 | })(data, options.type, [], function() { 306 | 307 | leaveLoop(); 308 | 309 | }); 310 | 311 | } 312 | 313 | })(responseArray); 314 | 315 | }); 316 | }; 317 | 318 | var writeResponse = function(github_api, next) { 319 | 320 | github_api.write.save(function() { 321 | 322 | next(github_api); 323 | }); 324 | 325 | }; 326 | 327 | var updateCache = function(github_api, next) { 328 | 329 | // Check to see if the cache status is true. If so we need to 330 | // Generate one more write. 331 | 332 | if (github_api.cache.status()) { 333 | 334 | var cacheData = github_api.cache.dump(); 335 | 336 | github_api.write.write(cacheData.contents, cacheData.location, 'data', github_api.options.output.format, function() { 337 | 338 | github_api.cache.saved(); 339 | 340 | grunt.log.writeln('Updated Cache!'); 341 | 342 | next(github_api); 343 | }); 344 | 345 | } else { 346 | 347 | next(github_api); 348 | } 349 | }; 350 | 351 | var process = github_api.init(this, grunt) 352 | .step(getAPILimits) 353 | .step(generateRequest) 354 | .step(processRequest) 355 | .step(writeResponse) 356 | .step(updateCache) 357 | .execute(function(err, results) { 358 | 359 | if (err) { 360 | console.log(err); 361 | done(false); 362 | } 363 | 364 | done(); 365 | 366 | }); 367 | 368 | }); 369 | 370 | }; 371 | -------------------------------------------------------------------------------- /tasks/lib/utils.js: -------------------------------------------------------------------------------- 1 | /*! 2 | * grunt-github-api 3 | * https://github.com/assemble/grunt-github-api 4 | * Authored by Jeffrey Herb 5 | * 6 | * Copyright (c) 2013 Jeffrey Herb, contributors 7 | * Licensed under the MIT license. 8 | */ 9 | 10 | 'use strict'; 11 | 12 | // Include libraries 13 | var https = require('https'); 14 | var fs = require('fs'); 15 | var crypto = require('crypto'); 16 | var mkdirp = require('mkdirp'); 17 | 18 | var github_api = function() { 19 | 20 | var steps = []; 21 | var requestQueue = []; 22 | var writeQueue = []; 23 | var cacheData = { 24 | location: "", 25 | contents: {}, 26 | changed: false 27 | }; 28 | 29 | var init = function(task, grunt) { 30 | 31 | // Pull the current task data and gurnt utilities 32 | this.task = task; 33 | this.grunt = grunt; 34 | 35 | // Set default project options 36 | var defaultOptions = { 37 | name: task.target, 38 | output: { 39 | path: "api-data", 40 | cache: ".cache.json", 41 | format: { 42 | indent: 4, 43 | encoding: 'utf8' 44 | } 45 | }, 46 | connection: { 47 | host: 'api.github.com', 48 | headers: { 49 | 'User-Agent': 'node-http/0.10.1', 50 | 'Content-Type': 'application/json' 51 | }, 52 | }, 53 | rateLimit: { 54 | warning: 10, 55 | }, 56 | type: "data", 57 | cache: true, 58 | concat: false, 59 | filters: false, 60 | oAuth: false, 61 | }; 62 | 63 | // Set the options 64 | this.options = (mergeObject(defaultOptions, task.options({}))); 65 | this.data = task.data; 66 | 67 | // Set the cache location from the options 68 | if (cacheData.location === "") { 69 | cacheData.location = this.options.output.path + "/" + this.options.output.cache; 70 | 71 | // Check to see if the cache directory path exists 72 | if (!grunt.file.isDir(this.options.output.path)) { 73 | grunt.file.mkdir(this.options.output.path); 74 | } 75 | 76 | if (grunt.file.exists(cacheData.location)) { 77 | 78 | cacheData.contents = grunt.file.readJSON(cacheData.location); 79 | 80 | } 81 | } 82 | 83 | // flush steps and request array 84 | steps = []; 85 | requestQueue = []; 86 | 87 | return this; 88 | 89 | }; 90 | 91 | var step = function (fn) { 92 | 93 | // Add step to the steps array 94 | steps.push(fn); 95 | 96 | return this; 97 | 98 | }; 99 | 100 | var execute = function(cb) { 101 | 102 | // Check to see if any steps have been implemented. 103 | if (steps.length === 0) { 104 | 105 | if (cb) { 106 | cb(null, true); 107 | } 108 | 109 | return true; 110 | 111 | } 112 | 113 | var step = 0, totalSteps = steps.length, self = this; 114 | 115 | // Step through all the defined steps. 116 | steps[step++](self, function next(github_api, terminate) { 117 | 118 | if (terminate) { 119 | 120 | if(cb) { 121 | 122 | cb(null, true); 123 | 124 | } else { 125 | 126 | return true; 127 | 128 | } 129 | 130 | } else { 131 | 132 | if (step < totalSteps) { 133 | 134 | steps[step++](self, next); 135 | 136 | } else { 137 | 138 | if(cb) { 139 | 140 | cb(null, true); 141 | 142 | } else { 143 | 144 | return true; 145 | 146 | } 147 | 148 | } 149 | 150 | } 151 | 152 | }); 153 | 154 | }; 155 | 156 | var request = { 157 | 158 | add: function (conObject, src, dest, options) { 159 | 160 | requestQueue.push([conObject, src, dest, options]); 161 | 162 | }, 163 | 164 | send: function(cb) { 165 | 166 | (function nextRequest(requestQueue, collection, response) { 167 | 168 | var request = requestQueue.shift(); 169 | 170 | // Add source to the connection header. 171 | request[0].path = request[1]; 172 | 173 | 174 | var req = https.request(request[0], function(res) { 175 | 176 | var data = ""; 177 | res.setEncoding('utf8'); 178 | 179 | res.on('data', function (chunk) { 180 | data += chunk; 181 | }); 182 | 183 | res.on('end', function() { 184 | 185 | // Parse the data into an object so it can be manipulated 186 | var reqData = JSON.parse(data); 187 | 188 | // Check for an error response from the GitHub API. 189 | if (reqData.message) { 190 | 191 | console.log("GitHub returned an error: " + reqData.message); 192 | 193 | } else { 194 | 195 | collection.push(reqData); 196 | 197 | } 198 | 199 | if (requestQueue.length === 0) { 200 | 201 | response.push([request[2], collection, request[3]]); 202 | 203 | // We have finished getting all of the requests send back the response array 204 | cb(response); 205 | 206 | } else { 207 | 208 | // Check to see if they multiple requests belong together 209 | if (request[3].concat) { 210 | 211 | // Requests are together, call nest request 212 | nextRequest(requestQueue, collection, response); 213 | 214 | } else { 215 | 216 | // These are individual request, so add current results to response buffer. 217 | response.push([request[2], collection, request[3]]); 218 | 219 | // Flush data array 220 | collection = []; 221 | 222 | // Call next request 223 | nextRequest(requestQueue, collection, response); 224 | 225 | } 226 | 227 | } 228 | 229 | }); 230 | 231 | }); 232 | 233 | req.on('error', function(e){ 234 | console.log(e); 235 | }); 236 | 237 | req.end(); 238 | 239 | })(requestQueue, [], []); 240 | 241 | } 242 | 243 | }; 244 | 245 | var write = { 246 | 247 | add: function(data, dest, type, format) { 248 | 249 | writeQueue.push([data, dest, type, format]); 250 | 251 | }, 252 | 253 | write: function(data, dest, type, format, cb) { 254 | 255 | var buffer; 256 | 257 | if (type === "data") { 258 | buffer = new Buffer(JSON.stringify(data, null, format.indent)); 259 | } else { 260 | buffer = new Buffer(data[0].content, 'base64').toString(format.encoding); 261 | } 262 | 263 | fs.writeFile(dest, buffer, function(err) { 264 | 265 | console.log(dest + " was written to disk."); 266 | 267 | if (err) { 268 | console.log(err); 269 | } 270 | 271 | cb(); 272 | 273 | }); 274 | 275 | }, 276 | 277 | save: function(cb) { 278 | 279 | if (writeQueue.length > 0) { 280 | 281 | (function nextFile(writeQueue, w) { 282 | 283 | var wq = writeQueue.shift(); 284 | 285 | // Figure out the directory path 286 | var dirPath = wq[1].split("/"); 287 | 288 | // Remove the last filename 289 | dirPath.pop(); 290 | 291 | // Reconstruct the file path based on the split array. 292 | dirPath = dirPath.join("/"); 293 | 294 | // Check to see if the path exists 295 | fs.exists(dirPath, function (exists) { 296 | 297 | if (!exists) { 298 | 299 | mkdirp(dirPath, function (err) { 300 | 301 | if (err) { 302 | console.log("Error: Creating data directory path - " + dirPath); 303 | } else { 304 | 305 | // Now the directory structure is in place write the file. 306 | write.write(wq[0], wq[1], wq[2], wq[3], function(){ 307 | 308 | if (writeQueue.length === 0) { 309 | 310 | cb(); 311 | 312 | } else { 313 | 314 | nextFile(writeQueue, w); 315 | 316 | } 317 | 318 | }); 319 | } 320 | 321 | }); 322 | 323 | } else { 324 | 325 | // The directories exists, write the file 326 | write.write(wq[0], wq[1], wq[2], wq[3], function(){ 327 | 328 | if (writeQueue.length === 0) { 329 | 330 | cb(); 331 | 332 | } else { 333 | 334 | nextFile(writeQueue, w); 335 | 336 | } 337 | 338 | }); 339 | } 340 | 341 | }); 342 | 343 | })(writeQueue, this); 344 | 345 | } else { 346 | 347 | cb(); 348 | } 349 | 350 | } 351 | 352 | }; 353 | 354 | var path = { 355 | 356 | cleaner: function(path, leadSlash, tailSlash) { 357 | 358 | // Check for a leading slash, then add or remove it as needed. 359 | if (path.charAt(0) === "/") { 360 | 361 | if (!leadSlash) { 362 | path = path.substring(1); 363 | } 364 | 365 | } else { 366 | 367 | if (leadSlash) { 368 | path = "/" + path; 369 | } 370 | 371 | } 372 | 373 | // Check for trailing slash, then add or remove it as needed. 374 | if (path.charAt(path.length - 1) === "/") { 375 | 376 | if (!tailSlash) { 377 | path = path.substring(0, path.length - 1); 378 | } 379 | 380 | } else { 381 | 382 | if (tailSlash) { 383 | path += "/"; 384 | } 385 | 386 | } 387 | 388 | return path; 389 | 390 | }, 391 | 392 | construct: function(array) { 393 | 394 | return array.join("/"); 395 | 396 | }, 397 | 398 | parameters: function() { 399 | 400 | var temp = []; 401 | 402 | for (var i=0; i