├── .gitignore ├── .jshintrc ├── .npmignore ├── README.md ├── app.js ├── config.sample.json ├── dashboards ├── comments.json └── issues.json ├── db.js ├── img ├── comment_trends.png └── issue_trends.png ├── package.json ├── reposync.js └── test └── reposync_tests.js /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | 3 | logs 4 | *.log 5 | npm-debug.log* 6 | 7 | *.orig 8 | 9 | .settings.xml 10 | .settings 11 | 12 | .c9revisions 13 | .DS_Store 14 | .idea 15 | 16 | testAuth.json 17 | .env 18 | 19 | data/ 20 | 21 | apidoc/ 22 | 23 | tsconfig.json 24 | 25 | config.json 26 | -------------------------------------------------------------------------------- /.jshintrc: -------------------------------------------------------------------------------- 1 | { 2 | "esnext": true, 3 | "browser": false, 4 | "bitwise":false, 5 | "curly": true, 6 | "eqnull": true, 7 | "strict": false, 8 | "devel": true, 9 | "eqeqeq": true, 10 | "forin": false, 11 | "immed": true, 12 | "supernew": true, 13 | "expr": true, 14 | "indent": 2, 15 | "latedef": false, 16 | "newcap": true, 17 | "noarg": true, 18 | "noempty": true, 19 | "undef": true, 20 | "boss": true, 21 | "trailing": true, 22 | "laxbreak": true, 23 | "laxcomma": true, 24 | "sub": true, 25 | "unused": true, 26 | "maxdepth": 6, 27 | "maxlen": 140, 28 | 29 | "globals": { 30 | "System": true, 31 | "Promise": true, 32 | "define": true, 33 | "require": true, 34 | "Chromath": false, 35 | "setTimeout": true, 36 | "setImmediate": true 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /.npmignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | .env 3 | 4 | logs 5 | *.log 6 | npm-debug.log* 7 | 8 | *.orig 9 | 10 | .settings.xml 11 | .settings 12 | 13 | .c9revisions 14 | .DS_Store 15 | .idea 16 | 17 | testAuth.json 18 | 19 | apidoc/ 20 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # GitHub Elasticsearch Analytics 2 | 3 | This node.js app will crawl the GitHub issues and comments API to then save them in Elasticsearch. 4 | 5 | ![](https://github.com/grafana/github-to-es/blob/master/img/issue_trends.png) 6 | 7 | ## Install 8 | 9 | ``` 10 | git clone https://github.com/grafana/github-to-es.git 11 | cd github-to-es 12 | npm install 13 | ``` 14 | 15 | ## Configure 16 | 17 | Copy config.sample.json to config.json. 18 | 19 | ```json 20 | { 21 | "github_token": "mytoken", 22 | "elasticsearch": { 23 | "host": "localhost", 24 | "port": 9200 25 | }, 26 | "repos": [ 27 | { 28 | "repo": "grafana/grafana-docker", 29 | "comments": true, 30 | "issues": true, 31 | "sinceDays": 2 32 | } 33 | ] 34 | } 35 | ``` 36 | 37 | Specify your Elasticsearch details (no auth options available at this point). For the repository entries you can specify if 38 | comments and/or issues should be fetched. If you want a complete (from the start) import of all issues and comments remove 39 | the sinceDays option or set it to 0. After the complete import is done you can do incremental updates by setting this to 1. 40 | 41 | ## Init & Create Elasticsearch index 42 | 43 | ``` 44 | node app.js init | bunyan 45 | ``` 46 | 47 | The above command will create an Elasticsearch index named `github`. The `| bunyan` part is optional. It's to get nicer console logging (instead of the default json logger). To use bunyan install 48 | it first using `npm install -g bunyan`. 49 | 50 | You can reset (remove and recreates) the index using: 51 | ``` 52 | node app.js reset | bunyan 53 | ``` 54 | 55 | ## Start import 56 | 57 | ``` 58 | node app.js start | bunyan 59 | ``` 60 | 61 | ## Add Elasticsearch Data source in Grafana 62 | 63 | When you add the data source specify `github` as index name and `created_at` in the Timestamp field. 64 | 65 | ### Grafana Dashboards 66 | 67 | - [GitHub Repo Issues](http://play.grafana.org/dashboard/db/github-repo-trends-issues). 68 | - [GitHub Repo Comments](http://play.grafana.org/dashboard/db/github-repo-trends-comments). 69 | 70 | 71 | # Limitations & Possible improvements 72 | 73 | Currently GitHub API limits the number of pages you can fetch to 400. So there is a limit for the initial complete 74 | import of issues & comments to 40000 issues and 40000 comments. 75 | 76 | It would be nice to get stars & other repo stats as well. 77 | 78 | 79 | 80 | 81 | 82 | 83 | -------------------------------------------------------------------------------- /app.js: -------------------------------------------------------------------------------- 1 | var Queue = require('promise-queue'); 2 | var program = require('commander'); 3 | var RepoSync = require('./reposync'); 4 | var db = require('./db'); 5 | 6 | // set promise 7 | Queue.configure(require('bluebird')); 8 | 9 | var queue = new Queue(1, 1000); 10 | 11 | function startRepoSync() { 12 | const config = require('./config.json'); 13 | 14 | for (let repoSyncOptions of config.repos) { 15 | const rs = new RepoSync(repoSyncOptions, queue); 16 | rs.start(); 17 | } 18 | } 19 | 20 | program 21 | .version('0.0.1') 22 | .command('start') 23 | .action(startRepoSync); 24 | 25 | program 26 | .command('reset') 27 | .action(function () { 28 | db.resetIndex(); 29 | }); 30 | 31 | program 32 | .command('init') 33 | .action(function () { 34 | db.createIndex(); 35 | }); 36 | 37 | program.parse(process.argv); 38 | 39 | // console.log("res", res); 40 | // for (var i = 0; i <= res.length; i++) { 41 | // var issue = res[i]; 42 | // console.log('number', issue.number); 43 | // } 44 | 45 | // if (github.hasNextPage(res)) { 46 | // github.getNextPage(res, customHeaders, function(err, res) { 47 | // showIssueIds(res); 48 | // }); 49 | // } 50 | -------------------------------------------------------------------------------- /config.sample.json: -------------------------------------------------------------------------------- 1 | { 2 | "github_token": "mytoken", 3 | "elasticsearch": { 4 | "host": "localhost", 5 | "port": 9200 6 | }, 7 | "repos": [ 8 | { 9 | "repo": "grafana/grafana-docker", 10 | "comments": true, 11 | "issues": true, 12 | "sinceDays": 2 13 | }, 14 | { 15 | "repo": "grafana/piechart-panel", 16 | "comments": true, 17 | "issues": true, 18 | "sinceDays": 2 19 | } 20 | ] 21 | } 22 | -------------------------------------------------------------------------------- /dashboards/comments.json: -------------------------------------------------------------------------------- 1 | { 2 | "__inputs": [ 3 | { 4 | "name": "DS_GITHUB", 5 | "label": "github", 6 | "description": "", 7 | "type": "datasource", 8 | "pluginId": "elasticsearch", 9 | "pluginName": "Elasticsearch" 10 | } 11 | ], 12 | "__requires": [ 13 | { 14 | "type": "datasource", 15 | "id": "elasticsearch", 16 | "name": "Elasticsearch", 17 | "version": "3.0.0" 18 | }, 19 | { 20 | "type": "grafana", 21 | "id": "grafana", 22 | "name": "Grafana", 23 | "version": "4.2.0" 24 | }, 25 | { 26 | "type": "panel", 27 | "id": "graph", 28 | "name": "Graph", 29 | "version": "" 30 | }, 31 | { 32 | "type": "panel", 33 | "id": "table", 34 | "name": "Table", 35 | "version": "" 36 | } 37 | ], 38 | "annotations": { 39 | "list": [] 40 | }, 41 | "editable": true, 42 | "gnetId": null, 43 | "graphTooltip": 0, 44 | "hideControls": false, 45 | "id": null, 46 | "links": [ 47 | { 48 | "icon": "dashboard", 49 | "includeVars": true, 50 | "keepTime": false, 51 | "tags": [], 52 | "title": "Issues ", 53 | "type": "link", 54 | "url": "http://play.grafana.org/dashboard/db/github-repo-trends-issues?orgId=1" 55 | } 56 | ], 57 | "refresh": false, 58 | "rows": [ 59 | { 60 | "collapse": false, 61 | "height": 373, 62 | "panels": [ 63 | { 64 | "aliasColors": { 65 | "open": "#6ED0E0" 66 | }, 67 | "bars": false, 68 | "dashLength": 10, 69 | "dashes": false, 70 | "datasource": "${DS_GITHUB}", 71 | "fill": 3, 72 | "id": 2, 73 | "legend": { 74 | "alignAsTable": true, 75 | "avg": false, 76 | "current": false, 77 | "max": false, 78 | "min": false, 79 | "rightSide": true, 80 | "show": true, 81 | "total": true, 82 | "values": true 83 | }, 84 | "lines": true, 85 | "linewidth": 1, 86 | "links": [], 87 | "nullPointMode": "null", 88 | "percentage": false, 89 | "pointradius": 5, 90 | "points": false, 91 | "renderer": "flot", 92 | "seriesOverrides": [], 93 | "spaceLength": 10, 94 | "span": 12, 95 | "stack": true, 96 | "steppedLine": false, 97 | "targets": [ 98 | { 99 | "bucketAggs": [ 100 | { 101 | "field": "created_at", 102 | "id": "2", 103 | "settings": { 104 | "interval": "$interval", 105 | "min_doc_count": 0, 106 | "trimEdges": 0 107 | }, 108 | "type": "date_histogram" 109 | } 110 | ], 111 | "dsType": "elasticsearch", 112 | "metrics": [ 113 | { 114 | "field": "select field", 115 | "id": "1", 116 | "type": "count" 117 | } 118 | ], 119 | "query": "repo:$repo AND _type:comment", 120 | "refId": "A", 121 | "target": "", 122 | "timeField": "created_at" 123 | } 124 | ], 125 | "thresholds": [], 126 | "timeFrom": null, 127 | "timeShift": null, 128 | "title": "Comments", 129 | "tooltip": { 130 | "shared": true, 131 | "sort": 0, 132 | "value_type": "individual" 133 | }, 134 | "type": "graph", 135 | "xaxis": { 136 | "buckets": null, 137 | "mode": "time", 138 | "name": null, 139 | "show": true, 140 | "values": [] 141 | }, 142 | "yaxes": [ 143 | { 144 | "format": "short", 145 | "label": null, 146 | "logBase": 1, 147 | "max": null, 148 | "min": null, 149 | "show": true 150 | }, 151 | { 152 | "format": "short", 153 | "label": null, 154 | "logBase": 1, 155 | "max": null, 156 | "min": null, 157 | "show": true 158 | } 159 | ] 160 | } 161 | ], 162 | "repeat": null, 163 | "repeatIteration": null, 164 | "repeatRowId": null, 165 | "showTitle": false, 166 | "title": "Dashboard Row", 167 | "titleSize": "h6" 168 | }, 169 | { 170 | "collapse": false, 171 | "height": 434, 172 | "panels": [ 173 | { 174 | "columns": [ 175 | { 176 | "text": "user_login", 177 | "value": "user_login" 178 | }, 179 | { 180 | "text": "Count", 181 | "value": "Count" 182 | } 183 | ], 184 | "datasource": "${DS_GITHUB}", 185 | "filterNull": false, 186 | "fontSize": "100%", 187 | "id": 5, 188 | "links": [], 189 | "pageSize": null, 190 | "scroll": true, 191 | "showHeader": true, 192 | "sort": { 193 | "col": 1, 194 | "desc": true 195 | }, 196 | "span": 4, 197 | "styles": [ 198 | { 199 | "dateFormat": "YYYY-MM-DD HH:mm:ss", 200 | "pattern": "Time", 201 | "type": "date" 202 | }, 203 | { 204 | "colorMode": null, 205 | "colors": [ 206 | "rgba(245, 54, 54, 0.9)", 207 | "rgba(237, 129, 40, 0.89)", 208 | "rgba(50, 172, 45, 0.97)" 209 | ], 210 | "decimals": 2, 211 | "pattern": "/.*/", 212 | "thresholds": [], 213 | "type": "number", 214 | "unit": "short" 215 | } 216 | ], 217 | "targets": [ 218 | { 219 | "bucketAggs": [ 220 | { 221 | "fake": true, 222 | "field": "user_login", 223 | "id": "3", 224 | "settings": { 225 | "min_doc_count": 1, 226 | "order": "desc", 227 | "orderBy": "_count", 228 | "size": "10" 229 | }, 230 | "type": "terms" 231 | } 232 | ], 233 | "dsType": "elasticsearch", 234 | "metrics": [ 235 | { 236 | "field": "select field", 237 | "id": "1", 238 | "type": "count" 239 | } 240 | ], 241 | "query": "_type:comment AND repo:$repo", 242 | "refId": "A", 243 | "target": "", 244 | "timeField": "created_at" 245 | } 246 | ], 247 | "title": "Top Commentators", 248 | "transform": "json", 249 | "type": "table" 250 | }, 251 | { 252 | "columns": [ 253 | { 254 | "text": "issue", 255 | "value": "issue" 256 | }, 257 | { 258 | "text": "Count", 259 | "value": "Count" 260 | } 261 | ], 262 | "datasource": "${DS_GITHUB}", 263 | "filterNull": false, 264 | "fontSize": "100%", 265 | "id": 6, 266 | "links": [], 267 | "pageSize": null, 268 | "scroll": true, 269 | "showHeader": true, 270 | "sort": { 271 | "col": 1, 272 | "desc": true 273 | }, 274 | "span": 4, 275 | "styles": [ 276 | { 277 | "dateFormat": "YYYY-MM-DD HH:mm:ss", 278 | "pattern": "Time", 279 | "type": "date" 280 | }, 281 | { 282 | "colorMode": null, 283 | "colors": [ 284 | "rgba(245, 54, 54, 0.9)", 285 | "rgba(237, 129, 40, 0.89)", 286 | "rgba(50, 172, 45, 0.97)" 287 | ], 288 | "decimals": 2, 289 | "pattern": "/.*/", 290 | "thresholds": [], 291 | "type": "number", 292 | "unit": "short" 293 | } 294 | ], 295 | "targets": [ 296 | { 297 | "bucketAggs": [ 298 | { 299 | "fake": true, 300 | "field": "issue", 301 | "id": "3", 302 | "settings": { 303 | "min_doc_count": 1, 304 | "order": "desc", 305 | "orderBy": "_count", 306 | "size": "10" 307 | }, 308 | "type": "terms" 309 | } 310 | ], 311 | "dsType": "elasticsearch", 312 | "metrics": [ 313 | { 314 | "field": "select field", 315 | "id": "1", 316 | "type": "count" 317 | } 318 | ], 319 | "query": "_type:comment AND repo:$repo", 320 | "refId": "A", 321 | "target": "", 322 | "timeField": "created_at" 323 | } 324 | ], 325 | "timeFrom": null, 326 | "title": "Top Commented Issues", 327 | "transform": "json", 328 | "type": "table" 329 | }, 330 | { 331 | "columns": [ 332 | { 333 | "text": "user_login", 334 | "value": "user_login" 335 | }, 336 | { 337 | "text": "Sum", 338 | "value": "Sum" 339 | } 340 | ], 341 | "datasource": "${DS_GITHUB}", 342 | "filterNull": false, 343 | "fontSize": "100%", 344 | "id": 7, 345 | "links": [], 346 | "pageSize": null, 347 | "scroll": true, 348 | "showHeader": true, 349 | "sort": { 350 | "col": 1, 351 | "desc": true 352 | }, 353 | "span": 4, 354 | "styles": [ 355 | { 356 | "dateFormat": "YYYY-MM-DD HH:mm:ss", 357 | "pattern": "Time", 358 | "type": "date" 359 | }, 360 | { 361 | "colorMode": null, 362 | "colors": [ 363 | "rgba(245, 54, 54, 0.9)", 364 | "rgba(237, 129, 40, 0.89)", 365 | "rgba(50, 172, 45, 0.97)" 366 | ], 367 | "decimals": 2, 368 | "pattern": "/.*/", 369 | "thresholds": [], 370 | "type": "number", 371 | "unit": "short" 372 | } 373 | ], 374 | "targets": [ 375 | { 376 | "bucketAggs": [ 377 | { 378 | "fake": true, 379 | "field": "user_login", 380 | "id": "3", 381 | "settings": { 382 | "min_doc_count": 1, 383 | "order": "desc", 384 | "orderBy": "_count", 385 | "size": "10" 386 | }, 387 | "type": "terms" 388 | } 389 | ], 390 | "dsType": "elasticsearch", 391 | "metrics": [ 392 | { 393 | "field": "reactions_total", 394 | "id": "1", 395 | "meta": {}, 396 | "settings": {}, 397 | "type": "sum" 398 | } 399 | ], 400 | "query": "_type:comment AND repo:$repo", 401 | "refId": "A", 402 | "target": "", 403 | "timeField": "created_at" 404 | } 405 | ], 406 | "timeFrom": null, 407 | "title": "Comment reaction totals", 408 | "transform": "json", 409 | "type": "table" 410 | } 411 | ], 412 | "repeat": null, 413 | "repeatIteration": null, 414 | "repeatRowId": null, 415 | "showTitle": false, 416 | "title": "Dashboard Row", 417 | "titleSize": "h6" 418 | } 419 | ], 420 | "schemaVersion": 14, 421 | "style": "dark", 422 | "tags": [], 423 | "templating": { 424 | "list": [ 425 | { 426 | "allValue": null, 427 | "current": {}, 428 | "datasource": "${DS_GITHUB}", 429 | "hide": 0, 430 | "includeAll": false, 431 | "label": "Repo", 432 | "multi": true, 433 | "name": "repo", 434 | "options": [], 435 | "query": "{\"find\": \"terms\", \"field\": \"repo\"} ", 436 | "refresh": 1, 437 | "regex": "", 438 | "sort": 0, 439 | "tagValuesQuery": "", 440 | "tags": [], 441 | "tagsQuery": "", 442 | "type": "query", 443 | "useTags": false 444 | }, 445 | { 446 | "auto": true, 447 | "auto_count": 30, 448 | "auto_min": "10s", 449 | "current": { 450 | "text": "1d", 451 | "value": "1d" 452 | }, 453 | "hide": 0, 454 | "label": null, 455 | "name": "interval", 456 | "options": [ 457 | { 458 | "selected": false, 459 | "text": "auto", 460 | "value": "$__auto_interval" 461 | }, 462 | { 463 | "selected": false, 464 | "text": "1m", 465 | "value": "1m" 466 | }, 467 | { 468 | "selected": false, 469 | "text": "10m", 470 | "value": "10m" 471 | }, 472 | { 473 | "selected": false, 474 | "text": "30m", 475 | "value": "30m" 476 | }, 477 | { 478 | "selected": false, 479 | "text": "1h", 480 | "value": "1h" 481 | }, 482 | { 483 | "selected": false, 484 | "text": "6h", 485 | "value": "6h" 486 | }, 487 | { 488 | "selected": false, 489 | "text": "12h", 490 | "value": "12h" 491 | }, 492 | { 493 | "selected": true, 494 | "text": "1d", 495 | "value": "1d" 496 | }, 497 | { 498 | "selected": false, 499 | "text": "7d", 500 | "value": "7d" 501 | }, 502 | { 503 | "selected": false, 504 | "text": "14d", 505 | "value": "14d" 506 | }, 507 | { 508 | "selected": false, 509 | "text": "1M", 510 | "value": "1M" 511 | } 512 | ], 513 | "query": "1m,10m,30m,1h,6h,12h,1d,7d,14d,1M", 514 | "refresh": 2, 515 | "type": "interval" 516 | }, 517 | { 518 | "datasource": "github", 519 | "filters": [], 520 | "hide": 0, 521 | "label": null, 522 | "name": "Filters", 523 | "type": "adhoc" 524 | } 525 | ] 526 | }, 527 | "time": { 528 | "from": "now-2y", 529 | "to": "now" 530 | }, 531 | "timepicker": { 532 | "refresh_intervals": [ 533 | "5s", 534 | "10s", 535 | "30s", 536 | "1m", 537 | "5m", 538 | "15m", 539 | "30m", 540 | "1h", 541 | "2h", 542 | "1d" 543 | ], 544 | "time_options": [ 545 | "5m", 546 | "15m", 547 | "1h", 548 | "6h", 549 | "12h", 550 | "24h", 551 | "2d", 552 | "7d", 553 | "30d" 554 | ] 555 | }, 556 | "timezone": "browser", 557 | "title": "Github Repo Trends Comments", 558 | "version": 13 559 | } -------------------------------------------------------------------------------- /dashboards/issues.json: -------------------------------------------------------------------------------- 1 | { 2 | "__inputs": [ 3 | { 4 | "name": "DS_GITHUB", 5 | "label": "github", 6 | "description": "", 7 | "type": "datasource", 8 | "pluginId": "elasticsearch", 9 | "pluginName": "Elasticsearch" 10 | } 11 | ], 12 | "__requires": [ 13 | { 14 | "type": "datasource", 15 | "id": "elasticsearch", 16 | "name": "Elasticsearch", 17 | "version": "3.0.0" 18 | }, 19 | { 20 | "type": "grafana", 21 | "id": "grafana", 22 | "name": "Grafana", 23 | "version": "4.2.0" 24 | }, 25 | { 26 | "type": "panel", 27 | "id": "graph", 28 | "name": "Graph", 29 | "version": "" 30 | }, 31 | { 32 | "type": "panel", 33 | "id": "table", 34 | "name": "Table", 35 | "version": "" 36 | } 37 | ], 38 | "annotations": { 39 | "list": [] 40 | }, 41 | "editable": true, 42 | "gnetId": null, 43 | "graphTooltip": 0, 44 | "hideControls": false, 45 | "id": null, 46 | "links": [ 47 | { 48 | "icon": "dashboard", 49 | "tags": [], 50 | "title": "Comments", 51 | "type": "link", 52 | "url": "http://play.grafana.org/dashboard/db/github-repo-trends-comments" 53 | } 54 | ], 55 | "refresh": false, 56 | "rows": [ 57 | { 58 | "collapse": false, 59 | "height": 378, 60 | "panels": [ 61 | { 62 | "aliasColors": { 63 | "open": "#6ED0E0" 64 | }, 65 | "bars": false, 66 | "dashLength": 10, 67 | "dashes": false, 68 | "datasource": "${DS_GITHUB}", 69 | "fill": 3, 70 | "id": 1, 71 | "legend": { 72 | "alignAsTable": true, 73 | "avg": false, 74 | "current": false, 75 | "max": false, 76 | "min": false, 77 | "rightSide": true, 78 | "show": true, 79 | "total": true, 80 | "values": true 81 | }, 82 | "lines": true, 83 | "linewidth": 1, 84 | "links": [], 85 | "nullPointMode": "null", 86 | "percentage": false, 87 | "pointradius": 5, 88 | "points": false, 89 | "renderer": "flot", 90 | "seriesOverrides": [], 91 | "spaceLength": 10, 92 | "span": 12, 93 | "stack": true, 94 | "steppedLine": false, 95 | "targets": [ 96 | { 97 | "bucketAggs": [ 98 | { 99 | "fake": true, 100 | "field": "state", 101 | "id": "3", 102 | "settings": { 103 | "min_doc_count": 0, 104 | "order": "asc", 105 | "orderBy": "_term", 106 | "size": "10" 107 | }, 108 | "type": "terms" 109 | }, 110 | { 111 | "field": "created_at", 112 | "id": "2", 113 | "settings": { 114 | "interval": "$interval", 115 | "min_doc_count": 0, 116 | "trimEdges": 0 117 | }, 118 | "type": "date_histogram" 119 | } 120 | ], 121 | "dsType": "elasticsearch", 122 | "metrics": [ 123 | { 124 | "field": "select field", 125 | "id": "1", 126 | "type": "count" 127 | } 128 | ], 129 | "query": "repo:$repo AND _type:issue", 130 | "refId": "A", 131 | "target": "", 132 | "timeField": "created_at" 133 | } 134 | ], 135 | "thresholds": [], 136 | "timeFrom": null, 137 | "timeShift": null, 138 | "title": "Created Issues", 139 | "tooltip": { 140 | "shared": true, 141 | "sort": 0, 142 | "value_type": "individual" 143 | }, 144 | "type": "graph", 145 | "xaxis": { 146 | "buckets": null, 147 | "mode": "time", 148 | "name": null, 149 | "show": true, 150 | "values": [] 151 | }, 152 | "yaxes": [ 153 | { 154 | "format": "short", 155 | "label": null, 156 | "logBase": 1, 157 | "max": null, 158 | "min": null, 159 | "show": true 160 | }, 161 | { 162 | "format": "short", 163 | "label": null, 164 | "logBase": 1, 165 | "max": null, 166 | "min": null, 167 | "show": true 168 | } 169 | ] 170 | } 171 | ], 172 | "repeat": null, 173 | "repeatIteration": null, 174 | "repeatRowId": null, 175 | "showTitle": false, 176 | "title": "Dashboard Row", 177 | "titleSize": "h6" 178 | }, 179 | { 180 | "collapse": false, 181 | "height": 356, 182 | "panels": [ 183 | { 184 | "aliasColors": { 185 | "open": "#6ED0E0" 186 | }, 187 | "bars": false, 188 | "dashLength": 10, 189 | "dashes": false, 190 | "datasource": "${DS_GITHUB}", 191 | "decimals": 0, 192 | "fill": 3, 193 | "id": 3, 194 | "legend": { 195 | "alignAsTable": true, 196 | "avg": false, 197 | "current": false, 198 | "max": false, 199 | "min": false, 200 | "rightSide": true, 201 | "show": true, 202 | "sort": "total", 203 | "sortDesc": true, 204 | "total": true, 205 | "values": true 206 | }, 207 | "lines": true, 208 | "linewidth": 1, 209 | "links": [], 210 | "nullPointMode": "null", 211 | "percentage": false, 212 | "pointradius": 5, 213 | "points": false, 214 | "renderer": "flot", 215 | "seriesOverrides": [], 216 | "spaceLength": 10, 217 | "span": 12, 218 | "stack": true, 219 | "steppedLine": false, 220 | "targets": [ 221 | { 222 | "alias": "{{labels}}", 223 | "bucketAggs": [ 224 | { 225 | "fake": true, 226 | "field": "labels", 227 | "id": "3", 228 | "settings": { 229 | "min_doc_count": 1, 230 | "missing": "no_labels", 231 | "order": "desc", 232 | "orderBy": "_count", 233 | "size": "10" 234 | }, 235 | "type": "terms" 236 | }, 237 | { 238 | "field": "created_at", 239 | "id": "2", 240 | "settings": { 241 | "interval": "$interval", 242 | "min_doc_count": 0, 243 | "trimEdges": 0 244 | }, 245 | "type": "date_histogram" 246 | } 247 | ], 248 | "dsType": "elasticsearch", 249 | "metrics": [ 250 | { 251 | "field": "select field", 252 | "id": "1", 253 | "type": "count" 254 | } 255 | ], 256 | "query": "repo:$repo AND _type:issue", 257 | "refId": "A", 258 | "target": "", 259 | "timeField": "created_at" 260 | } 261 | ], 262 | "thresholds": [], 263 | "timeFrom": null, 264 | "timeShift": null, 265 | "title": "Issues by label", 266 | "tooltip": { 267 | "shared": true, 268 | "sort": 0, 269 | "value_type": "individual" 270 | }, 271 | "type": "graph", 272 | "xaxis": { 273 | "buckets": null, 274 | "mode": "time", 275 | "name": null, 276 | "show": true, 277 | "values": [] 278 | }, 279 | "yaxes": [ 280 | { 281 | "format": "short", 282 | "label": null, 283 | "logBase": 1, 284 | "max": null, 285 | "min": null, 286 | "show": true 287 | }, 288 | { 289 | "format": "short", 290 | "label": null, 291 | "logBase": 1, 292 | "max": null, 293 | "min": null, 294 | "show": true 295 | } 296 | ] 297 | } 298 | ], 299 | "repeat": null, 300 | "repeatIteration": null, 301 | "repeatRowId": null, 302 | "showTitle": false, 303 | "title": "Dashboard Row", 304 | "titleSize": "h6" 305 | }, 306 | { 307 | "collapse": false, 308 | "height": 430, 309 | "panels": [ 310 | { 311 | "columns": [ 312 | { 313 | "text": "labels", 314 | "value": "labels" 315 | }, 316 | { 317 | "text": "Count", 318 | "value": "Count" 319 | } 320 | ], 321 | "datasource": "${DS_GITHUB}", 322 | "filterNull": false, 323 | "fontSize": "100%", 324 | "id": 8, 325 | "links": [], 326 | "pageSize": null, 327 | "scroll": true, 328 | "showHeader": true, 329 | "sort": { 330 | "col": 1, 331 | "desc": true 332 | }, 333 | "span": 3, 334 | "styles": [ 335 | { 336 | "dateFormat": "YYYY-MM-DD HH:mm:ss", 337 | "pattern": "Time", 338 | "type": "date" 339 | }, 340 | { 341 | "colorMode": null, 342 | "colors": [ 343 | "rgba(245, 54, 54, 0.9)", 344 | "rgba(237, 129, 40, 0.89)", 345 | "rgba(50, 172, 45, 0.97)" 346 | ], 347 | "decimals": 2, 348 | "pattern": "/.*/", 349 | "thresholds": [], 350 | "type": "number", 351 | "unit": "short" 352 | } 353 | ], 354 | "targets": [ 355 | { 356 | "bucketAggs": [ 357 | { 358 | "fake": true, 359 | "field": "labels", 360 | "id": "3", 361 | "settings": { 362 | "min_doc_count": 1, 363 | "missing": "no_labels", 364 | "order": "desc", 365 | "orderBy": "_count", 366 | "size": "10" 367 | }, 368 | "type": "terms" 369 | } 370 | ], 371 | "dsType": "elasticsearch", 372 | "metrics": [ 373 | { 374 | "field": "select field", 375 | "id": "1", 376 | "type": "count" 377 | } 378 | ], 379 | "query": "_type:issue AND repo:$repo", 380 | "refId": "A", 381 | "target": "", 382 | "timeField": "created_at" 383 | } 384 | ], 385 | "title": "Top Labels", 386 | "transform": "json", 387 | "type": "table" 388 | }, 389 | { 390 | "columns": [ 391 | { 392 | "text": "assignee", 393 | "value": "assignee" 394 | }, 395 | { 396 | "text": "Count", 397 | "value": "Count" 398 | } 399 | ], 400 | "datasource": "${DS_GITHUB}", 401 | "filterNull": false, 402 | "fontSize": "100%", 403 | "id": 9, 404 | "links": [], 405 | "pageSize": null, 406 | "scroll": true, 407 | "showHeader": true, 408 | "sort": { 409 | "col": 1, 410 | "desc": true 411 | }, 412 | "span": 3, 413 | "styles": [ 414 | { 415 | "dateFormat": "YYYY-MM-DD HH:mm:ss", 416 | "pattern": "Time", 417 | "type": "date" 418 | }, 419 | { 420 | "colorMode": null, 421 | "colors": [ 422 | "rgba(245, 54, 54, 0.9)", 423 | "rgba(237, 129, 40, 0.89)", 424 | "rgba(50, 172, 45, 0.97)" 425 | ], 426 | "decimals": 2, 427 | "pattern": "/.*/", 428 | "thresholds": [], 429 | "type": "number", 430 | "unit": "short" 431 | } 432 | ], 433 | "targets": [ 434 | { 435 | "bucketAggs": [ 436 | { 437 | "fake": true, 438 | "field": "assignee", 439 | "id": "3", 440 | "settings": { 441 | "min_doc_count": 1, 442 | "missing": null, 443 | "order": "desc", 444 | "orderBy": "_count", 445 | "size": "10" 446 | }, 447 | "type": "terms" 448 | } 449 | ], 450 | "dsType": "elasticsearch", 451 | "metrics": [ 452 | { 453 | "field": "select field", 454 | "id": "1", 455 | "type": "count" 456 | } 457 | ], 458 | "query": "_type:issue AND repo:$repo", 459 | "refId": "A", 460 | "target": "", 461 | "timeField": "created_at" 462 | } 463 | ], 464 | "timeFrom": null, 465 | "title": "Assigned Most Issues", 466 | "transform": "json", 467 | "type": "table" 468 | }, 469 | { 470 | "columns": [ 471 | { 472 | "text": "number", 473 | "value": "number" 474 | }, 475 | { 476 | "text": "Sum", 477 | "value": "Sum" 478 | } 479 | ], 480 | "datasource": "${DS_GITHUB}", 481 | "filterNull": false, 482 | "fontSize": "100%", 483 | "id": 10, 484 | "links": [], 485 | "pageSize": null, 486 | "scroll": true, 487 | "showHeader": true, 488 | "sort": { 489 | "col": 1, 490 | "desc": true 491 | }, 492 | "span": 3, 493 | "styles": [ 494 | { 495 | "dateFormat": "YYYY-MM-DD HH:mm:ss", 496 | "pattern": "Time", 497 | "type": "date" 498 | }, 499 | { 500 | "colorMode": null, 501 | "colors": [ 502 | "rgba(245, 54, 54, 0.9)", 503 | "rgba(237, 129, 40, 0.89)", 504 | "rgba(50, 172, 45, 0.97)" 505 | ], 506 | "decimals": 2, 507 | "pattern": "/.*/", 508 | "thresholds": [], 509 | "type": "number", 510 | "unit": "short" 511 | } 512 | ], 513 | "targets": [ 514 | { 515 | "bucketAggs": [ 516 | { 517 | "fake": true, 518 | "field": "number", 519 | "id": "3", 520 | "settings": { 521 | "min_doc_count": 1, 522 | "missing": null, 523 | "order": "desc", 524 | "orderBy": "1", 525 | "size": "10" 526 | }, 527 | "type": "terms" 528 | } 529 | ], 530 | "dsType": "elasticsearch", 531 | "metrics": [ 532 | { 533 | "field": "reactions_total", 534 | "id": "1", 535 | "meta": {}, 536 | "settings": {}, 537 | "type": "sum" 538 | } 539 | ], 540 | "query": "_type:issue AND repo:$repo", 541 | "refId": "A", 542 | "target": "", 543 | "timeField": "created_at" 544 | } 545 | ], 546 | "timeFrom": null, 547 | "title": "Most Reactions", 548 | "transform": "json", 549 | "type": "table" 550 | }, 551 | { 552 | "columns": [ 553 | { 554 | "text": "number", 555 | "value": "number" 556 | }, 557 | { 558 | "text": "Sum", 559 | "value": "Sum" 560 | } 561 | ], 562 | "datasource": "${DS_GITHUB}", 563 | "filterNull": false, 564 | "fontSize": "100%", 565 | "id": 11, 566 | "links": [], 567 | "pageSize": null, 568 | "scroll": true, 569 | "showHeader": true, 570 | "sort": { 571 | "col": 1, 572 | "desc": true 573 | }, 574 | "span": 3, 575 | "styles": [ 576 | { 577 | "dateFormat": "YYYY-MM-DD HH:mm:ss", 578 | "pattern": "Time", 579 | "type": "date" 580 | }, 581 | { 582 | "colorMode": null, 583 | "colors": [ 584 | "rgba(245, 54, 54, 0.9)", 585 | "rgba(237, 129, 40, 0.89)", 586 | "rgba(50, 172, 45, 0.97)" 587 | ], 588 | "decimals": 2, 589 | "pattern": "/.*/", 590 | "thresholds": [], 591 | "type": "number", 592 | "unit": "short" 593 | } 594 | ], 595 | "targets": [ 596 | { 597 | "bucketAggs": [ 598 | { 599 | "fake": true, 600 | "field": "number", 601 | "id": "3", 602 | "settings": { 603 | "min_doc_count": 1, 604 | "missing": null, 605 | "order": "desc", 606 | "orderBy": "1", 607 | "size": "10" 608 | }, 609 | "type": "terms" 610 | } 611 | ], 612 | "dsType": "elasticsearch", 613 | "metrics": [ 614 | { 615 | "field": "comments", 616 | "id": "1", 617 | "meta": {}, 618 | "settings": {}, 619 | "type": "sum" 620 | } 621 | ], 622 | "query": "_type:issue AND repo:$repo", 623 | "refId": "A", 624 | "target": "", 625 | "timeField": "created_at" 626 | } 627 | ], 628 | "timeFrom": null, 629 | "title": "Most Comments", 630 | "transform": "json", 631 | "type": "table" 632 | } 633 | ], 634 | "repeat": null, 635 | "repeatIteration": null, 636 | "repeatRowId": null, 637 | "showTitle": false, 638 | "title": "Dashboard Row", 639 | "titleSize": "h6" 640 | } 641 | ], 642 | "schemaVersion": 14, 643 | "style": "dark", 644 | "tags": [], 645 | "templating": { 646 | "list": [ 647 | { 648 | "allValue": null, 649 | "current": {}, 650 | "datasource": "${DS_GITHUB}", 651 | "hide": 0, 652 | "includeAll": false, 653 | "label": "Repo", 654 | "multi": true, 655 | "name": "repo", 656 | "options": [], 657 | "query": "{\"find\": \"terms\", \"field\": \"repo\"} ", 658 | "refresh": 1, 659 | "regex": "", 660 | "sort": 0, 661 | "tagValuesQuery": "", 662 | "tags": [], 663 | "tagsQuery": "", 664 | "type": "query", 665 | "useTags": false 666 | }, 667 | { 668 | "auto": true, 669 | "auto_count": 30, 670 | "auto_min": "10s", 671 | "current": { 672 | "text": "1w", 673 | "value": "1w" 674 | }, 675 | "hide": 0, 676 | "label": null, 677 | "name": "interval", 678 | "options": [ 679 | { 680 | "selected": false, 681 | "text": "auto", 682 | "value": "$__auto_interval" 683 | }, 684 | { 685 | "selected": false, 686 | "text": "1m", 687 | "value": "1m" 688 | }, 689 | { 690 | "selected": false, 691 | "text": "10m", 692 | "value": "10m" 693 | }, 694 | { 695 | "selected": false, 696 | "text": "30m", 697 | "value": "30m" 698 | }, 699 | { 700 | "selected": false, 701 | "text": "1h", 702 | "value": "1h" 703 | }, 704 | { 705 | "selected": false, 706 | "text": "6h", 707 | "value": "6h" 708 | }, 709 | { 710 | "selected": false, 711 | "text": "12h", 712 | "value": "12h" 713 | }, 714 | { 715 | "selected": false, 716 | "text": "1d", 717 | "value": "1d" 718 | }, 719 | { 720 | "selected": true, 721 | "text": "1w", 722 | "value": "1w" 723 | }, 724 | { 725 | "selected": false, 726 | "text": "14d", 727 | "value": "14d" 728 | }, 729 | { 730 | "selected": false, 731 | "text": "1M", 732 | "value": "1M" 733 | } 734 | ], 735 | "query": "1m,10m,30m,1h,6h,12h,1d,1w,14d,1M", 736 | "refresh": 2, 737 | "type": "interval" 738 | }, 739 | { 740 | "datasource": "github", 741 | "filters": [], 742 | "hide": 0, 743 | "label": null, 744 | "name": "Filters", 745 | "type": "adhoc" 746 | } 747 | ] 748 | }, 749 | "time": { 750 | "from": "now-2y", 751 | "to": "now" 752 | }, 753 | "timepicker": { 754 | "refresh_intervals": [ 755 | "5s", 756 | "10s", 757 | "30s", 758 | "1m", 759 | "5m", 760 | "15m", 761 | "30m", 762 | "1h", 763 | "2h", 764 | "1d" 765 | ], 766 | "time_options": [ 767 | "5m", 768 | "15m", 769 | "1h", 770 | "6h", 771 | "12h", 772 | "24h", 773 | "2d", 774 | "7d", 775 | "30d" 776 | ] 777 | }, 778 | "timezone": "browser", 779 | "title": "Github Repo Trends Issues", 780 | "version": 7 781 | } -------------------------------------------------------------------------------- /db.js: -------------------------------------------------------------------------------- 1 | 2 | var bunyan = require('bunyan'); 3 | var log = bunyan.createLogger({name: 'app'}); 4 | var elasticsearch = require('elasticsearch'); 5 | var config = require('./config.json'); 6 | 7 | var client = new elasticsearch.Client({ 8 | host: config.elasticsearch.host + ':' + config.elasticsearch.port, 9 | log: 'error' 10 | }); 11 | 12 | function createIndex() { 13 | log.info("creating index"); 14 | 15 | return client.indices.create({ 16 | index: "github", 17 | body: { 18 | "mappings": { 19 | "issue": { 20 | "properties": { 21 | "title": { "type": "text" }, 22 | "state": { "type": "keyword" }, 23 | "repo": { "type": "keyword" }, 24 | "labels": { "type": "keyword" }, 25 | "number": { "type": "keyword" }, 26 | "comments": { "type": "long" }, 27 | "assignee": { "type": "keyword" }, 28 | "user_login": { "type": "keyword" }, 29 | "closed_by": { "type": "keyword" }, 30 | "milestone": { "type": "keyword" }, 31 | "created_at": { "type": "date" }, 32 | "closed_at": { "type": "date" }, 33 | "updated_at": { "type": "date" }, 34 | "is_pull_request": { "type": "boolean" }, 35 | "minutes_open": { "type": "long" }, 36 | "created_iso_week_day": { "type": "integer" }, 37 | } 38 | }, 39 | "comment": { 40 | "properties": { 41 | "issue": { "type": "keyword" }, 42 | "repo": { "type": "keyword" }, 43 | "user_login": { "type": "keyword" }, 44 | "created_at": { "type": "date" }, 45 | "created_iso_week_day": { "type": "integer" }, 46 | } 47 | } 48 | } 49 | } 50 | }).then(res => { 51 | log.info(res, 'ES index created'); 52 | }).catch(err => { 53 | log.error(err, "index creation failed"); 54 | }); 55 | } 56 | 57 | function resetIndex() { 58 | log.info('deleting old ES index'); 59 | client.indices.delete({index: 'github'}) 60 | .then(res => { 61 | log.info(res, "index deleted"); 62 | return createIndex(); 63 | }).catch(err => { 64 | log.error("index deletion failed"); 65 | return createIndex(); 66 | }); 67 | } 68 | 69 | function handleEsError(err, resp) { 70 | log.error('ES error', err, resp); 71 | } 72 | 73 | function saveIssue(issue) { 74 | return client.index({ 75 | index: 'github', 76 | type: 'issue', 77 | id: issue.id, 78 | body: issue, 79 | }).then(res => { 80 | log.info('Saved issue', {number: issue.number, repo: issue.repo}); 81 | }).catch(handleEsError); 82 | } 83 | 84 | function saveComment(comment) { 85 | return client.index({ 86 | index: 'github', 87 | type: 'comment', 88 | id: comment.id, 89 | body: comment, 90 | }).then(res => { 91 | log.info('Saved comment', {created_at: comment.created_at, by: comment.user_login, repo: comment.repo}); 92 | }).catch(handleEsError); 93 | } 94 | 95 | module.exports = { 96 | saveIssue: saveIssue, 97 | saveComment: saveComment, 98 | resetIndex: resetIndex, 99 | createIndex: createIndex, 100 | }; 101 | -------------------------------------------------------------------------------- /img/comment_trends.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/grafana/github-to-es/3d412cb291ffcb47e0aeabd3a31fe87fadca2f5c/img/comment_trends.png -------------------------------------------------------------------------------- /img/issue_trends.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/grafana/github-to-es/3d412cb291ffcb47e0aeabd3a31fe87fadca2f5c/img/issue_trends.png -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "github-to-es", 3 | "version": "1.0.0", 4 | "description": "", 5 | "main": "index.js", 6 | "scripts": { 7 | "test": "echo \"Error: no test specified\" && exit 1" 8 | }, 9 | "author": "", 10 | "license": "ISC", 11 | "dependencies": { 12 | "bluebird": "^3.5.0", 13 | "bunyan": "^1.8.10", 14 | "commander": "^2.9.0", 15 | "diskdb": "^0.1.17", 16 | "elasticsearch": "^13.0.0-rc2", 17 | "github": "^9.2.0", 18 | "lodash": "^4.17.4", 19 | "moment": "^2.18.1", 20 | "node-env-file": "^0.1.8", 21 | "promise-queue": "^2.2.3" 22 | }, 23 | "scripts": { 24 | "test": "./node_modules/mocha/bin/mocha" 25 | }, 26 | "devDependencies": { 27 | "mocha": "^3.3.0" 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /reposync.js: -------------------------------------------------------------------------------- 1 | var _ = require('lodash'); 2 | var db = require('./db'); 3 | var bunyan = require('bunyan'); 4 | var log = bunyan.createLogger({name: 'RepoSync'}); 5 | var config = require('./config.json'); 6 | var GitHubApi = require("github"); 7 | var moment = require('moment'); 8 | 9 | var github = new GitHubApi({ 10 | // optional 11 | debug: false, 12 | protocol: "https", 13 | host: "api.github.com", // should be api.github.com for GitHub 14 | pathPrefix: "", // for some GHEs; none for GitHub 15 | headers: { 16 | "user-agent": "github-es-exporter", 17 | "Accept": "application/vnd.github.squirrel-girl-preview", 18 | }, 19 | Promise: require('bluebird'), 20 | followRedirects: false, // default: true; there's currently an issue with non-get redirects, so allow ability to disable follow-redirects 21 | timeout: 50000 22 | }); 23 | 24 | log.info("github_token", config.github_token); 25 | 26 | // user token 27 | github.authenticate({ 28 | type: "token", 29 | token: config.github_token, 30 | }); 31 | 32 | class RepoSync { 33 | 34 | constructor(options, queue) { 35 | this.options = options; 36 | this.queue = queue; 37 | 38 | const repoParts = options.repo.split('/'); 39 | this.owner = repoParts[0]; 40 | this.repo = repoParts[1]; 41 | } 42 | 43 | start() { 44 | log.info('Staring repo sync instance', {repo: this.options.repo}); 45 | 46 | if (this.options.issues) { 47 | this.queue.add(() => this.startIssueSync()); 48 | } 49 | 50 | if (this.options.comments) { 51 | this.queue.add(() => this.startCommentsSync()); 52 | } 53 | } 54 | 55 | startIssueSync() { 56 | let params = { 57 | owner: this.owner, 58 | repo: this.repo, 59 | direction: "asc", 60 | state: 'all', 61 | page: this.options.page || 0, 62 | per_page: 100, 63 | }; 64 | 65 | if (this.options.sinceDays) { 66 | params.since = moment().subtract(parseInt(this.options.sinceDays), 'days').utc().format(); 67 | } 68 | 69 | return github.issues.getForRepo(params).then(this.issueListHandler.bind(this)); 70 | } 71 | 72 | startCommentsSync() { 73 | log.info('Comments sync started'); 74 | 75 | let params = { 76 | owner: this.owner, 77 | repo: this.repo, 78 | sort: 'updated', 79 | direction: "asc", 80 | page: this.options.page || 0, 81 | per_page: 100, 82 | }; 83 | 84 | if (this.options.sinceDays) { 85 | params.since = moment().subtract(parseInt(this.options.sinceDays), 'days').utc().format(); 86 | } 87 | 88 | return github.issues.getCommentsForRepo(params).then(this.commentsListHandler.bind(this)); 89 | } 90 | 91 | saveComment(comment) { 92 | this.queue.add(() => db.saveComment(comment)); 93 | } 94 | 95 | commentsListHandler(res) { 96 | log.info('Got comments', { 97 | 'rate-limit-remaining': res.meta['x-ratelimit-remaining'], 98 | count: res.data.length 99 | }); 100 | 101 | for (let gc of res.data) { 102 | let issueNr = gc.issue_url.substr(gc.issue_url.lastIndexOf('/') + 1); 103 | var comment = { 104 | id: gc.id, 105 | issue: issueNr, 106 | repo: this.options.repo, 107 | created_at: gc.created_at, 108 | user_login: gc.user.login, 109 | reactions_total: gc.reactions.total_count, 110 | reactions_plus1: gc.reactions['+1'], 111 | reactions_minus1: gc.reactions['-1'], 112 | reactions_heart: gc.reactions.heart, 113 | } 114 | this.saveComment(comment); 115 | } 116 | 117 | if (github.hasNextPage(res)) { 118 | this.nextCommentsPage(res); 119 | } else { 120 | log.info("Got last comments page", res.meta.link); 121 | } 122 | } 123 | 124 | nextCommentsPage(res) { 125 | setTimeout(() => { 126 | this.queue.add(() => { 127 | log.info('Github comments getting next page'); 128 | return github.getNextPage(res).then(this.commentsListHandler.bind(this)); 129 | }); 130 | }, 1000); 131 | } 132 | 133 | nextIssuePage(res) { 134 | setTimeout(() => { 135 | this.queue.add(() => { 136 | log.info('Github issues getting next page'); 137 | return github.getNextPage(res).then(this.issueListHandler.bind(this)); 138 | }); 139 | }, 1000); 140 | } 141 | 142 | saveIssue(issue) { 143 | this.queue.add(() => db.saveIssue(issue)); 144 | } 145 | 146 | issueListHandler(res) { 147 | log.info('Got issues', { 148 | 'rate-limit-remainign': res.meta['x-ratelimit-remaining'], 149 | count: res.data.length 150 | }); 151 | 152 | for (let gi of res.data) { 153 | var issue = this.transformIssueToElasticDoc(gi); 154 | this.saveIssue(issue); 155 | } 156 | 157 | if (github.hasNextPage(res)) { 158 | this.nextIssuePage(res); 159 | } else { 160 | log.info("Got last issues page", res.meta.link); 161 | } 162 | } 163 | 164 | transformIssueToElasticDoc(gi) { 165 | var issue = { 166 | number: gi.number, 167 | title: gi.title, 168 | state: gi.state, 169 | repo: this.options.repo, 170 | comments: gi.comments, 171 | labels: _.map(gi.labels, 'name'), 172 | milestone: _.get(gi, "milestone.title", null), 173 | created_at: gi.created_at, 174 | updated_at: gi.updated_at, 175 | closed_at: gi.closed_at, 176 | user_login: _.get(gi, "user.login", null), 177 | assignee: _.get(gi, 'assignee.login', null), 178 | is_pull_request: gi.pull_request !== undefined, 179 | id: gi.id, 180 | reactions_total: gi.reactions.total_count, 181 | reactions_plus1: gi.reactions['+1'], 182 | reactions_minus1: gi.reactions['-1'], 183 | reactions_heart: gi.reactions.heart, 184 | created_iso_week_day: moment(gi.created_at).isoWeekday(), 185 | }; 186 | 187 | // calculate time opnen 188 | if (gi.closed_at) { 189 | var span = moment.duration(moment(gi.closed_at).diff(moment(gi.created_at))); 190 | issue.minutes_open = span.asMinutes(); 191 | } 192 | 193 | if (gi.closed_by) { 194 | issue.closed_by = gi.closed_by.login; 195 | } 196 | 197 | 198 | return issue; 199 | } 200 | } 201 | 202 | module.exports = RepoSync; 203 | -------------------------------------------------------------------------------- /test/reposync_tests.js: -------------------------------------------------------------------------------- 1 | const assert = require('assert'); 2 | const RepoSync = require('../reposync'); 3 | 4 | describe('RepoSync', function() { 5 | 6 | describe('creating a new repo sync', function() { 7 | 8 | it('should init options', function() { 9 | const rs = new RepoSync({ 10 | queue: {}, 11 | repo: 'grafana/grafana' 12 | }); 13 | 14 | assert.equal(rs.owner, 'grafana'); 15 | assert.equal(rs.repo, 'grafana'); 16 | }); 17 | }); 18 | 19 | }); 20 | --------------------------------------------------------------------------------