├── .gitignore ├── .jshintrc ├── LICENSE ├── Procfile ├── README.md ├── app.js ├── cleanDb.js ├── config ├── development.config.js ├── index.js ├── production.config.js └── staging.config.js ├── monitor.js ├── newrelic.js ├── package.json ├── runTests.js ├── source ├── db │ └── index.js ├── elastic │ └── index.js ├── engine │ ├── connectors │ │ ├── behance.js │ │ ├── dribbble.js │ │ ├── facebook.js │ │ ├── flickr.js │ │ ├── gist.js │ │ ├── github.js │ │ ├── index.js │ │ ├── instagram.js │ │ ├── pocket.js │ │ ├── stackoverflow.js │ │ ├── tumblr.js │ │ ├── twitter.js │ │ ├── vimeo.js │ │ ├── vk.js │ │ └── youtube.js │ ├── disableNetworks.js │ ├── executor.js │ ├── handleUnexpected.js │ ├── scheduleTo.js │ └── scheduler.js ├── models │ ├── items.js │ ├── networks.js │ └── users.js └── utils │ ├── helpers.js │ ├── logger.js │ └── memwatch.js └── test ├── connectors ├── github.specs.js ├── replies │ ├── github.connector.init.json │ ├── github.connector.normal.json │ ├── github.connector.normal.new.json │ ├── github.connector.normal.nonew.json │ ├── stackoverflow.connector.init.json │ ├── stackoverflow.connector.init.json.gz │ ├── stackoverflow.connector.new.json │ ├── stackoverflow.connector.new.json.gz │ ├── stackoverflow.connector.normal.json │ ├── stackoverflow.connector.normal.json.gz │ ├── twitter.connector.init.json │ ├── twitter.connector.normal.json │ └── twitter.connector.second.json ├── stackoverflow.specs.js └── twitter.specs.js ├── engine ├── balancer.spec.js └── scheduleTo.specs.js ├── fakes ├── logger.js └── moment.js └── mocha.opts /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | *.log -------------------------------------------------------------------------------- /.jshintrc: -------------------------------------------------------------------------------- 1 | { 2 | "predef": [ 3 | "after", 4 | "afterEach", 5 | "afterEach", 6 | "before", 7 | "beforeEach", 8 | "chai", 9 | "describe", 10 | "expect", 11 | "iit", 12 | "it", 13 | "runs", 14 | "sinon", 15 | "spyOn", 16 | "waits", 17 | "waitsFor", 18 | "xit", 19 | "xdescribe", 20 | "define", 21 | "module", 22 | "exports", 23 | "require", 24 | "window", 25 | "process", 26 | "console", 27 | "__dirname", 28 | "Buffer" 29 | ], 30 | 31 | "asi" : false, 32 | "bitwise" : true, 33 | "boss" : false, 34 | "browser" : true, 35 | "curly" : true, 36 | "debug": false, 37 | "devel": false, 38 | "eqeqeq": true, 39 | "evil": false, 40 | "expr": true, 41 | "forin": false, 42 | "immed": true, 43 | "latedef" : false, 44 | "laxbreak": false, 45 | "multistr": true, 46 | "newcap": true, 47 | "noarg": true, 48 | "noempty": false, 49 | "nonew": true, 50 | "onevar": false, 51 | "plusplus": false, 52 | "regexp": false, 53 | "strict": false, 54 | "globalstrict": true, 55 | "sub": false, 56 | "trailing" : true, 57 | "undef": true, 58 | "unused": "vars" 59 | } -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2013 Alexander Beletsky 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy 4 | of this software and associated documentation files (the "Software"), to deal 5 | in the Software without restriction, including without limitation the rights 6 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | copies of the Software, and to permit persons to whom the Software is 8 | furnished to do so, subject to the following conditions: 9 | 10 | The above copyright notice and this permission notice shall be included in 11 | all copies or substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | THE SOFTWARE. -------------------------------------------------------------------------------- /Procfile: -------------------------------------------------------------------------------- 1 | web: node monitor.js -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Collector 2 | 3 | Collector is core component of current *Likeastore* architecture. It gathers application data through different open API's. Initially it was just dumb script of calling API wrappers, processing data and storing it to MongoDB, but after awhile it turned to be an interesting piece of code. 4 | 5 | ## You don't need API wrapper 6 | 7 | The point is. In many cases of accessing some HTTP API you don't need any wrapper library. I personally was always looking for some ready to take wrapper, since I thought it would be easier. It still true, in some way. Now I see that all you actually need is [request](https://github.com/mikeal/request). 8 | 9 | [request](https://github.com/mikeal/request) is brilliant library. It simplifies access to HTTP recourses, having very minimal and nice interface. 10 | 11 | I'm not saying wrappers are wrong, but there are few problems with them, as for me: 12 | 13 | * A lot of them (could be difficult to pick up) 14 | * Not up to date (new version of API is out and previous in unsupported) 15 | * Brings some overhead (you need to access few API method, but getting library) 16 | 17 | So, in my case wrappers are wrong. Going with pure `request` object, was much more easier and fun. 18 | 19 | ## API's are different 20 | 21 | API's differs from each other, especially in terms of authorization and rate limits. 22 | 23 | We had to integrate 3 API: Github, Twitter, Google+. All of them claims OAuth support, but all of them implement it differently (bit differently). 24 | 25 | Fortunately there are 2 things that come to help [passport](http://passportjs.org/) and.. again [request](https://github.com/mikeal/request). First helps OAuth integration and token retrieval, second has built-in OAuth support, so as you understood the meanings of all OAuth signature attributes, you can go. 26 | 27 | ## More than HTTP requests 28 | 29 | All famous API are restrictive. Meant, all of them have *quotes* on a number of requests you can issue to HTTP for a given period of time. If you need to collect data for a bunch of users, you can easily overuse limits and be blocked. So, there should be a kind of scheduler that would respect quotes and do optimal number of calls. 30 | 31 | Each *connector* is function with state. State contains information about current connector mode and data to access API optimally. The mode, is kind of knowledge that if connector is running *initially* it have to collect all possible data and after switch to *normal* mode and request the latest one. 32 | 33 | ## Basic pieces 34 | 35 | So, collector contains of few pieces: `engine`, `tasksBuilder`, `connectors`, `stater` and few `utils`. 36 | 37 | ### Engine 38 | 39 | [source/engine/index.js](source/engine/index.js) is the heart of collector. It creates a lists of *tasks* that have to be executed on next engine run, executes them and repeats. 40 | 41 | It might sound as a simple task.. and it's quite simple. But in synchronous world, not asynchronous one. 42 | 43 | In asynchronous world you could not run `while(true)` cycle and do some job inside. Node.js single threaded, event-log driven application, based on asynchronous IO. If you run `while(true)` you would simply consume all CPU, not allowing async operations to conclude. 44 | 45 | ["How to run infinite number of tasks in Node.js application?"](http://stackoverflow.com/questions/15886096/infinite-execution-of-tasks-for-nodejs-application]). 46 | 47 | Stackoverflow, was helpful as usual. Very useful [answer](http://stackoverflow.com/a/15886384/386751) pointing to [async.queue](https://github.com/caolan/async/#queue). That's very powerful control flow algorithm. It allows to put array of tasks for execution, each are executing one-by-one and then eventing then all tasks are done. After that event you can run procedure again. 48 | 49 | ## Tasks builder 50 | 51 | [source/engine/tasks/builder.js](source/engine/tasks/index.js) is the one that creates list of tasks. It knows about quotas. So, it tracks could the current task be run in exact type of moment or better to wait, to do not exceed rate limits. 52 | 53 | It uses another great component, called [moment.js](http://momentjs.com/) - the best library to work with dates. 54 | 55 | ## Task 56 | 57 | The actual task is [executor](/source/engine/connectors/factory.js) function. Executor owns the connector instance, reads it's state from db, runs it. After it updates the state (since connector changes it's state almost on each run) and storing the gathered data. 58 | 59 | The useful library here [mongojs](https://github.com/gett/mongojs), simplifies MongoDB access as much as it's even possible. 60 | 61 | ## Connector 62 | 63 | Each connector implement access to API. All of them has some aspects, briefly go through. 64 | 65 | ### Github API 66 | 67 | [connectors/github.js](/source/engine/connectors/github.js) extracts all repositories starred by user. 68 | 69 | To authorize, github provides `access_token`. `access_token` is received after user [approves](http://developer.github.com/v3/oauth/#web-application-flow) the access the application to users data. For github API all you need to do, just to add `access_token` as query parameter to request. Simple and easy. 70 | 71 | Github API have paging and support hypermedia, response headers contains links for further recourse. 72 | 73 | ### Twitter API 74 | 75 | [Twitter REST API](https://dev.twitter.com/docs/api) is truly powerful, but a bit harder to use. 76 | 77 | First of all, it has very tough rate limits. Further more, it distinguish between 'user access API' and 'application access API' (or combination of 'application' the access on behalf of user). 78 | 79 | As we need to collect data for a lot of users, 'application access API' would rather quickly go out of rates and `taskBuilder` would have to always wait. 80 | 81 | [connector/twitter.js](/source/engine/connectors/twitter.js) implements access to Twitter API. 82 | 83 | Instead of simple usage of token, twitter uses more complicated OAuth request [signature](https://dev.twitter.com/docs/auth/authorizing-request). I was very unsure who to generate one, but fortunately `request` supports OAuth [signature](https://github.com/mikeal/request#oauth-signing). 84 | 85 | ### Google+ API 86 | 87 | [Google+ API](https://developers.google.com/+/api/) was next. We wanted to get posts used '+1ed'. 88 | 89 | Very quickly it became obvious, that it is not possible. Google+ simple does not provide API for that. 90 | 91 | That's a shame and that was the reason to drop [connectors/googleplus.js](/source/engine/connectors/googleplus.js). 92 | 93 | Tried to find a replacement for that service. And the answer was really close - [stackoverflow.com](http://stackoverflow.com/) is the place where questions and [favorites](http://stackoverflow.com/users/386751/alexanderb?tab=favorites) born, and keeping track on them is good idea for product. 94 | 95 | ### Stackoverflow API 96 | 97 | [Stackoverflow API](http://api.stackoverflow.com/1.0/usage) appeared to be as good as Github one. OAuth support, but without OAuth signature, easy register of application etc. 98 | 99 | [connector/stackoverflow.js](/source/engine/connectors/stackoverflow.js) implements access to Stackoverlow. 100 | 101 | Documentation is nice, I quickly found the way to access users favorites. But, I was really suprised that I issued HTTP request and got response. 102 | 103 | ``` 104 | Additionally, all API responses are GZIP'd. The Content-Encoding header is always set, but some proxies will strip this out. The proper way to decode API responses can be found here. 105 | ``` 106 | 107 | Everything is GZIP'd. 108 | 109 | This time `request.js` did not appear as magic stick. It did not provide gzip support from the box. Fortunately, I found this [blog post](http://apptakk.blogspot.com/2012/05/handling-gzip-encoded-http-responses-in.html). Using of MemoryStream worked really nice, I was able to decompress requests and read data from it. 110 | 111 | ## Stater 112 | 113 | [source/utils/stater.js](/source/utils/stater.js) is utility to move connector from one state to another. It uses the list of `stateChages` which contains the function of checking state change condition and actual change functor. 114 | 115 | ## Utils 116 | 117 | [source/utils/logger.js](/source/utils/logger.js) will help you for logging different application activities. To make it more readable from console, it uses [colors](https://github.com/marak/colors.js/) library. 118 | 119 | For logging connectors stuff, you can use special factory method: 120 | 121 | ```js 122 | var log = logger.connector('github'); 123 | log.info('retrieved ' + data.length + ' stars'); 124 | ``` 125 | 126 | It also contains few functions that I used from SO answers (links are inside). One that helped me to substite ugly-looking `for-cycle` another one that helps to head with Twitter API ids [problem](http://stackoverflow.com/questions/9717488/using-since-id-and-max-id-in-twitter-api) in JavaScript. 127 | 128 | ## Tests 129 | 130 | All (almost all) aspects of `collectors` functionality are covered by [unit tests](/test). 131 | 132 | There are few things that helped me to create those tests. I use [Mocha](https://github.com/visionmedia/mocha) as primary test framework, to run the tests: 133 | 134 | ``` 135 | $ mocha 136 | ``` 137 | 138 | Connectors are issuing HTTP request and unit tests have to mock it. [Nock](https://github.com/flatiron/nock) is a simple and nice library for that. Good thing is that's possible to read response content from files. All tests responses are placed in [/test/replies](/test/replies) folder. To mock HTTP request, 139 | 140 | ```js 141 | beforeEach(function (done) { 142 | nock('http://api.stackoverflow.com') 143 | .get('/1.1/users/12345/favorites?access_token=fakeToken&pagesize=100&sort=creation&page=1') 144 | .replyWithFile(200, __dirname + '/replies/stackoverflow.connector.init.json.gz'); 145 | 146 | connector(state, function (err, state, favorites) { 147 | updatedState = state; 148 | returnedFavorites = favorites; 149 | 150 | done(); 151 | }); 152 | }); 153 | ``` 154 | 155 | Some modules, as connector or task builder are using logger inside. If you run the test the output would be polluted with non-required logs. Traditionally, it is solved by injection the dependency in module and being able to substitute with stub. [Rewire](https://github.com/jhnns/rewire) is a module to help. 156 | 157 | All connectors tests are stubbing logger, 158 | 159 | ```js 160 | beforeEach(function () { 161 | connector = rewire('./../source/engine/connectors/stackoverflow'); 162 | connector.__set__('logger', loggerFake); 163 | }); 164 | ``` 165 | 166 | # License 167 | 168 | MIT License -------------------------------------------------------------------------------- /app.js: -------------------------------------------------------------------------------- 1 | var argv = require('optimist').argv; 2 | 3 | var env = process.env.NODE_ENV = process.env.NODE_ENV || 'development'; 4 | var mode = process.env.COLLECTOR_MODE = process.env.COLLECTOR_MODE || argv.mode || 'normal'; 5 | 6 | require('./source/utils/memwatch'); 7 | 8 | var http = require('http'); 9 | var https = require('https'); 10 | http.globalAgent.maxSockets = 128; 11 | https.globalAgent.maxSockets = 128; 12 | 13 | var config = require('./config'); 14 | var logger = require('./source/utils/logger'); 15 | var scheduler = require('./source/engine/scheduler'); 16 | var appName = 'collector-' + process.env.COLLECTOR_MODE; 17 | 18 | logger.success(appName + ' started env:' + env + ' mongodb: ' + config.connection + ' mode: ' + mode); 19 | scheduler(mode).run(); -------------------------------------------------------------------------------- /cleanDb.js: -------------------------------------------------------------------------------- 1 | var db = require('./source/db/dbConnector').db; 2 | 3 | console.log('cleaning up likestore db...'); 4 | db.dropDatabase(); 5 | db.close(function (err) { 6 | if (err) { 7 | return console.log('clean up failed: ' + err); 8 | } 9 | 10 | return console.log('likestore db is clean now.'); 11 | }); -------------------------------------------------------------------------------- /config/development.config.js: -------------------------------------------------------------------------------- 1 | var config = { 2 | connection: 'mongodb://localhost:27017/likeastoredb', 3 | options: { auto_reconnect: true }, 4 | 5 | applicationUrl: 'http://localhost:3001', 6 | siteUrl: 'http://localhost:3000', 7 | 8 | elastic: { 9 | host: { 10 | host: 'localhost', 11 | port: 9200 12 | }, 13 | 14 | requestTimeout: 5000 15 | }, 16 | 17 | // api keys 18 | services: { 19 | github: { 20 | appId: 'dc3c7a7050dccee24ed3', 21 | appSecret: 'c18dde90f5e928a39b0f0432d5125a3e0a31a23d' 22 | }, 23 | 24 | twitter: { 25 | consumerKey: 'dgwuxgGb07ymueGJF0ug', 26 | consumerSecret: 'eusoZYiUldYqtI2SwK9MJNbiygCWOp9lQX7i5gnpWU' 27 | }, 28 | 29 | facebook: { 30 | appId: '394024317362081', 31 | appSecret: 'bc86f2ab9afcb1227227146e5ea9ad44' 32 | }, 33 | 34 | stackoverflow: { 35 | clientId: '1533', 36 | clientKey: 'J2wyheThU5jYFiOpGG22Eg((', 37 | clientSecret: 'KOCBFY4OUP6OE7Q1xNw1wA((' 38 | }, 39 | 40 | vimeo: { 41 | clientId: 'd445a0de20a3b178b0422ad0c6d5891bdfd00b97', 42 | clientSecret: 'e8e0008413ae1d1ed3e45d8c89d7943ad3937167' 43 | }, 44 | 45 | youtube: { 46 | clientId: '955769903356-5f1407fo9efvljm3hhl5b8mbhos61blq.apps.googleusercontent.com', 47 | clientSecret: 'QtlyTnCusfX7G7fbjaEkdmHK' 48 | }, 49 | 50 | behance: { 51 | clientId: 'JyyJsEZRbcqTXcukjnq8ivQMb7BfAIUd', 52 | clientSecret: 'L2s8uQl3s7G5uy2ECeRp9dHeWuyA6mrj' 53 | }, 54 | 55 | pocket: { 56 | consumerKey: '24341-1a1bc9c0ad0f3ffa9eb3194b' 57 | }, 58 | 59 | tumblr: { 60 | consumerKey: '6vUnFztIzNd6ISG8kBn7UyhGkHA8a49UjXUx9rCYbrWBnbFZBr', 61 | consumerSecret: 'pnUrbwgmLHubWqaBxRIzD216FxAq8wZCzf2hXysL9huV1Sfq9R' 62 | }, 63 | 64 | flickr: { 65 | consumerKey: 'de1be7a4d307073deca73ad46d9faf40', 66 | consumerSecret: '6103498d0db1c48a' 67 | } 68 | }, 69 | 70 | mandrill: { 71 | token: '2kXX0stV1Hf56y9DYZts3A' 72 | }, 73 | 74 | logentries: { 75 | token: null 76 | }, 77 | 78 | newrelic: { 79 | application: 'likeastore-collector-dev-' + process.env.COLLECTOR_MODE, 80 | licenseKey: null 81 | }, 82 | 83 | logging: { 84 | level: 'debug' 85 | }, 86 | 87 | collector: { 88 | // scheduler cycle 89 | schedulerRestartShort: 1000, 90 | 91 | // scheduler cycle (in case of zero tasks executed previously) 92 | schedulerRestartLong: 1000, 93 | 94 | // after collector got to normal mode, next scheduled run in 10 sec 95 | nextNormalRunAfter: 1000 * 10, 96 | 97 | // after collector got to rateLimit mode, next scheduled run in 15 min 98 | nextRateLimitRunAfter: 1000 * 60 * 15, 99 | 100 | // initial mode quotes 101 | quotes: { 102 | facebook: { 103 | runAfter: 5000 104 | }, 105 | 106 | github: { 107 | runAfter: 5000 108 | }, 109 | 110 | gist: { 111 | runAfter: 5000 112 | }, 113 | 114 | twitter: { 115 | runAfter: 60000 116 | }, 117 | 118 | stackoverflow: { 119 | runAfter: 5000 120 | }, 121 | 122 | vimeo: { 123 | runAfter: 5000 124 | }, 125 | 126 | youtube: { 127 | runAfter: 5000 128 | }, 129 | 130 | dribbble: { 131 | runAfter: 5000 132 | }, 133 | 134 | behance: { 135 | runAfter: 5000 136 | }, 137 | 138 | pocket: { 139 | runAfter: 5000 140 | }, 141 | 142 | tumblr: { 143 | runAfter: 5000 144 | }, 145 | 146 | instagram: { 147 | runAfter: 5000 148 | }, 149 | 150 | flickr: { 151 | runAfter: 5000 152 | } 153 | }, 154 | 155 | request: { 156 | timeout: 5000 157 | } 158 | } 159 | }; 160 | 161 | module.exports = config; -------------------------------------------------------------------------------- /config/index.js: -------------------------------------------------------------------------------- 1 | var util = require('util'); 2 | 3 | var env = process.env.NODE_ENV || 'development'; 4 | var config = util.format('./%s.config.js', env); 5 | 6 | module.exports = require(config); -------------------------------------------------------------------------------- /config/production.config.js: -------------------------------------------------------------------------------- 1 | var config = { 2 | connection: process.env.MONGO_CONNECTION, 3 | options: { auto_reconnect: true }, 4 | 5 | applicationUrl: 'https://app.likeastore.com', 6 | siteUrl: 'https://likeastore.com', 7 | 8 | elastic: { 9 | host: { 10 | protocol: 'https', 11 | host: 'search.likeastore.com', 12 | port: 443, 13 | query: { 14 | access_token: process.env.ELASTIC_ACCESS_TOKEN 15 | } 16 | }, 17 | 18 | requestTimeout: 5000 19 | }, 20 | 21 | // api keys 22 | services: { 23 | github: { 24 | appId: process.env.GITHUB_APP_ID, 25 | appSecret: process.env.GITHUB_APP_SECRET 26 | }, 27 | 28 | twitter: { 29 | consumerKey: process.env.TWITTER_CONSUMER_KEY, 30 | consumerSecret: process.env.TWITTER_CONSUMER_SECRET 31 | }, 32 | 33 | facebook: { 34 | appId: process.env.FACEBOOK_APP_ID, 35 | appSecret: process.env.FACEBOOK_APP_SECRET 36 | }, 37 | 38 | stackoverflow: { 39 | clientId: process.env.STACKOVERFLOW_CLIENT_ID, 40 | clientKey: process.env.STACKOVERFLOW_CLIENT_KEY, 41 | clientSecret: process.env.STACKOVERFLOW_CLIENT_SECRET 42 | }, 43 | 44 | vimeo: { 45 | clientId: process.env.VIMEO_CLIENT_ID, 46 | clientSecret: process.env.VIMEO_CLIENT_SECRET 47 | }, 48 | 49 | youtube: { 50 | clientId: process.env.YOUTUBE_CLIENT_ID, 51 | clientSecret: process.env.YOUTUBE_CLIENT_SECRET 52 | }, 53 | 54 | behance: { 55 | clientId: process.env.BAHANCE_CLIENT_ID, 56 | clientSecret: process.env.BAHANCE_CLIENT_SECRET 57 | }, 58 | 59 | pocket: { 60 | consumerKey: process.env.POCKET_CONSUMER_KEY 61 | }, 62 | 63 | tumblr: { 64 | consumerKey: process.env.TUMBLR_CONSUMER_KEY, 65 | consumerSecret: process.env.TUMBLR_CONSUMER_SECRET, 66 | }, 67 | 68 | instagram: { 69 | clientId: process.env.INSTAGRAM_CLIENT_ID, 70 | clientSecret: process.env.INSTAGRAM_CLIENT_SECRET 71 | }, 72 | 73 | flickr: { 74 | consumerKey: process.env.FLICKR_CONSUMER_KEY, 75 | consumerSecret: process.env.FLICKR_CONSUMER_SECRET 76 | } 77 | }, 78 | 79 | mandrill: { 80 | token: process.env.MANDRILL_TOKEN 81 | }, 82 | 83 | logentries: { 84 | token: process.env.LOGENTRIES_TOKEN 85 | }, 86 | 87 | newrelic: { 88 | application: 'likeastore-collector-' + process.env.COLLECTOR_MODE, 89 | licenseKey: 'e5862474ee62b99898c861dddfbfa8a89ac54f49' 90 | }, 91 | 92 | logging: { 93 | level: 'err' 94 | }, 95 | 96 | collector: { 97 | // scheduler cycle 98 | schedulerRestart: 1000, 99 | 100 | // scheduler cycle (in case of zero tasks executed previously) 101 | schedulerRestartLong: 1000 * 60, 102 | 103 | // after collector got to normal mode, next scheduled run in 15 mins 104 | nextNormalRunAfter: 1000 * 60 * 15, 105 | 106 | // after collector got to rateLimit mode, next scheduled run in hour 107 | nextRateLimitRunAfter: 1000 * 60 * 60, 108 | 109 | // initial mode quotes 110 | quotes: { 111 | facebook: { 112 | runAfter: 5000 113 | }, 114 | 115 | github: { 116 | runAfter: 5000 117 | }, 118 | 119 | gist: { 120 | runAfter: 5000 121 | }, 122 | 123 | twitter: { 124 | runAfter: 60000 125 | }, 126 | 127 | stackoverflow: { 128 | runAfter: 5000 129 | }, 130 | 131 | vimeo: { 132 | runAfter: 5000 133 | }, 134 | 135 | youtube: { 136 | runAfter: 5000 137 | }, 138 | 139 | dribbble: { 140 | runAfter: 5000 141 | }, 142 | 143 | behance: { 144 | runAfter: 5000 145 | }, 146 | 147 | pocket: { 148 | runAfter: 5000 149 | }, 150 | 151 | tumblr: { 152 | runAfter: 5000 153 | }, 154 | 155 | instagram: { 156 | runAfter: 5000 157 | }, 158 | 159 | flickr: { 160 | runAfter: 5000 161 | } 162 | }, 163 | 164 | request: { 165 | timeout: 10000 166 | } 167 | } 168 | }; 169 | 170 | module.exports = config; -------------------------------------------------------------------------------- /config/staging.config.js: -------------------------------------------------------------------------------- 1 | var config = { 2 | connection: process.env.MONGO_CONNECTION, 3 | options: { auto_reconnect: true }, 4 | 5 | applicationUrl: 'https://app-stage.likeastore.com', 6 | siteUrl: 'https://stage.likeastore.com', 7 | 8 | elastic: { 9 | host: { 10 | protocol: 'https', 11 | host: 'stage.likeastore.com', 12 | port: 443, 13 | query: { 14 | access_token: process.env.ELASTIC_ACCESS_TOKEN 15 | } 16 | }, 17 | 18 | requestTimeout: 5000 19 | }, 20 | 21 | // api keys 22 | services: { 23 | github: { 24 | appId: '47974c5d6fefbe07881e', 25 | appSecret: 'f1008ace415b3892bd36ef97443452a39dd7c29f' 26 | }, 27 | 28 | twitter: { 29 | consumerKey: 'XDCQAahVo1EjhFqGoh5c2Q', 30 | consumerSecret: 'LppQuUU5FDTRwFJRwnlhfGj3IMDDTKmVCUm1JTHkA' 31 | }, 32 | 33 | facebook: { 34 | appId: '554634024574376', 35 | appSecret: 'a8d2c5e643b67cdf80ed8b8832634b2c' 36 | }, 37 | 38 | stackoverflow: { 39 | clientId: '1801', 40 | clientKey: 'L)KUpw85QEW105j43oik8g((', 41 | clientSecret: 'DadJ5kAh3YWlj0wv7EHqDg((' 42 | }, 43 | 44 | vimeo: { 45 | clientId: 'c83157e81d0bd1f4a20ffed96c1b9f8d4d97a9dd', 46 | clientSecret: '34a99e44d55d78cad9e842caf376501a9547028d' 47 | }, 48 | 49 | youtube: { 50 | clientId: '448353031199-vm7a5vrs3m0frtm7rrpnnsson3cha3a2.apps.googleusercontent.com', 51 | clientSecret: 'nag018PB5ijVec9ZWcpsnyRd' 52 | }, 53 | 54 | behance: { 55 | clientId: 'uzBD0GX9Q0CwE3tE9hw69wTpI6P1VBu1', 56 | clientSecret: 'baScisHRs3Af11.E7OlC4q2jyrD7fj._' 57 | }, 58 | 59 | pocket: { 60 | consumerKey: '24374-c7057a9cd0bb40642cd4ff97' 61 | }, 62 | 63 | tumblr: { 64 | consumerKey: 'KajWjUFEKXe9yuMFgvYdE5RYXoGGHkq6NLOgUU98eZCOzy3oH3', 65 | consumerSecret: 'IOh4nVo7IW05r9xm4efjOKiOadoBdTxfLsIsibOpfKnfrS7nCm' 66 | }, 67 | 68 | instagram: { 69 | clientId: '12df39bad4f5459d92d014393a5df29f', 70 | clientSecret: '60cf98a5b54b4abab22d6c0b9b0f088f' 71 | }, 72 | 73 | flickr: { 74 | consumerKey: '6eecff3104c9001f4ae0febcbb78f652', 75 | consumerSecret: 'b3eef2a9981d9894' 76 | } 77 | }, 78 | 79 | mandrill: { 80 | token: '2kXX0stV1Hf56y9DYZts3A' 81 | }, 82 | 83 | logentries: { 84 | token: 'ee7930a7-7950-491f-b3aa-6657a928dbdb' 85 | }, 86 | 87 | newrelic: { 88 | application: 'likeastore-collector-stage-' + process.env.COLLECTOR_MODE, 89 | licenseKey: 'e5862474ee62b99898c861dddfbfa8a89ac54f49' 90 | }, 91 | 92 | logging: { 93 | level: 'debug' 94 | }, 95 | 96 | collector: { 97 | // scheduler cycle 98 | schedulerRestart: 1000, 99 | 100 | // scheduler cycle (in case of zero tasks executed previously) 101 | schedulerRestartLong: 1000 * 60, 102 | 103 | // after collector got to normal mode, next scheduled run in 15 mins 104 | nextNormalRunAfter: 1000 * 60 * 15, 105 | 106 | // after collector got to rateLimit mode, next scheduled run in hour 107 | nextRateLimitRunAfter: 1000 * 60 * 60, 108 | 109 | // initial mode quotes 110 | quotes: { 111 | facebook: { 112 | runAfter: 5000 113 | }, 114 | 115 | github: { 116 | runAfter: 5000 117 | }, 118 | 119 | gist: { 120 | runAfter: 5000 121 | }, 122 | 123 | twitter: { 124 | runAfter: 60000 125 | }, 126 | 127 | stackoverflow: { 128 | runAfter: 5000 129 | }, 130 | 131 | vimeo: { 132 | runAfter: 5000 133 | }, 134 | 135 | youtube: { 136 | runAfter: 5000 137 | }, 138 | 139 | dribbble: { 140 | runAfter: 5000 141 | }, 142 | 143 | behance: { 144 | runAfter: 5000 145 | }, 146 | 147 | pocket: { 148 | runAfter: 5000 149 | }, 150 | 151 | tumblr: { 152 | runAfter: 5000 153 | }, 154 | 155 | instagram: { 156 | runAfter: 5000 157 | }, 158 | 159 | flickr: { 160 | runAfter: 5000 161 | } 162 | }, 163 | 164 | request: { 165 | timeout: 10000 166 | } 167 | } 168 | }; 169 | 170 | module.exports = config; -------------------------------------------------------------------------------- /monitor.js: -------------------------------------------------------------------------------- 1 | var respawn = require('respawn'); 2 | var util = require('util'); 3 | var logger = require('./source/utils/logger'); 4 | 5 | function monitor(mode) { 6 | var proc = respawn(['node', 'app.js', '--mode', mode], { 7 | cwd: '.', 8 | maxRestarts: 10, 9 | sleep: 1000, 10 | }); 11 | 12 | proc.on('spawn', function () { 13 | util.print('application monitor started...'); 14 | }); 15 | 16 | proc.on('exit', function (code, signal) { 17 | logger.fatal({msg: 'process exited, code: ' + code + ' signal: ' + signal + ' mode: ' + mode}); 18 | }); 19 | 20 | proc.on('stdout', function (data) { 21 | util.print(data.toString()); 22 | }); 23 | 24 | proc.on('stderr', function (data) { 25 | logger.error({msg: 'process error', data: data.toString()}); 26 | }); 27 | 28 | return proc; 29 | } 30 | 31 | [monitor('initial'), monitor('normal')].forEach(function (m) { 32 | m.start(); 33 | }); 34 | 35 | -------------------------------------------------------------------------------- /newrelic.js: -------------------------------------------------------------------------------- 1 | /** 2 | * New Relic agent configuration. 3 | * 4 | * See lib/config.defaults.js in the agent distribution for a more complete 5 | * description of configuration variables and their potential values. 6 | */ 7 | var config = require('./config'); 8 | 9 | exports.config = { 10 | /** 11 | * Array of application names. 12 | */ 13 | app_name : [config.newrelic.application], 14 | /** 15 | * Your New Relic license key. 16 | */ 17 | license_key : config.newrelic.licenseKey, 18 | logging : { 19 | /** 20 | * Level at which to log. 'trace' is most useful to New Relic when diagnosing 21 | * issues with the agent, 'info' and higher will impose the least overhead on 22 | * production applications. 23 | */ 24 | level : 'trace' 25 | } 26 | }; 27 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "likeastore-collector", 3 | "version": "0.0.14", 4 | "private": true, 5 | "scripts": { 6 | "start": "node monitor.js", 7 | "test": "node runTests.js --reporter spec", 8 | "watch": "nodemon --watch source --watch node_modules monitor.js", 9 | "watch-test": "nodemon --watch source --watch test runTests.js --reporter min" 10 | }, 11 | "dependencies": { 12 | "mongojs": "~0.9.8", 13 | "underscore": "~1.4.4", 14 | "async": "~0.2.10", 15 | "chai": "~1.5.0", 16 | "moment": "~2.0.0", 17 | "colors": "~0.6.2", 18 | "nock": "~0.18.2", 19 | "request": "~2.21.0", 20 | "rewire": "~1.1.3", 21 | "node-logentries": "~0.1.2", 22 | "optimist": "~0.6.0", 23 | "memwatch": "~0.2.2", 24 | "respawn": "~0.1.8", 25 | "newrelic": "~1.2.0", 26 | "mocha": "~1.11.0", 27 | "elasticsearch": "~2.1.4" 28 | }, 29 | "engines": { 30 | "node": "~0.10.x" 31 | }, 32 | "subdomain": "likeastore-collector", 33 | "devDependencies": { 34 | "mocha": "~1.11.0" 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /runTests.js: -------------------------------------------------------------------------------- 1 | require('./node_modules/mocha/bin/_mocha'); -------------------------------------------------------------------------------- /source/db/index.js: -------------------------------------------------------------------------------- 1 | var mongo = require('mongojs'); 2 | 3 | var connection; 4 | 5 | module.exports = function (config) { 6 | if (!connection) { 7 | connection = mongo.connect(config.connection, ['users', 'networks', 'items', 'subscribers', 'tests']); 8 | if (!connection) { 9 | throw new Error('could not connect to ' + config.connection); 10 | } 11 | } 12 | 13 | return connection; 14 | }; -------------------------------------------------------------------------------- /source/elastic/index.js: -------------------------------------------------------------------------------- 1 | var elasticsearch = require('elasticsearch'); 2 | 3 | module.exports = function (config) { 4 | var client = elasticsearch.Client(config.elastic); 5 | 6 | return client; 7 | }; -------------------------------------------------------------------------------- /source/engine/connectors/behance.js: -------------------------------------------------------------------------------- 1 | var request = require('request'); 2 | var moment = require('moment'); 3 | var util = require('util'); 4 | 5 | var scheduleTo = require('../scheduleTo'); 6 | var logger = require('./../../utils/logger'); 7 | var handleUnexpected = require('../handleUnexpected'); 8 | var config = require('../../../config'); 9 | 10 | var API = 'https://www.behance.net/v2'; 11 | 12 | function connector(state, user, callback) { 13 | var accessToken = state.accessToken; 14 | var username = state.username; 15 | var log = logger.connector('behance'); 16 | 17 | if (!accessToken) { 18 | return callback('missing accessToken for user: ' + state.user); 19 | } 20 | 21 | if (!username) { 22 | return callback('missing username for user: ' + state.user); 23 | } 24 | 25 | initState(state); 26 | 27 | log.info('prepearing request in (' + state.mode + ') mode for user: ' + state.user); 28 | 29 | var uri = formatRequestUri(accessToken, username, state); 30 | var headers = { 'Content-Type': 'application/json', 'User-Agent': 'likeastore/collector'}; 31 | 32 | request({uri: uri, headers: headers, timeout: config.collector.request.timeout, json: true}, function (err, response, body) { 33 | if (failed(err, response, body)) { 34 | return handleUnexpected(response, body, state, err, function (err) { 35 | callback (err, state); 36 | }); 37 | } 38 | 39 | return handleResponse(response, body); 40 | }); 41 | 42 | function failed(err, response, body) { 43 | return err || response.statusCode !== 200 || !body; 44 | } 45 | 46 | function initState(state) { 47 | if (!state.mode) { 48 | state.mode = 'initial'; 49 | } 50 | 51 | if (!state.errors) { 52 | state.errors = 0; 53 | } 54 | 55 | if (state.mode === 'initial' && !state.page) { 56 | state.page = 1; 57 | } 58 | 59 | if (state.mode === 'rateLimit') { 60 | state.mode = state.prevMode; 61 | } 62 | } 63 | 64 | function formatRequestUri(accessToken, username, state) { 65 | var base = util.format('%s/users/%s/appreciations?access_token=%s', API, username, accessToken); 66 | return state.mode === 'initial' && state.page ? 67 | util.format('%s&page=%s', base, state.page) : 68 | base; 69 | } 70 | 71 | function handleResponse(response, body) { 72 | var rateLimit = response.statusCode === 429 ? 0 : 9999; 73 | var appreciations = body.appreciations; 74 | 75 | log.info('rate limit remaining: ' + rateLimit + ' for user: ' + state.user); 76 | 77 | if (!Array.isArray(appreciations)) { 78 | return handleUnexpected(response, body, state, function (err) { 79 | callback(err, scheduleTo(updateState(state, [], rateLimit, true))); 80 | }); 81 | } 82 | 83 | var stars = appreciations.map(function (r) { 84 | return { 85 | itemId: r.project.id.toString(), 86 | idInt: r.project.id, 87 | user: state.user, 88 | userData: user, 89 | created: moment.unix(r.project.created_on).toDate(), 90 | title: r.project.name, 91 | authorName: first(r.project.owners).display_name, 92 | authorUrl: first(r.project.owners).url, 93 | avatarUrl: best(first(r.project.owners).images), 94 | source: r.project.url, 95 | thumbnail: best(r.project.covers), 96 | type: 'behance' 97 | }; 98 | }); 99 | 100 | log.info('retrieved ' + stars.length + ' appreciations for user: ' + state.user); 101 | 102 | return callback(null, scheduleTo(updateState(state, stars, rateLimit, false)), stars); 103 | 104 | function first(array) { 105 | return array[0]; 106 | } 107 | 108 | function best(object) { 109 | var sorted = Object.keys(object).sort(); 110 | return object[sorted[sorted.length - 1]]; 111 | } 112 | } 113 | 114 | function updateState(state, data, rateLimit, failed) { 115 | state.lastExecution = moment().toDate(); 116 | 117 | if (!failed) { 118 | if (state.mode === 'initial' && data.length > 0) { 119 | state.page += 1; 120 | } 121 | 122 | if (state.mode === 'initial' && data.length === 0) { 123 | state.mode = 'normal'; 124 | delete state.page; 125 | } 126 | 127 | if (rateLimit <= 1) { 128 | var currentState = state.mode; 129 | state.mode = 'rateLimit'; 130 | state.prevMode = currentState; 131 | } 132 | } 133 | 134 | return state; 135 | } 136 | } 137 | 138 | module.exports = connector; -------------------------------------------------------------------------------- /source/engine/connectors/dribbble.js: -------------------------------------------------------------------------------- 1 | var request = require('request'); 2 | var moment = require('moment'); 3 | var util = require('util'); 4 | 5 | var scheduleTo = require('../scheduleTo'); 6 | var logger = require('./../../utils/logger'); 7 | var handleUnexpected = require('../handleUnexpected'); 8 | var config = require('../../../config'); 9 | 10 | var API = 'http://api.dribbble.com'; 11 | 12 | function connector(state, user, callback) { 13 | var log = logger.connector('dribbble'); 14 | var username = state.username; 15 | 16 | if (!username) { 17 | return callback('missing username for user: ' + state.user); 18 | } 19 | 20 | initState(state); 21 | 22 | log.info('prepearing request in (' + state.mode + ') mode for user: ' + state.user); 23 | 24 | var uri = formatRequestUri(username, state); 25 | var headers = { 'Content-Type': 'application/json', 'User-Agent': 'likeastore/collector'}; 26 | 27 | request({uri: uri, headers: headers, timeout: config.collector.request.timeout, json: true}, function (err, response, body) { 28 | if (failed(err, response, body)) { 29 | return handleUnexpected(response, body, state, err, function (err) { 30 | callback (err, state); 31 | }); 32 | } 33 | 34 | return handleResponse(response, body); 35 | }); 36 | 37 | function failed(err, response, body) { 38 | return err || response.statusCode !== 200 || !body; 39 | } 40 | 41 | function initState(state) { 42 | if (!state.mode) { 43 | state.mode = 'initial'; 44 | } 45 | 46 | if (!state.errors) { 47 | state.errors = 0; 48 | } 49 | 50 | if (state.mode === 'initial' && !state.page) { 51 | state.page = 1; 52 | } 53 | 54 | if (state.mode === 'rateLimit') { 55 | state.mode = state.prevMode; 56 | } 57 | } 58 | 59 | function formatRequestUri(username, state) { 60 | var base = util.format('%s/players/%s/shots/likes?per_page=30', API, username); 61 | return state.mode === 'initial' && state.page ? 62 | util.format('%s&page=%s', base, state.page) : 63 | base; 64 | } 65 | 66 | function handleResponse(response, body) { 67 | var shots = body.shots; 68 | 69 | if (!Array.isArray(shots)) { 70 | return handleUnexpected(response, body, state, function (err) { 71 | callback(err, scheduleTo(updateState(body, state, [], 9999, true))); 72 | }); 73 | } 74 | 75 | var likes = shots.map(function (r) { 76 | return { 77 | itemId: r.id.toString(), 78 | idInt: r.id, 79 | user: state.user, 80 | userData: user, 81 | created: moment(r.created_at).toDate(), 82 | title: r.title, 83 | authorName: r.player.name, 84 | authorUrl: r.player.url, 85 | avatarUrl: r.player.avatar_url, 86 | source: r.url, 87 | thumbnail: r.image_400_url || r.image_url, 88 | type: 'dribbble' 89 | }; 90 | }); 91 | 92 | log.info('retrieved ' + likes.length + ' likes for user: ' + state.user); 93 | 94 | return callback(null, scheduleTo(updateState(body, state, likes, 9999, false)), likes); 95 | } 96 | 97 | function updateState(body, state, data, rateLimit, failed) { 98 | state.lastExecution = moment().toDate(); 99 | 100 | if (!failed) { 101 | if (state.mode === 'initial' && +body.page < body.total) { 102 | state.page += 1; 103 | } 104 | 105 | if (state.mode === 'initial' && +body.page < body.total) { 106 | state.mode = 'normal'; 107 | delete state.page; 108 | } 109 | 110 | if (rateLimit <= 1) { 111 | var currentState = state.mode; 112 | state.mode = 'rateLimit'; 113 | state.prevMode = currentState; 114 | } 115 | } 116 | 117 | return state; 118 | } 119 | } 120 | 121 | module.exports = connector; -------------------------------------------------------------------------------- /source/engine/connectors/facebook.js: -------------------------------------------------------------------------------- 1 | var request = require('request'); 2 | var moment = require('moment'); 3 | var util = require('util'); 4 | 5 | var logger = require('../../utils/logger'); 6 | var scheduleTo = require('../scheduleTo'); 7 | var handleUnexpected = require('../handleUnexpected'); 8 | var config = require('../../../config'); 9 | 10 | var API = 'https://graph.facebook.com'; 11 | var FIELDS = 'links.limit(%s).offset(%s).fields(id,caption,from,icon,message,name,link,created_time,picture),likes.limit(%s).offset(%s).fields(link,name,website,description,id,created_time,picture.type(square)),name,username'; 12 | 13 | function connector(state, user, callback) { 14 | var accessToken = state.accessToken; 15 | var log = logger.connector('facebook'); 16 | 17 | if (!accessToken) { 18 | return callback('missing accessToken for user: ' + state.user); 19 | } 20 | 21 | initState(state); 22 | 23 | log.info('prepearing request in (' + state.mode + ') mode for user: ' + state.user); 24 | 25 | var uri = formatRequestUri(accessToken, state); 26 | var headers = { 'Content-Type': 'application/json', 'User-Agent': 'likeastore/collector'}; 27 | 28 | request({uri: uri, headers: headers, timeout: config.collector.request.timeout, json: true}, function (err, response, body) { 29 | if (failed(err, response, body)) { 30 | return handleUnexpected(response, body, state, err, function (err) { 31 | callback (err, state); 32 | }); 33 | } 34 | 35 | return handleResponse(response, body); 36 | }); 37 | 38 | function failed(err, response, body) { 39 | return err || response.statusCode !== 200 || !body; 40 | } 41 | 42 | function initState(state) { 43 | if (!state.mode) { 44 | state.mode = 'initial'; 45 | } 46 | 47 | if (state.mode === 'initial' && !state.page) { 48 | state.page = 0; 49 | } 50 | 51 | if (!state.errors) { 52 | state.errors = 0; 53 | } 54 | 55 | if (state.mode === 'rateLimit') { 56 | state.mode = state.prevMode; 57 | } 58 | } 59 | 60 | function formatRequestUri(accessToken, state) { 61 | var pageSize = state.mode === 'initial' ? 500 : 50; 62 | var offset = state.mode === 'initial' ? state.page * pageSize : 0; 63 | var fields = formatFields(pageSize, offset); 64 | 65 | return util.format('%s/me?fields=%s&access_token=%s', API, fields, accessToken); 66 | } 67 | 68 | function formatFields(pageSize, offset) { 69 | return util.format(FIELDS, pageSize, offset, pageSize, offset); 70 | } 71 | 72 | function formatDescription (message, name) { 73 | return message ? util.format('%s - %s', message, name) : name; 74 | } 75 | 76 | function handleResponse(response, body) { 77 | var likesData = body.likes && body.likes.data || []; 78 | var linksData = body.links && body.links.data || []; 79 | var error = body.error; 80 | var authorName = body.username || body.name; 81 | 82 | if (error) { 83 | return handleUnexpected(response, body, state, error, function (err) { 84 | callback(err, scheduleTo(updateState(state, [], 9999, true))); 85 | }); 86 | } 87 | 88 | var likes = likesData.map(function (r) { 89 | return { 90 | itemId: r.id.toString(), 91 | idInt: r.id, 92 | user: state.user, 93 | userData: user, 94 | name: r.name, 95 | source: r.link, 96 | avatarUrl: r.picture.data.url || 'https://www.gravatar.com/avatar?d=mm', 97 | authorName: authorName, 98 | created: moment(r.created_time).toDate(), 99 | description: formatDescription(r.description, r.name), 100 | kind: 'like', 101 | type: 'facebook' 102 | }; 103 | }); 104 | 105 | var links = linksData.map(function (r) { 106 | return { 107 | itemId: r.id.toString(), 108 | idInt: r.id, 109 | user: state.user, 110 | userData: user, 111 | name: r.from.name, 112 | source: r.link, 113 | avatarUrl: r.picture || 'https://www.gravatar.com/avatar?d=mm', 114 | authorName: authorName, 115 | created: moment(r.created_time).toDate(), 116 | description: formatDescription(r.message, r.name), 117 | kind: 'link', 118 | type: 'facebook' 119 | }; 120 | }); 121 | 122 | likes = likes.concat(links); 123 | 124 | log.info('retrieved ' + likes.length + ' likes and links for user: ' + state.user); 125 | 126 | return callback(null, scheduleTo(updateState(state, likes, 9999, false)), likes); 127 | } 128 | 129 | function updateState(state, data, rateLimit, failed) { 130 | state.lastExecution = moment().toDate(); 131 | 132 | if (!failed) { 133 | if (state.mode === 'initial' && data.length > 0) { 134 | state.page += 1; 135 | } 136 | 137 | if (state.mode === 'initial' && data.length === 0) { 138 | state.mode = 'normal'; 139 | delete state.page; 140 | } 141 | } 142 | 143 | return state; 144 | } 145 | } 146 | 147 | module.exports = connector; -------------------------------------------------------------------------------- /source/engine/connectors/flickr.js: -------------------------------------------------------------------------------- 1 | var request = require('request'); 2 | var moment = require('moment'); 3 | var scheduleTo = require('../scheduleTo'); 4 | var util = require('util'); 5 | 6 | var handleUnexpected = require('../handleUnexpected'); 7 | var helpers = require('./../../utils/helpers'); 8 | var logger = require('./../../utils/logger'); 9 | var config = require('../../../config'); 10 | 11 | var API = 'https://api.flickr.com/services/rest'; 12 | 13 | function connector(state, user, callback) { 14 | var accessToken = state.accessToken; 15 | var accessTokenSecret = state.accessTokenSecret; 16 | var log = logger.connector('flickr'); 17 | 18 | if (!accessToken) { 19 | return callback('missing accessToken for user: ' + state.user); 20 | } 21 | 22 | if (!accessTokenSecret) { 23 | return callback('missing accessTokenSecret for user: ' + state.user); 24 | } 25 | 26 | initState(state); 27 | 28 | var uri = formatRequestUri(state); 29 | var headers = { 'Content-Type': 'application/json', 'User-Agent': 'likeastore/collector'}; 30 | 31 | var oauth = { 32 | consumer_key: config.services.flickr.consumerKey, 33 | consumer_secret: config.services.flickr.consumerSecret, 34 | token: accessToken, 35 | token_secret: accessTokenSecret 36 | }; 37 | 38 | log.info('prepearing request in (' + state.mode + ') mode for user: ' + state.user); 39 | 40 | request({uri: uri, headers: headers, oauth: oauth, timeout: config.collector.request.timeout, json: true}, function (err, response, body) { 41 | if (failed(err, response, body)) { 42 | return handleUnexpected(response, body, state, err, function (err) { 43 | callback (err, state); 44 | }); 45 | } 46 | 47 | return handleResponse(response, body); 48 | }); 49 | 50 | function failed(err, response, body) { 51 | return err || response.statusCode !== 200 || !body; 52 | } 53 | 54 | function initState(state) { 55 | if (!state.mode) { 56 | state.mode = 'initial'; 57 | } 58 | 59 | if (!state.errors) { 60 | state.errors = 0; 61 | } 62 | 63 | if (!state.page) { 64 | state.page = 1; 65 | } 66 | 67 | if (state.mode === 'rateLimit') { 68 | state.mode = state.prevMode; 69 | } 70 | } 71 | 72 | function formatRequestUri(state) { 73 | var base = util.format( 74 | '%s?api_key=%s&format=json&nojsoncallback=1&method=flickr.favorites.getList&extras=description,owner_name,url_c&page=%s', 75 | API, 76 | config.services.flickr.consumerKey, 77 | state.page); 78 | 79 | return base; 80 | } 81 | 82 | function handleResponse(response, body) { 83 | var photos = body.photos && body.photos.photo; 84 | 85 | if (!Array.isArray(photos)) { 86 | return handleUnexpected(response, body, state, function (err) { 87 | callback(err, scheduleTo(updateState(state, body, [], 9999, true))); 88 | }); 89 | } 90 | 91 | var favorites = photos.map(function (fav) { 92 | return { 93 | itemId: fav.id, 94 | user: state.user, 95 | userData: user, 96 | created: moment.unix(fav.date_faved).toDate(), 97 | description: fav.description && fav.description._content, 98 | authorName: fav.ownername, 99 | source: fav.url_c, 100 | thumbnail: fav.url_c, 101 | type: 'flickr' 102 | }; 103 | }); 104 | 105 | log.info('retrieved ' + favorites.length + ' favorites for user: ' + state.user); 106 | 107 | return callback(null, scheduleTo(updateState(state, body, favorites, 9999, false)), favorites); 108 | } 109 | 110 | function updateState(state, body, data, rateLimit, failed) { 111 | state.lastExecution = moment().toDate(); 112 | 113 | if (!failed) { 114 | if (state.mode === 'initial' && state.page < body.pages) { 115 | state.page += 1; 116 | } else { 117 | state.mode = 'normal'; 118 | state.page = 1; 119 | } 120 | 121 | if (rateLimit <= 1) { 122 | var currentState = state.mode; 123 | state.mode = 'rateLimit'; 124 | state.prevMode = currentState; 125 | } 126 | } 127 | 128 | return state; 129 | } 130 | } 131 | 132 | module.exports = connector; -------------------------------------------------------------------------------- /source/engine/connectors/gist.js: -------------------------------------------------------------------------------- 1 | var request = require('request'); 2 | var moment = require('moment'); 3 | var util = require('util'); 4 | 5 | var scheduleTo = require('../scheduleTo'); 6 | var logger = require('./../../utils/logger'); 7 | var handleUnexpected = require('../handleUnexpected'); 8 | var config = require('../../../config'); 9 | 10 | var API = 'https://api.github.com'; 11 | 12 | function connect(state, user, callback) { 13 | var accessToken = state.accessToken; 14 | var log = logger.connector('gist'); 15 | 16 | if (!accessToken) { 17 | return callback('missing accessToken for user: ' + state.user); 18 | } 19 | 20 | initState(state); 21 | 22 | log.info('prepearing request in (' + state.mode + ') mode for user: ' + state.user); 23 | 24 | var uri = formatRequestUri(accessToken, state); 25 | var headers = { 'Content-Type': 'application/json', 'User-Agent': 'likeastore/collector'}; 26 | 27 | request({uri: uri, headers: headers, timeout: config.collector.request.timeout, json: true}, function (err, response, body) { 28 | if (failed(err, response, body)) { 29 | return handleUnexpected(response, body, state, err, function (err) { 30 | callback (err, state); 31 | }); 32 | } 33 | 34 | return handleResponse(response, body); 35 | }); 36 | 37 | function failed(err, response, body) { 38 | return err || response.statusCode !== 200 || !body; 39 | } 40 | 41 | function initState(state) { 42 | if (!state.mode) { 43 | state.mode = 'initial'; 44 | } 45 | 46 | if (!state.errors) { 47 | state.errors = 0; 48 | } 49 | 50 | if (state.mode === 'initial' && !state.page) { 51 | state.page = 1; 52 | } 53 | 54 | if (state.mode === 'rateLimit') { 55 | state.mode = state.prevMode; 56 | } 57 | } 58 | 59 | function formatRequestUri(accessToken, state) { 60 | var pageSize = state.mode === 'initial' ? 100 : 50; 61 | var base = util.format('%s/gists/starred?access_token=%s&per_page=%s', API, accessToken, pageSize); 62 | return state.mode === 'initial' || state.page ? 63 | util.format('%s&page=%s', base, state.page) : 64 | base; 65 | } 66 | 67 | function handleResponse(response, body) { 68 | var rateLimit = +response.headers['x-ratelimit-remaining']; 69 | log.info('rate limit remaining: ' + rateLimit + ' for user: ' + state.user); 70 | 71 | if (!Array.isArray(body)) { 72 | return handleUnexpected(response, body, state, function (err) { 73 | callback(err, scheduleTo(updateState(state, [], rateLimit, true))); 74 | }); 75 | } 76 | 77 | var stars = body.map(function (r) { 78 | var gistOwner = r.user || { 79 | login: 'anonymous', 80 | html_url: null, 81 | gravatar_id: 'anon' 82 | }; 83 | 84 | return { 85 | itemId: r.id.toString(), 86 | idInt: r.id, 87 | user: state.user, 88 | userData: user, 89 | repo: 'gist', 90 | authorName: gistOwner.login, 91 | authorUrl: gistOwner.html_url, 92 | authorGravatar: gistOwner.gravatar_id, 93 | avatarUrl: 'https://www.gravatar.com/avatar/' + gistOwner.gravatar_id + '?d=mm', 94 | source: r.html_url, 95 | created: moment(r.created_at).toDate(), 96 | description: r.description, 97 | gist: true, 98 | type: 'github' 99 | }; 100 | }); 101 | 102 | log.info('retrieved ' + stars.length + ' gists for user: ' + state.user); 103 | 104 | return callback(null, scheduleTo(updateState(state, stars, rateLimit, false)), stars); 105 | } 106 | 107 | function updateState(state, data, rateLimit, failed) { 108 | state.lastExecution = moment().toDate(); 109 | 110 | if (!failed) { 111 | if (state.mode === 'initial' && data.length > 0) { 112 | state.page += 1; 113 | } 114 | 115 | if (state.mode === 'initial' && data.length === 0) { 116 | state.mode = 'normal'; 117 | delete state.page; 118 | } 119 | 120 | if (rateLimit <= 1) { 121 | var currentState = state.mode; 122 | state.mode = 'rateLimit'; 123 | state.prevMode = currentState; 124 | } 125 | } 126 | 127 | return state; 128 | } 129 | } 130 | 131 | module.exports = connect; -------------------------------------------------------------------------------- /source/engine/connectors/github.js: -------------------------------------------------------------------------------- 1 | var request = require('request'); 2 | var moment = require('moment'); 3 | var util = require('util'); 4 | 5 | var scheduleTo = require('../scheduleTo'); 6 | var logger = require('./../../utils/logger'); 7 | var handleUnexpected = require('../handleUnexpected'); 8 | var config = require('../../../config'); 9 | 10 | var API = 'https://api.github.com'; 11 | 12 | function connector(state, user, callback) { 13 | var accessToken = state.accessToken; 14 | var log = logger.connector('github'); 15 | 16 | if (!accessToken) { 17 | return callback('missing accessToken for user: ' + state.user); 18 | } 19 | 20 | initState(state); 21 | 22 | log.info('prepearing request in (' + state.mode + ') mode for user: ' + state.user); 23 | 24 | var uri = formatRequestUri(accessToken, state); 25 | var headers = { 'Content-Type': 'application/json', 'User-Agent': 'likeastore/collector'}; 26 | 27 | request({uri: uri, headers: headers, timeout: config.collector.request.timeout, json: true}, function (err, response, body) { 28 | if (failed(err, response, body)) { 29 | return handleUnexpected(response, body, state, err, function (err) { 30 | callback (err, state); 31 | }); 32 | } 33 | 34 | return handleResponse(response, body); 35 | }); 36 | 37 | function failed(err, response, body) { 38 | return err || response.statusCode !== 200 || !body; 39 | } 40 | 41 | function initState(state) { 42 | if (!state.mode) { 43 | state.mode = 'initial'; 44 | } 45 | 46 | if (!state.errors) { 47 | state.errors = 0; 48 | } 49 | 50 | if (state.mode === 'initial' && !state.page) { 51 | state.page = 1; 52 | } 53 | 54 | if (state.mode === 'rateLimit') { 55 | state.mode = state.prevMode; 56 | } 57 | } 58 | 59 | function formatRequestUri(accessToken, state) { 60 | var pageSize = state.mode === 'initial' ? 100 : 50; 61 | var base = util.format('%s/user/starred?access_token=%s&per_page=%s', API, accessToken, pageSize); 62 | return state.mode === 'initial' || state.page ? 63 | util.format('%s&page=%s', base, state.page) : 64 | base; 65 | } 66 | 67 | function handleResponse(response, body) { 68 | var rateLimit = +response.headers['x-ratelimit-remaining']; 69 | log.info('rate limit remaining: ' + rateLimit + ' for user: ' + state.user); 70 | 71 | if (!Array.isArray(body)) { 72 | return handleUnexpected(response, body, state, function (err) { 73 | callback(err, scheduleTo(updateState(state, [], rateLimit, true))); 74 | }); 75 | } 76 | 77 | var stars = body.map(function (r) { 78 | return { 79 | itemId: r.id.toString(), 80 | idInt: r.id, 81 | user: state.user, 82 | userData: user, 83 | name: r.full_name, 84 | repo: r.name, 85 | authorName: r.owner.login, 86 | authorUrl: r.owner.html_url, 87 | authorGravatar: r.owner.gravatar_id, 88 | avatarUrl: 'https://www.gravatar.com/avatar/' + r.owner.gravatar_id + '?d=mm', 89 | source: r.html_url, 90 | created: moment(r.created_at).toDate(), 91 | description: r.description, 92 | type: 'github' 93 | }; 94 | }); 95 | 96 | log.info('retrieved ' + stars.length + ' stars for user: ' + state.user); 97 | 98 | return callback(null, scheduleTo(updateState(state, stars, rateLimit, false)), stars); 99 | } 100 | 101 | function updateState(state, data, rateLimit, failed) { 102 | state.lastExecution = moment().toDate(); 103 | 104 | if (!failed) { 105 | if (state.mode === 'initial' && data.length > 0) { 106 | state.page += 1; 107 | } 108 | 109 | if (state.mode === 'initial' && data.length === 0) { 110 | state.mode = 'normal'; 111 | delete state.page; 112 | } 113 | 114 | if (rateLimit <= 1) { 115 | var currentState = state.mode; 116 | state.mode = 'rateLimit'; 117 | state.prevMode = currentState; 118 | } 119 | } 120 | 121 | return state; 122 | } 123 | } 124 | 125 | module.exports = connector; -------------------------------------------------------------------------------- /source/engine/connectors/index.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | github: require('./github'), 3 | twitter: require('./twitter'), 4 | stackoverflow: require('./stackoverflow'), 5 | gist: require('./gist'), 6 | facebook: require('./facebook'), 7 | vimeo: require('./vimeo'), 8 | youtube: require('./youtube'), 9 | dribbble: require('./dribbble'), 10 | behance: require('./behance'), 11 | pocket: require('./pocket'), 12 | tumblr: require('./tumblr'), 13 | instagram: require('./instagram'), 14 | flickr: require('./flickr') 15 | }; -------------------------------------------------------------------------------- /source/engine/connectors/instagram.js: -------------------------------------------------------------------------------- 1 | var request = require('request'); 2 | var moment = require('moment'); 3 | var util = require('util'); 4 | 5 | var scheduleTo = require('../scheduleTo'); 6 | var logger = require('./../../utils/logger'); 7 | var handleUnexpected = require('../handleUnexpected'); 8 | var config = require('../../../config'); 9 | 10 | var API = 'https://api.instagram.com/v1'; 11 | 12 | function connector(state, user, callback) { 13 | var accessToken = state.accessToken; 14 | var log = logger.connector('instagram'); 15 | 16 | if (!accessToken) { 17 | return callback('missing accessToken for user: ' + state.user); 18 | } 19 | 20 | initState(state); 21 | 22 | log.info('prepearing request in (' + state.mode + ') mode for user: ' + state.user); 23 | 24 | var uri = formatRequestUri(accessToken, state); 25 | var headers = { 'Content-Type': 'application/json', 'User-Agent': 'likeastore/collector'}; 26 | 27 | request({uri: uri, headers: headers, timeout: config.collector.request.timeout, json: true}, function (err, response, body) { 28 | if (failed(err, response, body)) { 29 | return handleUnexpected(response, body, state, err, function (err) { 30 | callback (err, state); 31 | }); 32 | } 33 | 34 | return handleResponse(response, body); 35 | }); 36 | 37 | function failed(err, response, body) { 38 | return err || response.statusCode !== 200 || !body; 39 | } 40 | 41 | function initState(state) { 42 | if (!state.mode) { 43 | state.mode = 'initial'; 44 | } 45 | 46 | if (!state.errors) { 47 | state.errors = 0; 48 | } 49 | 50 | if (state.mode === 'initial' && !state.page) { 51 | state.page = 0; 52 | } 53 | 54 | if (state.mode === 'rateLimit') { 55 | state.mode = state.prevMode; 56 | } 57 | } 58 | 59 | function formatRequestUri(accessToken, state) { 60 | var base = util.format('%s/users/self/media/liked?access_token=%s', API, accessToken); 61 | return state.maxId ? util.format('%s&max_like_id=%s', base, state.maxId) : base; 62 | } 63 | 64 | function handleResponse(response, body) { 65 | var rateLimit = +response.headers['x-ratelimit-remaining']; 66 | log.info('rate limit remaining: ' + rateLimit + ' for user: ' + state.user); 67 | 68 | if (!Array.isArray(body.data)) { 69 | return handleUnexpected(response, body, state, 'unexpected response', function (err) { 70 | callback(err, scheduleTo(updateState(state, body, [], rateLimit, true))); 71 | }); 72 | } 73 | 74 | var likes = body.data.map(function (r) { 75 | return { 76 | itemId: r.id, 77 | user: state.user, 78 | userData: user, 79 | authorName: r.user && r.user.username, 80 | authorUrl: 'https://instagram.com/' + (r.user && r.user.username), 81 | avatarUrl: r.user && r.user.profile_picture, 82 | source: r.link, 83 | created: moment.unix(r.created_time).toDate(), 84 | description: r.caption && r.caption.text, 85 | thumbnail: r.images && r.images.standard_resolution && r.images.standard_resolution.url, 86 | type: 'instagram' 87 | }; 88 | }); 89 | 90 | log.info('retrieved ' + likes.length + ' likes for user: ' + state.user); 91 | 92 | return callback(null, scheduleTo(updateState(state, body, likes, rateLimit, false)), likes); 93 | } 94 | 95 | function updateState(state, body, data, rateLimit, failed) { 96 | state.lastExecution = moment().toDate(); 97 | 98 | if (!failed) { 99 | if (state.mode === 'initial' && body.pagination && body.pagination.next_max_like_id) { 100 | state.maxId = body.pagination.next_max_like_id; 101 | } else { 102 | state.mode = 'normal'; 103 | delete state.maxId; 104 | } 105 | 106 | if (rateLimit <= 1) { 107 | var currentState = state.mode; 108 | state.mode = 'rateLimit'; 109 | state.prevMode = currentState; 110 | } 111 | } 112 | 113 | return state; 114 | } 115 | } 116 | 117 | module.exports = connector; -------------------------------------------------------------------------------- /source/engine/connectors/pocket.js: -------------------------------------------------------------------------------- 1 | var request = require('request'); 2 | var moment = require('moment'); 3 | 4 | var scheduleTo = require('../scheduleTo'); 5 | var logger = require('./../../utils/logger'); 6 | var handleUnexpected = require('../handleUnexpected'); 7 | var config = require('../../../config'); 8 | 9 | var API = 'https://getpocket.com/v3/get'; 10 | 11 | function connector(state, user, callback) { 12 | var accessToken = state.accessToken; 13 | var log = logger.connector('pocket'); 14 | 15 | if (!accessToken) { 16 | return callback('missing accessToken for user: ' + state.user); 17 | } 18 | 19 | initState(state); 20 | 21 | log.info('prepearing request in (' + state.mode + ') mode for user: ' + state.user); 22 | 23 | var body = formatRequestBody(accessToken, state); 24 | var headers = { 'Content-Type': 'application/json', 'User-Agent': 'likeastore/collector'}; 25 | 26 | request.post({uri: API, headers: headers, timeout: config.collector.request.timeout, form: body}, function (err, response, body) { 27 | if (failed(err, response, body)) { 28 | return handleUnexpected(response, body, state, err, function (err) { 29 | callback (err, state); 30 | }); 31 | } 32 | 33 | try { 34 | body = JSON.parse(body); 35 | } catch (e) { 36 | return handleUnexpected(response, body, state, e, function (err) { 37 | callback (err, state); 38 | }); 39 | } 40 | 41 | return handleResponse(response, body); 42 | 43 | }); 44 | 45 | function failed(err, response, body) { 46 | return err || response.statusCode !== 200 || !body; 47 | } 48 | 49 | function initState(state) { 50 | if (!state.mode) { 51 | state.mode = 'initial'; 52 | } 53 | 54 | if (!state.errors) { 55 | state.errors = 0; 56 | } 57 | 58 | if (state.mode === 'initial' && !state.page) { 59 | state.page = 0; 60 | } 61 | 62 | if (state.mode === 'rateLimit') { 63 | state.mode = state.prevMode; 64 | } 65 | } 66 | 67 | function formatRequestBody(accessToken, state) { 68 | var pageSize = 500; 69 | 70 | var requestBody = { 71 | consumer_key: config.services.pocket.consumerKey, 72 | access_token: accessToken, 73 | favorite: 1, 74 | detailType: 'complete', 75 | sort: 'oldest' 76 | }; 77 | 78 | if (state.mode === 'initial') { 79 | requestBody.count = pageSize; 80 | requestBody.offset = pageSize * state.page; 81 | } else if (state.mode === 'normal') { 82 | requestBody.since = state.since; 83 | } 84 | 85 | return requestBody; 86 | } 87 | 88 | function handleResponse(response, body) { 89 | var rateLimit = +response.headers['x-limit-user-remaining']; 90 | log.info('rate limit remaining: ' + rateLimit + ' for user: ' + state.user); 91 | 92 | var list = Object.keys(body.list).map(function (key) { 93 | return body.list[key]; 94 | }); 95 | 96 | if (!Array.isArray(list)) { 97 | return handleUnexpected(response, body, state, 'unexpected response', function (err) { 98 | callback(err, scheduleTo(updateState(state, body, [], rateLimit, true))); 99 | }); 100 | } 101 | 102 | var stars = list.map(function (r) { 103 | return { 104 | itemId: r.item_id, 105 | idInt: +r.item_id, 106 | user: state.user, 107 | userData: user, 108 | title: r.resolved_title, 109 | authorName: r.authors && r.authors[0] && r.authors[0].name, 110 | authorUrl: r.authors && r.authors[0] && r.authors[0].url, 111 | source: r.resolved_url, 112 | created: moment.unix(r.time_favorited).toDate(), 113 | description: r.excerpt, 114 | thumbnail: r.image && r.image.src, 115 | type: 'pocket' 116 | }; 117 | }); 118 | 119 | log.info('retrieved ' + stars.length + ' stars for user: ' + state.user); 120 | 121 | return callback(null, scheduleTo(updateState(state, body, stars, rateLimit, false)), stars); 122 | } 123 | 124 | function updateState(state, body, data, rateLimit, failed) { 125 | state.lastExecution = moment().toDate(); 126 | state.since = body.since; 127 | 128 | if (!failed) { 129 | if (state.mode === 'initial' && data.length > 0) { 130 | state.page += 1; 131 | } 132 | 133 | if (state.mode === 'initial' && data.length === 0) { 134 | state.mode = 'normal'; 135 | delete state.page; 136 | } 137 | 138 | if (rateLimit <= 1) { 139 | var currentState = state.mode; 140 | state.mode = 'rateLimit'; 141 | state.prevMode = currentState; 142 | } 143 | } 144 | 145 | return state; 146 | } 147 | } 148 | 149 | module.exports = connector; -------------------------------------------------------------------------------- /source/engine/connectors/stackoverflow.js: -------------------------------------------------------------------------------- 1 | var request = require('request'); 2 | var zlib = require('zlib'); 3 | var moment = require('moment'); 4 | var scheduleTo = require('../scheduleTo'); 5 | var util = require('util'); 6 | 7 | var handleUnexpected = require('../handleUnexpected'); 8 | var config = require('../../../config'); 9 | var logger = require('../../utils/logger'); 10 | 11 | var API = 'https://api.stackexchange.com/2.1'; 12 | 13 | function connector(state, user, callback) { 14 | var accessToken = state.accessToken; 15 | var log = logger.connector('stackoverflow'); 16 | 17 | if (!accessToken) { 18 | return callback('missing accessToken for user: ' + state.user); 19 | } 20 | 21 | initState(state); 22 | 23 | log.info('prepearing request in (' + state.mode + ') mode for user: ' + state.user); 24 | 25 | var uri = formatRequestUri(accessToken, state); 26 | var headers = { 'Content-Type': 'application/json', 'Accept-Encoding': 'gzip', 'User-Agent': 'likeastore/collector'}; 27 | 28 | var stream = zlib.createGunzip(); 29 | var unzippedResponse = ''; 30 | 31 | stream.on('data', function (data) { 32 | unzippedResponse += data; 33 | }); 34 | 35 | stream.on('error', function (err) { 36 | logger.error({message: 'failed to deflate stream', err: err, unzippedResponse: unzippedResponse}); 37 | callback(err); 38 | }); 39 | 40 | stream.on('finish', function () { 41 | var response = JSON.parse(unzippedResponse); 42 | var rateLimit = +response.quota_remaining; 43 | log.info('rate limit remaining: ' + rateLimit + ' for user: ' + state.user); 44 | 45 | return handleResponse(response, rateLimit); 46 | }); 47 | 48 | request({uri: uri, headers: headers, timeout: config.collector.request.timeout}, function (err, response, body) { 49 | if (failed(err, response, body)) { 50 | return handleUnexpected(response, body, state, err, function (err) { 51 | callback (err, state); 52 | }); 53 | } 54 | }).pipe(stream); 55 | 56 | function failed(err, response, body) { 57 | return err || response.statusCode !== 200 || !body; 58 | } 59 | 60 | function initState(state) { 61 | if (!state.mode) { 62 | state.mode = 'initial'; 63 | } 64 | 65 | if (!state.errors) { 66 | state.errors = 0; 67 | } 68 | 69 | if (state.mode === 'initial' && !state.page) { 70 | state.page = 1; 71 | } 72 | 73 | if (state.mode === 'rateLimit') { 74 | state.mode = state.prevMode; 75 | } 76 | } 77 | 78 | function formatRequestUri(accessToken, state) { 79 | var pageSize = state.mode === 'initial' ? 100 : 50; 80 | var base = util.format('%s/me/favorites?access_token=%s&key=%s&pagesize=%s&sort=activity&order=desc&site=stackoverflow', API, accessToken, config.services.stackoverflow.clientKey, pageSize); 81 | return state.mode === 'initial' || state.page ? 82 | util.format('%s&page=%s', base, state.page) : 83 | base; 84 | } 85 | 86 | function handleResponse(body, rateLimit) { 87 | if (!Array.isArray(body.items)) { 88 | return handleUnexpected(null, body, state, function (err) { 89 | callback(err, scheduleTo(updateState(state, [], rateLimit, true))); 90 | }); 91 | } 92 | 93 | var favorites = body.items.map(function (fav) { 94 | return { 95 | itemId: fav.question_id.toString(), 96 | idInt: fav.question_id, 97 | user: state.user, 98 | userData: user, 99 | dateInt: fav.creation_date, 100 | created: moment.unix(fav.creation_date).toDate(), 101 | description: fav.title, 102 | authorName: fav.owner.display_name, 103 | avatarUrl: fav.owner.profile_image && fav.owner.profile_image.replace(/^http:\/\//i, 'https://'), 104 | source: 'http://stackoverflow.com/questions/' + fav.question_id, 105 | type: 'stackoverflow' 106 | }; 107 | }); 108 | 109 | log.info('retrieved ' + favorites.length + ' favorites for user: ' + state.user); 110 | 111 | return callback(null, scheduleTo(updateState(state, favorites, rateLimit, false)), favorites); 112 | } 113 | 114 | function updateState(state, data, rateLimit, failed) { 115 | state.lastExecution = moment().toDate(); 116 | 117 | if (!failed) { 118 | if (state.mode === 'initial' && data.length > 0) { 119 | state.page += 1; 120 | } 121 | 122 | if (state.mode === 'initial' && data.length === 0) { 123 | state.mode = 'normal'; 124 | delete state.page; 125 | } 126 | 127 | if (rateLimit <= 1) { 128 | var currentState = state.mode; 129 | state.mode = 'rateLimit'; 130 | state.prevMode = currentState; 131 | } 132 | } 133 | 134 | return state; 135 | } 136 | } 137 | 138 | module.exports = connector; -------------------------------------------------------------------------------- /source/engine/connectors/tumblr.js: -------------------------------------------------------------------------------- 1 | var request = require('request'); 2 | var moment = require('moment'); 3 | var scheduleTo = require('../scheduleTo'); 4 | var util = require('util'); 5 | 6 | var handleUnexpected = require('../handleUnexpected'); 7 | var logger = require('./../../utils/logger'); 8 | var config = require('../../../config'); 9 | 10 | var API = 'http://api.tumblr.com/v2'; 11 | 12 | function connector(state, user, callback) { 13 | var accessToken = state.accessToken; 14 | var accessTokenSecret = state.accessTokenSecret; 15 | var log = logger.connector('tumblr'); 16 | 17 | if (!accessToken) { 18 | return callback('missing accessToken for user: ' + state.user); 19 | } 20 | 21 | if (!accessTokenSecret) { 22 | return callback('missing accessTokenSecret for user: ' + state.user); 23 | } 24 | 25 | initState(state); 26 | 27 | var uri = formatRequestUri(state); 28 | var headers = { 'Content-Type': 'application/json', 'User-Agent': 'likeastore/collector'}; 29 | 30 | var oauth = { 31 | consumer_key: config.services.tumblr.consumerKey, 32 | consumer_secret: config.services.tumblr.consumerSecret, 33 | token: accessToken, 34 | token_secret: accessTokenSecret 35 | }; 36 | 37 | log.info('prepearing request in (' + state.mode + ') mode for user: ' + state.user); 38 | 39 | request({uri: uri, headers: headers, oauth: oauth, timeout: config.collector.request.timeout, json: true}, function (err, response, body) { 40 | if (failed(err, response, body)) { 41 | return handleUnexpected(response, body, state, err, function (err) { 42 | callback (err, state); 43 | }); 44 | } 45 | 46 | return handleResponse(response, body); 47 | }); 48 | 49 | function failed(err, response, body) { 50 | return err || response.statusCode !== 200 || !body; 51 | } 52 | 53 | function formatRequestUri(state) { 54 | var pageSize = 20; 55 | var base = util.format('%s/user/likes?limit=%s', API, pageSize); 56 | return state.mode === 'initial' ? util.format('%s&offset=%s', base, state.page * pageSize) : base; 57 | } 58 | 59 | function initState(state) { 60 | if (!state.mode) { 61 | state.mode = 'initial'; 62 | } 63 | 64 | if (!state.errors) { 65 | state.errors = 0; 66 | } 67 | 68 | if (!state.page) { 69 | state.page = 0; 70 | } 71 | 72 | if (state.mode === 'rateLimit') { 73 | state.mode = state.prevMode; 74 | } 75 | } 76 | 77 | function handleResponse(response, body) { 78 | var list = body.response.liked_posts; 79 | 80 | if (!Array.isArray(list)) { 81 | return handleUnexpected(response, body, state, 'unexpected response', function (err) { 82 | callback(err, scheduleTo(updateState(state, body.response, [], 9999, true))); 83 | }); 84 | } 85 | 86 | var likes = list.map(function (fav) { 87 | return { 88 | itemId: fav.id.toString(), 89 | idInt: fav.id, 90 | user: state.user, 91 | userData: user, 92 | created: moment(fav.date).toDate(), 93 | authorName: fav.blog_name, 94 | source: fav.post_url, 95 | title: fav.title, 96 | thumbnail: fav.photos && fav.photos[0] && fav.photos[0].original_size.url, 97 | type: 'tumblr' 98 | }; 99 | }); 100 | 101 | log.info('retrieved ' + likes.length + ' likes for user: ' + state.user); 102 | 103 | return callback(null, scheduleTo(updateState(state, body.response, likes, 9999, false)), likes); 104 | } 105 | 106 | function updateState(state, response, data, rateLimit, failed) { 107 | state.lastExecution = moment().toDate(); 108 | 109 | if (!failed) { 110 | if (state.mode === 'initial' && data.length > 0) { 111 | state.page += 1; 112 | } 113 | 114 | if (state.mode === 'initial' && (data.length === 0 || state.page * 20 > response.liked_count)) { 115 | state.mode = 'normal'; 116 | delete state.page; 117 | } 118 | 119 | if (rateLimit <= 1) { 120 | var currentState = state.mode; 121 | state.mode = 'rateLimit'; 122 | state.prevMode = currentState; 123 | } 124 | } 125 | 126 | return state; 127 | } 128 | } 129 | 130 | module.exports = connector; -------------------------------------------------------------------------------- /source/engine/connectors/twitter.js: -------------------------------------------------------------------------------- 1 | var request = require('request'); 2 | var moment = require('moment'); 3 | var scheduleTo = require('../scheduleTo'); 4 | var util = require('util'); 5 | 6 | var handleUnexpected = require('../handleUnexpected'); 7 | var helpers = require('./../../utils/helpers'); 8 | var logger = require('./../../utils/logger'); 9 | var config = require('../../../config'); 10 | 11 | var API = 'https://api.twitter.com/1.1'; 12 | 13 | function connector(state, user, callback) { 14 | var accessToken = state.accessToken; 15 | var accessTokenSecret = state.accessTokenSecret; 16 | var log = logger.connector('twitter'); 17 | 18 | if (!accessToken) { 19 | return callback('missing accessToken for user: ' + state.user); 20 | } 21 | 22 | if (!accessTokenSecret) { 23 | return callback('missing accessTokenSecret for user: ' + state.user); 24 | } 25 | 26 | initState(state); 27 | 28 | var uri = formatRequestUri(state); 29 | var headers = { 'Content-Type': 'application/json', 'User-Agent': 'likeastore/collector'}; 30 | 31 | var oauth = { 32 | consumer_key: config.services.twitter.consumerKey, 33 | consumer_secret: config.services.twitter.consumerSecret, 34 | token: accessToken, 35 | token_secret: accessTokenSecret 36 | }; 37 | 38 | log.info('prepearing request in (' + state.mode + ') mode for user: ' + state.user); 39 | 40 | request({uri: uri, headers: headers, oauth: oauth, timeout: config.collector.request.timeout, json: true}, function (err, response, body) { 41 | if (failed(err, response, body)) { 42 | return handleUnexpected(response, body, state, err, function (err) { 43 | callback (err, state); 44 | }); 45 | } 46 | 47 | return handleResponse(response, body); 48 | }); 49 | 50 | function failed(err, response, body) { 51 | return err || response.statusCode !== 200 || !body; 52 | } 53 | 54 | function initState(state) { 55 | if (!state.mode) { 56 | state.mode = 'initial'; 57 | } 58 | 59 | if (!state.errors) { 60 | state.errors = 0; 61 | } 62 | 63 | if (state.mode === 'rateLimit') { 64 | state.mode = state.prevMode; 65 | } 66 | } 67 | 68 | function formatRequestUri(state) { 69 | var base = util.format('%s/favorites/list.json?count=200&include_entities=false', API); 70 | return state.maxId ? 71 | util.format('%s&max_id=%s', base, state.maxId) : 72 | state.mode === 'normal' && state.sinceId ? 73 | util.format('%s&since_id=%s', base, state.sinceId) : 74 | base; 75 | } 76 | 77 | function handleResponse(response, body) { 78 | var rateLimit = +response.headers['x-rate-limit-remaining']; 79 | log.info('rate limit remaining: ' + rateLimit + ' for user: ' + state.user); 80 | 81 | if (!Array.isArray(body)) { 82 | return handleUnexpected(response, body, state, function (err) { 83 | callback(err, scheduleTo(updateState(state, [], rateLimit, true))); 84 | }); 85 | } 86 | 87 | var favorites = body.map(function (fav) { 88 | return { 89 | itemId: fav.id_str, 90 | user: state.user, 91 | userData: user, 92 | created: moment(fav.created_at).toDate(), 93 | description: fav.text, 94 | avatarUrl: fav.user.profile_image_url_https, 95 | authorName: fav.user.screen_name, 96 | source: util.format('%s/%s/status/%s', 'https://twitter.com', fav.user.screen_name, fav.id_str), 97 | type: 'twitter' 98 | }; 99 | }); 100 | 101 | log.info('retrieved ' + favorites.length + ' favorites for user: ' + state.user); 102 | 103 | return callback(null, scheduleTo(updateState(state, favorites, rateLimit, false)), favorites); 104 | } 105 | 106 | function updateState(state, data, rateLimit, failed) { 107 | state.lastExecution = moment().toDate(); 108 | 109 | if (!failed) { 110 | if (state.mode === 'initial' && data.length > 0 && !state.sinceId) { 111 | state.sinceId = data[0].itemId; 112 | } 113 | 114 | if (state.mode === 'normal' && data.length > 0) { 115 | state.sinceId = data[0].itemId; 116 | } 117 | 118 | if (state.mode === 'initial' && data.length > 0) { 119 | state.maxId = helpers.decrementStringId(data[data.length - 1].itemId); 120 | } 121 | 122 | if (state.mode === 'initial' && data.length === 0) { 123 | state.mode = 'normal'; 124 | delete state.maxId; 125 | } 126 | 127 | if (rateLimit <= 1) { 128 | var currentState = state.mode; 129 | state.mode = 'rateLimit'; 130 | state.prevMode = currentState; 131 | } 132 | } 133 | 134 | return state; 135 | } 136 | } 137 | 138 | module.exports = connector; -------------------------------------------------------------------------------- /source/engine/connectors/vimeo.js: -------------------------------------------------------------------------------- 1 | var _ = require('underscore'); 2 | var request = require('request'); 3 | var moment = require('moment'); 4 | var scheduleTo = require('../scheduleTo'); 5 | var util = require('util'); 6 | 7 | var handleUnexpected = require('../handleUnexpected'); 8 | var logger = require('./../../utils/logger'); 9 | var config = require('../../../config'); 10 | 11 | var API = 'http://vimeo.com/api/rest/v2?format=json&method=vimeo.videos.getLikes'; 12 | 13 | function connector(state, user, callback) { 14 | var accessToken = state.accessToken; 15 | var accessTokenSecret = state.accessTokenSecret; 16 | var log = logger.connector('vimeo'); 17 | 18 | if (!accessToken) { 19 | return callback('missing accessToken for user: ' + state.user); 20 | } 21 | 22 | if (!accessTokenSecret) { 23 | return callback('missing accessTokenSecret for user: ' + state.user); 24 | } 25 | 26 | initState(state); 27 | 28 | var uri = formatRequestUri(state); 29 | var headers = { 'Content-Type': 'application/json', 'User-Agent': 'likeastore/collector'}; 30 | 31 | var oauth = { 32 | consumer_key: config.services.vimeo.clientId, 33 | consumer_secret: config.services.vimeo.clientSecret, 34 | token: accessToken, 35 | token_secret: accessTokenSecret 36 | }; 37 | 38 | log.info('prepearing request in (' + state.mode + ') mode for user: ' + state.user); 39 | 40 | request({uri: uri, headers: headers, oauth: oauth, timeout: config.collector.request.timeout, json: true}, function (err, response, body) { 41 | if (failed(err, response, body)) { 42 | return handleUnexpected(response, body, state, err, function (err) { 43 | callback (err, state); 44 | }); 45 | } 46 | 47 | return handleResponse(response, body); 48 | }); 49 | 50 | function formatRequestUri(state) { 51 | var base = util.format('%s&user_id=%s&sort=newest&per_page=50&full_response=1', API, state.accessToken); 52 | return state.mode === 'initial' || state.page ? 53 | util.format('%s&page=%s', base, state.page) : 54 | base; 55 | } 56 | 57 | function failed(err, response, body) { 58 | return err || response.statusCode !== 200 || !body; 59 | } 60 | 61 | function initState(state) { 62 | if (!state.mode) { 63 | state.mode = 'initial'; 64 | } 65 | 66 | if (!state.errors) { 67 | state.errors = 0; 68 | } 69 | 70 | if (state.mode === 'initial' && !state.page) { 71 | state.page = 1; 72 | } 73 | 74 | if (state.mode === 'rateLimit') { 75 | state.mode = state.prevMode; 76 | } 77 | } 78 | 79 | function handleResponse(response, body) { 80 | var videos = body.videos && body.videos.video; 81 | 82 | if (!Array.isArray(videos)) { 83 | return handleUnexpected(response, body, state, function (err) { 84 | callback(err, scheduleTo(updateState(state, [], 9999, true))); 85 | }); 86 | } 87 | 88 | var favorites = videos.map(function (video) { 89 | return { 90 | itemId: video.id, 91 | user: state.user, 92 | userData: user, 93 | created: moment(video.upload_date).toDate(), 94 | description: video.description, 95 | title: video.title, 96 | authorName: video.owner.display_name, 97 | authorUrl: video.owner.profileurl, 98 | avatarUrl: video.owner.portraits && video.owner.portraits.portrait && findWith('width', '100', video.owner.portraits.portrait, '_content'), 99 | source: video.urls && video.urls.url && findWith('type', 'video', video.urls.url, '_content'), 100 | thumbnail: video.thumbnails && video.thumbnails.thumbnail && findWith('width', '640', video.thumbnails.thumbnail, '_content'), 101 | type: 'vimeo' 102 | }; 103 | }); 104 | 105 | log.info('retrieved ' + favorites.length + ' favorites for user: ' + state.user); 106 | 107 | return callback(null, scheduleTo(updateState(state, favorites, 9999, false)), favorites); 108 | 109 | function findWith(prop, val, array, ret) { 110 | var found = _.find(array, function (item) { 111 | return item[prop] === val; 112 | }); 113 | 114 | return found && found[ret]; 115 | } 116 | } 117 | 118 | function updateState(state, data, rateLimit, failed) { 119 | state.lastExecution = moment().toDate(); 120 | 121 | if (!failed) { 122 | if (state.mode === 'initial' && data.length === 50) { 123 | state.page += 1; 124 | } 125 | 126 | if (state.mode === 'initial' && data.length < 50) { 127 | state.mode = 'normal'; 128 | delete state.page; 129 | } 130 | } 131 | 132 | return state; 133 | } 134 | } 135 | 136 | module.exports = connector; -------------------------------------------------------------------------------- /source/engine/connectors/vk.js: -------------------------------------------------------------------------------- 1 | var _ = require('underscore'); 2 | var async = require('async'); 3 | var request = require('request'); 4 | var moment = require('moment'); 5 | var util = require('util'); 6 | 7 | var scheduleTo = require('../scheduleTo'); 8 | var logger = require('./../../utils/logger'); 9 | var handleUnexpected = require('../handleUnexpected'); 10 | var config = require('../../../config'); 11 | 12 | var API = 'https://api.vk.com'; 13 | 14 | function connector(state, user, callback) { 15 | async.series([posts], function (err, results) { 16 | if (err) { 17 | return callback(err, state); 18 | } 19 | 20 | return callback(null, state, _.flatten(results)); 21 | }); 22 | 23 | function posts(callback) { 24 | var accessToken = state.accessToken; 25 | var log = logger.connector('github'); 26 | 27 | if (!accessToken) { 28 | return callback('missing accessToken for user: ' + state.user); 29 | } 30 | 31 | initState(state); 32 | 33 | log.info('prepearing request in (' + state.mode + ') mode for user: ' + state.user); 34 | 35 | var uri = formatRequestUri(accessToken, state); 36 | var headers = { 'Content-Type': 'application/json', 'User-Agent': 'likeastore/collector'}; 37 | 38 | console.log(uri); 39 | 40 | request({uri: uri, headers: headers, timeout: config.collector.request.timeout, json: true}, function (err, response, body) { 41 | if (err) { 42 | return handleUnexpected(response, body, state, err, function (err) { 43 | callback (err, state); 44 | }); 45 | } 46 | 47 | return handleResponse(response, body); 48 | }); 49 | 50 | function initState(state) { 51 | if (!state.mode) { 52 | state.mode = 'initial'; 53 | } 54 | 55 | if (!state.errors) { 56 | state.errors = 0; 57 | } 58 | 59 | if (state.mode === 'initial' && !state.postsPage) { 60 | state.postsPage = 0; 61 | } 62 | 63 | if (state.mode === 'rateLimit') { 64 | state.mode = state.prevMode; 65 | } 66 | } 67 | 68 | function formatRequestUri(accessToken, state) { 69 | var pageSize = state.mode === 'initial' ? 100 : 50; 70 | var base = util.format('%s/method/fave.getPosts?count=%s', API, pageSize); 71 | var url = state.mode === 'initial' || state.postsPage ? 72 | util.format('%s&offset=%s', base, state.postsPage * pageSize) : 73 | base; 74 | 75 | return util.format('%s&access_token=%s', url, accessToken); 76 | } 77 | 78 | function handleResponse(response, body) { 79 | console.dir(body); 80 | 81 | // var rateLimit = +response.headers['x-ratelimit-remaining']; 82 | // log.info('rate limit remaining: ' + rateLimit + ' for user: ' + state.user); 83 | 84 | // if (!Array.isArray(body)) { 85 | // return handleUnexpected(response, body, state, function (err) { 86 | // callback(err, scheduleTo(updateState(state, [], rateLimit, true))); 87 | // }); 88 | // } 89 | 90 | // var stars = body.map(function (r) { 91 | // return { 92 | // itemId: r.id.toString(), 93 | // idInt: r.id, 94 | // user: state.user, 95 | // userData: user, 96 | // name: r.full_name, 97 | // repo: r.name, 98 | // authorName: r.owner.login, 99 | // authorUrl: r.owner.html_url, 100 | // authorGravatar: r.owner.gravatar_id, 101 | // avatarUrl: 'https://www.gravatar.com/avatar/' + r.owner.gravatar_id + '?d=mm', 102 | // source: r.html_url, 103 | // created: moment(r.created_at).toDate(), 104 | // description: r.description, 105 | // type: 'github' 106 | // }; 107 | // }); 108 | 109 | // log.info('retrieved ' + stars.length + ' stars for user: ' + state.user); 110 | 111 | return callback(null, scheduleTo(updateState(state, [], 9999, false)), null); 112 | } 113 | 114 | function updateState(state, data, rateLimit, failed) { 115 | state.lastExecution = moment().toDate(); 116 | 117 | if (!failed) { 118 | if (state.mode === 'initial' && data.length > 0) { 119 | state.page += 1; 120 | } 121 | 122 | if (state.mode === 'initial' && data.length === 0) { 123 | state.mode = 'normal'; 124 | delete state.page; 125 | } 126 | 127 | if (rateLimit <= 1) { 128 | var currentState = state.mode; 129 | state.mode = 'rateLimit'; 130 | state.prevMode = currentState; 131 | } 132 | } 133 | 134 | return state; 135 | } 136 | } 137 | } 138 | 139 | module.exports = connector; -------------------------------------------------------------------------------- /source/engine/connectors/youtube.js: -------------------------------------------------------------------------------- 1 | var request = require('request'); 2 | var moment = require('moment'); 3 | var util = require('util'); 4 | 5 | var scheduleTo = require('../scheduleTo'); 6 | var logger = require('./../../utils/logger'); 7 | var handleUnexpected = require('../handleUnexpected'); 8 | var config = require('../../../config'); 9 | 10 | var API = 'https://www.googleapis.com/youtube/v3'; 11 | 12 | function connector(state, user, callback) { 13 | var log = logger.connector('youtube'); 14 | 15 | if (state.unauthorized && state.refreshToken) { 16 | return refreshAccessToken(state, connect); 17 | } 18 | 19 | connect(null, state); 20 | 21 | function connect(err, state) { 22 | if (err) { 23 | return callback(err); 24 | } 25 | 26 | var accessToken = state.accessToken; 27 | 28 | if (!accessToken) { 29 | return callback('missing accessToken for user: ' + state.user); 30 | } 31 | 32 | initState(state); 33 | 34 | log.info('prepearing request in (' + state.mode + ') mode for user: ' + state.user); 35 | 36 | var uri = formatRequestUri(accessToken, state); 37 | var headers = { 'Content-Type': 'application/json', 'User-Agent': 'likeastore/collector'}; 38 | 39 | request({uri: uri, headers: headers, timeout: config.collector.request.timeout, json: true}, function (err, response, body) { 40 | if (failed(err, response, body)) { 41 | return handleUnexpected(response, body, state, err, function (err) { 42 | callback (err, state); 43 | }); 44 | } 45 | 46 | return handleResponse(response, body); 47 | }); 48 | } 49 | 50 | function failed(err, response, body) { 51 | return err || response.statusCode !== 200 || !body; 52 | } 53 | 54 | function initState(state) { 55 | if (!state.mode) { 56 | state.mode = 'initial'; 57 | } 58 | 59 | if (!state.errors) { 60 | state.errors = 0; 61 | } 62 | 63 | if (state.mode === 'rateLimit') { 64 | state.mode = state.prevMode; 65 | } 66 | } 67 | 68 | function formatRequestUri(accessToken, state) { 69 | var base = util.format('%s/activities?part=contentDetails%2Csnippet&maxResults=50&mine=true&access_token=%s', API, accessToken); 70 | return state.mode === 'initial' && state.nextPageToken ? 71 | util.format('%s&pageToken=%s', base, state.nextPageToken) : 72 | base; 73 | } 74 | 75 | function refreshAccessToken(state, callback) { 76 | var refreshToken = state.refreshToken; 77 | 78 | if (!refreshToken) { 79 | return callback('missing refreshToken for user: ' + state.user); 80 | } 81 | 82 | var refreshTokenUrl = 'https://accounts.google.com/o/oauth2/token'; 83 | var headers = { 'Content-Type': 'application/x-www-form-urlencoded', 'User-Agent': 'likeastore/collector'}; 84 | var data = { 85 | 'client_id': config.services.youtube.clientId, 86 | 'client_secret': config.services.youtube.clientSecret, 87 | 'refresh_token': refreshToken, 88 | 'grant_type': 'refresh_token' 89 | }; 90 | 91 | logger.important('refreshing accessToken for user: ' + state.user); 92 | 93 | request.post({url: refreshTokenUrl, headers: headers, form: data, json: true}, function (err, response, body) { 94 | if (err || !body.access_token) { 95 | logger.error('failed to refresh accessToken for user: ' + state.user); 96 | 97 | return callback(err, state); 98 | } 99 | 100 | state.accessToken = body.access_token; 101 | delete state.unauthorized; 102 | delete state.errors; 103 | 104 | logger.important('accessToken refreshed for user: ' + state.user); 105 | 106 | callback(null, state); 107 | }); 108 | } 109 | 110 | function handleResponse(response, body) { 111 | var items = body.items; 112 | 113 | if (!Array.isArray(items)) { 114 | return handleUnexpected(response, body, state, function (err) { 115 | callback(err, scheduleTo(updateState(body, state, [], 9999, true))); 116 | }); 117 | } 118 | 119 | var likes = items.filter(function (item) { 120 | return item.snippet.type === 'like'; 121 | }).map(function (video) { 122 | return { 123 | itemId: video.contentDetails.like.resourceId.videoId, 124 | user: state.user, 125 | userData: user, 126 | created: moment(video.snippet.publishedAt).toDate(), 127 | description: video.snippet.description, 128 | title: video.snippet.title, 129 | authorName: null, 130 | authorUrl: null, 131 | avatarUrl: null, 132 | source: youtubeLink(video.contentDetails), 133 | thumbnail: video.snippet.thumbnails && video.snippet.thumbnails.high && video.snippet.thumbnails.high.url, 134 | type: 'youtube' 135 | }; 136 | }); 137 | 138 | log.info('retrieved ' + likes.length + ' likes for user: ' + state.user); 139 | 140 | return callback(null, scheduleTo(updateState(body, state, likes, 9999, false)), likes); 141 | 142 | function youtubeLink(details) { 143 | var id = details && details.like && details.like.resourceId && details.like.resourceId.videoId; 144 | return id && 'http://youtube.com/watch?v=' + id; 145 | } 146 | } 147 | 148 | function updateState(body, state, data, rateLimit, failed) { 149 | state.lastExecution = moment().toDate(); 150 | 151 | if (!failed) { 152 | if (state.mode === 'initial' && body.nextPageToken) { 153 | state.nextPageToken = body.nextPageToken; 154 | } 155 | 156 | if (state.mode === 'initial' && !body.nextPageToken) { 157 | state.mode = 'normal'; 158 | delete state.nextPageToken; 159 | } 160 | 161 | if (rateLimit <= 1) { 162 | var currentState = state.mode; 163 | state.mode = 'rateLimit'; 164 | state.prevMode = currentState; 165 | } 166 | } 167 | 168 | return state; 169 | } 170 | } 171 | 172 | module.exports = connector; -------------------------------------------------------------------------------- /source/engine/disableNetworks.js: -------------------------------------------------------------------------------- 1 | var networks = require('../models/networks'); 2 | 3 | function disableNetworks(state, callback) { 4 | return networks.disable(state, callback); 5 | } 6 | 7 | module.exports = disableNetworks; -------------------------------------------------------------------------------- /source/engine/executor.js: -------------------------------------------------------------------------------- 1 | var util = require('util'); 2 | var async = require('async'); 3 | var moment = require('moment'); 4 | var items = require('../models/items'); 5 | var users = require('../models/users'); 6 | var networks = require('../models/networks'); 7 | var logger = require('../utils/logger'); 8 | 9 | function executor(state, connectors, callback) { 10 | var executorStarted = moment(); 11 | 12 | async.waterfall([ 13 | readUser, 14 | executeConnector, 15 | findNew, 16 | saveToMongo, 17 | saveToEleastic, 18 | saveState 19 | ], function (err, results) { 20 | var executorFinished = moment(); 21 | var duration = moment.duration(executorFinished.diff(executorStarted)); 22 | 23 | logger.important( 24 | util.format('connector: %s (%s), items: %s, executed: %d sec.', 25 | state.service, 26 | state.user, 27 | formatLength(results), 28 | duration.asSeconds().toFixed(2))); 29 | 30 | callback(err); 31 | }); 32 | 33 | function readUser(next) { 34 | logger.info('[executor]: reading user info'); 35 | 36 | users.findByEmail(state.user, function(err, user) { 37 | next(logErrorAndProceed(err, '[executor]: failed to find user'), user); 38 | }); 39 | } 40 | 41 | function executeConnector(user, next) { 42 | logger.info('[executor]: executing connector for (' + state.user + ') service: ' + state.service); 43 | 44 | var connector = connectors[state.service]; 45 | connector(state, user, function (err, state, collected) { 46 | next(logErrorAndProceed(err, '[executor]: failed to execute connector'), user, collected); 47 | }); 48 | } 49 | 50 | function findNew(user, collected, next) { 51 | logger.info('[executor]: finding new (' + state.user + ') collected: ' + formatLength(collected)); 52 | 53 | items.findNew(collected, state, function (err, detected) { 54 | next(logErrorAndProceed(err, '[executor]: failed to find new items'), user, detected); 55 | }); 56 | } 57 | 58 | function saveToMongo(user, detected, next) { 59 | logger.info('[executor]: saving to mongo (' + state.user + ') detected: ' + formatLength(detected)); 60 | 61 | items.insert(detected, state, function(err, saved) { 62 | next(logErrorAndProceed(err, '[executor]: failed to save items'), user, saved); 63 | }); 64 | } 65 | 66 | function saveToEleastic(user, saved, next) { 67 | logger.info('[executor]: saving to elastic (' + state.user + ') saved: ' + formatLength(saved)); 68 | 69 | items.index(saved, state, function (err) { 70 | next(logErrorAndProceed(err, '[executor]: failed to index items'), user, saved); 71 | }); 72 | } 73 | 74 | function saveState(user, saved, next) { 75 | logger.info('[executor]: saving state (' + state.user + ')'); 76 | 77 | networks.update(state, user, function (err) { 78 | next(logErrorAndProceed(err, '[executor]: failed to save state'), saved); 79 | }); 80 | } 81 | 82 | function logErrorAndProceed(err, message) { 83 | if (err) { 84 | logger.error({message: message, err: err}); 85 | } 86 | } 87 | 88 | function formatLength(items) { 89 | return items ? items.length : '[NOT COLLECTED]'; 90 | } 91 | } 92 | 93 | module.exports = executor; -------------------------------------------------------------------------------- /source/engine/handleUnexpected.js: -------------------------------------------------------------------------------- 1 | var logger = require('../utils/logger'); 2 | 3 | var MAX_ERRORS_ALLOWED = 5; 4 | 5 | function handleUnexpected(response, body, state, err, callback) { 6 | if (typeof err === 'function') { 7 | callback = err; 8 | } 9 | 10 | state.errors += 1; 11 | 12 | var status = response ? response.statusCode : body ? body.error_id : err; 13 | 14 | if (status === 401) { 15 | state.unauthorized = true; 16 | } 17 | 18 | state.lastError = { err: err, status: status, body: body}; 19 | 20 | if (state.errors >= MAX_ERRORS_ALLOWED) { 21 | delete state.errors; 22 | state.disabled = true; 23 | 24 | logger.warning({ 25 | message: 'Connector disabled, due to max errors count.', 26 | body: body, 27 | status: status, 28 | err: err 29 | }); 30 | } 31 | 32 | callback(null); 33 | } 34 | 35 | module.exports = handleUnexpected; 36 | -------------------------------------------------------------------------------- /source/engine/scheduleTo.js: -------------------------------------------------------------------------------- 1 | var moment = require('moment'); 2 | var config = require('../../config'); 3 | 4 | function scheduleTo(state) { 5 | var currentMoment = moment(); 6 | var service = state.service; 7 | 8 | var scheduleForMode = { 9 | initial: config.collector.quotes[service].runAfter, 10 | normal: config.collector.nextNormalRunAfter, 11 | rateLimit: config.collector.nextRateLimitRunAfter 12 | }; 13 | 14 | var next = scheduleForMode[state.mode]; 15 | var scheduledTo = currentMoment.add(next, 'milliseconds'); 16 | 17 | state.scheduledTo = scheduledTo.toDate(); 18 | 19 | return state; 20 | } 21 | 22 | module.exports = scheduleTo; 23 | -------------------------------------------------------------------------------- /source/engine/scheduler.js: -------------------------------------------------------------------------------- 1 | var util = require('util'); 2 | var moment = require('moment'); 3 | var async = require('async'); 4 | 5 | var executor = require('./executor'); 6 | var disableNetworks = require('./disableNetworks'); 7 | var networks = require('../models/networks'); 8 | var logger = require('../utils/logger'); 9 | var config = require('../../config'); 10 | var connectors = require('./connectors'); 11 | 12 | function scheduler (mode) { 13 | 14 | function schedulerLoop() { 15 | // var collectorSteps = [runCollectingTasks, runCleaningTasks]; 16 | var collectorSteps = [runCollectingTasks]; 17 | async.series(collectorSteps, restartScheduler); 18 | } 19 | 20 | function restartScheduler (err, results) { 21 | if (err) { 22 | logger.error(err); 23 | } 24 | 25 | var totalTasks = results.reduce(function (memo, value) { 26 | return memo + value; 27 | }); 28 | 29 | // http://stackoverflow.com/questions/16072699/nodejs-settimeout-memory-leak 30 | var timeout = setTimeout(schedulerLoop, totalTasks > 0 ? config.collector.schedulerRestartShort : config.collector.schedulerRestartLong); 31 | } 32 | 33 | function runCollectingTasks(callback) { 34 | prepareCollectingTasks(function (err, tasks) { 35 | if (err) { 36 | return callback(err); 37 | } 38 | 39 | runTasks(tasks, 'collecting', callback); 40 | }); 41 | } 42 | 43 | function runCleaningTasks(callback) { 44 | prepareCleaningTasks(function (err, tasks) { 45 | if (err) { 46 | return callback(err); 47 | } 48 | 49 | runTasks(tasks, 'cleaning', callback); 50 | }); 51 | } 52 | 53 | function prepareCollectingTasks(callback) { 54 | networks.findByMode(mode, function (err, states) { 55 | if (err) { 56 | return callback({message: 'error during networks query', err: err}); 57 | } 58 | 59 | if (!states) { 60 | return callback({message: 'failed to read networks states'}); 61 | } 62 | 63 | callback(null, createCollectingTasks(states)); 64 | }); 65 | } 66 | 67 | function prepareCleaningTasks(callback) { 68 | networks.findByMode(mode, function (err, states) { 69 | if (err) { 70 | return callback({message: 'error during networks query', err: err}); 71 | } 72 | 73 | if (!states) { 74 | return callback({message: 'failed to read networks states'}); 75 | } 76 | 77 | callback(null, createCleaningTasks(states)); 78 | }); 79 | } 80 | 81 | function runTasks(tasks, type, callback) { 82 | tasks.length > 0 && logger.important(util.format('%s currently allowed to run: %s', type, tasks.length)); 83 | 84 | var started = moment(); 85 | async.series(tasks, function (err) { 86 | if (err) { 87 | // report error but continue execution to do not break execution chain.. 88 | logger.error(err); 89 | } 90 | 91 | var finished = moment(); 92 | var duration = moment.duration(finished.diff(started)); 93 | 94 | logger.important(util.format('%s tasks processed: %s, duration: %d sec. (%d mins.)', type, tasks.length, duration.asSeconds().toFixed(2), duration.asMinutes().toFixed(2))); 95 | 96 | callback(null, tasks.length); 97 | }); 98 | } 99 | 100 | function createCollectingTasks(states) { 101 | var tasks = states.map(function (state) { 102 | return allowedToExecute(state) ? collectingTask(state) : null; 103 | }).filter(function (task) { 104 | return task !== null; 105 | }); 106 | 107 | return tasks; 108 | } 109 | 110 | function createCleaningTasks(states) { 111 | var tasks = states.map(function (state) { 112 | return allowToDisable(state) ? disableNetworksTask(state) : null; 113 | }).filter(function (task) { 114 | return task !== null; 115 | }); 116 | 117 | return tasks; 118 | } 119 | 120 | function allowedToExecute (state) { 121 | if (state.skip || state.disabled) { 122 | return false; 123 | } 124 | 125 | if (!state.scheduledTo) { 126 | return true; 127 | } 128 | 129 | return moment().diff(state.scheduledTo) > 0; 130 | } 131 | 132 | function allowToDisable (state) { 133 | if (state.disabled || state.skip || !state.userData) { 134 | return false; 135 | } 136 | 137 | return moment().diff(state.userData.loginLastDate, 'months') > 1; 138 | } 139 | 140 | function collectingTask(state) { 141 | return function (callback) { return executor(state, connectors, callback); }; 142 | } 143 | 144 | function disableNetworksTask(state) { 145 | return function (callback) { return disableNetworks(state, callback); }; 146 | } 147 | 148 | return { 149 | run: function () { 150 | schedulerLoop(); 151 | } 152 | }; 153 | } 154 | 155 | module.exports = scheduler; 156 | -------------------------------------------------------------------------------- /source/models/items.js: -------------------------------------------------------------------------------- 1 | var _ = require('underscore'); 2 | var async = require('async'); 3 | var moment = require('moment'); 4 | 5 | var config = require('../../config'); 6 | var db = require('../db')(config); 7 | var elastic = require('../elastic')(config); 8 | 9 | module.exports = { 10 | findNew: function (items, state, callback) { 11 | if (!items || items.length === 0) { 12 | return callback(null, items); 13 | } 14 | 15 | var check = function (item, callback) { 16 | db.items.findOne({user: state.user, itemId: item.itemId}, function(err, found) { 17 | callback(!found); 18 | }); 19 | }; 20 | 21 | async.filter(items, check, function (detected) { 22 | callback(null, detected); 23 | }); 24 | }, 25 | 26 | insert: function (items, state, callback) { 27 | if (!items || items.length === 0) { 28 | return callback(null, items); 29 | } 30 | 31 | items = items.map(function (item) { 32 | return _.extend(item, {date: moment().toDate()}); 33 | }); 34 | 35 | db.items.insert(items, callback); 36 | }, 37 | 38 | index: function (items, state, callback) { 39 | if (!items || items.length === 0) { 40 | return callback(null, items); 41 | } 42 | 43 | var commands = []; 44 | items.forEach(function (item) { 45 | commands.push({'index': {'_index': 'items', '_type': 'item', '_id': item._id.toString()}}); 46 | commands.push(item); 47 | }); 48 | 49 | elastic.bulk({body: commands}, callback); 50 | } 51 | }; -------------------------------------------------------------------------------- /source/models/networks.js: -------------------------------------------------------------------------------- 1 | var config = require('../../config'); 2 | var db = require('../db')(config); 3 | var users = require('./users'); 4 | 5 | var queries = { 6 | initial: { 7 | $or: [ {mode: {$exists: false }}, { mode: 'initial'}, {mode: 'rateLimit'}] 8 | }, 9 | 10 | normal: { 11 | mode: 'normal' 12 | } 13 | }; 14 | 15 | module.exports = { 16 | findAll: function (query, callback) { 17 | return db.networks.find(query, callback); 18 | }, 19 | 20 | find: function (query, callback) { 21 | return db.networks.findOne(query, callback); 22 | }, 23 | 24 | findByMode: function (mode, callback) { 25 | return db.networks.find(queries[mode], callback); 26 | }, 27 | 28 | update: function (obj, user, callback) { 29 | obj.userData = user; 30 | 31 | db.networks.findAndModify({ 32 | query: { _id: obj._id }, 33 | update: obj 34 | }, callback); 35 | }, 36 | 37 | disable: function(state, callback) { 38 | return db.networks.findAndModify({ 39 | query: {_id: state._id}, 40 | update: {$set: {disabled: true, lastError: 'disabled due to user inactivity'}} 41 | }, callback); 42 | }, 43 | 44 | stream: function (query) { 45 | return db.networks.find(query || {}); 46 | } 47 | }; -------------------------------------------------------------------------------- /source/models/users.js: -------------------------------------------------------------------------------- 1 | var _ = require('underscore'); 2 | var moment = require('moment'); 3 | 4 | var config = require('../../config'); 5 | var db = require('../db')(config); 6 | 7 | var userPickFields = ['_id', 'avatar', 'bio', 'displayName', 'email', 'location', 'name', 'username', 'website']; 8 | 9 | function findByEmail(email, callback) { 10 | db.users.findOne({email: email}, function (err, user) { 11 | if (err) { 12 | return callback(err); 13 | } 14 | 15 | if (!user) { 16 | return callback({message: 'user not found', email: email}); 17 | } 18 | 19 | callback(null, _.pick(user, userPickFields)); 20 | }); 21 | } 22 | 23 | function findNonActive(callback) { 24 | var query = {$or: [ {loginLastDate: {$exists: false}}, {loginLastDate: {$lt: moment().subtract('month', 2).toDate() }}]}; 25 | db.users.find(query, callback); 26 | } 27 | 28 | module.exports = { 29 | findByEmail: findByEmail, 30 | findNonActive: findNonActive 31 | }; -------------------------------------------------------------------------------- /source/utils/helpers.js: -------------------------------------------------------------------------------- 1 | // http://stackoverflow.com/questions/16494663/rewrite-cycle-in-functional-style 2 | function takeWhile(array, predicate) { 3 | var pos = -1; 4 | var all = array.every(function(x, n) { 5 | return (pos = n), predicate(x); 6 | }); 7 | return array.slice(0, pos + all); 8 | } 9 | 10 | function dropWhile(array, predicate) { 11 | var pos = -1; 12 | var all = array.every(function(x, n) { 13 | return (pos = n), predicate(x); 14 | }); 15 | return array.slice(pos + all); 16 | } 17 | 18 | // http://stackoverflow.com/questions/9717488/using-since-id-and-max-id-in-twitter-api 19 | function decrementStringId (n) { 20 | n = n.toString(); 21 | var result = n; 22 | var i = n.length-1; 23 | while (i > -1) { 24 | if (n[i] === '0') { 25 | result = result.substring(0, i) + '9' + result.substring(i + 1); 26 | i--; 27 | } else { 28 | result = result.substring(0, i) + (parseInt(n[i], 10) - 1).toString() + result.substring(i + 1); 29 | return result; 30 | } 31 | } 32 | return result; 33 | } 34 | 35 | 36 | module.exports = { 37 | takeWhile: takeWhile, 38 | dropWhile: dropWhile, 39 | decrementStringId: decrementStringId 40 | }; -------------------------------------------------------------------------------- /source/utils/logger.js: -------------------------------------------------------------------------------- 1 | var util = require('util'); 2 | var colors = require('colors'); 3 | var moment = require('moment'); 4 | var logentries = require('node-logentries'); 5 | var config = require('../../config'); 6 | 7 | var mode = process.env.COLLECTOR_MODE; 8 | 9 | var log = logentries.logger({ 10 | token: config.logentries.token, 11 | printerror: false 12 | }); 13 | 14 | log.level(config.logging.level); 15 | 16 | module.exports = { 17 | success: function (message) { 18 | message = typeof message === 'string' ? message : JSON.stringify(message); 19 | console.log(this.timestamptMessage(util.format('SUCCESS: %s', message)).green); 20 | log.log('info', util.format('[%s mode] %s', mode, message)); 21 | }, 22 | 23 | warning: function (message) { 24 | message = typeof message === 'string' ? message : JSON.stringify(message); 25 | console.log(this.timestamptMessage(util.format('WARNING: %s', message)).yellow); 26 | log.log('warning', util.format('[%s mode] %s', mode, message)); 27 | }, 28 | 29 | error: function (message) { 30 | message = typeof message === 'string' ? message : JSON.stringify(message); 31 | console.log(this.timestamptMessage(util.format('ERROR: %s', message)).red); 32 | log.log('err', util.format('[%s mode] %s', mode, message)); 33 | }, 34 | 35 | fatal: function (message) { 36 | message = typeof message === 'string' ? message : JSON.stringify(message); 37 | console.log(this.timestamptMessage(util.format('ERROR: %s', message)).red); 38 | log.log('emerg', util.format('[%s mode] %s', mode, message)); 39 | }, 40 | 41 | important: function (message) { 42 | message = typeof message === 'string' ? message : JSON.stringify(message); 43 | console.log(this.timestamptMessage(util.format('IMPORTANT: %s', message)).cyan); 44 | log.log('crit', util.format('[%s mode] %s', mode, message)); 45 | }, 46 | 47 | info: function (message) { 48 | message = typeof message === 'string' ? message : JSON.stringify(message); 49 | console.log(this.timestamptMessage(message)); 50 | log.log('info', util.format('[%s mode] %s', mode, message)); 51 | }, 52 | 53 | connector: function (name) { 54 | var me = this; 55 | 56 | return { 57 | info: function (message) { 58 | me.info('connector ' + name + ': ' + message); 59 | }, 60 | warning: function (message) { 61 | me.warning('connector ' + name + ': ' + message); 62 | }, 63 | error: function (message) { 64 | me.error('connector ' + name + ': ' + message); 65 | }, 66 | success: function (message) { 67 | me.success('connector ' + name + ': ' + message); 68 | } 69 | }; 70 | }, 71 | 72 | timestamptMessage: function (message) { 73 | return util.format('[%s] [%s mode] %s', moment(), mode, message); 74 | } 75 | }; -------------------------------------------------------------------------------- /source/utils/memwatch.js: -------------------------------------------------------------------------------- 1 | var memwatch = require('memwatch'); 2 | var logger = require('./logger'); 3 | var appName = 'collector-' + process.env.COLLECTOR_MODE; 4 | 5 | if (process.env.NODE_ENV === 'production' && appName === 'collector-normal') { 6 | memwatch.on('leak', function(info) { 7 | logger.warning({msg: 'Memory leak detected', app: appName, info: info}); 8 | }); 9 | 10 | memwatch.on('stats', function(stats) { 11 | var trending = stats.usage_trend > 0; 12 | if (trending) { 13 | return logger.warning({msg: 'V8 stats (usage trending)', app: appName, stats: stats}); 14 | } 15 | 16 | logger.info({msg: 'V8 stats', app: appName, stats: stats}); 17 | }); 18 | } 19 | -------------------------------------------------------------------------------- /test/connectors/github.specs.js: -------------------------------------------------------------------------------- 1 | var expect = require('chai').expect; 2 | var nock = require('nock'); 3 | var rewire = require('rewire'); 4 | var loggerFake = require('../fakes/logger'); 5 | 6 | xdescribe('engine/connectors/github.js', function () { 7 | var state, connector; 8 | 9 | beforeEach(function () { 10 | connector = rewire('../../source/engine/connectors/github'); 11 | connector.__set__('logger', loggerFake); 12 | }); 13 | 14 | describe('when running', function () { 15 | var error; 16 | 17 | describe('and accessToken is missing', function () { 18 | beforeEach(function () { 19 | state = { 20 | userId: 'user', 21 | service: 'github' 22 | }; 23 | }); 24 | 25 | beforeEach(function (done) { 26 | connector(state, function (err, state, stars) { 27 | error = err; 28 | done(); 29 | }); 30 | }); 31 | 32 | it('should fail with error', function () { 33 | expect(error).to.equal('missing accessToken for user: ' + state.userId); 34 | }); 35 | }); 36 | 37 | describe('and username is missing', function () { 38 | beforeEach(function () { 39 | state = { 40 | userId: 'userId', 41 | accessToken: 'fakeAccessToken', 42 | service: 'github' 43 | }; 44 | }); 45 | 46 | beforeEach(function (done) { 47 | connector(state, function (err, state, stars) { 48 | error = err; 49 | done(); 50 | }); 51 | }); 52 | 53 | it('should fail with error', function () { 54 | expect(error).to.equal('missing username for user: ' + state.userId); 55 | }); 56 | }); 57 | 58 | describe('in initial mode', function () { 59 | var updatedState, returnedStars; 60 | 61 | describe('first run', function () { 62 | beforeEach(function () { 63 | state = { 64 | userId: 'userId', 65 | username: 'fakeGithubUser', 66 | accessToken: 'fakeAccessToken', 67 | service: 'github' 68 | }; 69 | }); 70 | 71 | beforeEach(function (done) { 72 | nock('https://api.github.com') 73 | .defaultReplyHeaders({ 'x-ratelimit-remaining': 100}) 74 | .get('/users/fakeGithubUser/starred?access_token=fakeAccessToken&per_page=100&page=1') 75 | .replyWithFile(200, __dirname + '/replies/github.connector.init.json'); 76 | 77 | connector(state, function (err, state, stars) { 78 | updatedState = state; 79 | returnedStars = stars; 80 | 81 | done(); 82 | }); 83 | }); 84 | 85 | it ('still in initial mode', function () { 86 | expect(updatedState.mode).to.equal('initial'); 87 | }); 88 | 89 | it ('retrieves data from first page', function () { 90 | expect(returnedStars.length).to.equal(30); 91 | }); 92 | 93 | describe ('updates state', function () { 94 | it('with next page', function () { 95 | expect(updatedState.page).to.equal(2); 96 | }); 97 | 98 | it('with id of first retrieved item', function () { 99 | expect(updatedState.sinceId).to.equal(6522993); 100 | }); 101 | }); 102 | }); 103 | 104 | describe('second run', function () { 105 | beforeEach(function () { 106 | state = { 107 | userId: 'userId', 108 | username: 'fakeGithubUser', 109 | accessToken: 'fakeAccessToken', 110 | service: 'github', 111 | mode: 'initial', 112 | page: 2 113 | }; 114 | }); 115 | 116 | beforeEach(function (done) { 117 | nock('https://api.github.com') 118 | .defaultReplyHeaders({ 'x-ratelimit-remaining': 100}) 119 | .get('/users/fakeGithubUser/starred?access_token=fakeAccessToken&per_page=100&page=2') 120 | .replyWithFile(200, __dirname + '/replies/github.connector.init.json'); 121 | 122 | connector(state, function (err, state, stars) { 123 | updatedState = state; 124 | returnedStars = stars; 125 | 126 | done(); 127 | }); 128 | }); 129 | 130 | it ('still in initial mode', function () { 131 | expect(updatedState.mode).to.equal('initial'); 132 | }); 133 | 134 | it ('retrieves data from second page', function () { 135 | expect(returnedStars.length).to.equal(30); 136 | }); 137 | 138 | describe ('updates state', function () { 139 | it('with next page', function () { 140 | expect(updatedState.page).to.equal(3); 141 | }); 142 | }); 143 | }); 144 | 145 | describe('third run', function () { 146 | beforeEach(function () { 147 | state = { 148 | userId: 'userId', 149 | username: 'fakeGithubUser', 150 | accessToken: 'fakeAccessToken', 151 | service: 'github', 152 | mode: 'initial', 153 | page: 3 154 | }; 155 | }); 156 | 157 | beforeEach(function (done) { 158 | nock('https://api.github.com') 159 | .defaultReplyHeaders({ 'x-ratelimit-remaining': 100}) 160 | .get('/users/fakeGithubUser/starred?access_token=fakeAccessToken&per_page=100&page=3') 161 | .reply(200, []); 162 | 163 | connector(state, function (err, state, stars) { 164 | updatedState = state; 165 | returnedStars = stars; 166 | 167 | done(); 168 | }); 169 | }); 170 | 171 | it ('goes to normal mode', function () { 172 | expect(updatedState.mode).to.equal('normal'); 173 | }); 174 | 175 | it ('no data on third page', function () { 176 | expect(returnedStars.length).to.equal(0); 177 | }); 178 | 179 | describe ('updates state', function () { 180 | it('removes page from state', function () { 181 | expect(updatedState.page).to.not.be.ok; 182 | }); 183 | }); 184 | }); 185 | }); 186 | 187 | describe('in normal mode', function () { 188 | var updatedState, returnedStars; 189 | 190 | describe('no new stars', function () { 191 | beforeEach(function () { 192 | state = { 193 | userId: 'userId', 194 | username: 'fakeGithubUser', 195 | accessToken: 'fakeAccessToken', 196 | service: 'github', 197 | mode: 'normal', 198 | sinceId: '6522993' 199 | }; 200 | }); 201 | 202 | beforeEach(function (done) { 203 | nock('https://api.github.com') 204 | .defaultReplyHeaders({ 'x-ratelimit-remaining': 100}) 205 | .get('/users/fakeGithubUser/starred?access_token=fakeAccessToken&per_page=100') 206 | .replyWithFile(200, __dirname + '/replies/github.connector.normal.nonew.json'); 207 | 208 | connector(state, function (err, state, stars) { 209 | updatedState = state; 210 | returnedStars = stars; 211 | 212 | done(); 213 | }); 214 | }); 215 | 216 | it ('retrieves no data', function () { 217 | expect(returnedStars.length).to.equal(0); 218 | }); 219 | 220 | describe ('updates state', function () { 221 | it('still in normal mode', function () { 222 | expect(updatedState.mode).to.equal('normal'); 223 | }); 224 | }); 225 | }); 226 | 227 | describe('new stars appeared', function () { 228 | beforeEach(function () { 229 | state = { 230 | userId: 'userId', 231 | username: 'fakeGithubUser', 232 | accessToken: 'fakeAccessToken', 233 | service: 'github', 234 | mode: 'normal', 235 | sinceId: '6522993' 236 | }; 237 | }); 238 | 239 | beforeEach(function (done) { 240 | nock('https://api.github.com') 241 | .defaultReplyHeaders({ 'x-ratelimit-remaining': 100}) 242 | .get('/users/fakeGithubUser/starred?access_token=fakeAccessToken&per_page=100') 243 | .replyWithFile(200, __dirname + '/replies/github.connector.normal.new.json'); 244 | 245 | connector(state, function (err, state, stars) { 246 | updatedState = state; 247 | returnedStars = stars; 248 | 249 | done(); 250 | }); 251 | }); 252 | 253 | it ('retrieves new data', function () { 254 | expect(returnedStars.length).to.equal(1); 255 | }); 256 | 257 | describe ('updates state', function () { 258 | it('still in normal mode', function () { 259 | expect(updatedState.mode).to.equal('normal'); 260 | }); 261 | 262 | it('stores sinceId', function () { 263 | expect(updatedState.sinceId).to.equal(6522994); 264 | }); 265 | }); 266 | }); 267 | 268 | describe('when meeting rate limit', function () { 269 | var rateLimitToStop; 270 | 271 | beforeEach(function () { 272 | rateLimitToStop = 1; 273 | }); 274 | 275 | beforeEach(function () { 276 | state = { 277 | userId: 'userId', 278 | username: 'fakeGithubUser', 279 | accessToken: 'fakeAccessToken', 280 | service: 'github', 281 | mode: 'normal', 282 | sinceId: '6522993' 283 | }; 284 | }); 285 | 286 | beforeEach(function (done) { 287 | nock('https://api.github.com') 288 | .get('/users/fakeGithubUser/starred?access_token=fakeAccessToken&per_page=100') 289 | .reply(200, [], { 'x-ratelimit-remaining': rateLimitToStop}); 290 | 291 | connector(state, function (err, state, stars) { 292 | updatedState = state; 293 | returnedStars = stars; 294 | 295 | done(); 296 | }); 297 | }); 298 | 299 | it ('should set rate limit exceed flag', function () { 300 | expect(updatedState.mode).to.equal('rateLimit'); 301 | }); 302 | 303 | it ('should store previous state', function () { 304 | expect(updatedState.prevMode).to.equal('normal'); 305 | }); 306 | 307 | describe('and run in rateLimit state', function () { 308 | var updatedUpdatedState; 309 | 310 | beforeEach(function (done) { 311 | nock('https://api.github.com') 312 | .defaultReplyHeaders({ 'x-ratelimit-remaining': 100}) 313 | .get('/users/fakeGithubUser/starred?access_token=fakeAccessToken&per_page=100') 314 | .replyWithFile(200, __dirname + '/replies/github.connector.normal.new.json'); 315 | 316 | connector(updatedState, function (err, state, stars) { 317 | updatedUpdatedState = state; 318 | returnedStars = stars; 319 | 320 | done(); 321 | }); 322 | }); 323 | 324 | it ('should go back to previous mode', function () { 325 | expect(updatedUpdatedState.mode).to.equal('normal'); 326 | }); 327 | }); 328 | }); 329 | }); 330 | }); 331 | }); -------------------------------------------------------------------------------- /test/connectors/replies/github.connector.normal.json: -------------------------------------------------------------------------------- 1 | [ 2 | { 3 | "id": 6522993, 4 | "name": "TasteApp", 5 | "full_name": "tastejs/TasteApp", 6 | "owner": { 7 | "login": "tastejs", 8 | "id": 1733746, 9 | "avatar_url": "https://secure.gravatar.com/avatar/9d0fa095b1d0462af883ffd0bb01e68d?d=https://a248.e.akamai.net/assets.github.com%2Fimages%2Fgravatars%2Fgravatar-org-420.png", 10 | "gravatar_id": "9d0fa095b1d0462af883ffd0bb01e68d", 11 | "url": "https://api.github.com/users/tastejs", 12 | "html_url": "https://github.com/tastejs", 13 | "followers_url": "https://api.github.com/users/tastejs/followers", 14 | "following_url": "https://api.github.com/users/tastejs/following{/other_user}", 15 | "gists_url": "https://api.github.com/users/tastejs/gists{/gist_id}", 16 | "starred_url": "https://api.github.com/users/tastejs/starred{/owner}{/repo}", 17 | "subscriptions_url": "https://api.github.com/users/tastejs/subscriptions", 18 | "organizations_url": "https://api.github.com/users/tastejs/orgs", 19 | "repos_url": "https://api.github.com/users/tastejs/repos", 20 | "events_url": "https://api.github.com/users/tastejs/events{/privacy}", 21 | "received_events_url": "https://api.github.com/users/tastejs/received_events", 22 | "type": "Organization" 23 | }, 24 | "private": false, 25 | "html_url": "https://github.com/tastejs/TasteApp", 26 | "description": "Taking TodoMVC further.", 27 | "fork": false, 28 | "url": "https://api.github.com/repos/tastejs/TasteApp", 29 | "forks_url": "https://api.github.com/repos/tastejs/TasteApp/forks", 30 | "keys_url": "https://api.github.com/repos/tastejs/TasteApp/keys{/key_id}", 31 | "collaborators_url": "https://api.github.com/repos/tastejs/TasteApp/collaborators{/collaborator}", 32 | "teams_url": "https://api.github.com/repos/tastejs/TasteApp/teams", 33 | "hooks_url": "https://api.github.com/repos/tastejs/TasteApp/hooks", 34 | "issue_events_url": "https://api.github.com/repos/tastejs/TasteApp/issues/events{/number}", 35 | "events_url": "https://api.github.com/repos/tastejs/TasteApp/events", 36 | "assignees_url": "https://api.github.com/repos/tastejs/TasteApp/assignees{/user}", 37 | "branches_url": "https://api.github.com/repos/tastejs/TasteApp/branches{/branch}", 38 | "tags_url": "https://api.github.com/repos/tastejs/TasteApp/tags", 39 | "blobs_url": "https://api.github.com/repos/tastejs/TasteApp/git/blobs{/sha}", 40 | "git_tags_url": "https://api.github.com/repos/tastejs/TasteApp/git/tags{/sha}", 41 | "git_refs_url": "https://api.github.com/repos/tastejs/TasteApp/git/refs{/sha}", 42 | "trees_url": "https://api.github.com/repos/tastejs/TasteApp/git/trees{/sha}", 43 | "statuses_url": "https://api.github.com/repos/tastejs/TasteApp/statuses/{sha}", 44 | "languages_url": "https://api.github.com/repos/tastejs/TasteApp/languages", 45 | "stargazers_url": "https://api.github.com/repos/tastejs/TasteApp/stargazers", 46 | "contributors_url": "https://api.github.com/repos/tastejs/TasteApp/contributors", 47 | "subscribers_url": "https://api.github.com/repos/tastejs/TasteApp/subscribers", 48 | "subscription_url": "https://api.github.com/repos/tastejs/TasteApp/subscription", 49 | "commits_url": "https://api.github.com/repos/tastejs/TasteApp/commits{/sha}", 50 | "git_commits_url": "https://api.github.com/repos/tastejs/TasteApp/git/commits{/sha}", 51 | "comments_url": "https://api.github.com/repos/tastejs/TasteApp/comments{/number}", 52 | "issue_comment_url": "https://api.github.com/repos/tastejs/TasteApp/issues/comments/{number}", 53 | "contents_url": "https://api.github.com/repos/tastejs/TasteApp/contents/{+path}", 54 | "compare_url": "https://api.github.com/repos/tastejs/TasteApp/compare/{base}...{head}", 55 | "merges_url": "https://api.github.com/repos/tastejs/TasteApp/merges", 56 | "archive_url": "https://api.github.com/repos/tastejs/TasteApp/{archive_format}{/ref}", 57 | "downloads_url": "https://api.github.com/repos/tastejs/TasteApp/downloads", 58 | "issues_url": "https://api.github.com/repos/tastejs/TasteApp/issues{/number}", 59 | "pulls_url": "https://api.github.com/repos/tastejs/TasteApp/pulls{/number}", 60 | "milestones_url": "https://api.github.com/repos/tastejs/TasteApp/milestones{/number}", 61 | "notifications_url": "https://api.github.com/repos/tastejs/TasteApp/notifications{?since,all,participating}", 62 | "labels_url": "https://api.github.com/repos/tastejs/TasteApp/labels{/name}", 63 | "created_at": "2012-11-03T18:56:27Z", 64 | "updated_at": "2013-05-07T09:51:37Z", 65 | "pushed_at": "2012-11-03T18:56:27Z", 66 | "git_url": "git://github.com/tastejs/TasteApp.git", 67 | "ssh_url": "git@github.com:tastejs/TasteApp.git", 68 | "clone_url": "https://github.com/tastejs/TasteApp.git", 69 | "svn_url": "https://github.com/tastejs/TasteApp", 70 | "homepage": "http://tastejs.com", 71 | "size": 136, 72 | "watchers_count": 54, 73 | "language": null, 74 | "has_issues": true, 75 | "has_downloads": true, 76 | "has_wiki": false, 77 | "forks_count": 0, 78 | "mirror_url": null, 79 | "open_issues_count": 1, 80 | "forks": 0, 81 | "open_issues": 1, 82 | "watchers": 54, 83 | "master_branch": "master", 84 | "default_branch": "master" 85 | }, 86 | { 87 | "id": 7268697, 88 | "name": "drywall", 89 | "full_name": "jedireza/drywall", 90 | "owner": { 91 | "login": "jedireza", 92 | "id": 979929, 93 | "avatar_url": "https://secure.gravatar.com/avatar/8aa48befef107b01994ec57fadf2d465?d=https://a248.e.akamai.net/assets.github.com%2Fimages%2Fgravatars%2Fgravatar-user-420.png", 94 | "gravatar_id": "8aa48befef107b01994ec57fadf2d465", 95 | "url": "https://api.github.com/users/jedireza", 96 | "html_url": "https://github.com/jedireza", 97 | "followers_url": "https://api.github.com/users/jedireza/followers", 98 | "following_url": "https://api.github.com/users/jedireza/following{/other_user}", 99 | "gists_url": "https://api.github.com/users/jedireza/gists{/gist_id}", 100 | "starred_url": "https://api.github.com/users/jedireza/starred{/owner}{/repo}", 101 | "subscriptions_url": "https://api.github.com/users/jedireza/subscriptions", 102 | "organizations_url": "https://api.github.com/users/jedireza/orgs", 103 | "repos_url": "https://api.github.com/users/jedireza/repos", 104 | "events_url": "https://api.github.com/users/jedireza/events{/privacy}", 105 | "received_events_url": "https://api.github.com/users/jedireza/received_events", 106 | "type": "User" 107 | }, 108 | "private": false, 109 | "html_url": "https://github.com/jedireza/drywall", 110 | "description": "A website and user system for Node.js", 111 | "fork": false, 112 | "url": "https://api.github.com/repos/jedireza/drywall", 113 | "forks_url": "https://api.github.com/repos/jedireza/drywall/forks", 114 | "keys_url": "https://api.github.com/repos/jedireza/drywall/keys{/key_id}", 115 | "collaborators_url": "https://api.github.com/repos/jedireza/drywall/collaborators{/collaborator}", 116 | "teams_url": "https://api.github.com/repos/jedireza/drywall/teams", 117 | "hooks_url": "https://api.github.com/repos/jedireza/drywall/hooks", 118 | "issue_events_url": "https://api.github.com/repos/jedireza/drywall/issues/events{/number}", 119 | "events_url": "https://api.github.com/repos/jedireza/drywall/events", 120 | "assignees_url": "https://api.github.com/repos/jedireza/drywall/assignees{/user}", 121 | "branches_url": "https://api.github.com/repos/jedireza/drywall/branches{/branch}", 122 | "tags_url": "https://api.github.com/repos/jedireza/drywall/tags", 123 | "blobs_url": "https://api.github.com/repos/jedireza/drywall/git/blobs{/sha}", 124 | "git_tags_url": "https://api.github.com/repos/jedireza/drywall/git/tags{/sha}", 125 | "git_refs_url": "https://api.github.com/repos/jedireza/drywall/git/refs{/sha}", 126 | "trees_url": "https://api.github.com/repos/jedireza/drywall/git/trees{/sha}", 127 | "statuses_url": "https://api.github.com/repos/jedireza/drywall/statuses/{sha}", 128 | "languages_url": "https://api.github.com/repos/jedireza/drywall/languages", 129 | "stargazers_url": "https://api.github.com/repos/jedireza/drywall/stargazers", 130 | "contributors_url": "https://api.github.com/repos/jedireza/drywall/contributors", 131 | "subscribers_url": "https://api.github.com/repos/jedireza/drywall/subscribers", 132 | "subscription_url": "https://api.github.com/repos/jedireza/drywall/subscription", 133 | "commits_url": "https://api.github.com/repos/jedireza/drywall/commits{/sha}", 134 | "git_commits_url": "https://api.github.com/repos/jedireza/drywall/git/commits{/sha}", 135 | "comments_url": "https://api.github.com/repos/jedireza/drywall/comments{/number}", 136 | "issue_comment_url": "https://api.github.com/repos/jedireza/drywall/issues/comments/{number}", 137 | "contents_url": "https://api.github.com/repos/jedireza/drywall/contents/{+path}", 138 | "compare_url": "https://api.github.com/repos/jedireza/drywall/compare/{base}...{head}", 139 | "merges_url": "https://api.github.com/repos/jedireza/drywall/merges", 140 | "archive_url": "https://api.github.com/repos/jedireza/drywall/{archive_format}{/ref}", 141 | "downloads_url": "https://api.github.com/repos/jedireza/drywall/downloads", 142 | "issues_url": "https://api.github.com/repos/jedireza/drywall/issues{/number}", 143 | "pulls_url": "https://api.github.com/repos/jedireza/drywall/pulls{/number}", 144 | "milestones_url": "https://api.github.com/repos/jedireza/drywall/milestones{/number}", 145 | "notifications_url": "https://api.github.com/repos/jedireza/drywall/notifications{?since,all,participating}", 146 | "labels_url": "https://api.github.com/repos/jedireza/drywall/labels{/name}", 147 | "created_at": "2012-12-21T04:15:26Z", 148 | "updated_at": "2013-05-08T14:22:39Z", 149 | "pushed_at": "2013-05-04T17:37:21Z", 150 | "git_url": "git://github.com/jedireza/drywall.git", 151 | "ssh_url": "git@github.com:jedireza/drywall.git", 152 | "clone_url": "https://github.com/jedireza/drywall.git", 153 | "svn_url": "https://github.com/jedireza/drywall", 154 | "homepage": "http://jedireza.github.io/drywall/", 155 | "size": 224, 156 | "watchers_count": 76, 157 | "language": "JavaScript", 158 | "has_issues": true, 159 | "has_downloads": true, 160 | "has_wiki": true, 161 | "forks_count": 17, 162 | "mirror_url": null, 163 | "open_issues_count": 0, 164 | "forks": 17, 165 | "open_issues": 0, 166 | "watchers": 76, 167 | "master_branch": "master", 168 | "default_branch": "master" 169 | } 170 | ] -------------------------------------------------------------------------------- /test/connectors/replies/stackoverflow.connector.init.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/likeastore/collector/850f3835334790d61f0c8d5045fcd0f702fcbc62/test/connectors/replies/stackoverflow.connector.init.json -------------------------------------------------------------------------------- /test/connectors/replies/stackoverflow.connector.init.json.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/likeastore/collector/850f3835334790d61f0c8d5045fcd0f702fcbc62/test/connectors/replies/stackoverflow.connector.init.json.gz -------------------------------------------------------------------------------- /test/connectors/replies/stackoverflow.connector.new.json: -------------------------------------------------------------------------------- 1 | { 2 | "total": 34, 3 | "page": 2, 4 | "pagesize": 100, 5 | "questions": [ 6 | { 7 | "tags": [ 8 | "javascript", 9 | "html5", 10 | "offline-caching", 11 | "offline-mode" 12 | ], 13 | "answer_count": 5, 14 | "accepted_answer_id": 9786599, 15 | "favorite_count": 4, 16 | "question_timeline_url": "/questions/9785903/timeline", 17 | "question_comments_url": "/questions/9785903/comments", 18 | "question_answers_url": "/questions/9785903/answers", 19 | "question_id": 9785903, 20 | "owner": { 21 | "user_id": 386751, 22 | "user_type": "registered", 23 | "display_name": "alexanderb", 24 | "reputation": 4732, 25 | "email_hash": "f32f547e66f3b6528376d67fdb67008f" 26 | }, 27 | "creation_date": 1332242922, 28 | "last_edit_date": 1332244680, 29 | "last_activity_date": 1332245887, 30 | "up_vote_count": 8, 31 | "down_vote_count": 0, 32 | "view_count": 1363, 33 | "score": 8, 34 | "community_owned": false, 35 | "title": "HTML5 / JS - check that application is offline" 36 | }] 37 | } -------------------------------------------------------------------------------- /test/connectors/replies/stackoverflow.connector.new.json.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/likeastore/collector/850f3835334790d61f0c8d5045fcd0f702fcbc62/test/connectors/replies/stackoverflow.connector.new.json.gz -------------------------------------------------------------------------------- /test/connectors/replies/stackoverflow.connector.normal.json: -------------------------------------------------------------------------------- 1 | { 2 | "total": 34, 3 | "page": 2, 4 | "pagesize": 100, 5 | "questions": [] 6 | } -------------------------------------------------------------------------------- /test/connectors/replies/stackoverflow.connector.normal.json.gz: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/likeastore/collector/850f3835334790d61f0c8d5045fcd0f702fcbc62/test/connectors/replies/stackoverflow.connector.normal.json.gz -------------------------------------------------------------------------------- /test/connectors/replies/twitter.connector.init.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/likeastore/collector/850f3835334790d61f0c8d5045fcd0f702fcbc62/test/connectors/replies/twitter.connector.init.json -------------------------------------------------------------------------------- /test/connectors/replies/twitter.connector.normal.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/likeastore/collector/850f3835334790d61f0c8d5045fcd0f702fcbc62/test/connectors/replies/twitter.connector.normal.json -------------------------------------------------------------------------------- /test/connectors/replies/twitter.connector.second.json: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/likeastore/collector/850f3835334790d61f0c8d5045fcd0f702fcbc62/test/connectors/replies/twitter.connector.second.json -------------------------------------------------------------------------------- /test/connectors/stackoverflow.specs.js: -------------------------------------------------------------------------------- 1 | var expect = require('chai').expect; 2 | var nock = require('nock'); 3 | var rewire = require('rewire'); 4 | var loggerFake = require('../fakes/logger'); 5 | 6 | xdescribe('engine/connectors/stackoverflow.js', function () { 7 | var state, connector; 8 | 9 | beforeEach(function () { 10 | connector = rewire('../../source/engine/connectors/stackoverflow'); 11 | connector.__set__('logger', loggerFake); 12 | }); 13 | 14 | describe('when running', function () { 15 | var error; 16 | 17 | describe('and username is missing', function () { 18 | beforeEach(function () { 19 | state = { 20 | userId: 'user', 21 | service: 'stackoverflow' 22 | }; 23 | }); 24 | 25 | beforeEach(function (done) { 26 | connector(state, function (err, state, stars) { 27 | error = err; 28 | done(); 29 | }); 30 | }); 31 | 32 | it('should fail with error', function () { 33 | expect(error).to.equal('missing username for user: ' + state.userId); 34 | }); 35 | }); 36 | 37 | describe('and accessToken is missing', function () { 38 | beforeEach(function () { 39 | state = { 40 | userId: 'user', 41 | service: 'stackoverflow', 42 | username: 12345 43 | }; 44 | }); 45 | 46 | beforeEach(function (done) { 47 | connector(state, function (err, state, stars) { 48 | error = err; 49 | done(); 50 | }); 51 | }); 52 | 53 | it('should fail with error', function () { 54 | expect(error).to.equal('missing accessToken for user: ' + state.userId); 55 | }); 56 | }); 57 | 58 | describe('in initial mode', function () { 59 | var updatedState, returnedFavorites; 60 | 61 | describe('first run', function () { 62 | beforeEach(function () { 63 | state = { 64 | userId: 'user', 65 | service: 'stackoverflow', 66 | username: 12345, 67 | accessToken: 'fakeToken' 68 | }; 69 | }); 70 | 71 | beforeEach(function (done) { 72 | nock('http://api.stackoverflow.com') 73 | .defaultReplyHeaders({'x-ratelimit-current': 100}) 74 | .get('/1.1/users/12345/favorites?access_token=fakeToken&pagesize=100&sort=creation&page=1') 75 | .replyWithFile(200, __dirname + '/replies/stackoverflow.connector.init.json.gz'); 76 | 77 | connector(state, function (err, state, favorites) { 78 | updatedState = state; 79 | returnedFavorites = favorites; 80 | 81 | done(); 82 | }); 83 | }); 84 | 85 | it('should update state with initial', function () { 86 | expect(updatedState.mode).to.equal('initial'); 87 | }); 88 | 89 | it('retrieves data from first page', function () { 90 | expect(returnedFavorites.length).to.equal(34); 91 | }); 92 | 93 | describe ('updates state', function () { 94 | it('with next page', function () { 95 | expect(updatedState.page).to.equal(2); 96 | }); 97 | 98 | it('stores from fromdate of top item (incremented)', function () { 99 | expect(updatedState.fromdate).to.equal(1332242921); 100 | }); 101 | }); 102 | }); 103 | 104 | describe('second run', function () { 105 | beforeEach(function () { 106 | state = { 107 | userId: 'user', 108 | service: 'stackoverflow', 109 | username: 12345, 110 | accessToken: 'fakeToken', 111 | fromdate: 1332242921, 112 | mode: 'initial', 113 | page: 2 114 | }; 115 | }); 116 | 117 | beforeEach(function (done) { 118 | nock('http://api.stackoverflow.com') 119 | .defaultReplyHeaders({'x-ratelimit-current': 100}) 120 | .get('/1.1/users/12345/favorites?access_token=fakeToken&pagesize=100&sort=creation&page=2') 121 | .replyWithFile(200, __dirname + '/replies/stackoverflow.connector.normal.json.gz'); 122 | 123 | connector(state, function (err, state, favorites) { 124 | updatedState = state; 125 | returnedFavorites = favorites; 126 | 127 | done(); 128 | }); 129 | }); 130 | 131 | it('goes to normal', function () { 132 | expect(updatedState.mode).to.equal('normal'); 133 | }); 134 | 135 | it('retrieves no data', function () { 136 | expect(returnedFavorites.length).to.equal(0); 137 | }); 138 | 139 | describe ('updates state', function () { 140 | it('removes next page', function () { 141 | expect(updatedState.page).to.not.be.ok; 142 | }); 143 | }); 144 | }); 145 | }); 146 | 147 | describe('in normal mode', function () { 148 | var updatedState, returnedFavorites; 149 | 150 | describe('no new data', function () { 151 | beforeEach(function () { 152 | state = { 153 | userId: 'user', 154 | service: 'stackoverflow', 155 | username: 12345, 156 | accessToken: 'fakeToken', 157 | fromdate: 1332242921, 158 | mode: 'normal' 159 | }; 160 | }); 161 | 162 | beforeEach(function (done) { 163 | nock('http://api.stackoverflow.com') 164 | .defaultReplyHeaders({'x-ratelimit-current': 100}) 165 | .get('/1.1/users/12345/favorites?access_token=fakeToken&pagesize=100&sort=creation&fromdate=1332242921') 166 | .replyWithFile(200, __dirname + '/replies/stackoverflow.connector.normal.json.gz'); 167 | 168 | connector(state, function (err, state, favorites) { 169 | updatedState = state; 170 | returnedFavorites = favorites; 171 | 172 | done(); 173 | }); 174 | }); 175 | 176 | it('retrieves no data', function () { 177 | expect(returnedFavorites.length).to.equal(0); 178 | }); 179 | }); 180 | 181 | describe('one new favorite', function () { 182 | beforeEach(function () { 183 | state = { 184 | userId: 'user', 185 | service: 'stackoverflow', 186 | username: 12345, 187 | accessToken: 'fakeToken', 188 | fromdate: 1332242921, 189 | mode: 'normal' 190 | }; 191 | }); 192 | 193 | beforeEach(function (done) { 194 | nock('http://api.stackoverflow.com') 195 | .defaultReplyHeaders({'x-ratelimit-current': 100}) 196 | .get('/1.1/users/12345/favorites?access_token=fakeToken&pagesize=100&sort=creation&fromdate=1332242921') 197 | .replyWithFile(200, __dirname + '/replies/stackoverflow.connector.new.json.gz'); 198 | 199 | connector(state, function (err, state, favorites) { 200 | updatedState = state; 201 | returnedFavorites = favorites; 202 | 203 | done(); 204 | }); 205 | }); 206 | 207 | it('retrieves new favorite', function () { 208 | expect(returnedFavorites.length).to.equal(1); 209 | }); 210 | 211 | describe ('updates state', function () { 212 | it('updates fromdate (incremented)', function () { 213 | expect(updatedState.fromdate).to.equal(1332242923); 214 | }); 215 | }); 216 | }); 217 | 218 | describe('when meeting rate limit', function () { 219 | var rateLimitToStop; 220 | 221 | beforeEach(function () { 222 | rateLimitToStop = 1; 223 | }); 224 | 225 | beforeEach(function () { 226 | state = { 227 | userId: 'user', 228 | service: 'stackoverflow', 229 | username: 12345, 230 | accessToken: 'fakeToken', 231 | fromdate: 1332242921, 232 | mode: 'normal' 233 | }; 234 | }); 235 | 236 | beforeEach(function (done) { 237 | nock('http://api.stackoverflow.com') 238 | .defaultReplyHeaders({'x-ratelimit-current': rateLimitToStop}) 239 | .get('/1.1/users/12345/favorites?access_token=fakeToken&pagesize=100&sort=creation&fromdate=1332242921') 240 | .replyWithFile(200, __dirname + '/replies/stackoverflow.connector.new.json.gz'); 241 | 242 | connector(state, function (err, state, favorites) { 243 | updatedState = state; 244 | returnedFavorites = favorites; 245 | 246 | done(); 247 | }); 248 | }); 249 | 250 | it ('should set rate limit exceed flag', function () { 251 | expect(updatedState.mode).to.equal('rateLimit'); 252 | }); 253 | 254 | it ('should store previous state', function () { 255 | expect(updatedState.prevMode).to.equal('normal'); 256 | }); 257 | 258 | describe('and run in rateLimit state', function () { 259 | var updatedUpdatedState; 260 | 261 | beforeEach(function (done) { 262 | nock('http://api.stackoverflow.com') 263 | .defaultReplyHeaders({'x-ratelimit-current': 100}) 264 | .get('/1.1/users/12345/favorites?access_token=fakeToken&pagesize=100&sort=creation&fromdate=1332242923') 265 | .replyWithFile(200, __dirname + '/replies/stackoverflow.connector.new.json.gz'); 266 | 267 | connector(updatedState, function (err, state, favorites) { 268 | updatedUpdatedState = state; 269 | returnedFavorites = favorites; 270 | 271 | done(); 272 | }); 273 | }); 274 | 275 | it ('should go back to previous mode', function () { 276 | expect(updatedUpdatedState.mode).to.equal('normal'); 277 | }); 278 | }); 279 | }); 280 | }); 281 | }); 282 | }); -------------------------------------------------------------------------------- /test/connectors/twitter.specs.js: -------------------------------------------------------------------------------- 1 | var expect = require('chai').expect; 2 | var nock = require('nock'); 3 | var rewire = require('rewire'); 4 | var loggerFake = require('../fakes/logger'); 5 | 6 | xdescribe('engine/connectors/twitter.js', function () { 7 | var state, connector; 8 | 9 | beforeEach(function () { 10 | connector = rewire('../../source/engine/connectors/twitter'); 11 | connector.__set__('logger', loggerFake); 12 | }); 13 | 14 | describe('when running', function () { 15 | var error; 16 | 17 | describe('and accessToken is missing', function () { 18 | beforeEach(function () { 19 | state = { 20 | userId: 'user', 21 | service: 'twitter' 22 | }; 23 | }); 24 | 25 | beforeEach(function (done) { 26 | connector(state, function (err, state, stars) { 27 | error = err; 28 | done(); 29 | }); 30 | }); 31 | 32 | it('should fail with error', function () { 33 | expect(error).to.equal('missing accessToken for user: ' + state.userId); 34 | }); 35 | }); 36 | 37 | describe('and accessTokenSecret is missing', function () { 38 | beforeEach(function () { 39 | state = { 40 | userId: 'user', 41 | accessToken: 'fakeAccessToken', 42 | service: 'twitter' 43 | }; 44 | }); 45 | 46 | beforeEach(function (done) { 47 | connector(state, function (err, state, stars) { 48 | error = err; 49 | done(); 50 | }); 51 | }); 52 | 53 | it('should fail with error', function () { 54 | expect(error).to.equal('missing accessTokenSecret for user: ' + state.userId); 55 | }); 56 | }); 57 | 58 | describe('and username is missing', function () { 59 | beforeEach(function () { 60 | state = { 61 | userId: 'userId', 62 | accessToken: 'fakeAccessToken', 63 | accessTokenSecret: 'fakeAccessToken', 64 | service: 'twitter' 65 | }; 66 | }); 67 | 68 | beforeEach(function (done) { 69 | connector(state, function (err, state, stars) { 70 | error = err; 71 | done(); 72 | }); 73 | }); 74 | 75 | it('should fail with error', function () { 76 | expect(error).to.equal('missing username for user: ' + state.userId); 77 | }); 78 | }); 79 | 80 | describe('in initial mode', function () { 81 | var updatedState, returnedFavorites; 82 | 83 | describe('first run', function () { 84 | beforeEach(function () { 85 | state = { 86 | userId: 'userId', 87 | accessToken: 'fakeAccessToken', 88 | accessTokenSecret: 'fakeAccessToken', 89 | username: 'fakeTwitterUser', 90 | service: 'twitter' 91 | }; 92 | }); 93 | 94 | beforeEach(function (done) { 95 | nock('https://api.twitter.com') 96 | .defaultReplyHeaders({'x-rate-limit-remaining': 100}) 97 | .get('/1.1/favorites/list.json?screen_name=fakeTwitterUser&count=200&include_entities=false') 98 | .replyWithFile(200, __dirname + '/replies/twitter.connector.init.json'); 99 | 100 | connector(state, function (err, state, favorites) { 101 | updatedState = state; 102 | returnedFavorites = favorites; 103 | 104 | done(); 105 | }); 106 | }); 107 | it ('still in initial mode', function () { 108 | expect(updatedState.mode).to.equal('initial'); 109 | }); 110 | 111 | it ('retrieves data from first page', function () { 112 | expect(returnedFavorites.length).to.equal(2); 113 | }); 114 | 115 | describe ('updates state', function () { 116 | it('initialize sinceId with first retrieved favorite id', function () { 117 | expect(updatedState.sinceId).to.equal('332570459445018627'); 118 | }); 119 | 120 | it('initialize maxId with last retrieved favorite id - 1', function () { 121 | expect(updatedState.maxId).to.equal('332542318055919616'); 122 | }); 123 | }); 124 | }); 125 | 126 | describe('second run', function () { 127 | beforeEach(function () { 128 | state = { 129 | userId: 'userId', 130 | accessToken: 'fakeAccessToken', 131 | accessTokenSecret: 'fakeAccessToken', 132 | username: 'fakeTwitterUser', 133 | service: 'twitter', 134 | maxId: '332542318055919617' 135 | }; 136 | }); 137 | 138 | beforeEach(function (done) { 139 | nock('https://api.twitter.com') 140 | .defaultReplyHeaders({'x-rate-limit-remaining': 100}) 141 | .get('/1.1/favorites/list.json?screen_name=fakeTwitterUser&count=200&include_entities=false&max_id=332542318055919617') 142 | .replyWithFile(200, __dirname + '/replies/twitter.connector.second.json'); 143 | 144 | connector(state, function (err, state, favorites) { 145 | updatedState = state; 146 | returnedFavorites = favorites; 147 | 148 | done(); 149 | }); 150 | }); 151 | 152 | it ('still in initial mode', function () { 153 | expect(updatedState.mode).to.equal('initial'); 154 | }); 155 | 156 | it ('retrieves data from first page', function () { 157 | expect(returnedFavorites.length).to.equal(2); 158 | }); 159 | 160 | describe ('updates state', function () { 161 | it('initialize sinceId with first retrieved favorite id', function () { 162 | expect(updatedState.sinceId).to.equal('332542318055919615'); 163 | }); 164 | 165 | it('initialize maxId with last retrieved favorite id - 1', function () { 166 | expect(updatedState.maxId).to.equal('332542318055919613'); 167 | }); 168 | }); 169 | }); 170 | 171 | describe('third run', function () { 172 | beforeEach(function () { 173 | state = { 174 | userId: 'userId', 175 | accessToken: 'fakeAccessToken', 176 | accessTokenSecret: 'fakeAccessToken', 177 | username: 'fakeTwitterUser', 178 | service: 'twitter', 179 | sinceId: '332570459445018627', 180 | maxId: '332542318055919614' 181 | }; 182 | }); 183 | 184 | beforeEach(function (done) { 185 | nock('https://api.twitter.com') 186 | .defaultReplyHeaders({'x-rate-limit-remaining': 100}) 187 | .get('/1.1/favorites/list.json?screen_name=fakeTwitterUser&count=200&include_entities=false&max_id=332542318055919614') 188 | .reply(200, []); 189 | 190 | connector(state, function (err, state, favorites) { 191 | updatedState = state; 192 | returnedFavorites = favorites; 193 | 194 | done(); 195 | }); 196 | }); 197 | 198 | it ('goes to normal mode', function () { 199 | expect(updatedState.mode).to.equal('normal'); 200 | }); 201 | 202 | it ('retrieves no data', function () { 203 | expect(returnedFavorites.length).to.equal(0); 204 | }); 205 | 206 | describe ('updates state', function () { 207 | it('removes maxId from state', function () { 208 | expect(updatedState.maxId).to.not.be.ok; 209 | }); 210 | }); 211 | 212 | }); 213 | }); 214 | 215 | describe('in normal mode', function () { 216 | var updatedState, returnedFavorites; 217 | 218 | beforeEach(function () { 219 | state = { 220 | userId: 'userId', 221 | accessToken: 'fakeAccessToken', 222 | accessTokenSecret: 'fakeAccessToken', 223 | username: 'fakeTwitterUser', 224 | service: 'twitter', 225 | sinceId: '332570459445018627', 226 | mode: 'normal' 227 | }; 228 | }); 229 | 230 | beforeEach(function (done) { 231 | nock('https://api.twitter.com') 232 | .defaultReplyHeaders({'x-rate-limit-remaining': 100}) 233 | .get('/1.1/favorites/list.json?screen_name=fakeTwitterUser&count=200&include_entities=false&since_id=332570459445018627') 234 | .replyWithFile(200, __dirname + '/replies/twitter.connector.normal.json'); 235 | 236 | connector(state, function (err, state, favorites) { 237 | updatedState = state; 238 | returnedFavorites = favorites; 239 | 240 | done(); 241 | }); 242 | }); 243 | 244 | it ('retrieves data if any', function () { 245 | expect(returnedFavorites.length).to.equal(2); 246 | }); 247 | 248 | describe ('updates state', function () { 249 | it('still in normal mode', function () { 250 | expect(updatedState.mode).to.equal('normal'); 251 | }); 252 | 253 | it('updates sinceId', function () { 254 | expect(updatedState.sinceId).to.equal('332542318055919620'); 255 | }); 256 | }); 257 | 258 | describe('when meeting rate limit', function () { 259 | var rateLimitToStop; 260 | 261 | beforeEach(function () { 262 | rateLimitToStop = 1; 263 | }); 264 | 265 | beforeEach(function () { 266 | state = { 267 | userId: 'userId', 268 | accessToken: 'fakeAccessToken', 269 | accessTokenSecret: 'fakeAccessToken', 270 | username: 'fakeTwitterUser', 271 | service: 'twitter', 272 | sinceId: '332570459445018627', 273 | mode: 'normal' 274 | }; 275 | }); 276 | 277 | beforeEach(function (done) { 278 | nock('https://api.twitter.com') 279 | .get('/1.1/favorites/list.json?screen_name=fakeTwitterUser&count=200&include_entities=false&since_id=332570459445018627') 280 | .reply(200, [], {'x-rate-limit-remaining': rateLimitToStop}); 281 | 282 | connector(state, function (err, state, favorites) { 283 | updatedState = state; 284 | returnedFavorites = favorites; 285 | 286 | done(); 287 | }); 288 | }); 289 | 290 | it ('should set rate limit exceed mode', function () { 291 | expect(updatedState.mode).to.equal('rateLimit'); 292 | }); 293 | 294 | it ('should store previous state', function () { 295 | expect(updatedState.prevMode).to.equal('normal'); 296 | }); 297 | 298 | describe('and run in rateLimit state', function () { 299 | var updatedUpdatedState; 300 | 301 | beforeEach(function (done) { 302 | nock('https://api.twitter.com') 303 | .defaultReplyHeaders({'x-rate-limit-remaining': 100}) 304 | .get('/1.1/favorites/list.json?screen_name=fakeTwitterUser&count=200&include_entities=false&since_id=332570459445018627') 305 | .replyWithFile(200, __dirname + '/replies/twitter.connector.normal.json'); 306 | 307 | connector(updatedState, function (err, state, favorites) { 308 | updatedUpdatedState = state; 309 | returnedFavorites = favorites; 310 | 311 | done(); 312 | }); 313 | }); 314 | 315 | it ('should go back to previous mode', function () { 316 | expect(updatedUpdatedState.mode).to.equal('normal'); 317 | }); 318 | }); 319 | }); 320 | }); 321 | }); 322 | }); 323 | -------------------------------------------------------------------------------- /test/engine/balancer.spec.js: -------------------------------------------------------------------------------- 1 | describe('balancer.spec.js', function () { 2 | var networks; 3 | }); -------------------------------------------------------------------------------- /test/engine/scheduleTo.specs.js: -------------------------------------------------------------------------------- 1 | var expect = require('chai').expect; 2 | var rewire = require('rewire'); 3 | var moment = require('moment'); 4 | var config = require('../../config'); 5 | var momentFake = require('../fakes/moment'); 6 | 7 | describe('engine/executor.js', function () { 8 | var scheduleTo, scheduledTo, current, state; 9 | 10 | beforeEach(function () { 11 | current = moment(); 12 | }); 13 | 14 | beforeEach(function () { 15 | scheduleTo = rewire('../../source/engine/scheduleTo'); 16 | scheduleTo.__set__('moment', momentFake(current)); 17 | }); 18 | 19 | describe('when running connector in initial mode', function () { 20 | beforeEach(function () { 21 | state = { service: 'github', mode: 'initial' }; 22 | }); 23 | 24 | beforeEach(function () { 25 | scheduleTo(state); 26 | }); 27 | 28 | it ('should schedule to nearest time, according to rate limits', function () { 29 | var next = current.add(config.collector.quotes.github.runAfter, 'milliseconds'); 30 | expect(next.diff(state.scheduledTo)).to.equal(0); 31 | }); 32 | }); 33 | 34 | describe('when running in normal', function () { 35 | beforeEach(function () { 36 | state = { service: 'github', mode: 'normal' }; 37 | }); 38 | 39 | beforeEach(function () { 40 | scheduleTo(state); 41 | }); 42 | 43 | it ('should schedule to next allowed time, according to config', function () { 44 | var next = current.add(config.collector.nextNormalRunAfter, 'milliseconds'); 45 | expect(next.diff(state.scheduledTo)).to.equal(0); 46 | }); 47 | }); 48 | 49 | describe('when running in rate limit mode', function () { 50 | beforeEach(function () { 51 | state = { service: 'github', mode: 'rateLimit' }; 52 | }); 53 | 54 | beforeEach(function () { 55 | scheduleTo(state); 56 | }); 57 | 58 | it ('should schedule to prevent rate limit, according to config', function () { 59 | var next = current.add(config.collector.nextRateLimitRunAfter, 'milliseconds'); 60 | expect(next.diff(state.scheduledTo)).to.equal(0); 61 | }); 62 | }); 63 | }); -------------------------------------------------------------------------------- /test/fakes/logger.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | success: function () { 3 | }, 4 | warning: function () { 5 | }, 6 | error: function () { 7 | }, 8 | info: function () { 9 | }, 10 | connector: function () { 11 | return { 12 | success: function () { 13 | }, 14 | warning: function () { 15 | }, 16 | error: function () { 17 | }, 18 | info: function () { 19 | } 20 | }; 21 | } 22 | }; -------------------------------------------------------------------------------- /test/fakes/moment.js: -------------------------------------------------------------------------------- 1 | function moment (dateToReturn) { 2 | return function () { 3 | return dateToReturn.clone(); 4 | }; 5 | } 6 | 7 | module.exports = moment; -------------------------------------------------------------------------------- /test/mocha.opts: -------------------------------------------------------------------------------- 1 | --reporter spec 2 | --ui bdd 3 | --recursive 4 | --colors 5 | --timeout 60000 6 | --slow 300 --------------------------------------------------------------------------------