├── server.js ├── .vscode └── launch.json ├── .gitignore ├── package.json ├── README.md ├── utils └── utils.js └── routes ├── fbTrigger.js └── githubTrigger.js /server.js: -------------------------------------------------------------------------------- 1 | var express = require('express'); 2 | var path = require('path'); 3 | var bodyParser = require('body-parser'); 4 | 5 | 6 | var githubTrigger = require('./routes/githubTrigger.js'); 7 | var fbTrigger = require('./routes/fbTrigger.js'); 8 | 9 | var app = express(); 10 | 11 | 12 | app.use(bodyParser.json()); 13 | app.use(bodyParser.urlencoded({ extended: false })); 14 | 15 | app.use('/api/githubTrigger', githubTrigger); 16 | app.use('/api/fbTrigger', fbTrigger); 17 | 18 | 19 | 20 | var port = process.env.PORT || 3000; 21 | app.listen(port, function () { 22 | console.log('server started on port ' + port); 23 | }) 24 | -------------------------------------------------------------------------------- /.vscode/launch.json: -------------------------------------------------------------------------------- 1 | { 2 | // Use IntelliSense to learn about possible Node.js debug attributes. 3 | // Hover to view descriptions of existing attributes. 4 | // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 5 | "version": "0.2.0", 6 | "configurations": [ 7 | { 8 | "type": "node", 9 | "request": "launch", 10 | "name": "Launch Program", 11 | "program": "${workspaceRoot}/server.js", 12 | "cwd": "${workspaceRoot}" 13 | }, 14 | { 15 | "type": "node", 16 | "request": "attach", 17 | "name": "Attach to Process", 18 | "port": 5858 19 | } 20 | ] 21 | } -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Logs 2 | logs 3 | *.log 4 | npm-debug.log* 5 | 6 | # Runtime data 7 | pids 8 | *.pid 9 | *.seed 10 | 11 | # Directory for instrumented libs generated by jscoverage/JSCover 12 | lib-cov 13 | 14 | # Coverage directory used by tools like istanbul 15 | coverage 16 | 17 | # nyc test coverage 18 | .nyc_output 19 | 20 | # Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files) 21 | .grunt 22 | 23 | # node-waf configuration 24 | .lock-wscript 25 | 26 | # Compiled binary addons (http://nodejs.org/api/addons.html) 27 | build/Release 28 | 29 | # Dependency directories 30 | node_modules 31 | jspm_packages 32 | 33 | # Optional npm cache directory 34 | .npm 35 | 36 | # Optional REPL history 37 | .node_repl_history 38 | 39 | #ignore this file to load some secret info locally 40 | secretGitIgnore.js -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "fb-github-trends", 3 | "version": "1.0.0", 4 | "description": "Posts trending repositories to the facebook page", 5 | "main": "server.js", 6 | "scripts": { 7 | "test": "echo \"Error: no test specified\" && exit 1" 8 | }, 9 | "repository": { 10 | "type": "git", 11 | "url": "git+https://github.com/bumbeishvili/fb-github-trends.git" 12 | }, 13 | "keywords": [ 14 | "fb", 15 | "github", 16 | "trending", 17 | "trend", 18 | "repo", 19 | "repositoy", 20 | "page" 21 | ], 22 | "author": "Dato Bumbeishvili", 23 | "license": "ISC", 24 | "bugs": { 25 | "url": "https://github.com/bumbeishvili/fb-github-trends/issues" 26 | }, 27 | "homepage": "https://github.com/bumbeishvili/fb-github-trends#readme", 28 | "dependencies": { 29 | "body-parser": "^1.15.2", 30 | "express": "^4.14.0", 31 | "fb": "^1.1.1", 32 | "mongojs": "^2.4.0" 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Github trends on Facebook 2 | 3 | 4 | Automatically post Github trends on Facebook page 5 | 6 | 7 | ## Pages 8 | [Github Trends](https://www.facebook.com/github.trends/) 9 | [Github Trends C#](https://www.facebook.com/github.trends.c.sharp/) 10 | [Github Trends Javascript](https://www.facebook.com/github.trends.javascript/) 11 | [Github Trends Css](https://www.facebook.com/github.trends.css.bot/) 12 | [Github Trends Html](https://www.facebook.com/github.trends.html.bot/ ) 13 | [Github Trends Java](https://www.facebook.com/github.trends.java.bot/ ) 14 | [Github Trends Php](https://www.facebook.com/github.trends.p.hipertext.prepocessor/ ) 15 | [Github Trends Python](https://www.facebook.com/github.trends.python/ ) 16 | [Github Trends Swift](https://www.facebook.com/Github-Trends-Swift-371312076570996/ ) 17 | 18 | 19 | 20 | 21 | 22 | # Project Details: 23 | 24 | 25 | ## Flow 26 | We have 2 trigger: 27 | 28 | `/api/githubTrigger` - scrapes trending github repo, saves new repos in mongodb and removes duplicates 29 | 30 | `/api/fbTrigger` - gets data from mongodb and posts it to the fb page 31 | 32 | [uptimeRobot](https://uptimerobot.com) - pings each urls in every 3 hours and renews information 33 | 34 | 35 | ## Links I have used extensively during development 36 | [Automating posts on FB page](http://stackoverflow.com/questions/26605805/automatic-post-to-my-facebook-page-from-node-js-server ) 37 | [developers.facebook.com](https://developers.facebook.com/apps/813508885415765/settings/) 38 | [Acces token debug](https://developers.facebook.com/tools/debug/accesstoken ) 39 | [Deploy node.js app to heroku](https://scotch.io/tutorials/how-to-deploy-a-node-js-app-to-heroku ) 40 | [Heroku configuration variables](https://devcenter.heroku.com/articles/config-vars) 41 | 42 | 43 | -------------------------------------------------------------------------------- /utils/utils.js: -------------------------------------------------------------------------------- 1 | var mongojs = require('mongojs'); 2 | 3 | var supportedLanguages = [ 4 | { 5 | lang: "" // Empty string means 'all languages' 6 | }, 7 | { 8 | lang: "csharp" 9 | }, 10 | { 11 | lang: "javascript" 12 | }, 13 | { 14 | lang: "css" 15 | }, 16 | { 17 | lang: "html" 18 | }, 19 | { 20 | lang: "java" 21 | }, 22 | { 23 | lang: "php" 24 | }, 25 | { 26 | lang: "swift" 27 | }, 28 | { 29 | lang: "python" 30 | } 31 | ]; 32 | 33 | var getClosestMonday = function getMonday(d) { 34 | d = new Date(d); 35 | var day = d.getDay(), 36 | diff = d.getDate() - day + (day == 0 ? -6 : 1); // adjust when day is sunday 37 | return new Date(d.setDate(diff)); 38 | }; 39 | 40 | var getRepoCompositeId = function getRepoCompositeId(trendingRepo) { 41 | var monday = getClosestMonday(trendingRepo.scrapeTime); 42 | var date = monday.getDate() + '/' + monday.getMonth(); 43 | var result = trendingRepo.owner + trendingRepo.name + trendingRepo.langCode + trendingRepo.type + date; 44 | return result; 45 | } 46 | 47 | 48 | var getDB = function getDB() { 49 | var db; 50 | if (process.env.mongoDBConnection) { 51 | db = mongojs(process.env.mongoDBConnection); 52 | } else { 53 | var secret = require('./secretGitIgnore'); 54 | db = mongojs(secret.mongoDBConnection); 55 | } 56 | return db; 57 | } 58 | 59 | var getRandomItem = function getRandomItem(items) { 60 | var item = items[Math.floor(Math.random() * items.length)]; 61 | return item; 62 | } 63 | 64 | var getAccessTokenByRepo = function getAccessTokenByRepo(repo) { 65 | var secret; 66 | if (!process.env.PORT) { 67 | secret = require('./secretGitIgnore'); 68 | } 69 | 70 | var prefix = repo.langCode || "allLang"; 71 | var accessToken = process.env[prefix + "AccessToken"] || secret[prefix + "AccessToken"]; 72 | return accessToken; 73 | } 74 | 75 | 76 | var removeByAttr = function removeByAttr(arr, attr, value) { 77 | var i = arr.length; 78 | while (i--) { 79 | if (arr[i] 80 | && arr[i].hasOwnProperty(attr) 81 | && (arguments.length > 2 && arr[i][attr] === value)) { 82 | 83 | arr.splice(i, 1); 84 | 85 | } 86 | } 87 | return arr; 88 | } 89 | 90 | var logReposByLang = function logReposByLang(repos) { 91 | var langCodes = repos.map(repo => repo.langCode); 92 | var countPerLang = langCodes.reduce((counts, item) => { 93 | if (item in counts) { 94 | counts[item]++; 95 | } 96 | else { 97 | counts[item] = 1; 98 | } 99 | return counts; 100 | }, {}); 101 | 102 | var distinctLangs = Object.keys(countPerLang); 103 | distinctLangs.forEach((lang) => { 104 | console.log(' ', countPerLang[lang], lang || 'top'); 105 | }); 106 | console.log(' ', 'TOTAL ', repos.length); 107 | return countPerLang; 108 | } 109 | 110 | module.exports.getClosestMonday = getClosestMonday; 111 | module.exports.getRepoCompositeId = getRepoCompositeId; 112 | module.exports.getDB = getDB; 113 | module.exports.getRandomItem = getRandomItem; 114 | module.exports.getAccessTokenByRepo = getAccessTokenByRepo; 115 | module.exports.removeByAttr = getAccessTokenByRepo; 116 | module.exports.logReposByLang = logReposByLang; 117 | module.exports.supportedLanguages = supportedLanguages; 118 | -------------------------------------------------------------------------------- /routes/fbTrigger.js: -------------------------------------------------------------------------------- 1 | var utils = require('../utils/utils'); 2 | var express = require('express'); 3 | var router = express.Router(); 4 | var FB = require('fb'); 5 | 6 | var db = utils.getDB(); 7 | 8 | var statuses = { 9 | processing:"processing", 10 | idle:"idle" 11 | } 12 | var status = statuses.idle; 13 | router.get('/status/:newValue',(req,res,next)=>{ 14 | const newValue = req.params.newValue; 15 | status=newValue; 16 | res.send(status); 17 | }) 18 | 19 | 20 | router.get('/log',(req,res,next)=>{ 21 | db.repos.find({ posted: { $ne: true }} , (err, repos) => { 22 | res.send(utils.logReposByLang(repos)); 23 | }) 24 | }) 25 | 26 | router.get('/extendAccessToken/:appId/:appSecret/:token', function (req, nodeRes, next) { 27 | 28 | const client_id = req.params.appId; 29 | const client_secret = req.params.appSecret; 30 | const existing_access_token = req.params.token; 31 | 32 | FB.api('oauth/access_token', { 33 | client_id: client_id, 34 | client_secret: client_secret, 35 | grant_type: 'fb_exchange_token', 36 | fb_exchange_token: existing_access_token 37 | }, function (res) { 38 | if (!res || res.error) { 39 | console.log(!res ? 'error occurred' : res.error); 40 | nodeRes.send('error + ' + res.error.message); 41 | return; 42 | } 43 | nodeRes.send('success - ' + res.access_token); 44 | var accessToken = res.access_token; 45 | var expires = res.expires ? res.expires : 0; 46 | }); 47 | }); 48 | 49 | 50 | 51 | router.get('/', function (req, res, next) { 52 | if(status==statuses.processing){ 53 | res.send('Already Processing'); 54 | return; 55 | } 56 | status=statuses.processing; 57 | res.send('Processing started'); 58 | 59 | //load unposted 60 | db.repos.find({ posted: { $ne: true } }, (err, repos) => { 61 | 62 | if (err) { 63 | console.log(err); 64 | } 65 | 66 | console.log('------------------------ LOADED UNPOSTED FROM DB ---------------------'); 67 | utils.logReposByLang(repos); 68 | var filteredLangCodes = []; 69 | 70 | var interval = setInterval(() => { 71 | //get repo, which was not posted yet 72 | // c sharp and css page is blocked, so :( 73 | db.repos.findOne({ posted: { $ne: true },langCode:{$nin: filteredLangCodes } }, (err, repo) => { 74 | console.log('Trying to post ', repo.name,repo.langCode); 75 | console.log('Filtered Langs ', filteredLangCodes) 76 | if (err) { 77 | console.log(err); 78 | clearInterval(interval); 79 | return; 80 | } 81 | if (!repo) { 82 | console.log('Done!'); 83 | clearInterval(interval); 84 | status=statuses.idle; 85 | return; 86 | } 87 | 88 | postToFB(repo, res, filteredLangCodes); 89 | }); 90 | 91 | 92 | 93 | }, 60000) 94 | 95 | }); 96 | 97 | 98 | }); 99 | 100 | function postToFB(repo, res, filteredLangCodes) { 101 | 102 | FB.setAccessToken(utils.getAccessTokenByRepo(repo)); 103 | 104 | var fbPost = { 105 | message: (repo.description || " ") + " \r\n(" + (repo.todaysStars||"") + (repo.todaysStars?", ":"") + repo.allStars + " total, written with " + (repo.language || "markdown") + " )", 106 | link: "http://www.github.com/" + repo.owner + "/" + repo.name, 107 | name: repo.name 108 | } 109 | var body = 110 | FB.api('me/feed', 'post', fbPost, function (res) { 111 | if (!res || res.error) { 112 | filteredLangCodes.push(repo.langCode); 113 | console.log(!res ? 'error occurred' : res.error); 114 | return; 115 | } 116 | console.log('New Post - : ', res.id, ' - ', repo.langCode || "Top", repo.name); 117 | 118 | repo.posted = true; 119 | db.repos.update({ "_id": repo._id }, repo); 120 | }); 121 | } 122 | 123 | 124 | 125 | module.exports = router; 126 | -------------------------------------------------------------------------------- /routes/githubTrigger.js: -------------------------------------------------------------------------------- 1 | var utils = require('../utils/utils'); 2 | var express = require('express'); 3 | var mongojs = require('mongojs'); 4 | var router = express.Router(); 5 | 6 | 7 | var Trending = require("github-trend"); 8 | var scraper = new Trending.Scraper(); 9 | 10 | var db = utils.getDB(); 11 | 12 | const configs = utils.supportedLanguages; 13 | 14 | const trendingTypes = { 15 | DAILY: 'DAILY', 16 | WEEKLY: 'WEEKLY', 17 | MONTHLY: 'MONTHLY' 18 | } 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | router.get('/', function (req, res, next) { 28 | var data = []; 29 | var langs = configs.map(c => c.lang); 30 | 31 | fetchTrendingRepos(langs, data) 32 | .then(processRepos.bind(this, res, data)) 33 | .catch(function (err) { 34 | console.log(err.message); 35 | }); 36 | 37 | }); 38 | 39 | 40 | function processRepos(res, data) { 41 | console.log('------------------------ FETCHED FROM GITHUB ---------------------'); 42 | utils.logReposByLang(data); 43 | 44 | data.forEach(trendingRepo => { 45 | trendingRepo.scrapeTime = new Date(); 46 | trendingRepo.postTime = new Date(); 47 | trendingRepo.compositeId = utils.getRepoCompositeId(trendingRepo); 48 | }); 49 | 50 | /* IF DATA IS MUDDLED, THE UNCOMMENT */ 51 | // //remove all first 52 | // db.repos.remove({}, (err, repos) => { 53 | // console.log('removed'); 54 | // }) 55 | 56 | 57 | //load all 58 | db.repos.find(function (err, repos) { 59 | if (err) { 60 | resp.send(err); 61 | } 62 | 63 | console.log('------------------------ LOADED FROM DB ---------------------'); 64 | utils.logReposByLang(repos); 65 | 66 | var newData = []; 67 | var ids = repos.map(repo => repo.compositeId); 68 | 69 | // add repos,which are new, or was not added more than one week 70 | data.forEach(newRepo => { 71 | if (ids.indexOf(newRepo.compositeId) == -1) { 72 | newData.push(newRepo); 73 | }; 74 | }); 75 | 76 | console.log('------------------------ NEW ---------------------'); 77 | utils.logReposByLang(newData); 78 | if (newData.length) { 79 | db.repos.insert(newData, () => { 80 | removeOldAndDuplicates(); 81 | }) 82 | } else { 83 | removeOldAndDuplicates(); 84 | } 85 | 86 | 87 | var toBePostedRepos = newData.concat(repos).filter(r => !r.posted); 88 | 89 | console.log('------------------------ WILL BE POSTED ON FB ---------------------'); 90 | utils.logReposByLang(toBePostedRepos); 91 | 92 | 93 | res.json(toBePostedRepos); 94 | }) 95 | 96 | } 97 | 98 | 99 | function removeOldAndDuplicates() { 100 | console.log(' removing duplicates ...'); 101 | db.repos.find(function (err, repos) { 102 | if (err) { 103 | resp.send(err); 104 | } 105 | repos.sort((a, b) => (a.compositeId > b.compositeId) ? 1 : ((b.compositeId > a.compositeId) ? -1 : 0)); 106 | 107 | 108 | 109 | //remove duplicates 110 | if (repos.length > 1) { 111 | for (var i = 1; i < repos.length; i++) { 112 | if (repos[i].compositeId == repos[i - 1].compositeId) { 113 | 114 | console.log('removing duplicate ' + repos[i].name); 115 | db.repos.remove({ _id: repos[i]._id }); 116 | } 117 | } 118 | } 119 | 120 | //remove old 121 | var date = new Date(); 122 | date.setMonth(date.getMonth() - 1); 123 | db.repos.remove({ scrapeTime: { $lte: date } }, () => { 124 | console.log('Done'); 125 | }); 126 | 127 | 128 | }) 129 | 130 | } 131 | 132 | function fetchTrendingRepos(languageArray, data) { 133 | return languageArray.reduce((promise, language) => { 134 | return promise.then(() => { 135 | return scraper.scrapeTrendingReposFullInfo(language) 136 | .then(repos => { 137 | var extended = repos.map(repo => { 138 | repo.langCode = language; 139 | repo.type = trendingTypes.DAILY; 140 | return repo; 141 | }); 142 | data.push.apply(data, extended); 143 | }); 144 | }); 145 | }, Promise.resolve()); 146 | } 147 | 148 | module.exports = router; --------------------------------------------------------------------------------