├── .gitignore
├── package.json
├── downloader.js
├── feedbot.js
├── runner.js
├── feed.js
├── rss.json
└── article.js


/.gitignore:
--------------------------------------------------------------------------------
1 | node_modules


--------------------------------------------------------------------------------
/package.json:
--------------------------------------------------------------------------------
 1 | {
 2 |     "name": "bfclient",
 3 |     "version": "0.1.0",
 4 |     "author" : "Andris Reinman",
 5 |     "maintainers":[
 6 |         {
 7 |             "name":"andris",
 8 |             "email":"andris@node.ee"
 9 |         }
10 |     ],
11 |     "dependencies": {
12 |         "fetch": "*",
13 |         "nodepie": "*",
14 |         "node-gearman":"*",
15 |         "redis": "*",
16 |         "iconv": "*",
17 |         "rai": "*",
18 |         "node-ffi": "*"
19 |     }
20 | }


--------------------------------------------------------------------------------
/downloader.js:
--------------------------------------------------------------------------------
 1 | 
 2 | // handle parent death
 3 | (function(){
 4 |     var FFI = require('node-ffi');
 5 |     var current = new FFI.Library(null, {"prctl": ["int32", ["int32", "uint32"]]})
 6 |     
 7 |     //1: PR_SET_PDEATHSIG, 15: SIGTERM
 8 |     var returned = current.prctl(1,15);
 9 |     
10 |     process.on('SIGTERM',function(){
11 |             //do something interesting
12 |             process.exit(1);
13 |     });
14 | })();
15 | 
16 | 
17 | var redis = require("redis").createClient(),
18 |     articlelib = require("./article");
19 | 
20 | mainloop();
21 | 
22 | function mainloop(){
23 |     
24 |     articlelib.fetchArticle(function(err, article){
25 |         if(err){
26 |             console.log(err);
27 |             setTimeout(mainloop, 300);
28 |             return;
29 |         }
30 |         if(!article){
31 |             //nothing found
32 |             setTimeout(mainloop, 1*1000);
33 |             return;
34 |         }
35 |         
36 |         console.log("Publishing "+article.title);
37 |         redis.publish("article", JSON.stringify(article));
38 | 
39 |         /*        
40 |         console.log(article.title);
41 |         console.log(new Array(article.title.length+1).join("="));
42 |         console.log("["+article.url+"]");
43 |         console.log("["+article.domain+"]");
44 |         console.log(article.content);
45 |         console.log(article.lemma);
46 |         */ 
47 |         
48 |         setTimeout(mainloop, 300);
49 |     });
50 | 
51 | }


--------------------------------------------------------------------------------
/feedbot.js:
--------------------------------------------------------------------------------
 1 | // handle parent death
 2 | (function(){
 3 |     var FFI = require('node-ffi');
 4 |     var current = new FFI.Library(null, {"prctl": ["int32", ["int32", "uint32"]]})
 5 |     
 6 |     //1: PR_SET_PDEATHSIG, 15: SIGTERM
 7 |     var returned = current.prctl(1,15);
 8 |     
 9 |     process.on('SIGTERM',function(){
10 |             //do something interesting
11 |             process.exit(1);
12 |     });
13 | })();
14 | 
15 | var fs = require("fs"),
16 |     feeder = require("./feed"),
17 |     rssList = JSON.parse(fs.readFileSync(__dirname + "/rss.json"));
18 | 
19 | // start the bot
20 | initialize();
21 | 
22 | function initialize(){
23 |     // Initial start
24 |     for(var i=0, len = rssList.length; i<len; i++){
25 |         startListening(rssList[i]);
26 |     }    
27 | }
28 | 
29 | function startListening(rss){
30 |     delay = nextInterval();
31 |     console.log("└── " + rss.title + " - checking in " + prettyTime(delay));
32 |     setTimeout(function(){
33 |         feeder.checkNewFeedItems(rss, function(err, added){
34 |             if(err){
35 |                 console.log(rss.title + " yielded in error: "+err.message);
36 |             }else{
37 |                 console.log("Checked "+rss.title+" (added "+added.length+")");
38 |             }
39 |             process.nextTick(startListening.bind(this, rss));
40 |         });
41 |     }, delay);
42 | };
43 | 
44 | function nextInterval(){
45 |     var max_delay = 7 * 60 * 1000, // 7 min
46 |         min_delay = 0.5 * 60 * 1000;   //  0.5 min
47 |     return Math.round(Math.random() * (max_delay - min_delay) + min_delay);
48 | }
49 | 
50 | function prettyTime(time){
51 |     var hours, minutes, seconds, resp = [];
52 | 
53 |     hours = Math.floor(time / (3600*1000));
54 |     time = time - hours*(3600*1000);
55 |     if(hours)resp.push(hours+" hour"+(hours>1?"s":""));
56 | 
57 |     minutes = Math.floor(time / (60*1000));
58 |     time = time - minutes*(60*1000);
59 |     if(minutes)resp.push(minutes+" minute"+(minutes>1?"s":""));
60 | 
61 |     seconds = Math.floor(time / (1*1000));
62 |     time = time - seconds*(1*1000);
63 |     resp.push(seconds+" second"+(!seconds || seconds>1?"s":""));
64 | 
65 |     return resp.join(", ");
66 | }


--------------------------------------------------------------------------------
/runner.js:
--------------------------------------------------------------------------------
  1 | var spawn = require("child_process").spawn,
  2 |     RAIServer = require("rai").RAIServer;
  3 | 
  4 | process.nextTick(createFeedbot);
  5 | process.nextTick(createDownloader);
  6 | process.nextTick(createStore);
  7 | process.nextTick(createAPI);
  8 | 
  9 | function createFeedbot(){
 10 |     console.log("Starting feedbot");
 11 |     
 12 |     var feedbot = spawn('/usr/local/bin/node', [__dirname+'/feedbot.js'], {cwd: __dirname});
 13 |     
 14 |     feedbot.stdout.on('data', function (data) {
 15 |         Tail('feedbot stdout: ' + (data || "").toString("utf-8").trim())
 16 |     });
 17 |     
 18 |     feedbot.stderr.on('data', function (data) {
 19 |         Tail('feedbot stderr: ' + (data || "").toString("utf-8").trim());
 20 |     });
 21 |     
 22 |     feedbot.on('exit', function (code) {
 23 |         Tail('feedbot exited with code ' + code);
 24 |         setTimeout(createFeedbot, 1000);
 25 |     });
 26 | }
 27 | 
 28 | function createDownloader(){
 29 |     Tail("Starting downloader");
 30 |     
 31 |     var downloader = spawn('/usr/local/bin/node', [__dirname+'/downloader.js'], {cwd: __dirname});
 32 |     
 33 |     downloader.stdout.on('data', function (data) {
 34 |         Tail('downloader stdout: ' + (data || "").toString("utf-8").trim());
 35 |     });
 36 |     
 37 |     downloader.stderr.on('data', function (data) {
 38 |         Tail('downloader stderr: ' + (data || "").toString("utf-8").trim());
 39 |     });
 40 |     
 41 |     downloader.on('exit', function (code) {
 42 |         Tail('downloader exited with code ' + code);
 43 |         setTimeout(createDownloader, 1000);
 44 |     });
 45 | }
 46 | 
 47 | function createStore(){
 48 |     var store = spawn('/usr/local/bin/node', [__dirname+'/../bfstore/store.js'], {cwd: __dirname+"/../bfstore"});
 49 |     
 50 |     store.stdout.on('data', function (data) {
 51 |         Tail('store stdout: ' + (data || "").toString("utf-8").trim());
 52 |     });
 53 |     
 54 |     store.stderr.on('data', function (data) {
 55 |         Tail('store stderr: ' + (data || "").toString("utf-8").trim());
 56 |     });
 57 |     
 58 |     store.on('exit', function (code) {
 59 |         Tail('store exited with code ' + code);
 60 |         setTimeout(createStore, 1000);
 61 |     });
 62 | }
 63 | 
 64 | function createAPI(){
 65 |     var api = spawn('/usr/local/bin/node', [__dirname+'/../bfstore/api.js'], {cwd: __dirname+"/../bfapi"});
 66 |     
 67 |     api.stdout.on('data', function (data) {
 68 |         Tail('api stdout: ' + (data || "").toString("utf-8").trim());
 69 |     });
 70 |     
 71 |     api.stderr.on('data', function (data) {
 72 |         Tail('api stderr: ' + (data || "").toString("utf-8").trim());
 73 |     });
 74 |     
 75 |     api.on('exit', function (code) {
 76 |         Tail('api exited with code ' + code);
 77 |         setTimeout(createAPI, 1000);
 78 |     });
 79 | }
 80 | 
 81 | var tail = [];
 82 | function Tail(msg){
 83 |     tail.push(msg);
 84 |     if(tail.length > 250){
 85 |         tail.shift();
 86 |     }
 87 |     console.log(msg);
 88 | }
 89 | 
 90 | var server = new RAIServer();
 91 | server.listen(8082);
 92 | 
 93 | server.on("connect", function(client){
 94 | 
 95 |     // Greet the client
 96 |     client.send("Hello!");
 97 | 
 98 |     // Wait for a command
 99 |     client.on("command", function(command, payload){
100 | 
101 |         command = (command || "").toString("utf-8").trim().toUpperCase();
102 | 
103 |         if(command == "TAIL"){
104 |             for(var i=0, len=tail.length; i<len; i++){
105 |                 client.send(new Buffer(tail[i], "utf-8"));
106 |             }
107 |             client.send("OK");
108 |         }else if(command == "QUIT"){
109 |             client.send("Goodbye");
110 |             client.end();
111 |         }else{
112 |             client.send("Unknown command");
113 |         }
114 | 
115 |     });
116 | 
117 | });


--------------------------------------------------------------------------------
/feed.js:
--------------------------------------------------------------------------------
  1 | var fetch = require("fetch"),
  2 |     NodePie = require("nodepie"),
  3 |     redis = require("redis").createClient(),
  4 |     crypto = require("crypto");
  5 | 
  6 | module.exports.checkNewFeedItems = checkNewFeedItems;
  7 | module.exports.popItemFromList = popItemFromList;
  8 | 
  9 | function popItemFromList(callback){
 10 |     redis.spop("feed:check", function(err, hash){
 11 |         if(hash){
 12 |             redis.hgetall("feed:url:"+hash, function(err, url){
 13 |                 if(url){
 14 |                     redis.del("feed:url:"+hash, function(){
 15 |                         process.nextTick(callback.bind(this, null, url));
 16 |                     });
 17 |                 }else{
 18 |                     process.nextTick(callback.bind(this, new Error("No URL object found")));
 19 |                 }
 20 |             });
 21 |         }else{
 22 |             process.nextTick(callback.bind(this, new Error("No hash found")));
 23 |         }
 24 |     });
 25 | }
 26 | 
 27 | function checkNewFeedItems(rss, callback){
 28 |     feedFetcher(rss, function(err, list){
 29 |         if(err){
 30 |             return process.nextTick(callback.bind(this, err));
 31 |         }
 32 | 
 33 |         checkListForNewItems(list, function(err, added){
 34 |             if(err){
 35 |                 return process.nextTick(callback.bind(this, err));
 36 |             }
 37 |             
 38 |             process.nextTick(callback.bind(this, null, added));
 39 |         });
 40 |     });
 41 | }
 42 | 
 43 | 
 44 | function feedFetcher(rss, callback){
 45 | 
 46 |     fetch.fetchUrl(rss.url, function(error, meta, rssContent){
 47 | 
 48 |         if(error){
 49 |             return callback(error);
 50 |         }
 51 | 
 52 |         //FIXME: lisa toetus modified-since päringutele
 53 |         if(meta.status != 200){
 54 |             return callback(new Error("RSS Status: " + meta.status));
 55 |         }
 56 | 
 57 |         var list = [],
 58 |             match = rssContent.slice(0,1024).toString().match(/encoding\s*=\s*["']([^"']+)["']/i),
 59 |             encoding = (rss.charset || (match && match[1]) || "UTF-8").trim().toUpperCase(),
 60 |             iconv;
 61 |         
 62 |         if(encoding != "UTF-8"){
 63 |             iconv = new Iconv(encoding, 'UTF-8//TRANSLIT//IGNORE');
 64 |             rssContent = iconv.convert(rssContent);
 65 |         };
 66 | 
 67 |         feed = new NodePie(rssContent.toString().trim());
 68 |         feed.init();
 69 | 
 70 |         feed.getItems().forEach(function(item){
 71 | 
 72 |             if(item.getPermalink()){
 73 |                 list.push({
 74 |                     site: rss.id,
 75 |                     title: item.getTitle().replace(/&amp;(?:amp;)?/g,"&").replace(/&quot;/g,'"'),
 76 |                     url: item.getPermalink().replace(/&amp;(?:amp;)?/g,"&"),
 77 |                     date: Math.round((item.getDate() || new Date()).getTime()/1000),
 78 |                     language: rss.language || "et"
 79 |                 });
 80 |             }
 81 | 
 82 |         });
 83 | 
 84 |         feed = null;
 85 | 
 86 |         process.nextTick(callback.bind(this, null, list));
 87 | 
 88 |     });
 89 | }
 90 | 
 91 | function checkListForNewItems(list, callback, added){
 92 |     added = added || [];
 93 |     if(!list.length){
 94 |         return process.nextTick(callback.bind(this, null, added));
 95 |     }
 96 | 
 97 |     var url = list.pop(),
 98 |         hash = md5(url.url);
 99 | 
100 |     redis.sismember("feed:urls", hash, function(err, isMember){
101 |         if(Number(isMember)){
102 |             return process.nextTick(checkListForNewItems.bind(this, list, callback, added));
103 |         }else{
104 |             added.push(url);
105 |             redis.multi().
106 |                 sadd("feed:urls", hash).
107 |                 sadd("feed:check", hash).
108 |                 hmset("feed:url:" + hash, url).
109 |                 exec(function(err, replies){
110 |                     process.nextTick(checkListForNewItems.bind(this, list, callback, added));
111 |                 });
112 |         }
113 |     });
114 | }
115 | 
116 | function md5(str){
117 |     return crypto.createHash("md5").update(str).digest("hex");
118 | }


--------------------------------------------------------------------------------
/rss.json:
--------------------------------------------------------------------------------
  1 | [{
  2 |     "id": "delfi.ee",
  3 |     "web": "http://www.delfi.ee/",
  4 |     "url": "http://feeds2.feedburner.com/delfiuudised",
  5 |     "title": "Delfi",
  6 |     "language": "et",
  7 |     "domains": ["delfi.ee"]
  8 | }, {
  9 |     "id": "delfi.ee",
 10 |     "web": "http://www.delfi.ee/",
 11 |     "url": "http://feeds2.feedburner.com/delfimajandus",
 12 |     "language": "et",
 13 |     "title": "Delfi"
 14 | }, {
 15 |     "id": "e24.ee",
 16 |     "web": "http://www.e24.ee/",
 17 |     "url": "http://www.e24.ee/rss/",
 18 |     "language": "et",
 19 |     "title": "E24",
 20 |     "domains": ["e24.ee"]
 21 | }, {
 22 |     "id": "uudisvoog.postimees.ee",
 23 |     "web": "http://www.uudisvoog.postimees.ee/",
 24 |     "url": "http://otsing.tahvel.info/feeds/postimehe_uudisvoog.php",
 25 |     "language": "et",
 26 |     "title": "E24 - firmateated",
 27 |     "domains": ["uudisvoog.postimees.ee"]
 28 | }, {
 29 |     "id": "err.ee",
 30 |     "web": "http://www.err.ee/",
 31 |     "url": "http://uudised.err.ee/uudised_rss.php",
 32 |     "language": "et",
 33 |     "title": "ERR",
 34 |     "domains": ["err.ee"]
 35 | }, {
 36 |     "id": "sport.err.ee",
 37 |     "web": "http://sport.err.ee/",
 38 |     "title": "ERR Sport",
 39 |     "url": "http://sport.err.ee/rss",
 40 |     "language": "et",
 41 |     "domains": ["sport.err.ee"]
 42 | }, {
 43 |     "id": "teadus.err.ee",
 44 |     "web": "http://teadus.err.ee/",
 45 |     "title": "ERR Teadus & Haridus",
 46 |     "url": "http://teadus.err.ee/rss",
 47 |     "language": "et",
 48 |     "domains": ["teadus.err.ee"]
 49 | }, {
 50 |     "id": "ekspress.ee",
 51 |     "web": "http://www.ekspress.ee/",
 52 |     "url": "http://feeds.feedburner.com/EestiEkspressFeed",
 53 |     "language": "et",
 54 |     "title": "Eesti Ekspress",
 55 |     "domains": ["ekspress.ee"]
 56 | }, {
 57 |     "id": "epl.ee",
 58 |     "web": "http://www.epl.ee/",
 59 |     "url": "http://feeds.feedburner.com/eestipaevaleht",
 60 |     "language": "et",
 61 |     "title": "Eesti Päevaleht",
 62 |     "domains": ["epl.ee"]
 63 | }, {
 64 |     "id": "elu24.ee",
 65 |     "web": "http://www.elu24.ee/",
 66 |     "url": "http://www.elu24.ee/rss/",
 67 |     "title": "Elu24",
 68 |     "domains": ["elu24.ee"]
 69 | }, {
 70 |     "id": "forte.ee",
 71 |     "web": "http://www.forte.ee/",
 72 |     "url": "http://feeds2.feedburner.com/forteuudised",
 73 |     "language": "et",
 74 |     "title": "Forte",
 75 |     "domains": ["forte.ee", "forte.delfi.ee"]
 76 | }, {
 77 |     "id": "ilmajaam.ee",
 78 |     "web": "http://www.ilmajaam.ee/",
 79 |     "url": "http://www.ilmajaam.ee/rss/",
 80 |     "language": "et",
 81 |     "title": "Ilmajaam",
 82 |     "domains": ["ilmajaam.ee"]
 83 | }, {
 84 |     "id": "kes-kus.ee",
 85 |     "web": "http://www.kes-kus.ee/",
 86 |     "url": "http://otsing.tahvel.info/feeds/keskus.php",
 87 |     "language": "et",
 88 |     "title": "Kes-Kus",
 89 |     "domains": ["kes-kus.ee"]
 90 | }, {
 91 |     "id": "kuulutaja.ee",
 92 |     "web": "http://www.kuulutaja.ee/",
 93 |     "url": "http://otsing.tahvel.info/feeds/kuulutaja.php",
 94 |     "language": "et",
 95 |     "title": "Kuulutaja",
 96 |     "domains": ["kuulutaja.ee"]
 97 | }, {
 98 |     "id": "maaleht.ee",
 99 |     "web": "http://www.maaleht.ee/",
100 |     "url": "http://feeds2.feedburner.com/maaleht",
101 |     "language": "et",
102 |     "title": "Maaleht",
103 |     "domains": ["maaleht.ee"]
104 | }, {
105 |     "id": "memokraat.ee",
106 |     "web": "http://www.memokraat.ee/",
107 |     "url": "http://feeds2.feedburner.com/Memokraat",
108 |     "language": "et",
109 |     "title": "Memokraat",
110 |     "domains": ["memokraat.ee"]
111 | }, {
112 |     "id": "naine24.ee",
113 |     "web": "http://www.naine24.ee/",
114 |     "url": "http://www.naine24.ee/rss/",
115 |     "language": "et",
116 |     "title": "Naine24",
117 |     "domains": ["naine24.ee"]
118 | }, {
119 |     "id": "naistekas.ee",
120 |     "web": "http://www.naistekas.ee/",
121 |     "url": "http://feeds.feedburner.com/naistekas",
122 |     "language": "et",
123 |     "title": "Naistekas",
124 |     "domains": ["naistekas.ee", "naistekas.delfi.ee"]
125 | }, {
126 |     "id": "postimees.ee",
127 |     "web": "http://www.postimees.ee/",
128 |     "url": "http://www.postimees.ee/rss/",
129 |     "language": "et",
130 |     "title": "Postimees Online",
131 |     "domains": ["postimees.ee"]
132 | }, {
133 |     "id": "postimees.ee",
134 |     "web": "http://www.postimees.ee/",
135 |     "url": "http://arvamus.postimees.ee/rss/",
136 |     "language": "et",
137 |     "title": "Postimees Online",
138 |     "domains": ["arvamus.postimees.ee"]
139 | }, {
140 |     "id": "publik.ee",
141 |     "web": "http://www.publik.ee/",
142 |     "url": "http://feeds2.feedburner.com/publikuudised",
143 |     "language": "et",
144 |     "title": "Publik",
145 |     "domains": ["publik.ee", "publik.delfi.ee"]
146 | }, {
147 |     "id": "parnupostimees.ee",
148 |     "web": "http://www.parnupostimees.ee/",
149 |     "url": "http://www.parnupostimees.ee/rss/",
150 |     "language": "et",
151 |     "title": "Pärnu Postimees",
152 |     "domains": ["parnupostimees.ee"]
153 | }, {
154 |     "id": "sakala.ajaleht.ee",
155 |     "web": "http://www.sakala.ajaleht.ee/",
156 |     "url": "http://www.sakala.ajaleht.ee/rss/",
157 |     "language": "et",
158 |     "title": "Sakala",
159 |     "domains": ["sakala.ajaleht.ee"]
160 | }, {
161 |     "id": "sirp.ee",
162 |     "web": "http://www.sirp.ee/",
163 |     "url": "http://otsing.tahvel.info/feeds/sirp.php",
164 |     "language": "et",
165 |     "title": "Sirp",
166 |     "domains": ["sirp.ee"]
167 | }, {
168 |     "id": "tallinnapostimees.ee",
169 |     "web": "http://www.tallinnapostimees.ee/",
170 |     "url": "http://www.tallinnapostimees.ee/rss/",
171 |     "language": "et",
172 |     "title": "Tallinna Postimees",
173 |     "domains": ["tallinnapostimees.ee"]
174 | }, {
175 |     "id": "tarbija24.ee",
176 |     "web": "http://www.tarbija24.ee/",
177 |     "url": "http://www.tarbija24.ee/rss/",
178 |     "language": "et",
179 |     "title": "Tarbija24",
180 |     "domains": ["tarbija24.ee"]
181 | }, {
182 |     "id": "tartupostimees.ee",
183 |     "web": "http://www.tartupostimees.ee/",
184 |     "url": "http://www.tartupostimees.ee/rss/",
185 |     "language": "et",
186 |     "title": "Tartu Postimees",
187 |     "domains": ["tartupostimees.ee"]
188 | }, {
189 |     "id": "ohtuleht.ee",
190 |     "web": "http://www.ohtuleht.ee/",
191 |     "url": "http://www.ohtuleht.ee/rss",
192 |     "language": "et",
193 |     "title": "Õhtuleht",
194 |     "domains": ["ohtuleht.ee"]
195 | }, {
196 |     "id": "opleht.ee",
197 |     "web": "http://www.opleht.ee/",
198 |     "url": "http://www.opleht.ee/rss.php",
199 |     "language": "et",
200 |     "title": "Õpetajate leht",
201 |     "domains": ["opleht.ee"]
202 | }, {
203 |     "id": "virumaateataja.ee",
204 |     "web": "http://www.virumaateataja.ee/",
205 |     "url": "http://www.virumaateataja.ee/rss/",
206 |     "language": "et",
207 |     "title": "Virumaa Teataja",
208 |     "domains": ["virumaateataja.ee"]
209 | }, {
210 |     "id": "ap3.ee",
211 |     "web": "http://www.ap3.ee/",
212 |     "url": "http://aripaev.ee.feedsportal.com/c/33176/f/545915/index.rss",
213 |     "language": "et",
214 |     "title": "Äripäev",
215 |     "domains": ["ap3.ee"]
216 | }]


--------------------------------------------------------------------------------
/article.js:
--------------------------------------------------------------------------------
  1 | 
  2 | var Gearman = require("node-gearman"),
  3 |     feeder = require("./feed"),
  4 |     urllib = require("url"),
  5 |     gearman = new Gearman("pangalink.net");
  6 | 
  7 | module.exports.fetchArticle = fetchArticle;
  8 | 
  9 | function fetchArticle(callback){
 10 |     feeder.popItemFromList(function(err, article){
 11 |         if(err){
 12 |             process.nextTick(callback.bind(this, null));
 13 |             return;
 14 |         }
 15 |         
 16 |         console.log("Checking for "+article.title+" ...");
 17 |         
 18 |         var articleJSONStr = "",
 19 |             job = gearman.submitJob("article", article.url);
 20 |         
 21 |         job.on("data", function(data){
 22 |             articleJSONStr += data.toString("binary");
 23 |         });
 24 |     
 25 |         job.on("end", function(){
 26 |             var response, domain;
 27 |             try{
 28 |                 response = JSON.parse(new Buffer(articleJSONStr, "binary").toString("utf-8")),
 29 |                 domain = (urllib.parse(response.url, true, true).hostname || "").replace(/^www\./, "").trim();
 30 |             }catch(E){
 31 |                 return process.nextTick(callback.bind(this, E));
 32 |             }
 33 |             
 34 |             fetchLemma(article.title+" "+response.article, function(err, lemma){
 35 |                 process.nextTick(callback.bind(this, null, {
 36 |                     title: article.title,
 37 |                     time: article.date,
 38 |                     domain: domain,
 39 |                     url: parseUrl(response.url || article.url),
 40 |                     html: response.article,
 41 |                     contents: stripHTML(response.article),
 42 |                     lemma: lemma || ""
 43 |                 }));
 44 |             });
 45 |             
 46 |         });
 47 |     
 48 |         job.setTimeout(60 * 1000, function(){
 49 |             process.nextTick(callback.bind(this, new Error("Gearman worker timeout")));
 50 |         });
 51 |     
 52 |         job.on("error", function(err){
 53 |             process.nextTick(callback.bind(this, err));
 54 |         });
 55 |         
 56 |     });
 57 | }
 58 | 
 59 | function fetchLemma(text, callback){
 60 |     
 61 |     text = decodeHTMLEntities((text || "").toString()).
 62 |             replace(/\r?\n|\r/g, " ").
 63 |             replace(/\<[\/a-zA-Z][^>]*\>/g, " ").
 64 |             replace(/\&nbsp;/g, " ").
 65 |             replace(/[^A-Za-z0-9õäöüšžÕÄÖÜŽŠ]/g," ").
 66 |             trim().
 67 |             replace(/\s+/g, ", ").
 68 |             trim().
 69 |             toLowerCase();
 70 |                 
 71 |     var article = "",
 72 |         job = gearman.submitJob("lemma", text);
 73 | 
 74 |     job.on("data", function(data){
 75 |         article += data.toString("binary");
 76 |     });
 77 | 
 78 |     job.on("end", function(){
 79 |         
 80 |         article = new Buffer(article, "binary").toString("utf-8").
 81 |                     replace(/[,\s]+/g, " ").
 82 |                     trim();
 83 |                     
 84 |         process.nextTick(callback.bind(this, null, article));
 85 |     });
 86 | 
 87 |     job.setTimeout(10*1000, function(){
 88 |         process.nextTick(callback.bind(this, new Error("Gearman worker timeout")));
 89 |     });
 90 | 
 91 |     job.on("error", function(err){
 92 |         process.nextTick(callback.bind(this, err));
 93 |     });
 94 | }
 95 | 
 96 | function parseUrl(url){
 97 |     var urlparts = urllib.parse(url, true, true),
 98 |         keys = Object.keys(urlparts.query || {}),
 99 |         key;
100 |     
101 |     for(var i=0, len = keys.length; i<len; i++){
102 |         key = keys[i];
103 |         
104 |         // Google analytics
105 |         if(key.match(/^utm_/)){
106 |             delete urlparts.query[key];
107 |             continue;
108 |         }
109 |         
110 |         // generic
111 |         if(key.match(/SESSID/)){
112 |             delete urlparts.query[key];
113 |             continue;
114 |         }
115 |         
116 |         // AP3
117 |         if(key=="ref"){
118 |             delete urlparts.query[key];
119 |             continue;
120 |         }
121 |         
122 |         // opleht
123 |         if(key=="rsscount"){
124 |             delete urlparts.query[key];
125 |             continue;
126 |         }
127 |     }
128 |     
129 |     urlparts.search = false;
130 |     url = urllib.format(urlparts);
131 |     // remove trailing = from query params
132 |     url = url.replace(/\=(?=&)|\=$/g, "");
133 |     return url;
134 | }
135 | 
136 | var HTMLEntities = {
137 | apos:0x0027,quot:0x0022,amp:0x0026,lt:0x003C,gt:0x003E,nbsp:0x00A0,iexcl:0x00A1,cent:0x00A2,pound:0x00A3,
138 | curren:0x00A4,yen:0x00A5,brvbar:0x00A6,sect:0x00A7,uml:0x00A8,copy:0x00A9,ordf:0x00AA,laquo:0x00AB,
139 | not:0x00AC,shy:0x00AD,reg:0x00AE,macr:0x00AF,deg:0x00B0,plusmn:0x00B1,sup2:0x00B2,sup3:0x00B3,
140 | acute:0x00B4,micro:0x00B5,para:0x00B6,middot:0x00B7,cedil:0x00B8,sup1:0x00B9,ordm:0x00BA,raquo:0x00BB,
141 | frac14:0x00BC,frac12:0x00BD,frac34:0x00BE,iquest:0x00BF,Agrave:0x00C0,Aacute:0x00C1,Acirc:0x00C2,Atilde:0x00C3,
142 | Auml:0x00C4,Aring:0x00C5,AElig:0x00C6,Ccedil:0x00C7,Egrave:0x00C8,Eacute:0x00C9,Ecirc:0x00CA,Euml:0x00CB,
143 | Igrave:0x00CC,Iacute:0x00CD,Icirc:0x00CE,Iuml:0x00CF,ETH:0x00D0,Ntilde:0x00D1,Ograve:0x00D2,Oacute:0x00D3,
144 | Ocirc:0x00D4,Otilde:0x00D5,Ouml:0x00D6,times:0x00D7,Oslash:0x00D8,Ugrave:0x00D9,Uacute:0x00DA,Ucirc:0x00DB,
145 | Uuml:0x00DC,Yacute:0x00DD,THORN:0x00DE,szlig:0x00DF,agrave:0x00E0,aacute:0x00E1,acirc:0x00E2,atilde:0x00E3,
146 | auml:0x00E4,aring:0x00E5,aelig:0x00E6,ccedil:0x00E7,egrave:0x00E8,eacute:0x00E9,ecirc:0x00EA,euml:0x00EB,
147 | igrave:0x00EC,iacute:0x00ED,icirc:0x00EE,iuml:0x00EF,eth:0x00F0,ntilde:0x00F1,ograve:0x00F2,oacute:0x00F3,
148 | ocirc:0x00F4,otilde:0x00F5,ouml:0x00F6,divide:0x00F7,oslash:0x00F8,ugrave:0x00F9,uacute:0x00FA,ucirc:0x00FB,
149 | uuml:0x00FC,yacute:0x00FD,thorn:0x00FE,yuml:0x00FF,OElig:0x0152,oelig:0x0153,Scaron:0x0160,scaron:0x0161,
150 | Yuml:0x0178,fnof:0x0192,circ:0x02C6,tilde:0x02DC,Alpha:0x0391,Beta:0x0392,Gamma:0x0393,Delta:0x0394,
151 | Epsilon:0x0395,Zeta:0x0396,Eta:0x0397,Theta:0x0398,Iota:0x0399,Kappa:0x039A,Lambda:0x039B,Mu:0x039C,
152 | Nu:0x039D,Xi:0x039E,Omicron:0x039F,Pi:0x03A0,Rho:0x03A1,Sigma:0x03A3,Tau:0x03A4,Upsilon:0x03A5,
153 | Phi:0x03A6,Chi:0x03A7,Psi:0x03A8,Omega:0x03A9,alpha:0x03B1,beta:0x03B2,gamma:0x03B3,delta:0x03B4,
154 | epsilon:0x03B5,zeta:0x03B6,eta:0x03B7,theta:0x03B8,iota:0x03B9,kappa:0x03BA,lambda:0x03BB,mu:0x03BC,
155 | nu:0x03BD,xi:0x03BE,omicron:0x03BF,pi:0x03C0,rho:0x03C1,sigmaf:0x03C2,sigma:0x03C3,tau:0x03C4,
156 | upsilon:0x03C5,phi:0x03C6,chi:0x03C7,psi:0x03C8,omega:0x03C9,thetasym:0x03D1,upsih:0x03D2,piv:0x03D6,
157 | ensp:0x2002,emsp:0x2003,thinsp:0x2009,zwnj:0x200C,zwj:0x200D,lrm:0x200E,rlm:0x200F,ndash:0x2013,
158 | mdash:0x2014,lsquo:0x2018,rsquo:0x2019,sbquo:0x201A,ldquo:0x201C,rdquo:0x201D,bdquo:0x201E,dagger:0x2020,
159 | Dagger:0x2021,bull:0x2022,hellip:0x2026,permil:0x2030,prime:0x2032,Prime:0x2033,lsaquo:0x2039,rsaquo:0x203A,
160 | oline:0x203E,frasl:0x2044,euro:0x20AC,image:0x2111,weierp:0x2118,real:0x211C,trade:0x2122,alefsym:0x2135,
161 | larr:0x2190,uarr:0x2191,rarr:0x2192,darr:0x2193,harr:0x2194,crarr:0x21B5,lArr:0x21D0,uArr:0x21D1,
162 | rArr:0x21D2,dArr:0x21D3,hArr:0x21D4,forall:0x2200,part:0x2202,exist:0x2203,empty:0x2205,nabla:0x2207,
163 | isin:0x2208,notin:0x2209,ni:0x220B,prod:0x220F,sum:0x2211,minus:0x2212,lowast:0x2217,radic:0x221A,
164 | prop:0x221D,infin:0x221E,ang:0x2220,and:0x2227,or:0x2228,cap:0x2229,cup:0x222A,"int":0x222B,
165 | there4:0x2234,sim:0x223C,cong:0x2245,asymp:0x2248,ne:0x2260,equiv:0x2261,le:0x2264,ge:0x2265,
166 | sub:0x2282,sup:0x2283,nsub:0x2284,sube:0x2286,supe:0x2287,oplus:0x2295,otimes:0x2297,perp:0x22A5,
167 | sdot:0x22C5,lceil:0x2308,rceil:0x2309,lfloor:0x230A,rfloor:0x230B,lang:0x2329,rang:0x232A,loz:0x25CA,
168 | spades:0x2660,clubs:0x2663,hearts:0x2665,diams:0x2666
169 | };
170 | 
171 | 
172 | function stripHTML(html){
173 |     var html = html.replace(/\r?\n|\r/g, "\u0000").
174 |         replace(/<!--.*?-->/g, "").
175 |         replace(/<\/?(?:p|h|ul|ol|div|br|hr|table|tr)\d?( [^>]*)?>/gi,"\u0000").
176 |         replace(/<[^>]*>(?!<)/g, "").
177 |         replace(/<[^>]*>/g, " ");
178 |         
179 |     html = decodeHTMLEntities(html).
180 |         replace(/</g, "&lt;").
181 |         replace(/>/g, "&gt;").
182 |         replace(/\s+/g, " ").
183 |         replace(/(\s*\u0000\s*)+/g, "\n").trim();
184 |     
185 |     return html;
186 | }
187 | 
188 | function decodeHTMLEntities(text){
189 |     return text.replace(/&(.+?);/g, function(str, ent){
190 |         return String.fromCharCode( ent[0] !== '#' ? HTMLEntities[ent] : ent[1] === 'x' ? parseInt(ent.substr(2),16) : parseInt(ent.substr(1), 10) );
191 |       }
192 |     );
193 | };


--------------------------------------------------------------------------------