├── CONTRIBUTORS.md ├── stremio-addon ├── stremio-manifest.json └── addon.js ├── importers ├── generic.js ├── xml-rss.js ├── dump.js └── json.js ├── .jshintrc ├── .jsbeautifyrc ├── .gitignore ├── lib ├── log.js ├── cfg.js ├── utils.js ├── importer.js ├── replication.js ├── retriever.js ├── db.js └── indexer.js ├── Gruntfile.js ├── LICENSE ├── package.json ├── defaults.js ├── README.md ├── cli └── multipass.js └── test └── basic.js /CONTRIBUTORS.md: -------------------------------------------------------------------------------- 1 | Ivo Georgiev - author 2 | 3 | jaruba - name 4 | 5 | sammuel86 - json importer support (YTS and EZTV) 6 | -------------------------------------------------------------------------------- /stremio-addon/stremio-manifest.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "Multipass Torrents", 3 | "id": "org.multipass.torrentstream", 4 | "description": "Collects and indexes torrents from dumps, RSS feeds and websites", 5 | "types": ["series","movie"], 6 | "filter": { "infoHash": { "$exists": true }, "query.imdb_id":{"$exists": true}, "query.type": {"$in":["series","movie"]} } 7 | } 8 | -------------------------------------------------------------------------------- /importers/generic.js: -------------------------------------------------------------------------------- 1 | var byline = require("byline"); 2 | 3 | module.exports = function(stream, source) 4 | { 5 | return stream = stream.pipe(byline.createStream()).on("data", function(line) 6 | { 7 | /* Cut the string into RegEx. this is my last resort. */ 8 | var line = line.toString(), match, hashes = []; 9 | var regex = new RegExp("(?:%3A)?(([0-9A-Fa-f]){40})", "g"); 10 | while (match = regex.exec(line)) hashes.push(match[1]); 11 | if (hashes) hashes.forEach(function(hash) { stream.emit("infoHash", hash, source.addon) }); 12 | }); 13 | }; 14 | -------------------------------------------------------------------------------- /.jshintrc: -------------------------------------------------------------------------------- 1 | { 2 | // Environments 3 | "browser" : true, 4 | "node" : true, 5 | 6 | // Options 7 | "curly" : true, 8 | "bitwise" : false, 9 | "immed" : true, 10 | "latedef" : true, 11 | "nonbsp" : true, 12 | "nonew" : true, 13 | "validthis": true, 14 | "debug" : true, 15 | "boss" : true, 16 | "eqeqeq" : true, 17 | "expr" : true, 18 | "eqnull" : true, 19 | "quotmark" : "single", 20 | "trailing" : true, 21 | "sub" : true, 22 | "trailing" : true, 23 | "undef" : true, 24 | "laxbreak" : true, 25 | "loopfunc" : true, 26 | "indent" : 4, 27 | "newcap" : false 28 | } -------------------------------------------------------------------------------- /.jsbeautifyrc: -------------------------------------------------------------------------------- 1 | { 2 | "js": { 3 | "brace_style": "collapse", 4 | "break_chained_methods": false, 5 | "e4x": false, 6 | "eval_code": false, 7 | "indent_level": 0, 8 | "indent_with_tabs": false, 9 | "indent_size": 4, 10 | "indent_char": " ", 11 | "jslint_happy": true, 12 | "keep_array_indentation": false, 13 | "keep_function_indentation": false, 14 | "max_preserve_newlines": 3, 15 | "preserve_newlines": true, 16 | "space_before_conditional": true, 17 | "space-after-anon-function": true, 18 | "space_in_paren": false, 19 | "unescape_strings": true, 20 | "wrap_line_length": 0, 21 | "allowed_file_extensions": ["js", "json", "jshintrc", "jsbeautifyrc"] 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Logs 2 | logs 3 | *.log 4 | 5 | # Runtime data 6 | pids 7 | *.pid 8 | *.seed 9 | 10 | # Directory for instrumented libs generated by jscoverage/JSCover 11 | lib-cov 12 | 13 | # Coverage directory used by tools like istanbul 14 | coverage 15 | 16 | # Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files) 17 | .grunt 18 | 19 | # node-waf configuration 20 | .lock-wscript 21 | 22 | # Compiled binary addons (http://nodejs.org/api/addons.html) 23 | build/Release 24 | 25 | # Dependency directory 26 | # https://www.npmjs.org/doc/misc/npm-faq.html#should-i-check-my-node_modules-folder-into-git 27 | node_modules 28 | 29 | # Database path 30 | db 31 | 32 | cli/x3me.js 33 | -------------------------------------------------------------------------------- /lib/log.js: -------------------------------------------------------------------------------- 1 | var cfg = require("./cfg"); 2 | 3 | // 4 | function logHash(hash, type, message) 5 | { 6 | // TODO we can log hashes individually here, as we receive messages on what is happening with them 7 | // currently retire and delete cannot be received since we don't clean our DB 8 | }; 9 | 10 | function logOther() 11 | { 12 | if (cfg.logLevel >= this.level) console[this.error ? "error" : "log"].apply(console, arguments) 13 | }; 14 | 15 | module.exports = { 16 | hash: logHash, 17 | message: logOther.bind({ level: 3}), 18 | warning: logOther.bind({ level: 2 }), 19 | error: logOther.bind({ level: 1, error: true }), 20 | important: logOther.bind({ level: 0 }), 21 | }; -------------------------------------------------------------------------------- /Gruntfile.js: -------------------------------------------------------------------------------- 1 | module.exports = function(grunt) { 2 | 'use strict'; 3 | 4 | require('load-grunt-tasks')(grunt); 5 | 6 | grunt.registerTask('js', [ 7 | 'jsbeautifier:default' 8 | ]); 9 | 10 | grunt.registerTask('jsdry', [ 11 | 'jsbeautifier:verify' 12 | ]); 13 | 14 | grunt.initConfig({ 15 | jsbeautifier: { 16 | options: { 17 | config: '.jsbeautifyrc' 18 | }, 19 | 20 | default: { 21 | src: ['*.js', '*/*.js'], 22 | }, 23 | 24 | verify: { 25 | src: ['*.js', '*/*.js'], 26 | options: { 27 | mode: 'VERIFY_ONLY' 28 | } 29 | } 30 | }, 31 | jshint: { 32 | src: { 33 | options: { 34 | jshintrc: '.jshintrc' 35 | }, 36 | src: ['*.js', '*/*.js'] 37 | } 38 | } 39 | 40 | }); 41 | 42 | }; -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2015 Ivo Georgiev 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | 23 | -------------------------------------------------------------------------------- /lib/cfg.js: -------------------------------------------------------------------------------- 1 | var _ = require("lodash"); 2 | var hat = require("hat"); 3 | var os = require("os"); 4 | var path = require("path"); 5 | var crypto = require("crypto"); 6 | var events = require("events"); 7 | var argv = require("minimist")(process.argv.slice(2)); 8 | 9 | var cfg = _.extend(new events.EventEmitter(), { sources: [], dht: true, ssdp: true, argv: argv }); 10 | 11 | //console.log("reading default config from defaults.js"); 12 | _.merge(cfg, require("../defaults")); 13 | 14 | if (argv.config) { 15 | console.log("reading config from "+argv.config); 16 | try { 17 | _.merge(cfg, require(argv.config)); 18 | } catch(e) { console.error(e) } 19 | } 20 | 21 | (Array.isArray(argv.source) ? argv.source : [argv.source]).forEach(function(source) { 22 | if (source) cfg.sources.push({ url: source, category: ["tv", "movies"] }); 23 | }); 24 | 25 | cfg.logLevel = !isNaN(argv.log) ? parseInt(argv.log) : cfg.logLevel; 26 | 27 | cfg.dbPath = cfg.dbPath || argv["db-path"] || argv.path || path.join(require("os").tmpdir(), "multipass"); 28 | cfg.dbId = cfg.dbId || argv["db-id"] || argv.id || hat(160,16); // use minimist alias 29 | var isHex = (cfg.dbId && cfg.dbId.length==40 && parseInt(cfg.dbId, 16)); 30 | cfg.dbId = isHex ? cfg.dbId : crypto.createHash("sha1").update(cfg.dbId).digest("hex"); 31 | cfg.stremioAddon = cfg.stremioAddon || argv["stremio-addon"]; 32 | 33 | process.nextTick(function() { cfg.emit("ready") }); 34 | 35 | module.exports = cfg; 36 | -------------------------------------------------------------------------------- /lib/utils.js: -------------------------------------------------------------------------------- 1 | var _ = require("lodash"); 2 | var cfg = require("./cfg"); 3 | 4 | var utils = { }; 5 | 6 | /* Utilities 7 | * Those can be overriden 8 | */ 9 | utils.getMaxPopularity = function(torrent) { 10 | return Math.max.apply(Math, _.values(torrent.popularity).map(function(x) { return x[0] }).concat(0)); 11 | }; 12 | utils.isFileBlacklisted = function(file) { 13 | return file.tag.concat(tags(file)).some(function(tag) { return cfg.blacklisted[tag] }); 14 | }; 15 | utils.getSourcesForTorrent = function(torrent) { 16 | return ["dht:"+torrent.infoHash].concat(torrent.announce.map(function(x){ return "tracker:"+x })); 17 | }; 18 | utils.getAvailForTorrent = function(torrent) { 19 | var maxSeeders = utils.getMaxPopularity(torrent); 20 | if (maxSeeders >= 300) return 4; 21 | if (maxSeeders >= 90) return 3; 22 | if (maxSeeders >= 15) return 2; 23 | if (maxSeeders > 0) return 1; 24 | return 0; 25 | }; 26 | 27 | function tags(file) 28 | { 29 | var tags = []; 30 | tags.push(file.path.split(".").pop()); // file extension 31 | 32 | // Then tokenize into keywords; try against tagWords 33 | file.path.split("/").forEach(function(seg) { 34 | var tokens = seg.split(/\.| |-|;|_/).filter(function(x){return x}); 35 | tokens = seg.split(/ |\.|\(|\)|\[|\]/).map(function(x) { return x.toLowerCase() }); // split, this time for tokens 36 | _.each(cfg.tags, function(words, tag) { 37 | if (tokens.filter(function(token) { return words.indexOf(token.toLowerCase()) > -1 }).length) tags.push(tag); 38 | }); 39 | }); 40 | 41 | return _.uniq(tags); 42 | } 43 | utils.tags = tags; 44 | 45 | module.exports = utils; -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "multipass-torrent", 3 | "version": "0.8.9", 4 | "description": "Collects torrents from various sources (dump, RSS, HTML pages) and associates the video files within with IMDB ID", 5 | "main": "cli/multipass.js", 6 | "scripts": { 7 | "start": "node cli/multipass", 8 | "test": "tape test/*.js" 9 | }, 10 | "repository": { 11 | "type": "git", 12 | "url": "http://github.com/Ivshti/multipass-torrent" 13 | }, 14 | "keywords": [ 15 | "torrent", 16 | "crawling", 17 | "scraping", 18 | "indexing" 19 | ], 20 | "author": "Ivo Georgiev", 21 | "license": "MIT", 22 | "bugs": { 23 | "url": "https://github.com/Ivshti/multipass-torrent/issues" 24 | }, 25 | "dependencies": { 26 | "async": "0.2.9", 27 | "bagpipe": "Ivshti/bagpipe", 28 | "bencode": "^0.7.0", 29 | "binary-search-tree": "0.2.x", 30 | "bittorrent-dht": "3.x.x", 31 | "byline": "4.2.x", 32 | "duplexify": "3.x.x", 33 | "entry": "0.4.0", 34 | "es6-map": "0.1.x", 35 | "feedparser": "0.19.x", 36 | "gunzip-maybe": "1.2.x", 37 | "hat": "0.0.x", 38 | "ip": "^0.3.0", 39 | "level": "1.x.x", 40 | "level-sublevel": "6.4.x", 41 | "lodash": "3.10.x", 42 | "minimist": "1.x.x", 43 | "mkdirp": "0.5.x", 44 | "multi-master-merge": "2.4.x", 45 | "name-to-imdb": "^1.0.1", 46 | "nat-pmp": "0.0.x", 47 | "needle": "0.x.x", 48 | "node-ssdp": "2.5.x", 49 | "parse-torrent": "5.x.x", 50 | "peer-search": "0.6.x", 51 | "q": "^1.4.1", 52 | "sift": "^2.0.9", 53 | "stremio-addons": "^1.31.0", 54 | "torrent-stream": "^1.0.0", 55 | "unzip": ">=0.1.9", 56 | "video-name-parser": "^1.0.1", 57 | "xml-stream": ">=0.4.4" 58 | }, 59 | "devDependencies": { 60 | "grunt": "^0.4.5", 61 | "grunt-cli": "^0.1.13", 62 | "grunt-contrib-jshint": "^0.11.2", 63 | "grunt-jsbeautifier": "^0.2.10", 64 | "load-grunt-tasks": "^3.2.0", 65 | "tape": "4.x.x" 66 | }, 67 | "homepage": "https://github.com/Ivshti/multipass-torrent" 68 | } 69 | -------------------------------------------------------------------------------- /importers/xml-rss.js: -------------------------------------------------------------------------------- 1 | var FeedParser = require("feedparser"); 2 | //var collect = require("../lib/collector").collect; 3 | var log = require("../lib/log"); 4 | 5 | var _ = require("lodash"); 6 | 7 | // This should emit results up through an EventEmitter or a pipe, not use collect directly 8 | 9 | module.exports = function(stream, source) 10 | { 11 | return stream = stream.pipe(new FeedParser()) 12 | .on("error", function (error) { log.error("xml-rss", error, source) }) 13 | .on("readable", function(meta) 14 | { 15 | var item, stream = this; 16 | while(item = stream.read()) 17 | { 18 | var match = (item.link+"\n"+item.description+"\n"+item["rss:link"]).match(new RegExp("([0-9A-Fa-f]){40}", "g")); 19 | var hash = (item["rss:torrent"] && item["rss:torrent"]["infohash"]["#"]) 20 | || (item["torrent:infohash"] && item["torrent:infohash"]["#"]) 21 | || (match && match[0]); 22 | 23 | if (! (hash && hash.length == 40)) log.error("xml-rss - invalid hash: "+hash); 24 | 25 | /* Category filter - custom */ 26 | if (source.category && item.categories && item.categories[0] 27 | && !source.category.filter(function(cat) { return item.categories[0].match(new RegExp(cat, "i")) }).length 28 | ) return; 29 | 30 | // For now, skip porn - to avoid false positives when finding movies 31 | if (item.categories && item.categories[0] && item.categories[0].match("porn")) 32 | return; 33 | 34 | var addon = source.addon || { }; // Additional info 35 | if (item.hasOwnProperty("torrent:seeds")) _.extend(addon, { 36 | uploaders: item["torrent:seeds"]["#"], 37 | downloaders: item["torrent:peers"]["#"], 38 | verified: !!parseInt(item["torrent:verified"]["#"]) 39 | }); 40 | addon.verified = source.verified || addon.verified; // the source can be assumed verified 41 | 42 | // TODO: read seeds via regex matching from description (like from http://torrentz.eu/feed?q=) 43 | // TODO: read from rss:description for torrentproject - http://torrentproject.com/rss/tv/ 44 | 45 | stream.emit("infoHash", hash, addon); 46 | } 47 | }); 48 | } 49 | -------------------------------------------------------------------------------- /defaults.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | "logLevel": 1, 3 | "trackers": ["udp://tracker.leechers-paradise.org:6969/announce", "udp://tracker.coppersurfer.tk:6969/announce"], 4 | "trackerTimeout": 3000, 5 | "fetchMetaTimeout": 25000, 6 | 7 | // Different frm trackers - we have to update seed/leech for each torrents for all trackers, but this is for when we do retriever / fetchTorrent 8 | "fetchTorrentTrackers": ["udp://tracker.leechers-paradise.org:6969/announce", "udp://tracker.coppersurfer.tk:6969/announce"], 9 | 10 | "retrieverSources": [ 11 | { url: "http://itorrents.org/torrent/%s.torrent" }, 12 | { url: "http://torcache.net/torrent/%s.torrent" }, 13 | { url: "http://torrage.com/torrent/%s.torrent" }, 14 | ], 15 | 16 | "processingConcurrency": 6, 17 | 18 | "minSeedToIndex": 4, 19 | "minSeedImportant": 200, // if it's important, we'll try to get the meta from the DHT / peers even when caches fail 20 | 21 | // If the torrent hasn't been updated for that time and not seeded, delete it 22 | "nonSeededTTL": 25*24*60*60*1000, 23 | 24 | // How often we update popularity 25 | "popularityTTL": 6*60*60*1000, 26 | 27 | "matchFiles": /.mp4$|.avi$|.mkv$/i, 28 | "excludeFiles": /^RARBG|^Sample|Cam Rip/i, 29 | "excludeTorrents": null, 30 | "excludeNonAscii": true, 31 | 32 | "cinemeta": "http://cinemeta.strem.io/stremioget", 33 | 34 | "stremioAddon": 7000, 35 | "stremioSecret": "8417fe936f0374fbd16a699668e8f3c4aa405d9f", 36 | "stremioCentral": "http://api9.strem.io", 37 | 38 | // Tagging system; all tags must be lowercase 39 | "tags": { 40 | "screener": ["screener", "dvdscr", "dvdscreener", "bdscr", "scr"], 41 | "cam": ["cam", "hdcam", "camrip", "hqcam"], 42 | "telesync": ["ts", "hdts", "telesync", "pdvd"], 43 | "workprint": ["workprint", "wp"], 44 | "r5": ["tc", "telecine", "r5"], // r5 + telecine 45 | "web": ["webdl", "web", "webrip"], 46 | "dvd": ["dvdrip"], 47 | "hdtv": ["dsr", "dsrip", "dthrip", "dvbrip", "hdtv", "pdtv", "tvrip", "hdtvrip", "hdrip"], 48 | "vod": ["vodrip", "vodr"], 49 | "br": ["bdrip", "brrip", "bluray", "bdr", "bd5", "bd9"], 50 | "dts": ["dts"], 51 | "hd": ["1080p"], 52 | "720p": ["720p"], 53 | "1080p": ["1080p"], 54 | "ac3": ["ac3"], 55 | "nonenglish": ["french", "italian", "spanish", "ru", "russian", "swesub", "dublado"], 56 | "badripper": ["italian", "sparks","evo", "hc", "korsub", "douglasvip", "murd3r", "nkr"], 57 | "yify": ["yts","yify"], 58 | //"yts": ["yts","yify"], // reserve for directly indexing from yts 59 | "cd1": ["cd1"], "cd2": ["cd2"], "cd3": ["cd3"], 60 | "splitted": ["cd1","cd2","cd3","cd4","part1","part2","part3", "pt1", "pt2", "pt3"], 61 | }, 62 | "blacklisted": { "screener": 1, "cam": 1, "telesync": 1, "workprint": 1, "r5": 1, "splitted": 1, "badripper": 1, "nonenglish": 1 } 63 | }; 64 | -------------------------------------------------------------------------------- /lib/importer.js: -------------------------------------------------------------------------------- 1 | /* Collect info hashes to queue for indexing 2 | */ 3 | 4 | var needle = require("needle"); 5 | var gunzip = require("gunzip-maybe"); 6 | var url = require("url"); 7 | var _ = require("lodash"); 8 | 9 | var importers = { 10 | dump: require("../importers/dump"), 11 | json: require("../importers/json"), 12 | xmlRss: require("../importers/xml-rss"), 13 | generic: require("../importers/generic") 14 | }; 15 | 16 | function collect(source, callback, onHash) { 17 | var status = { 18 | found: 0, 19 | start: Date.now() 20 | }; 21 | 22 | getStream(source, function(err, stream, detectedType) { 23 | if (err) return callback(err); 24 | 25 | var type = status.type = importers[source.type] ? source.type : detectedType; 26 | 27 | // Pass on to the importer 28 | stream = importers[type](stream, source); 29 | 30 | // Collection results 31 | var unique = { }; 32 | stream.on("infoHash", function(hash, extra) { 33 | hash = hash.toLowerCase(); 34 | if (unique[hash]) return; 35 | unique[hash] = true; 36 | status.found++; 37 | if (onHash) onHash(hash.toLowerCase(), extra); 38 | }); 39 | 40 | stream.on("error", callback); 41 | 42 | stream.on("end", function() { 43 | stream.removeAllListeners(); 44 | status.end = Date.now(); 45 | callback(null, status) 46 | }); 47 | }); 48 | }; 49 | 50 | function getStream(source, callback) { 51 | var stream, response, callback = _.once(callback); 52 | stream = response = needle.get(source.url, { 53 | follow_max: 4, open_timeout: 5000, 54 | headers: { "user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.99 Safari/537.36" } 55 | }).on("headers", function(headers) { 56 | var filename = headers["content-disposition"] || url.parse(source.url).pathname; 57 | var detectedType = "generic"; 58 | if (headers["content-type"] && headers["content-type"].match("xml")) detectedType = "xmlRss"; 59 | if (headers["content-type"] && headers["content-type"].match("json")) detectedType = "json"; 60 | if (filename.match(".txt.gz$")) detectedType = "dump"; 61 | 62 | if (detectedType !="json") stream = stream.pipe(gunzip()).pipe(gunzip()); // Some sources can be gunzipped twice (one for request, another for being a .txt.gz) 63 | 64 | stream.on("end", function() { response.end() }); // make sure response is closed 65 | 66 | callback(null, stream, detectedType); 67 | }).on("error", function(e) { callback(e) }) 68 | .on("end", function() { 69 | // TODO: we can check statusCode / etc? 70 | callback(new Error("empty response / couldn't detect type")); 71 | }) 72 | }; 73 | 74 | module.exports = { 75 | collect: collect, 76 | getStream: getStream 77 | }; 78 | -------------------------------------------------------------------------------- /importers/dump.js: -------------------------------------------------------------------------------- 1 | var byline = require("byline"); 2 | var _ = require("lodash"); 3 | var log = require("../lib/log"); 4 | var events = require("events"); 5 | var needle = require("needle"); 6 | 7 | module.exports = function(stream, source) 8 | { 9 | var emitter = new events.EventEmitter(); 10 | var i = 0, j = 0; // i - how many streams, j - how many completed 11 | 12 | stream.pipe(byline.createStream()) 13 | .on("data", function(line) 14 | { 15 | var parts = line.toString().split("|"), 16 | infoHash = parts[0].toLowerCase(), cat = (parts[2] || "").toLowerCase(), 17 | additional = { }; // additional info found 18 | 19 | var infoUrl = parts[3]; 20 | if (infoUrl && infoUrl.match("kat.cr|kickass.to")) additional.hints = { url: infoUrl }; // OR any torrent website that we know contains IMDB ID on it's info page 21 | 22 | // URL to torrent file 23 | if (parts[4] && parts[4].match(".torrent$")) additional.download = parts[4]; 24 | 25 | // IMDB ID match 26 | var imdbMatch = (parts[5] && parts[5].match("(tt[0-9]+)")) || (parts[3] && parts[3].match("(tt[0-9]+)")); 27 | if (imdbMatch) additional.hints = { imdb_id: imdbMatch && imdbMatch[0] }; // no issue to override, because the hint url is for getting imdb_id anyway 28 | 29 | if (cat.match("porn") || cat.match("adult")) return; // exclude that for now 30 | 31 | // TODO: fluid category filter 32 | if (source.category===true || (cat.match("movie") || cat.match("dvd") || cat.match("rip") || cat.match("tv")/* || cat.match("video")*/)) { 33 | additional.category = cat; 34 | hashReady(infoHash, _.extend(additional, source.addon || { })); 35 | }; 36 | }) 37 | .on("error", function(err) { log.error("dump", err) }) 38 | .on("end", checkEnded()); 39 | 40 | var checkingSeeders = source.minSeedersUrl && source.minSeeders; 41 | if (checkingSeeders) require("../lib/importer").getStream({ url: source.minSeedersUrl }, function(err, stream) { 42 | if (err) return checkingSeeders = false; // warning - bug - some info hashes will never be flushed 43 | 44 | stream.pipe(byline.createStream()) 45 | .on("data", function(line) { 46 | var parts = line.toString().split("|"); 47 | var infoHash = parts[0].toLowerCase(), uploaders = parseInt(parts[1]), downloaders = parseInt(parts[2]); 48 | if (uploaders >= source.minSeeders) hashReady(infoHash, { uploaders: uploaders, downloaders: downloaders }); 49 | }) 50 | .on("error", function(err){ log.error("dump-seeders",err) }) 51 | .on("end", checkEnded()) 52 | }); 53 | 54 | // TODO: make sure this is cleaned up 55 | var hashes = { }; 56 | function hashReady(hash, extra) { 57 | if (!checkingSeeders) return emitter.emit("infoHash", hash, extra); 58 | 59 | hashes[hash] = hashes[hash] || { hit: 0 }; 60 | hashes[hash].hit++; 61 | _.extend(hashes[hash], extra || { }); 62 | if (hashes[hash].hit == 2) emitter.emit("infoHash", hash, hashes[hash]); 63 | }; 64 | 65 | function checkEnded() { 66 | i++; 67 | return function() { if (++j == i) emitter.emit("end") }; 68 | }; 69 | 70 | return emitter; 71 | } 72 | -------------------------------------------------------------------------------- /lib/replication.js: -------------------------------------------------------------------------------- 1 | var replication = { }; 2 | 3 | // hole punch for our replication interface 4 | var entry = require("entry"); 5 | var natPmp = require("nat-pmp"); 6 | 7 | var log = require("./log"); 8 | var cfg = require("./cfg"); 9 | var db = require("./db"); 10 | 11 | var DHT = require("bittorrent-dht"); 12 | var SSDP = require("node-ssdp"); 13 | var ip = require("ip"); 14 | var zlib = require("zlib"); 15 | var duplexify = require("duplexify"); 16 | var events = require("events"); 17 | var net = require("net"); 18 | 19 | var dht = new DHT(); 20 | var ssdp = new SSDP.Client(); 21 | 22 | 23 | /* Replication - server & swarm 24 | */ 25 | replication.syncStream = function() 26 | { 27 | var stream = db.sync(); 28 | // return stream; 29 | return stream.pipe(duplexify(zlib.createGzip(), zlib.createGunzip())).pipe(stream); 30 | }; 31 | 32 | var server; 33 | replication.listenReplications = function(id) { 34 | server = net.createServer(function(c) { 35 | log.important("DB replication connection established from "+c.remoteAddress+":"+c.remotePort); 36 | c.pipe(replication.syncStream()).pipe(c); 37 | c.on("error", function(err) { console.log(err) }); 38 | }); 39 | server.listen(function() { 40 | var port = server.address().port; 41 | 42 | log.important("DB replication server listening at "+port); 43 | dht.announce(id, port); 44 | 45 | // Hole punch so this server is accessible behind router firewalls 46 | // We can play around with public / private - maybe we want a consistent public port? 47 | try { 48 | // Both throw err in their async code which we cannot catch 49 | entry.map({ external: port, internal: port, name: require("../package").name }, function(e) { e && console.error("entry error (non-fatal)", e) }); 50 | //natPmp.connect("10.0.1.1").portMapping({ public: port, private: port, ttl: 1000, description: require("../package").name }, function(e) { e && console.error(e) }); 51 | } catch(e) { console.error("entry error (non-fatal)", e) } 52 | }); 53 | 54 | // Announce as an SSDP server 55 | var ssdpServer = new SSDP.Server({ location: ip.address() +":" + server.address().port }); 56 | ssdpServer.addUSN("upnp:rootdevice"); 57 | ssdpServer.addUSN("urn:schemas-upnp-org:service:MultiPassTorrent:"+id); 58 | ssdpServer.start("0.0.0.0"); 59 | }; 60 | 61 | replication.findReplications = function(id) { 62 | log.important(id+": finding other instances to replicate with through "+[cfg.ssdp ? "ssdp" : "", cfg.dht ? "dht" : ""].join(", ")); 63 | 64 | // WARNING: what if it emits beforehand? 65 | if (cfg.dht) dht.on("ready", function() { 66 | dht.lookup(id); 67 | }); 68 | 69 | if (cfg.ssdp) ssdp.search("urn:schemas-upnp-org:service:MultiPassTorrent:"+id); 70 | }; 71 | 72 | /* Swarm 73 | */ 74 | var peers = { }; 75 | function onpeer(addr) 76 | { 77 | if (ip.address()+":"+server.address().port == addr) return; // Do not connect to ourselves 78 | 79 | if (peers[addr]) return; 80 | peers[addr] = true; 81 | 82 | var spl = addr.split(":"); 83 | var c = net.connect(spl[1], spl[0]); 84 | c.on("connect", function() { 85 | log.message("connected to peer "+addr); // TODO: handle errs 86 | c.pipe(db.syncStream()).pipe(c); 87 | }).on("error", function(e) { 88 | c.destroy() 89 | }).on("end", function() { 90 | c.destroy(); 91 | delete peers[addr]; 92 | }).on("close", function() { 93 | // cleanup sync pipe? 94 | }); 95 | }; 96 | dht.on("peer", onpeer); 97 | ssdp.on("response", function(meta, status, matchine) { onpeer(meta.LOCATION) }); 98 | 99 | module.exports = replication; 100 | -------------------------------------------------------------------------------- /lib/retriever.js: -------------------------------------------------------------------------------- 1 | var request = require("needle"), 2 | async = require("async"), 3 | _ = require("lodash"), 4 | needle = require("needle"), 5 | util = require("util"), 6 | cfg = require("./cfg"), 7 | torrentStream = require("torrent-stream"), 8 | bencode = require("bencode"), 9 | parseTorrent = require("parse-torrent"); 10 | 11 | var downloadSources = cfg.retrieverSources || []; 12 | 13 | var defaultHeaders = { 14 | "accept-charset" : "ISO-8859-1,utf-8;q=0.7,*;q=0.3", 15 | "accept-language" : "en-US,en;q=0.8", 16 | "accept" : "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8", 17 | "user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.99 Safari/537.36" 18 | }; 19 | 20 | var downloader = async.queue(function(task, cb) 21 | { 22 | setTimeout(cb, 500); /* This async queue is simply a rate limiter */ 23 | 24 | var result, errors = [], sources = (task.url ? [{ formatted:task.url }] : []).concat(downloadSources); 25 | 26 | async.whilst(function() { return !result && sources.length }, function(innerCb) 27 | { 28 | var source = sources.shift(), 29 | innerCb = _.once(innerCb), 30 | url = source.formatted ? source.formatted : util.format(source.url, [ task.infoHash.toUpperCase() ]); 31 | 32 | needle.get(url, { 33 | open_timeout: 3500, read_timeout: 3500, 34 | agent: module.exports.getAgent(), 35 | follow_max: 3, compressed: true, 36 | headers: _.extend({ referer: source.url || source.formatted }, defaultHeaders) 37 | }, function(err, res, body) { 38 | if (err) { err.source = url; errors.push(err); return innerCb(); } 39 | else if (res && res.statusCode != 200) { errors.push(_.extend(new Error("status code "+res.statusCode), { source: url })); return innerCb() } 40 | 41 | if (!body) return innerCb(); 42 | 43 | try { 44 | if (typeof(body) == "string") body = JSON.parse(body); 45 | body = body["piece length"] ? bencode.encode({ info: _.extend({ pieces: [] }, body) }) : body; 46 | result = parseTorrent(body); 47 | result.infoHash = task.infoHash; 48 | if (! result.files) throw new Error("no files found in torrent"); 49 | } catch(e) { e.source = url; errors.push(e) }; 50 | innerCb(); 51 | }); 52 | }, function() 53 | { 54 | if (! (result && result.infoHash)) return task.callback(errors.concat([new Error("Did not manage to download parsable torrent")])); 55 | task.callback(null, result); 56 | }); 57 | }, 2); 58 | 59 | // TODO: ability to rate-limit that section of the code 60 | // OR just use a rate-limited peer-search instead of torrent-stream's peer searching 61 | function fetchTorrent(infoHash, opts, cb) { 62 | var engine = new torrentStream(infoHash, { 63 | connections: 30, 64 | trackers: cfg.fetchTorrentTrackers, 65 | }); 66 | var cb = _.once(cb); 67 | 68 | engine.ready(function() { 69 | cb(null, engine.torrent); 70 | engine.destroy(); 71 | }); 72 | setTimeout(function() { 73 | cb(new Error("fetchTorrent timed out")); 74 | engine.destroy(); 75 | }, cfg.fetchMetaTimeout || 25 * 1000); 76 | }; 77 | 78 | function downloadTorrent(infoHash, opts, cb) 79 | { 80 | if (typeof(opts) == "function") cb = opts; 81 | if (! (opts && typeof(opts) == "object")) opts = {}; 82 | 83 | downloader[opts.important ? "unshift" : "push"]({ infoHash: infoHash, callback: function(err, torrent) { 84 | if (err && opts.important) return fetchTorrent(infoHash, opts, cb); 85 | if (err) return cb(err); 86 | cb(null, torrent); 87 | }, url: opts.url }) 88 | } 89 | 90 | module.exports = { retrieve: downloadTorrent }; 91 | module.exports.getAgent = function() { return undefined }; // dummy, to replace if you want your own agent 92 | -------------------------------------------------------------------------------- /lib/db.js: -------------------------------------------------------------------------------- 1 | var mmm = require("multi-master-merge"); 2 | var level = require("level"); // LevelUP + LevelDOWN 3 | var path = require("path"); 4 | var url = require("url"); 5 | var mkdirp = require("mkdirp"); 6 | var sublevel = require("level-sublevel"); 7 | var Map = require("es6-map"); 8 | var _ = require("lodash"); 9 | var async = require("async"); 10 | var events = require("events"); 11 | 12 | var cfg = require("./cfg"); 13 | var utils = require("./utils"); 14 | 15 | mkdirp.sync(cfg.dbPath); 16 | var db = mmm(level(cfg.dbPath), { encoding: "json", gc: true, postupdate: updateIndexes }); 17 | var AVLTree = require("binary-search-tree").AVLTree; 18 | var mmmEnc = require("multi-master-merge/encoders")("json"); 19 | 20 | /* Indexes 21 | */ 22 | db.evs = new events.EventEmitter(); 23 | db.indexes = { meta: new AVLTree(), seeders: new Map(), updated: new Map() }; 24 | function getHashes(x) { 25 | return (Array.isArray(x.episode) ? x.episode : [x.episode]).map(function(ep) { // separate hashes for multi-episode files 26 | return [x.imdb_id, x.season, ep ].filter(function(x) { return x }).join(" ") 27 | }); 28 | } 29 | function updateIndexes(entry, callback) 30 | { 31 | var torrent = entry.value; 32 | if (! torrent) return callback && callback(); 33 | var maxSeeders = utils.getMaxPopularity(torrent); 34 | 35 | db.evs.emit("idxbuild", torrent, entry.peer, entry.seq); 36 | 37 | db.indexes.seeders.set(torrent.infoHash, maxSeeders); 38 | db.indexes.updated.set(torrent.infoHash, torrent.updated); 39 | if (torrent.files && maxSeeders) torrent.files.forEach(function(x) { 40 | if (utils.isFileBlacklisted(x)) return; 41 | getHashes(x).forEach(function(hash) { 42 | db.indexes.meta.delete(hash, torrent.infoHash); // always ensure there are no duplicates 43 | //if (maxSeeders > cfg.minSeedToIndex) db.indexes.meta.insert(hash, torrent.infoHash); 44 | db.indexes.meta.insert(hash, torrent.infoHash); // currently index everything 45 | // WARNING: if we use the maxSeeders check, we risk to run this codepath with a conflicted copy of the torrent which doesn't have seeds 46 | // for now this is un-solvable 47 | }); 48 | }); 49 | callback && callback(); 50 | } 51 | 52 | /* Build initial index 53 | */ 54 | db.createLogStream({ }).on("data", function(entry) { 55 | updateIndexes({ peer: entry.peer, seq: entry.seq, value: entry.value }); 56 | }).on("end", function() { db.evs.emit("idxready") }); 57 | 58 | /* Compact with old events API 59 | * this should maybe be obsoleted? 60 | */ 61 | Object.defineProperty(db, "onIdxBuild", { set: function(fn) { db.evs.on("idxbuild", fn) } }); 62 | Object.defineProperty(db, "onIdxReady", { set: function(fn) { db.evs.on("idxready", fn) } }); 63 | 64 | /* Querying 65 | */ 66 | db.lookup = function(query, count, callback) 67 | { 68 | callback(null, db.indexes.meta.search(getHashes(query)[0]) 69 | .map(function(id) { return { id: id, seeders: db.indexes.seeders.get(id) || 0 } }) 70 | .sort(function(a,b) { return b.seeders - a.seeders }) 71 | .slice(0, count)); 72 | }; 73 | 74 | // query - { imdb_id, season?, episode? } ; items - number of items to return, callback - function(err, items) 75 | db.find = function(query, items, callback) 76 | { 77 | db.lookup(query, items, function(err, matches) { 78 | if (err) return callback(err); 79 | async.map(matches, function(x, cb) { 80 | return db.get(x.id, function(err, res) { cb(err, res && res[0] && res[0].value) }); 81 | }, callback); 82 | }); 83 | }; 84 | 85 | // forEachMeta(iterator, callback) 86 | db.forEachMeta = function(fn, cb) { db.indexes.meta.executeOnEveryNode(fn); cb(); } 87 | 88 | // forEachTorrent(iterator, callback) 89 | db.forEachTorrent = function(fn, cb) { db.indexes.seeders.executeOnEveryNode(fn); cb(); } 90 | 91 | // count(function(err, count) { }) 92 | db.count = function(cb) { cb(null, db.indexes.seeders.size) }; 93 | 94 | // 95 | db.popularities = function(callback) { 96 | var popularities = { }; 97 | db.forEachMeta(function(n) { 98 | // value of db.indexes.seeders is equivalent to utils.getMaxPopularity 99 | if (n.key) popularities[n.key.split(" ")[0]] = Math.max.apply(null, n.data.map(function(k) { return db.indexes.seeders.get(k) })) || 0; 100 | }, function() { callback(null, popularities) }); 101 | } 102 | 103 | module.exports = db; 104 | -------------------------------------------------------------------------------- /lib/indexer.js: -------------------------------------------------------------------------------- 1 | /* Index torrents into our database 2 | */ 3 | 4 | var _ = require("lodash"); 5 | var async = require("async"); 6 | var Stremio = require("stremio-addons"); 7 | var path = require("path"); 8 | 9 | var nameToImdb = require("name-to-imdb"); 10 | var parseVideoName = require("video-name-parser"); 11 | 12 | var cfg = require("./cfg"); 13 | 14 | function index(task, options, callback) 15 | { 16 | var task = typeof(task) == "string" ? { infoHash: task } : task; 17 | var torrent = task.torrent || { }; 18 | 19 | // Core properties 20 | torrent.added = torrent.added || Date.now(); 21 | torrent.updated = Date.now(); 22 | torrent.sources = torrent.sources || { }; 23 | torrent.popularity = torrent.popularity || { }; 24 | 25 | // Update sources 26 | var isInSource = task.source && torrent.sources[task.source.url]; 27 | if (task.source) torrent.sources[task.source.url] = Date.now(); 28 | if (task.extra && task.source && task.extra.hasOwnProperty("uploaders")) torrent.popularity[task.source.url] = [task.extra.uploaders, task.extra.downloaders]; 29 | 30 | // Skip logic - if we're already hit from that source 31 | //if (!options.force && isInSource) return callback(null, torrent, true); 32 | 33 | // Skip logic - if we already have an indexed torrent with files / uninteresting 34 | if (!options.force && (torrent.files || torrent.uninteresting)) return callback(null, torrent, true); 35 | 36 | // Retrieve the torrent meta - namely .files 37 | (options.retrieve || require("../lib/retriever").retrieve)(task.infoHash, { 38 | // WARNING: what if seedleech is not called yet? 39 | important: (task.source && task.source.important) || task.important, 40 | url: task.extra && task.extra.download 41 | }, function(err, tor) { 42 | if (err) return callback(err); 43 | if (! tor.files) return callback(new Error("torrent has no files: "+task.infoHash)); 44 | 45 | _.extend(torrent, _.omit(tor, "pieces", "info", "infoBuffer")); 46 | 47 | torrent.files = torrent.files 48 | .map(function(file, idx) { 49 | if (! file.path) throw new Error('sanity check failed - file does not have path: '+JSON.stringify(torrent)); 50 | return _.extend(file, { idx: idx, name: path.basename(file.path) }); 51 | }) 52 | .filter(function(file) { return file.path.match(options.matchFiles || cfg.matchFiles) && !file.name.match(cfg.excludeFiles) }) 53 | .filter(function(file) { return !cfg.excludeNonAscii || /^[\000-\177]*$/.test(file.path) }); 54 | 55 | (function(next) { 56 | if (! torrent.files.length) return next(); 57 | if (cfg.excludeTorrents && torrent.name.match(cfg.excludeTorrents)) return next(); 58 | 59 | async.map(torrent.files, function(file, cb) { 60 | // Add parsed properties (name, season/episode, year, type) to the file, as well as if we got this info from the source 61 | _.extend(file, parseVideoName(file.path), _.pick(task.extra, "imdb_id", "type", "name")); 62 | 63 | if (! interestingFile(file)) return cb(); 64 | 65 | if (file.imdb_id) return cb(null, file); 66 | nameToImdb(_.extend({ hintUrl: task.hints && task.hints.url }, file), function(err, imdb_id) { 67 | if (err) return cb(err); 68 | file.imdb_id = imdb_id; 69 | cb(null, file); 70 | }); 71 | }, function(err, res) { 72 | if (err) return callback(err); 73 | 74 | if (task.hints && task.hints.url && !res.some(function(x) { return x && x.imdb_id })) 75 | return callback(new Error("hint URL passed but no imdb_id")); 76 | 77 | torrent.files = res.filter(function(x) { return x }); 78 | next(true); 79 | }); 80 | })(function(called) { 81 | torrent.updated = Date.now(); torrent.updatedCinemeta = called || false; // 'updatedMeta' should be more proper 82 | torrent.uninteresting = !torrent.files.length; 83 | callback(null, torrent); 84 | }); 85 | }); 86 | } 87 | 88 | function seedleech(infoHash, callback) 89 | { 90 | var Tracker = require("peer-search/tracker"); 91 | 92 | var popularity = { }, cb = _.once(function() { callback(null, popularity) }); 93 | setTimeout(cb, cfg.trackerTimeout); 94 | 95 | async.each(cfg.trackers, function(tracker, ready) { 96 | var t = new Tracker(tracker, { }, infoHash); 97 | t.run(); 98 | t.on("info", function(inf) { 99 | popularity[tracker] = [inf.seeders, inf.leechers]; 100 | ready(); 101 | }); 102 | }, cb); 103 | } 104 | 105 | // Conflict resolution logic is here 106 | function merge(torrents) 107 | { 108 | // NOTE: here, on the merge logic, we can set properties that should always be set 109 | // Or just rip out the model logic from LinvoDB into a separate module and use it 110 | return torrents.reduce(function(a, b) { 111 | return _.merge(a, b, function(x, y) { 112 | // this is for the files array, and we want more complicated behaviour 113 | if (_.isArray(a) && _.isArray(b)) return b; 114 | }) 115 | }) 116 | } 117 | 118 | function interestingFile(f) 119 | { 120 | return (f.type == "movie" || f.type == "series") && f.length > 85*1024*1024 121 | } 122 | 123 | module.exports = { index: index, seedleech: seedleech, merge: merge }; 124 | -------------------------------------------------------------------------------- /importers/json.js: -------------------------------------------------------------------------------- 1 | 'use strict'; 2 | 3 | var needle = require('needle'); 4 | var Q = require('q'); 5 | var url = require('url'); 6 | var async = require('async'); 7 | var _ = require('lodash'); 8 | var log = require('../lib/log'); 9 | var events = require('events'); 10 | var parseTorrent = require('parse-torrent'); 11 | 12 | module.exports = function(stream, source) { 13 | var emitter = new events.EventEmitter(); 14 | 15 | Q.all([getjson(source.url), parsesource(url.parse(addhttp(source.url)))]).spread(importjson); 16 | 17 | function importjson(json, host) { 18 | if (host.parsedName === 'unknown') { 19 | log.error('json unknown source', source); 20 | return; 21 | } 22 | switch (host.parsedName) { 23 | case 'eztv': 24 | processEztv(host); 25 | break; 26 | case 'yts': 27 | log.error('json detected yts; please use https://yts.to/tz_daily.txt.gz instead', source); 28 | break; 29 | } 30 | } 31 | 32 | function processEztv(host) { 33 | var queue = async.queue(function(task, next) { 34 | getjson(task).then(function(response) { 35 | if (response.episodes) { 36 | var returnObject = { 37 | show_name: response.title, 38 | imdb_id: response.imdb_id, 39 | verified: true 40 | } 41 | response.episodes.forEach(function(item) { 42 | _.assign(returnObject, { 43 | episode_tvdb: item.tvdb_id, 44 | episode_name: item.title 45 | }); 46 | _.forEach(_.omit(item.torrents, '0'), function(t, key) { // quality 0 === 420p we remove to avoid duplicates 47 | _.assign(returnObject, { 48 | quality: key 49 | }); 50 | emitter.emit('infoHash', parseTorrent(t.url).infoHash, returnObject); 51 | }); 52 | }); 53 | } else if (response[0].imdb_id) { 54 | response.forEach(function(item) { 55 | if (host.slashes) { 56 | queue.push(host.protocol + '//' + host.host + '/show/' + item.imdb_id) 57 | } else { 58 | queue.push(host.protocol + host.host + '/show/' + item.imdb_id) 59 | } 60 | }); 61 | } else if (response && response.length > 0) { 62 | response.forEach(function(url) { 63 | if (host.slashes) { 64 | queue.push(host.protocol + '//' + host.host + '/' + url) 65 | } else { 66 | queue.push(host.protocol + host.host + '/' + url) 67 | } 68 | }); 69 | } 70 | 71 | process.nextTick(next); 72 | }, function(err) { 73 | log.error(err); 74 | process.nextTick(next); 75 | }); 76 | }, 2); 77 | 78 | queue.drain = function() { emitter.emit('end') }; 79 | 80 | if (host.slashes) { 81 | queue.push(host.protocol + '//' + host.host + '/shows') 82 | } else { 83 | queue.push(host.protocol + host.host + '/shows') 84 | } 85 | }; 86 | 87 | function parsesource(s) { 88 | 89 | var eztvEndpoints = ['eztvapi.re', 'tv.ytspt.re', 'api.popcorntime.io', 'api.popcorntime.cc', 'http://7aa7xwqtxoft27r2.onion']; 90 | var ytsEndpoints = ['yts.to', 'yts.io', 'yts.sh', 'http://gm6gttbnl3bjulil.onion']; 91 | 92 | if (new RegExp(eztvEndpoints.join('|')).test(s.host)) { 93 | s.parsedName = 'eztv'; 94 | } else if (new RegExp(ytsEndpoints.join('|')).test(s.host)) { 95 | s.parsedName = 'yts'; 96 | } else { 97 | s.parsedName = 'unknown'; 98 | } 99 | 100 | return Q(s); 101 | } 102 | 103 | function addhttp(url) { 104 | if (!/^(?:f|ht)tps?\:\/\//.test(url)) { 105 | url = "http://" + url; 106 | } 107 | return url; 108 | } 109 | 110 | function getjson(url) { 111 | log.message("json getting: "+url); 112 | 113 | var defer = Q.defer(); 114 | var params = { 115 | compressed: true, // sets 'Accept-Encoding' to 'gzip,deflate' 116 | follow_max: 2, 117 | headers: { "user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/45.0.2454.99 Safari/537.36" } 118 | }; 119 | needle.get(url, params, function(error, response) { 120 | if (!error && response.statusCode == 200) { 121 | defer.resolve(response.body); 122 | } else { 123 | defer.reject(error || response.statusCode) 124 | } 125 | }); 126 | return defer.promise; 127 | } 128 | 129 | return emitter; 130 | } 131 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # multipass-torrent 2 | 3 | **multipass-torrent** indexes torrents by meta information such as IMDB ID, episode/season, so that it can deliver **beautiful catalogues** automatically. 4 | 5 | ## What is it? 6 | 7 | Collects torrents from various sources (dump, RSS, HTML pages) and associates the **video files** within with **IMDB ID** using Stremio's index.get Addon API. 8 | 9 | Runs on a multi-master replicated LevelDB, thanks to [mafintosh/multi-master-merge](http://github.com/mafintosh/multi-master-merge). The peers to replicate with are discovered via database ID, passed via ``--db-id=<16 byte hex string>``, discovered through **DHT and SSDP**. 10 | 11 | This means the system is **distributed** and you can run several instances with their own DB copy. This is useful for creating redundancy and even crawling from several machines to distribute the load. 12 | 13 | It also has a Stremio Addon front-end, allowing for the content you scraped to be used in [Stremio media center](http://strem.io). 14 | 15 | 16 | ## GUI 17 | multipass-torrent is compatible with the stremio addon protocol, so you can use [stremio-addons-client](http://github.com/Ivshti/stremio-addons-client) as a GUI. 18 | 19 | The end result looks like this: [screenshot](https://raw.githubusercontent.com/Ivshti/stremio-addons-client/master/screenshots/stremio-addons-client.png). 20 | 21 | You should see the multipass streams in the right bar under "**Streams**". 22 | 23 | ```bash 24 | # first, start multipass-torrent and keep it running 25 | 26 | npm install stremio-addons-client 27 | cd node_modules/stremio-addons-client 28 | npm start 29 | # open browser at http://localhost:9900/#?addonUrl=http%3A%2F%2Flocalhost%3A7000 30 | # you should see multipass add-on in the Add-ons panel 31 | ``` 32 | 33 | ## Examples 34 | ```bash 35 | node cli/multipass --db-id="identifier string of your replica set" --source="https://torrentz.eu/feed_verified?q=" --db-path=/tmp/test 36 | ``` 37 | 38 | Now you can run the same command on another computer (or terminal) to see the replication. Run it without ``--source`` so you see the replication instead of importing the same source again 39 | ```bash 40 | node cli/multipass --db-id="identifier string of your replica set" --db-path=/tmp/test-2 41 | ``` 42 | 43 | Output from first command: 44 | ``` 45 | ccb9a6f8a9af421809ad6b1f58a76f493fb30fb6: finding other instances to replicate with 46 | DB replication server listening at 50962 47 | importing from https://torrentz.eu/feed_verified?q= 48 | We have 0 torrents 49 | importing finished from https://torrentz.eu/feed_verified?q=, 96 infoHashes, undefined of them new, through xmlRss importer (380ms) 50 | We have 11 torrents 51 | We have 23 torrents 52 | We have 34 torrents 53 | We have 48 torrents 54 | We have 60 torrents 55 | We have 73 torrents 56 | We have 83 torrents 57 | We have 96 torrents 58 | We have 96 torrents 59 | ``` 60 | Output from second command: 61 | ``` 62 | ccb9a6f8a9af421809ad6b1f58a76f493fb30fb6: finding other instances to replicate with 63 | DB replication server listening at 51192 64 | We have 0 torrents 65 | connected to peer 192.168.0.103:50962 66 | We have 96 torrents 67 | ``` 68 | 69 | 70 | ## Querying 71 | Now that the data is in the DB, how to make use of it? 72 | 73 | Currently, no querying mechanism is implmeneted (that will change very soon), but you can see a simple dump by running (after you've populated DB at /tmp/test by one of the previous commands): 74 | ```bash 75 | node cli/multipass --db-dump --db-id=ccb9a6f8a9af421809ad6b1f58a76f493fb30fb6 --db-path=/tmp/test 76 | ``` 77 | 78 | ## Programatic usage 79 | Currently not documented, although the files in lib/ are clear enough on first read to see how to use it. 80 | You can also include cli/multipass as a module to take advantage of it's import and process queues. 81 | 82 | ## Command-line usage 83 | * ``--source`` - provide an URL to source to crawl - this can be in .txt.gz dump, RSS feed or simply an HTML page containing info hashes; you can use multiple ``--source`` arguments 84 | * ``--id`` or ``--db-id`` - the DB ID (16 bit hex string) to use for replication; can also be a plain string, which will be hashed to 16 bit hex; instances with the same DB ID will replicate the DB among them 85 | * ``--db-path`` - the filesystem path of the LevelDB database; default will be "multipass" inside OS's temporary directory 86 | * ``--log=level`` - level is a number from 0 to 3, 3 being most verbose - the logging level 87 | 88 | ## Stremio Addon 89 | multipass can be used as an Addon to [Stremio](http://strem.io) - just pass ``--stremio-addon=PORT`` to the CLI, and then start stremio with ``--services=http://localhost:PORT``. A GUI to add custom addons is soon to be added in Stremio. 90 | 91 | ## Testing / code standards 92 | ```bash 93 | npm test # testing 94 | 95 | # those aren't done by default by the testing yet 96 | grunt jshint # detect errors and potential problems in the js code. 97 | grunt js # beautifies all js files 98 | grunt jsdry # shows what files will be changed via grunt js 99 | ``` 100 | 101 | ## why multipass? 102 | [For anything else there's multipass](https://www.pinterest.com/pin/83738874291404469/) 103 | 104 | Also, a handy coincidence with the DB we ended up using - [multi-master-merge](http://github.com/mafintosh/multi-master-merge) 105 | -------------------------------------------------------------------------------- /cli/multipass.js: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | 3 | var url = require("url"); 4 | var net = require("net"); 5 | var _ = require("lodash"); 6 | var async = require("async"); 7 | var Tracker = require("peer-search/tracker"); 8 | var events = require('events'); 9 | 10 | var cfg = require("../lib/cfg"); 11 | var log = require("../lib/log"); 12 | var db = require("../lib/db"); 13 | var replication = require("../lib/replication"); 14 | var indexer = require("../lib/indexer"); 15 | var importer = require("../lib/importer"); 16 | var utils = require("../lib/utils"); 17 | 18 | var argv = module.parent ? { } : require("minimist")(process.argv.slice(2)); 19 | 20 | var mp = new events.EventEmitter(); 21 | var sources = { }, recurring = { }; 22 | mp.db = db; // expose db 23 | 24 | /* Config - dependant stuff 25 | */ 26 | cfg.on("ready", function() { 27 | // If DB supports replication (multi-master-merge) 28 | if (db.sync) { 29 | replication.listenReplications(cfg.dbId); // start our replication server 30 | replication.findReplications(cfg.dbId); // replicate to other instances 31 | } 32 | 33 | log.important("DB Path "+cfg.dbPath); 34 | log.important("we have "+cfg.sources.length+" sources"); 35 | 36 | if (cfg.sources) cfg.sources.forEach(mp.importQueue.push); 37 | }); 38 | cfg.on("updated", function() { 39 | if (cfg.sources) cfg.sources.forEach(function(source) { if (! recurring[source.url]) mp.importQueue.push(source) }); 40 | }); 41 | 42 | /* Collect infoHashes from source 43 | */ 44 | mp.importQueue = async.queue(mp.import = function(source, next) { 45 | source = typeof(source) == "string" ? { url: source } : source; 46 | 47 | if (argv["disable-collect"]) { log.important("skipping "+source.url+" because of --disable-collect"); return next(); } 48 | 49 | if (source.fn) return source.fn(mp, function() { 50 | if (source.interval) recurring[source.url] = setTimeout(function() { mp.importQueue.push(source) }, source.interval); // repeat at interval - re-push 51 | next(); 52 | }); 53 | 54 | log.important("importing from "+source.url); 55 | importer.collect(source, function(err, status) { 56 | if (err) log.error("importer.collect",err); 57 | else { 58 | log.important("importing finished from "+source.url+", "+status.found+" infoHashes, "+status.imported+" of them new, through "+status.type+" importer ("+(status.end-status.start)+"ms)"); 59 | buffering(source, status.found); 60 | } 61 | 62 | if (source.interval) recurring[source.url] = setTimeout(function() { mp.importQueue.push(source) }, source.interval); // repeat at interval - re-push 63 | 64 | next(); 65 | }, function(hash, extra) { 66 | log.hash(hash, "collect"); 67 | if (!argv["disable-process"]) mp.processQueue.push({ infoHash: hash, extra: extra, hints: extra && extra.hints, source: source }); 68 | // extra - collected from the source, can be info like uploaders/downloaders, category, etc. 69 | // hints - hints to particular meta information already found from the source, like imdb_id, season/episode 70 | }); 71 | }, 1); 72 | 73 | /* Process & index infoHashes 74 | */ 75 | mp.processQueue = async.queue(function(task, _next) { 76 | var next = _.once(function() { called = true; buffering(task.source); _next() }), called = false; 77 | setTimeout(function() { next(); if (!called) log.error("process timeout for "+task.infoHash) }, 10*1000); 78 | 79 | log.hash(task.infoHash, "processing"); 80 | 81 | // consider using db.indexes.seeders to figure out a skip case here; don't overcomplicate though 82 | db.get(task.infoHash, function(err, res) { 83 | if (err) { 84 | log.error(err); 85 | return next(); 86 | } 87 | 88 | // WARNING: no skip logic here, as we need at least to update .sources and seed/leech data 89 | // Pass a merge of existing torrent objects as a base for indexing 90 | var noChanges; 91 | task.torrent = res && res.length && indexer.merge(res.sort(function(a, b) { return a.seq - b.seq }).map(function(x) { return x.value })); 92 | task.important = task.torrent && (utils.getMaxPopularity(task.torrent) > cfg.minSeedImportant); 93 | async.auto({ 94 | index: function(cb) { indexer.index(task, { }, function(err, tor, nochanges) { noChanges = nochanges; cb(err, tor) }) }, 95 | seedleech: function(cb) { (task.torrent && task.torrent.popularityUpdated > (Date.now() - cfg.popularityTTL)) ? cb() : indexer.seedleech(task.infoHash, cb) } 96 | }, function(err, indexing) { 97 | if (err) { 98 | if (task.callback) task.callback(err); log.error("processQueue", task.infoHash, err); 99 | return next(); 100 | } 101 | 102 | // Note that this is a _.merge, popularity is not overriden 103 | var torrent = _.merge(indexing.index, indexing.seedleech ? { popularity: indexing.seedleech, popularityUpdated: Date.now() } : { }); 104 | // Don't save if we don't have changes and we've got only 1 revision 105 | if (! (res.length == 1 && noChanges)) db.merge(torrent.infoHash, res, torrent); 106 | 107 | mp.emit("found", task.source.url, torrent); 108 | 109 | next(); 110 | if (task.callback) task.callback(null, torrent); 111 | 112 | if (torrent.uninteresting && !res.length) log.warning(torrent.infoHash+" / "+torrent.name+" is non-interesting, no files indexed"); 113 | log.hash(task.infoHash, "processed"); 114 | }); 115 | }); 116 | }, cfg.processingConcurrency); 117 | 118 | db.evs.on("idxbuild", function(tor, peer, seq) { 119 | var updated = tor.sources && Math.max.apply(null, _.values(tor.sources)); 120 | if (cfg.nonSeededTTL && peer && updated && (Date.now()-updated > cfg.nonSeededTTL) && !utils.getMaxPopularity(tor)) 121 | db.log.del(peer, seq, function() { console.log("removed "+tor.infoHash) }); 122 | }); 123 | 124 | /* Emit buffering event 125 | */ 126 | function buffering(source, total) { 127 | if (! (source && source.url)) return; 128 | if (! sources[source.url]) sources[source.url] = { progress: 0, total: 0 }; 129 | if (!isNaN(total)) return sources[source.url].total = total; 130 | sources[source.url].progress++; 131 | var perc; 132 | perc = sources[source.url].progress/sources[source.url].total; 133 | perc = (Math.floor(perc * 100) / 100).toFixed(2); 134 | mp.emit("buffering", source, perc); 135 | if (perc == 1) { 136 | mp.emit("finished", source); 137 | delete sources[source.url]; 138 | } 139 | } 140 | 141 | /* Programatic usage of this 142 | */ 143 | if (module.parent) return module.exports = mp; 144 | 145 | /* Log number of torrents we have 146 | */ 147 | db.evs.on("idxready", function() { 148 | async.forever(function(next) { 149 | log.important("We have "+db.indexes.seeders.size+" torrents, "+mp.processQueue.length()+" queued"); 150 | setTimeout(next, 5000); 151 | }); 152 | }); 153 | 154 | /* Stremio Addon interface 155 | */ 156 | if (cfg.stremioAddon) require("../stremio-addon/addon")(db, utils, cfg)(cfg.stremioAddon); 157 | 158 | 159 | -------------------------------------------------------------------------------- /stremio-addon/addon.js: -------------------------------------------------------------------------------- 1 | var Stremio = require("stremio-addons"); 2 | var http = require("http"); 3 | var _ = require("lodash"); 4 | var async = require("async"); 5 | var sift = require("sift"); 6 | var bagpipe = require("bagpipe"); 7 | var events = require("events"); 8 | 9 | module.exports = function(db, utils, cfg) { 10 | 11 | // Keeping meta collection up to date; Algo here is 12 | // async.queue with concurrency = 1 / bagpipe(1) ; we push update requests and collectMeta() calls to it 13 | // updateMeta() function, called on idxbuild, debounced at half a second (or 300ms?) 14 | // sample test: 79.1mb / 74.8mb / 75.1mb RAM with 1000 lean meta ; without: 72.0mb, 74.4, 72.3 NO DIFF in memory usage 15 | var LID = cfg.LID || cfg.dbId.slice(0, 10); 16 | var metaQueryProps = ["imdb_id", "type", "name", "year", "genre", "director", "dvdRelease", "imdbRating", "poster", "popularities."+LID]; 17 | 18 | 19 | var CINEMETA_URL = process.env.CINEMETA || cfg.cinemeta || "http://cinemeta.strem.io/stremioget"; 20 | var addons = new Stremio.Client(); 21 | addons.add(CINEMETA_URL); 22 | 23 | var meta = { col: [], updated: 0, have: { } }; 24 | var metaPipe = new bagpipe(1); 25 | 26 | var lastPopularities; 27 | 28 | function updateMeta(ready) { 29 | db.popularities(function(err, popularities) { 30 | lastPopularities = popularities; 31 | 32 | var popSort = function(x) { return -popularities[x.imdb_id || x] }; 33 | var constructMeta = function(x) { 34 | x.imdbRating = parseFloat(x.imdbRating); 35 | x.popularities = { }; // reset that - don't know if it brings any benefits 36 | x.popularities[LID] = popularities[x.imdb_id] || 0; 37 | x.isPeered = !!popularities[x.imdb_id]; 38 | // figure out year? since for series it's like "2011-2015" we can sort by the first field, but we can't replace the value 39 | meta.have[x.imdb_id] = 1; 40 | }; 41 | 42 | var toGet = []; 43 | Object.keys(popularities).forEach(function(k) { if (! meta.have[k]) toGet.push(k) }); 44 | toGet = toGet.filter(function(x) { return x && x.match("^tt") }); 45 | toGet = toGet.sort(function(a, b) { return (popularities[b] || 0) - (popularities[a] || 0) }); 46 | toGet = toGet.slice(0, 500); 47 | 48 | if (toGet.length == 0) return process.nextTick(ready); 49 | 50 | addons.meta.find({ query: { imdb_id: { $in: toGet } }, limit: toGet.length }, function(err, res) { 51 | process.nextTick(ready); // ensure we don't dead-end (deadlock is not a right term, block is not the right term, terms have to figured out for async code) 52 | if (err) console.error("meta.find from "+CINEMETA_URL, err); 53 | meta.ready = true; 54 | meta.col = _.chain(meta.col).concat(res || []).sortBy(popSort).uniq("imdb_id").each(constructMeta).value(); 55 | db.evs.emit("catalogue-update", meta.col, popularities); 56 | }); 57 | }); 58 | }; 59 | 60 | db.evs.on("idxbuild", _.debounce(function() { if (! metaPipe.queue.length) metaPipe.push(updateMeta) }, 1000)); 61 | db.evs.on("idxready", function() { metaPipe.push(updateMeta) }); 62 | 63 | // Basic validation of args 64 | function validate(args) { 65 | var meta = args.query; 66 | if (! (args.query || args.infoHash)) return { code: 0, message: "query/infoHash required" }; 67 | if (meta && !meta.imdb_id) return { code: 1, message: "imdb_id required" }; 68 | if (meta && (meta.type == "series" && !(meta.hasOwnProperty("episode") && meta.hasOwnProperty("season")))) 69 | return { code: 2, message: "season and episode required for series type" }; 70 | return false; 71 | }; 72 | 73 | function query(args, callback) { 74 | var start = Date.now(); 75 | 76 | (function(next) { 77 | if (args.infoHash) return db.get(args.infoHash, function(err, res) { next(err, res && res[0] && res[0].value) }); 78 | if (! args.query) return next(new Error("must specify query or infoHash")); 79 | 80 | //var preferred = _.uniq(PREFERRED.concat(args.preferred || []), function(x) { return x.tag }); 81 | var preferred = args.preferred || []; 82 | var prio = function(resolution) { 83 | return preferred.map(function(pref) { 84 | return utils.getAvailForTorrent(resolution.torrent) >= pref.min_avail && resolution.file && resolution.file.tag.indexOf(pref.tag)!=-1 85 | }).reduce(function(a,b) { return a+b }, 0); 86 | }; 87 | 88 | var resolution = null; 89 | db.lookup(args.query, 3, function(err, matches) { 90 | if (err) return next(err); 91 | 92 | async.whilst( 93 | function() { return matches.length && (!resolution || prio(resolution) < preferred.length) }, 94 | function(callback) { 95 | var hash = matches.shift().id; 96 | db.get(hash, function(err, res) { 97 | if (err) return callback({ err: err }); 98 | 99 | var tor = res[0] && res[0].value; 100 | if (! tor) return callback({ err: "hash not found "+hash }); 101 | 102 | var file = _.find(tor.files, function(f) { 103 | return f.imdb_id == args.query.imdb_id && 104 | (args.query.season ? (f.season == args.query.season) : true) && 105 | (args.query.episode ? ((f.episode || []).indexOf(args.query.episode) != -1) : true) 106 | }); 107 | 108 | // TODO: error when no file is found? 109 | 110 | var res = { torrent: tor, file: file }; 111 | if (!resolution || prio(res) > prio(resolution)) resolution = res; 112 | 113 | callback(); 114 | }); 115 | }, 116 | function(err) { resolution ? next(resolution.err, resolution.torrent, resolution.file) : next(err) } 117 | ); 118 | }); 119 | })(function(err, torrent, file) { 120 | // Output according to Stremio Addon API for stream.get 121 | // http://strem.io/addons-api 122 | callback(err, torrent ? _.extend({ 123 | infoHash: torrent.infoHash.toLowerCase(), 124 | uploaders: utils.getMaxPopularity(torrent), // optional 125 | downloaders: Math.max.apply(Math, _.values(torrent.popularity).map(function(x) { return x[1] }).concat(0)), // optional 126 | //map: torrent.files, 127 | //pieceLength: torrent.pieceLength, 128 | availability: utils.getAvailForTorrent(torrent), 129 | isFree: true, 130 | sources: utils.getSourcesForTorrent(torrent), // optional but preferred 131 | runtime: Date.now()-start // optional 132 | }, file ? { 133 | mapIdx: file.idx, 134 | tag: file.tag, filename: file.name, 135 | } : { }) : null); 136 | }); 137 | }; 138 | 139 | var FILTER = _.object([ "sort.popularities."+LID,"query.popularities."+LID ], [{ "$exists": true },{ "$exists": true }]); 140 | var FILTER_OVERRIDE_CINEMETA = _.extend(FILTER, { 141 | "projection.imdb_id": {"$exists":true}, 142 | "popular": {"$exists":true}, 143 | "query.type": { $in: ["movie", "series"] }, 144 | }); 145 | 146 | var manifest = _.merge({ 147 | // this should be always overridable by stremio-manifest 148 | stremio_LID: LID, 149 | // set filter so that we intercept meta.find from cinemeta 150 | filter: FILTER_OVERRIDE_CINEMETA, 151 | }, require("./stremio-manifest"), _.pick(require("../package"), "version"), cfg.stremioManifest || {}); 152 | 153 | var methods; 154 | var service = new Stremio.Server(methods = { 155 | "stream.get": function(args, callback, user) { // OBSOLETE 156 | service.events.emit("stream.get", args, callback); 157 | 158 | var error = validate(args); 159 | if (error) return callback(error); 160 | query(args, callback); 161 | }, 162 | "stream.find": function(args, callback, user) { 163 | service.events.emit("stream.find", args, callback); 164 | 165 | if (args.query) { 166 | // New format ; same as stream.get, even returns the full result; no point to slim it down, takes same time 167 | var error = validate(args); 168 | if (error) return callback(error); 169 | async.map([ _.extend({ preferred: [{ tag: "hd", min_avail: 2 }] }, args), args ], query, function(err, res) { 170 | if (err) return callback(err); 171 | if (! res) return callback(null, res); 172 | 173 | var results = _.chain(res).filter(function(x) { return x }).uniq(function(x) { return x.infoHash }).value(); 174 | service.events.emit("stream.find.res", results); 175 | callback(null, results); 176 | }); 177 | } else return callback({code: 10, message: "unsupported arguments"}); 178 | }, 179 | "stream.popularities": function(args, callback, user) { // OBSOLETE 180 | service.events.emit("stream.popularities", args, callback); 181 | db.popularities(function(err, popularities) { callback(err, popularities ? { popularities: popularities } : null) }); 182 | }, 183 | "meta.find": function(args, callback, user) { 184 | service.events.emit("meta.find", args, callback); 185 | 186 | var firstSort = Object.keys(args.sort || { })[0]; 187 | if (firstSort !== "popularities."+LID) { 188 | addons.meta.find(args, function(err, res) { 189 | if (res && lastPopularities) res.forEach(function(x) { x.isPeered = !!lastPopularities[x.imdb_id] }); 190 | callback(err, res); 191 | }); 192 | return; 193 | } 194 | 195 | // Call this to wait for meta to be collected 196 | if (args.projection && ( args.projection=="full" ) ) return callback(new Error("full projection not supported by mp")); 197 | 198 | (meta.ready ? function(n) { n() } : metaPipe.push.bind(metaPipe))(function(ready) { 199 | if (typeof(ready) == "function") process.nextTick(ready); // ensure we don't lock 200 | 201 | args.query = _.pick.apply(null, [args.query || { }].concat(metaQueryProps)); 202 | args.sort = _.pick.apply(null, [args.sort || { }].concat(metaQueryProps)); 203 | //if (! _.keys(args.sort).length) args.sort["popularities."+LID] = -1; // no need as this is our default sort order 204 | 205 | var proj, projFn; 206 | if (args.projection && typeof(args.projection) == "object") { 207 | proj = _.keys(args.projection); 208 | projFn = _.values(args.projection)[0] ? _.pick : _.omit; 209 | } 210 | 211 | var res = _.chain(meta.col) 212 | .filter(args.query ? sift(args.query) : _.constant(true)) 213 | //.sortByOrder(_.keys(args.sort), _.values(args.sort).map(function(x) { return x>0 ? "asc" : "desc" })) 214 | .slice(args.skip || 0, Math.min(400, args.limit)) 215 | .map(function(x) { return projFn ? projFn(x, proj) : x }) 216 | .value(); 217 | 218 | service.events.emit("meta.find.res", res); 219 | callback(null, res); 220 | }); 221 | }, 222 | "stats.get": function(args, callback, user) { 223 | service.events.emit("stats.get", args, callback); 224 | 225 | var items = 0, episodes = 0, movies = 0; 226 | db.forEachMeta(function(n) { 227 | if (n.key.indexOf(" ") != -1) episodes++; else movies++; 228 | items++; 229 | }, function() { 230 | db.count(function(err, c) { 231 | callback(null, { statsNum: items+" movies and episodes", stats: [ 232 | { name: "number of items - "+items, count: items, colour: items > 20 ? "green" : (items > 10 ? "yellow" : "red") }, 233 | { name: "number of movies - "+movies, count: movies, colour: movies > 20 ? "green" : (movies > 10 ? "yellow" : "red") }, 234 | { name: "number of episodes - "+episodes, count: episodes, colour: episodes > 20 ? "green" : (episodes > 10 ? "yellow" : "red") }, 235 | { name: "number of torrents - "+c, count: c, colour: c > 50 ? "green" : (c > 20 ? "yellow" : "red") } 236 | ] }); 237 | }); 238 | }); 239 | }, 240 | }, { stremioget: true, allow: [cfg.stremioCentral,"http://api8.herokuapp.com","http://api9.strem.io","https://api9.strem.io"], secret: cfg.stremioSecret }, manifest); 241 | 242 | // Event emitter in case we want to intercept/plug-in to this 243 | service.events = new events.EventEmitter(); 244 | 245 | function listen(port, ip) { 246 | var server = http.createServer(function (req, res) { 247 | req.on("error", function(e) { console.error(e) }); 248 | service.middleware(req, res, function() { res.end() }); 249 | }) 250 | .on("error", function(e) { console.error("mp server", e) }) 251 | .on("listening", function() 252 | { 253 | console.log("Multipass Stremio Addon listening on "+server.address().port); 254 | }); 255 | return server.listen(port, ip); 256 | } 257 | 258 | module.exports.service = service; 259 | module.exports.methods = methods; 260 | return listen; 261 | 262 | } 263 | -------------------------------------------------------------------------------- /test/basic.js: -------------------------------------------------------------------------------- 1 | var tape = require("tape"); 2 | var async = require("async"); 3 | var _ = require("lodash"); 4 | 5 | var Stremio = require("stremio-addons"); 6 | 7 | var cfg = require("../lib/cfg"); 8 | cfg.dbPath = require("path").join(require("os").tmpdir(), Date.now()+""); 9 | var log = require("../lib/log"); 10 | var db = require("../lib/db"); 11 | var utils = require("../lib/utils"); 12 | var indexer = require("../lib/indexer"); 13 | var importer = require("../lib/importer"); 14 | var retriever = require("../lib/retriever"); 15 | 16 | cfg.logLevel = 0; 17 | 18 | var hashes = [ ]; // global so we can reuse it 19 | var movie_ids = { }; var series_ids = { }; // also global, so we can reuse those 20 | 21 | 22 | tape("importer with rss source", function(t) { 23 | /* WARNING: this entire test file depends on this source; if it fails, all tests will fail 24 | * write individual tests covering edge cases in all modules, not dependant on external influence 25 | */ 26 | t.timeoutAfter(10000); 27 | 28 | importer.collect({ url: "http://torrentz.eu/feed_verified?q=", category: ["tv", "movies"] }, function(err, status) { 29 | t.ok(!err, "no err from importer.collect"); 30 | t.ok(hashes.length > 20, "hashes collected ("+hashes.length+") are more than 20"); 31 | t.end(); 32 | }, function(hash) { 33 | hashes.push(hash); 34 | t.ok(typeof(hash)=="string" && hash.length==40, "valid infoHash"); 35 | }); 36 | }); 37 | 38 | tape("importer with dump source", function(t) { 39 | t.timeoutAfter(20000); 40 | 41 | var hashesDump = []; 42 | 43 | importer.collect({ url: "http://bitsnoop.com/api/latest_tz.php?t=verified", category: ["tv", "movies"], type: "dump" }, function(err, status) { 44 | t.ok(!err, "no err from importer.collect"); 45 | t.ok(hashesDump.length >= 5, "hashes collected ("+hashesDump.length+") are more than 5"); 46 | t.ok(status.type == "dump", "we've collected from a dump") 47 | t.end(); 48 | 49 | }, function(hash, extra) { 50 | hashesDump.push(hash); 51 | t.ok(typeof(hash)=="string" && hash.length==40, "valid infoHash"); 52 | t.ok(extra && extra.category.match("movie|tv"), "match movie/tv in category"); 53 | }); 54 | }); 55 | 56 | /* 57 | tape("importer with dump source - large with minseeders", function(t) { 58 | t.timeoutAfter(500*1000); 59 | 60 | var count = 0; 61 | 62 | importer.collect({ 63 | url: "http://ext.bitsnoop.com/export/b3_verified.txt.gz", 64 | minSeedersUrl: "http://ext.bitsnoop.com/export/b3_e003_torrents.txt.gz", 65 | minSeeders: 5, 66 | category: ["tv", "movies"], type: "dump" 67 | }, function(err, status) { 68 | t.ok(!err, "no err from importer.collect"); 69 | t.ok(status.type == "dump", "we've collected from a dump") 70 | console.log("found "+count+" hashes"); 71 | // around 3k with over 10 seeds 72 | // found 5k with over 5 seeds 73 | // around 74 | t.end(); 75 | }, function(hash, extra) { 76 | //t.ok(typeof(hash)=="string" && hash.length==40, "valid infoHash"); 77 | //t.ok(extra && extra.category.match("movie|tv"), "match movie/tv in category"); 78 | //t.ok(extra && extra.uploaders >= 10, "has min uploaders"); 79 | count++; 80 | }); 81 | }); 82 | */ 83 | 84 | 85 | tape("retriever", function(t) { 86 | t.timeoutAfter(5000); 87 | 88 | // try with 3 hashes, accept 2/6 success rate - some of them are simply not available 89 | var results = []; 90 | async.each(hashes.slice(0,6), function(hash, callback) { 91 | retriever.retrieve(hash, function(err, tor) { 92 | //if (err) console.error(err); 93 | if (tor) results.push(tor); 94 | callback(); 95 | }); 96 | }, function() { 97 | t.ok(results.length >= 2, "we have 2 or more results"); 98 | t.ok(results.every(function(x) { return x.infoHash }), "all of them have infohash"); 99 | t.ok(results.every(function(x) { return x.files }), "all of them have files"); 100 | t.end(); 101 | }); 102 | }); 103 | 104 | tape("retriever - catch errors", function(t) { 105 | t.timeoutAfter(10000); 106 | var hash = "230bb375188a9ecf57ba469fc8ec36cf5634a0382"; 107 | 108 | retriever.retrieve(hash, function(errs, tor) { 109 | t.ok(errs && errs.length, "has errors"); 110 | t.end(); 111 | }); 112 | }) 113 | 114 | tape("retriever - pass url", function(t) { 115 | t.timeoutAfter(3000); 116 | 117 | // try with 3 hashes, accept 2/6 success rate - some of them are simply not available 118 | var results = [ ]; 119 | async.each(hashes.slice(0,6), function(hash, callback) { 120 | retriever.retrieve(hash, { url: "http://torcache.net/torrent/"+hash.toUpperCase()+".torrent" },function(err, tor) { 121 | //if (err) console.error(err); 122 | if (tor) results.push(tor); 123 | callback(); 124 | }); 125 | }, function() { 126 | t.ok(results.length >= 2, "we have 2 or more results"); 127 | t.ok(results.every(function(x) { return x.infoHash }), "all of them have infohash"); 128 | t.ok(results.every(function(x) { return x.files }), "all of them have files"); 129 | t.end(); 130 | }); 131 | }); 132 | 133 | 134 | tape("retriever - fallback to DHT/peers fetching", function(t) { 135 | t.timeoutAfter(30000); 136 | 137 | // try with 3 hashes, accept 3/3 success rate - all metas should be there with peers 138 | var results = [ ]; 139 | async.each(hashes.slice(0,3), function(hash, callback) { 140 | retriever.retrieve(hash, { important: true, url: "http://notcache.net/"+hash.toUpperCase()+".torrent" }, function(err, tor) { 141 | //if (err) console.error(err); 142 | if (tor) results.push(tor); 143 | callback(); 144 | }); 145 | }, function() { 146 | t.ok(results.length >= 3, "we have 3 or more results"); 147 | t.ok(results.every(function(x) { return x.infoHash }), "all of them have infohash"); 148 | t.ok(results.every(function(x) { return x.files }), "all of them have files"); 149 | t.end(); 150 | }); 151 | }); 152 | 153 | 154 | // TODO this is extremely primitive 155 | var mp = require("../cli/multipass"); 156 | var successful = []; 157 | tape("processor - import torrents", function(t) { 158 | t.timeoutAfter(40000); // 40s for 50 torrents 159 | 160 | async.each(hashes.slice(0, 50), function(hash, callback) { 161 | mp.processQueue.push({ infoHash: hash, source: { url: "http://torrentz.eu/search?q=" }, callback: function(err, torrent) { 162 | //if (err) console.error(err); 163 | //if (err) return callback(err); 164 | if (torrent) { 165 | var maxSeed = utils.getMaxPopularity(torrent); 166 | 167 | t.ok(maxSeed, "popular torrent"); 168 | 169 | successful.push(torrent); 170 | // Collect those for later tests 171 | if (maxSeed) (torrent.files || []).forEach(function(f) { 172 | //console.log(f.imdb_id,f.type) 173 | if (f.length < 85*1024*1024) return; 174 | if (! f.imdb_id) return; 175 | if (f.type == "movie") movie_ids[f.imdb_id] = (movie_ids[f.imdb_id] || 0)+1; 176 | if (f.type == "series") series_ids[f.imdb_id] = [f.season,f.episode[0]]; // fill it with season / episode so we can use for testing later 177 | }); 178 | } 179 | callback(); 180 | } }) 181 | }, function(err) { 182 | t.ok(!err, "no error"); 183 | 184 | t.ok(successful.length > 20, "we have more than 20 results"); 185 | t.ok(Object.keys(movie_ids).length > 2, "we have more than two movies"); 186 | t.ok(Object.keys(series_ids).length > 2, "we have more than two series"); 187 | //console.log(movie_ids, series_ids) 188 | t.end(); 189 | }); 190 | }); 191 | 192 | 193 | /* 194 | tape("processor - skip behaviour", function(t) { 195 | 196 | }); 197 | */ 198 | 199 | 200 | tape("indexes - contain the imdb ids", function(t) { 201 | Object.keys(movie_ids).forEach(function(id) { 202 | t.ok(db.indexes.meta.search(id).length, "we have entries for id "+id); 203 | }); 204 | t.end(); 205 | }); 206 | 207 | 208 | tape("db - db.find works with movies", function(t) { 209 | var imdb_id = Object.keys(movie_ids)[0]; 210 | db.find({ imdb_id: imdb_id }, 1, function(err, torrents) { 211 | t.ok(!err, "no error"); 212 | t.ok(torrents[0], "has a result"); 213 | t.ok(torrents.length <= 1, "no more than 1 result"); 214 | t.ok(torrents[0] && torrents[0].infoHash, "infoHash for result"); 215 | t.ok(torrents[0] && _.find(torrents[0].files, function(f) { return f.imdb_id == imdb_id }), "we have a file with that imdb_id inside"); 216 | t.end(); 217 | }); 218 | }); 219 | 220 | tape("db - db.find works series", function(t) { 221 | var imdb_id = Object.keys(series_ids)[0]; 222 | if (!series_ids[imdb_id]) { t.error(new Error("internal test error - no series")); return t.end(); } 223 | var season = series_ids[imdb_id][0], episode = series_ids[imdb_id][1]; 224 | 225 | db.find({ imdb_id: imdb_id, season: season, episode: episode }, 1, function(err, torrents) { 226 | t.ok(!err, "no error"); 227 | t.ok(torrents.length <= 1, "no more than 1 result"); 228 | t.ok(torrents[0] && torrents[0].infoHash, "infoHash for result"); 229 | t.ok(torrents[0] && _.find(torrents[0].files, function(f) { return f.imdb_id == imdb_id && f.season == season && f.episode.indexOf(episode)!=-1 }), "we have a file with that imdb_id inside"); 230 | t.end(); 231 | }); 232 | }); 233 | 234 | // UNIT TESTS 235 | // TODO: test db.lookup 236 | // TODO: test db.forEachMeta 237 | // TODO: test db.forEachTorrent 238 | // TODO: test db.count 239 | // TODO: test db.popularities 240 | 241 | /* Addon tests 242 | */ 243 | var addonPort, addon; 244 | 245 | tape("addon - listening on port", function(t) { 246 | t.timeoutAfter(500); 247 | 248 | var server = require("../stremio-addon/addon")(db, utils, cfg)().on("listening", function() { 249 | addonPort = server.address().port; 250 | t.end(); 251 | }) 252 | }); 253 | 254 | tape("addon - initializes properly", function(t) { 255 | t.timeoutAfter(1000); 256 | 257 | addon = new Stremio.Client(); 258 | addon.setAuth(cfg.stremioCentral, cfg.stremioSecret); 259 | addon.add("http://localhost:"+addonPort); 260 | addon.on("addon-ready", function(service) { 261 | t.ok(service.manifest, "has manifest"); 262 | t.ok(service.manifest.name, "has name"); 263 | t.ok(service.manifest.methods && service.manifest.methods.length, "has methods"); 264 | t.ok(service.manifest.methods && service.manifest.methods.indexOf("stream.get")!=-1, "has stream.get method"); 265 | t.end(); 266 | }); 267 | }); 268 | 269 | tape("addon - stats.get", function(t) { 270 | t.timeoutAfter(1000); 271 | 272 | addon.call("stats.get", { }, function(err, resp) { 273 | t.ok(!err, "no error"); 274 | t.ok(resp && resp.statsNum, "has statsNum"); 275 | t.ok(resp && Array.isArray(resp.stats), "has stats"); 276 | t.end(); 277 | }); 278 | }); 279 | 280 | tape("addon - stream.popularities", function(t) { 281 | t.timeoutAfter(1000); 282 | 283 | addon.call("stream.popularities", { }, function(err, resp) { 284 | t.ok(!err, "no error"); 285 | t.ok(resp && resp.popularities && Object.keys(resp.popularities).length, "has popularities"); 286 | t.end(); 287 | }); 288 | }); 289 | 290 | tape("addon - sample query with a movie", function(t) { 291 | t.timeoutAfter(1000); 292 | 293 | var imdb_id = Object.keys(movie_ids)[0]; 294 | 295 | addon.stream.get({ query: { imdb_id: imdb_id, type: "movie" } }, function(err, resp) { 296 | t.ok(!err, "no error"); 297 | t.ok(resp && resp.infoHash && resp.infoHash.length == 40, "has infoHash"); 298 | //t.ok(resp && Array.isArray(resp.map), "has map"); 299 | //t.ok(resp && !isNaN(resp.mapIdx), "has mapIdx"); 300 | t.ok(resp && !isNaN(resp.availability), "has availability"); 301 | //t.ok(resp && !isNaN(resp.uploaders), "has uploaders"); 302 | 303 | t.end(); 304 | }); 305 | }); 306 | 307 | 308 | tape("addon - sample query with a movie - stream.find", function(t) { 309 | t.timeoutAfter(3000); 310 | 311 | var imdb_id = _.pairs(movie_ids).sort(function(b,a){ return a[1] - b[1] })[0]; 312 | if (! imdb_id) { t.error(new Error("internal test error - no movie")); return t.end(); } 313 | imdb_id = imdb_id[0]; 314 | 315 | addon.stream.find({ query: { imdb_id: imdb_id, type: "movie" } }, function(err, resp) { 316 | t.ok(!err, "no error"); 317 | t.ok(Array.isArray(resp), "returns an array of streams"); 318 | t.end(); 319 | }); 320 | }); 321 | 322 | tape("addon - sample query with an episode", function(t) { 323 | t.timeoutAfter(3000); 324 | 325 | var imdb_id = Object.keys(series_ids)[0]; 326 | if (! (imdb_id && series_ids[imdb_id])) { t.error(new Error("internal test error - no series")); return t.end(); } 327 | var season = series_ids[imdb_id][0], episode = series_ids[imdb_id][1]; 328 | 329 | addon.stream.get({ query: { imdb_id: imdb_id, season: season, episode: episode, type: "series" } }, function(err, resp) { 330 | t.ok(!err, "no error"); 331 | t.ok(resp && resp.infoHash && resp.infoHash.length == 40, "has infoHash"); 332 | //t.ok(resp && Array.isArray(resp.map), "has map"); 333 | //t.ok(resp && !isNaN(resp.mapIdx), "has mapIdx"); 334 | t.ok(resp && !isNaN(resp.availability), "has availability"); 335 | //t.ok(resp && !isNaN(resp.uploaders), "has uploaders"); 336 | 337 | /* 338 | var file = resp && resp.map[resp.mapIdx]; 339 | t.ok(file, "has selected file"); 340 | t.ok(file && file.season && file.episode, "selected file has season/episode"); 341 | t.ok(file && file.season==season && file.episode.indexOf(episode)!=-1, "selected file matches query"); 342 | */ 343 | 344 | t.end(); 345 | }); 346 | }); 347 | 348 | tape("addon - test preferrences", function(t) { 349 | t.skip("TEST NOT IMPLEMENTED - functionality is"); 350 | t.end(); 351 | }); 352 | 353 | 354 | tape("addon - get stream by infoHash", function(t) { 355 | t.timeoutAfter(1500); 356 | 357 | addon.stream.get({ infoHash: successful[0].infoHash }, function(err, resp) { 358 | t.ok(resp && resp.infoHash && resp.infoHash.length == 40, "has infoHash"); 359 | //t.ok(resp && Array.isArray(resp.map), "has map"); 360 | t.ok(resp && !isNaN(resp.availability), "has availability"); 361 | //t.ok(resp && !isNaN(resp.uploaders), "has uploaders"); 362 | 363 | t.end(); 364 | }); 365 | }); 366 | 367 | 368 | tape("addon - get popularities", function(t) { 369 | addon.call("stream.popularities", { }, function(err, res) { 370 | t.ok(!err, "no error"); 371 | t.ok(res && res.popularities, "has popularities object"); 372 | t.ok(Object.keys(res.popularities).length > 1, "popularities object full"); 373 | //t.ok() 374 | }); 375 | }); 376 | 377 | 378 | tape("addon - meta.find", function(t) { 379 | addon.call("meta.find", { limit: 5, query: {} }, function(err, res) { 380 | t.ok(!err, "no error"); 381 | t.ok(res && res.length === 5, "returns 5 results"); 382 | t.end(); 383 | }); 384 | 385 | }); 386 | 387 | tape("addon - meta.find by genre", function(t) { 388 | addon.call("meta.find", { limit: 3, query: { genre: "Comedy" } }, function(err, res) { 389 | t.ok(!err, "no error"); 390 | t.ok(res && res.length === 3, "returns 3 results"); 391 | res.forEach(function(r) { 392 | t.ok(r.genre.indexOf("Comedy")!=-1, "has Comedy in genre"); 393 | }); 394 | t.end(); 395 | }); 396 | 397 | }); 398 | --------------------------------------------------------------------------------