├── config.json ├── download.txt ├── README.md ├── .gitignore ├── package.json ├── index.js ├── pline ├── dispatch.js └── pline.js ├── extractor ├── youtube.js ├── sohu.js ├── iqiyi.js └── letv.js ├── transcoder ├── letv.js ├── youtube.js ├── sohu.js └── iqiyi.js └── util └── util.js /config.json: -------------------------------------------------------------------------------- 1 | { 2 | "download": "./", 3 | "thread": 5 4 | } -------------------------------------------------------------------------------- /download.txt: -------------------------------------------------------------------------------- 1 | http://www.iqiyi.com/v_19rrktkbho.html#vfrm=2-3-0-1 2 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # 运行 2 | 3 | ```npm start``` 4 | 5 | # 要求 6 | 7 | 1. 需要安装ffmpeg视频转码库。 8 | 9 | 10 | # 感谢 11 | 部分思路，借鉴自python3编写的同类项目 https://github.com/soimort/you-get 12 | 13 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Logs 2 | logs 3 | *.log 4 | npm-debug.log* 5 | 6 | # Runtime data 7 | pids 8 | *.pid 9 | *.seed 10 | 11 | # Directory for instrumented libs generated by jscoverage/JSCover 12 | lib-cov 13 | 14 | # Coverage directory used by tools like istanbul 15 | coverage 16 | 17 | # Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files) 18 | .grunt 19 | 20 | # node-waf configuration 21 | .lock-wscript 22 | 23 | # Compiled binary addons (http://nodejs.org/api/addons.html) 24 | build/Release 25 | 26 | # Dependency directory 27 | # https://docs.npmjs.com/misc/faq#should-i-check-my-node-modules-folder-into-git 28 | node_modules 29 | 30 | # Other files 31 | download.txt -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "ngetter", 3 | "version": "1.0.1", 4 | "description": "下载视频网站视频流，并且转码", 5 | "main": "index.js", 6 | "scripts": { 7 | "start": "node index.js .", 8 | "test": "echo \"Error: no test specified\" && exit 1" 9 | }, 10 | "author": "liuguanyu", 11 | "license": "MIT", 12 | "dependencies": { 13 | "big-integer": "^1.5.6", 14 | "commander": "^2.8.1", 15 | "crypto": "0.0.3", 16 | "download": "^4.4.3", 17 | "http": "0.0.0", 18 | "iconv-lite": "^0.4.11", 19 | "immutable": "^3.7.4", 20 | "md5": "^2.0.0", 21 | "qs": "^4.0.0", 22 | "request": "^2.58.0", 23 | "rimraf": "^2.4.2", 24 | "url": "^0.10.3", 25 | "uuid": "^2.0.1", 26 | "xml2js": "^0.4.9" 27 | }, 28 | "bin": { 29 | "nget": "./index.js" 30 | }, 31 | "devDependencies": {}, 32 | "repository": { 33 | "type": "git", 34 | "url": "https://github.com/liuguanyu/nget.git" 35 | }, 36 | "keywords": [ 37 | "video", 38 | "download" 39 | ], 40 | "bugs": { 41 | "url": "https://github.com/liuguanyu/nget/issues" 42 | }, 43 | "homepage": "https://github.com/liuguanyu/nget" 44 | } 45 | -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | var cli = require("commander"), 3 | fs = require("fs"), 4 | path = require("path"); 5 | 6 | cli. 7 | allowUnknownOption(). 8 | version( require("./package.json").version ). 9 | option("-f, --file [value]", "download urls' file"). 10 | option("-u, --url [value]", "download url"). 11 | option("-d, --download [value]", "download folder"). 12 | option("-t, --thread [value]", "download thread number"). 13 | parse( process.argv ); 14 | 15 | // 载入配置 16 | var config = require("./config.json"); 17 | if(cli.thread && typeof(cli.thread)=="string") config.thread = parseInt(cli.thread); 18 | if(cli.download && typeof(cli.download)=="string") config.download = cli.download; 19 | fs.writeFile( 20 | path.resolve(__dirname,"config.json"), 21 | JSON.stringify(config, null, "\t") 22 | ); 23 | 24 | var dispatch = require("./pline/dispatch.js")(config); 25 | //检测是否是管道 26 | if(!process.stdin.isTTY) return dispatch.pipe(); 27 | 28 | //非管道检测有下载列表，有下载URL，无下载列表和下载URL三种情况，优先级下载列表大于URL 29 | if(cli.file && typeof(cli.file)=="string") return dispatch.file(cli.file); 30 | else if(cli.url && typeof(cli.url)=="string") return dispatch.url(cli.url); 31 | else return dispatch.file("./download.txt"); -------------------------------------------------------------------------------- /pline/dispatch.js: -------------------------------------------------------------------------------- 1 | var PLine = require("./pline.js"); 2 | var path = require("path"); 3 | 4 | function Dispatch(config) { 5 | this.folder = path.resolve(config.download); 6 | this.thread = config.thread; 7 | } 8 | Dispatch.prototype.pipe = function() { 9 | return new Promise(function(resolve) { 10 | var data = ""; 11 | process.stdin.resume(); //Compatible with old stream 12 | process.stdin.setEncoding("utf-8"); 13 | process.stdin.on("data", function(chunk) { data += chunk.toString() }); 14 | process.stdin.on("end", function() { resolve(data.split("\n")) }); 15 | }).then(this.download); 16 | } 17 | Dispatch.prototype.file = function(file) { 18 | return new Promise(function(resolve, reject) { 19 | require("fs").readFile(file, function(err, data) { 20 | if(err) reject(err); 21 | resolve(data.toString().split("\n")); 22 | }) 23 | }).then(this.download.bind(this)).catch(function(error) { console.log(error) }); 24 | } 25 | Dispatch.prototype.url = function(url) { 26 | return this.download([url]); 27 | } 28 | Dispatch.prototype.download = function(urls) { 29 | var self = this; 30 | return urls.filter(function(el) { return el.trim() !== "" }) 31 | .forEach(function(url) { 32 | (new PLine(url, self.folder)).run(); 33 | }) 34 | } 35 | 36 | module.exports = function(config) { return new Dispatch(config) } -------------------------------------------------------------------------------- /extractor/youtube.js: -------------------------------------------------------------------------------- 1 | var postfix = "mp4"; 2 | var crypto = require('crypto'); 3 | var urlparse = require('url').parse; 4 | var qs = require('qs'); 5 | 6 | var util = require("../util/util.js"); 7 | 8 | var getVidFromUrl = function (url){ 9 | var matches = url.match(/youtu\.be\/([^/]+)/) 10 | || url.match(/youtube\.com\/embed\/([^/?]+)/) 11 | || url.match(/youtube\.com\/v\/([^\/?]+)/) ; 12 | 13 | if (matches) return matches[1]; 14 | 15 | var urlInfo = urlparse(url); 16 | var qsList = qs.parse(urlInfo["query"]); 17 | 18 | if (qsList["v"]){ 19 | return qsList["v"]; 20 | } 21 | }; 22 | 23 | var youtube = function (){}; 24 | 25 | youtube.prototype = { 26 | extract : function (url){ 27 | var infoUrl = "http://www.youtube.com/get_video_info?video_id=" + getVidFromUrl(url); 28 | var ua = 'Mozilla/5.0 (Linux; Android 4.2.1; en-us; Nexus 5 Build/JOP40D) AppleWebKit/535.19 (KHTML, like Gecko) Chrome/18.0.1025.166 Mobile Safari/535.19'; 29 | 30 | return util.httpUtil.getHtml(infoUrl, ua).then(function (data){ 31 | var info = qs.parse(data), 32 | title = info["title"], 33 | streams = qs.parse(info["url_encoded_fmt_stream_map"]), 34 | urls = streams["url"]; 35 | 36 | return { 37 | "title" : title, 38 | "urls" : (Array.isArray(urls) ? [urls[0]] : [urls]), 39 | "size" : "N/A", 40 | "site_postfix" : postfix 41 | }; 42 | }) 43 | } 44 | }; 45 | 46 | module.exports = youtube; -------------------------------------------------------------------------------- /transcoder/letv.js: -------------------------------------------------------------------------------- 1 | var path = require("path"); 2 | var fs = require("fs"); 3 | var exec = require('child_process').exec; 4 | 5 | module.exports = { 6 | transcode : function (node){ 7 | var targetFile = path.resolve(node.path, '.', node.idx + ".ts"); 8 | var cmd = "ffmpeg -i " + node.file + " -vcodec copy -acodec copy -vbsf h264_mp4toannexb " + targetFile + " -loglevel -8"; 9 | 10 | return new Promise(function (resolve, reject){ 11 | exec(cmd, function (err, stdout, stderr){ 12 | if (err){ 13 | reject(err); 14 | } 15 | else{ 16 | fs.unlink(node.file, function(){ 17 | resolve({ 18 | path : node.path, 19 | i : node.idx, 20 | file : targetFile 21 | }); 22 | }); 23 | } 24 | }); 25 | }); 26 | }, 27 | 28 | mergeAndTranscode : function (nodes, finalFile){ 29 | var rets = [] , myPath; 30 | nodes.forEach(function (el){ 31 | rets.push(el.file); 32 | }); 33 | 34 | myPath = nodes[0]["path"]; 35 | 36 | var cmd = 'ffmpeg -i concat:\"' + rets.join("|") + '\" -acodec copy -vcodec copy -absf aac_adtstoasc \"' + finalFile + '\" -loglevel -8'; 37 | 38 | return new Promise(function (resolve, reject){ 39 | exec(cmd, function (err, stdout, stderr){ 40 | if (err){ 41 | reject(err); 42 | } 43 | else{ 44 | resolve({ 45 | finalFile : finalFile, 46 | workPath : myPath 47 | }); 48 | } 49 | }); 50 | }); 51 | } 52 | }; -------------------------------------------------------------------------------- /transcoder/youtube.js: -------------------------------------------------------------------------------- 1 | var path = require("path"); 2 | var fs = require("fs"); 3 | var exec = require('child_process').exec; 4 | 5 | module.exports = { 6 | transcode : function (node){ 7 | var targetFile = path.resolve(node.path, '.', node.idx + ".ts"); 8 | var cmd = "ffmpeg -i " + node.file + " -vcodec copy -acodec copy -vbsf h264_mp4toannexb " + targetFile + " -loglevel -8"; 9 | 10 | return new Promise(function (resolve, reject){ 11 | exec(cmd, function (err, stdout, stderr){ 12 | if (err){ 13 | reject(err); 14 | } 15 | else{ 16 | fs.unlink(node.file, function(){ 17 | resolve({ 18 | path : node.path, 19 | i : node.idx, 20 | file : targetFile 21 | }); 22 | }); 23 | } 24 | }); 25 | }); 26 | }, 27 | 28 | mergeAndTranscode : function (nodes, finalFile){ 29 | var rets = [] , myPath; 30 | nodes.forEach(function (el){ 31 | rets.push(el.file); 32 | }); 33 | 34 | myPath = nodes[0]["path"]; 35 | 36 | // var targetFile = path.resolve(myPath, '.', "output.mp4"); 37 | 38 | var cmd = 'ffmpeg -i concat:\"' + rets.join("|") + '\" -acodec copy -vcodec copy -absf aac_adtstoasc \"' + finalFile + '\" -loglevel -8'; 39 | 40 | return new Promise(function (resolve, reject){ 41 | exec(cmd, function (err, stdout, stderr){ 42 | if (err){ 43 | reject(err); 44 | } 45 | else{ 46 | resolve({ 47 | finalFile : finalFile, 48 | workPath : myPath 49 | }); 50 | } 51 | }); 52 | }); 53 | } 54 | }; -------------------------------------------------------------------------------- /transcoder/sohu.js: -------------------------------------------------------------------------------- 1 | var path = require("path"); 2 | var fs = require("fs"); 3 | var exec = require('child_process').exec; 4 | 5 | module.exports = { 6 | transcode : function (node){ 7 | var targetFile = path.resolve(node.path, '.', node.idx + ".ts"); 8 | var cmd = "ffmpeg -i " + node.file + " -vcodec copy -acodec copy -vbsf h264_mp4toannexb " + targetFile + " -loglevel -8"; 9 | 10 | return new Promise(function (resolve, reject){ 11 | exec(cmd, function (err, stdout, stderr){ 12 | if (err){ 13 | reject(err); 14 | } 15 | else{ 16 | fs.unlink(node.file, function(){ 17 | resolve({ 18 | path : node.path, 19 | i : node.idx, 20 | file : targetFile 21 | }); 22 | }); 23 | } 24 | }); 25 | }); 26 | }, 27 | 28 | mergeAndTranscode : function (nodes, finalFile){ 29 | var rets = [] , myPath; 30 | nodes.forEach(function (el){ 31 | rets.push(el.file); 32 | }); 33 | 34 | myPath = nodes[0]["path"]; 35 | 36 | // var targetFile = path.resolve(myPath, '.', "output.mp4"); 37 | 38 | var cmd = 'ffmpeg -i concat:\"' + rets.join("|") + '\" -acodec copy -vcodec copy -absf aac_adtstoasc \"' + finalFile + '\" -loglevel -8'; 39 | 40 | return new Promise(function (resolve, reject){ 41 | exec(cmd, function (err, stdout, stderr){ 42 | if (err){ 43 | reject(err); 44 | } 45 | else{ 46 | resolve({ 47 | finalFile : finalFile, 48 | workPath : myPath 49 | }); 50 | } 51 | }); 52 | }); 53 | } 54 | }; -------------------------------------------------------------------------------- /transcoder/iqiyi.js: -------------------------------------------------------------------------------- 1 | var path = require("path"); 2 | var fs = require("fs"); 3 | var exec = require('child_process').exec; 4 | 5 | module.exports = { 6 | transcode : function (node){ 7 | var targetFile = path.resolve(node.path, '.', node.idx + ".ts"); 8 | var cmd = "ffmpeg -i " + node.file + " -vcodec copy -acodec copy -vbsf h264_mp4toannexb " + targetFile + " -loglevel -8"; 9 | 10 | return new Promise(function (resolve, reject){ 11 | exec(cmd, function (err, stdout, stderr){ 12 | if (err){ 13 | reject(err); 14 | } 15 | else{ 16 | fs.unlink(node.file, function(){ 17 | resolve({ 18 | path : node.path, 19 | i : node.idx, 20 | file : targetFile 21 | }); 22 | }); 23 | } 24 | }); 25 | }); 26 | }, 27 | 28 | mergeAndTranscode : function (nodes, finalFile){ 29 | var rets = [] , myPath; 30 | nodes.forEach(function (el){ 31 | rets.push(el.file); 32 | }); 33 | 34 | myPath = nodes[0]["path"]; 35 | 36 | // var targetFile = path.resolve(myPath, '.', "output.mp4"); 37 | 38 | var cmd = 'ffmpeg -i concat:\"' + rets.join("|") + '\" -acodec copy -vcodec copy -absf aac_adtstoasc \"' + finalFile + '\" -loglevel -8'; 39 | 40 | return new Promise(function (resolve, reject){ 41 | exec(cmd, function (err, stdout, stderr){ 42 | if (err){ 43 | reject(err); 44 | } 45 | else{ 46 | resolve({ 47 | finalFile : finalFile, 48 | workPath : myPath 49 | }); 50 | } 51 | }); 52 | });/*.then(function (){ 53 | return new Promise(function (resolve, reject){ 54 | var cmd = "ffmpeg -y -i " + targetFile + " \"" + finalFile + "\" -loglevel -8"; 55 | 56 | exec(cmd, function (err, stdout, stderr){ 57 | if (err){ 58 | reject(err); 59 | } 60 | 61 | resolve({ 62 | finalFile : finalFile, 63 | workPath : myPath 64 | }); 65 | }); 66 | }); 67 | });*/ 68 | } 69 | }; -------------------------------------------------------------------------------- /pline/pline.js: -------------------------------------------------------------------------------- 1 | var util = require("../util/util.js"); 2 | var path = require("path"); 3 | var rimraf = require("rimraf"); 4 | 5 | var getSiteNameByUrl = function (url){ 6 | var match = url.match(/https?:\/\/([^\/]+)\//); 7 | 8 | if (match === null){ 9 | throw new Exception("Error url!"); 10 | } 11 | 12 | var siteInfo = match[1].split("."); 13 | 14 | return siteInfo[siteInfo.length - 2]; 15 | }; 16 | 17 | var PLine = function (url, folder){ 18 | this.url = url; 19 | this.workPath = folder; 20 | }; 21 | 22 | PLine.prototype = { 23 | getExtractor : function (){ 24 | var extractor = require("../extractor/" + getSiteNameByUrl(this.url) + ".js"); 25 | 26 | this.extractor = new extractor(); 27 | 28 | return this.extractor; 29 | }, 30 | 31 | download : function (data){ 32 | console.log("正在下载：" + data.title); 33 | console.log("总大小：" + util.spaceUtil.getSize(data.size)); 34 | console.log("分块数：" + data.urls.length); 35 | 36 | return util.downloadUtil.download(data.urls, data.site_postfix); 37 | }, 38 | 39 | transcode : function (data, postfix){ 40 | var self = this; 41 | var promises = []; 42 | var transcoder = require("../transcoder/" + getSiteNameByUrl(this.url) + ".js"); 43 | 44 | data.forEach(function (el){ 45 | promises.push(el.then(function (node){ 46 | return transcoder.transcode(node); 47 | })); 48 | }); 49 | 50 | return Promise.all(promises).then(function (){ 51 | var rets = []; 52 | 53 | arguments[0].forEach(function(el){ 54 | rets.push(el); 55 | }); 56 | 57 | var finalFile = path.resolve( self.workPath, [self.title, postfix].join(".")); 58 | return transcoder.mergeAndTranscode(rets, finalFile); 59 | }); 60 | }, 61 | 62 | clean : function (data){ 63 | console.log("下载文件到：" + data.finalFile); 64 | 65 | return new Promise(function (resolve, reject){ 66 | rimraf(data.workPath, function (){ 67 | resolve(); 68 | }); 69 | }); 70 | }, 71 | 72 | run : function (){ 73 | var self = this; 74 | 75 | this.getExtractor().extract(this.url).then(function (data){ 76 | self.title = data.title; 77 | return self.download(data); 78 | }).then(function (data){ 79 | return self.transcode(data, "mp4"); // 暂时只加mov 80 | }).then(function (data){ 81 | self.clean(data); 82 | }).then(function (data){ 83 | console.log("任务完成"); 84 | }).catch(function (err){ 85 | console.log(err.stack); 86 | }); 87 | } 88 | }; 89 | 90 | module.exports = PLine; -------------------------------------------------------------------------------- /extractor/sohu.js: -------------------------------------------------------------------------------- 1 | var postfix = "mp4"; 2 | var util = require("../util/util.js"); 3 | var urlparse = require('url').parse; 4 | 5 | var dealWithVideoInfo = function (info){ 6 | info["host"] = info['allot']; 7 | info["title"] = info['data']['tvName']; 8 | info["size"] = info['data']['clipsBytes'].reduce(function (previous, current){ 9 | return previous + parseInt(current, 10); 10 | }); 11 | 12 | return info; 13 | } 14 | 15 | var getRealUrl = function (host, vid, tvid, newClip, clip, ck){ 16 | var clipURL = urlparse(clip).path; 17 | var url = 'http://' + host + '/?prot=9&prod=flash&pt=1&file=' + clipURL + '&new=' + newClip + '&key=' + ck + '&vid=' + vid + '&uid=' + (+(new Date())) + '&t=' + Math.random(); 18 | 19 | return util.httpUtil.getHtml(url).then(function(data){ 20 | return JSON.parse(data)["url"]; 21 | }); 22 | } 23 | 24 | var getRealUrls = function (info){ 25 | var data = info.data; 26 | 27 | return data["su"].reduce(function (prev, current, idx){ 28 | var newClip = current, clip = data['clipsURL'][idx], ck = data['ck'][idx]; 29 | prev.push(getRealUrl(info["host"], info["vid"], info["tvid"], newClip, clip, ck)); 30 | 31 | return prev; 32 | }, []); 33 | } 34 | 35 | var getVidByUrl = function (url){ 36 | return new Promise(function (resolve, reject){ 37 | if (/http:\/\/share.vrs.sohu.com/.test(url)){ 38 | resolve(url.match(/id=(\d+)/)[1]); 39 | } 40 | 41 | return util.httpUtil.getHtml(url).then(function(data){ 42 | resolve(data.match(/\Wvid\s*[\:=]\s*[\'"]?(\d+)[\'"]?/)[1]); 43 | }); 44 | }); 45 | } 46 | 47 | var getVideoInfo = function (type, vid){ 48 | var prefixUrl = { 49 | "tv" : "http://hot.vrs.sohu.com/vrs_flash.action?vid=%s", 50 | "not_tv" : "http://my.tv.sohu.com/play/videonew.do?vid=%s&referer=http://my.tv.sohu.com" 51 | }, url = prefixUrl[type].replace(/%s/, vid); 52 | 53 | return util.httpUtil.getHtml(url).then(function (data){ 54 | var info = dealWithVideoInfo(JSON.parse(data)); 55 | 56 | info.vid = vid; 57 | return Promise.all(getRealUrls(info)).then(function(){ 58 | info.urls = arguments[0]; 59 | info.site_postfix = postfix; 60 | 61 | return info; 62 | }); 63 | }); 64 | } 65 | 66 | var sohu = function (){}; 67 | 68 | sohu.prototype = { 69 | extract : function (url){ 70 | return getVidByUrl(url).then(function (vid){ 71 | if (/http:\/\/tv.sohu.com\//.test(url)){ 72 | return getVideoInfo("tv", vid); 73 | } 74 | else{ 75 | return getVideoInfo("not_tv", vid); 76 | } 77 | }); 78 | } 79 | }; 80 | 81 | module.exports = sohu; -------------------------------------------------------------------------------- /extractor/iqiyi.js: -------------------------------------------------------------------------------- 1 | var postfix = "f4v"; 2 | var uuid = require("uuid"); 3 | var md5 = require("md5"); 4 | 5 | var util = require("../util/util.js"); 6 | 7 | var mix = function (tvid){ 8 | var enc = [], src="eknas", tm = Math.floor(Math.random() * (4000 - 2000) + 2000); 9 | 10 | enc.push("8ed797d224d043e7ac23d95b70227d32"); 11 | enc.push(tm); 12 | enc.push(tvid); 13 | 14 | return [tm, md5(enc.join("")), src]; 15 | }; 16 | 17 | 18 | var getVMS = function (tvid, vid, uid){ 19 | var params = mix(tvid); 20 | 21 | var vmsreq = "http://cache.video.qiyi.com/vms?key=fvip&src=1702633101b340d8917a69cf8a4b8c7"; 22 | 23 | vmsreq += "&tvId=" + tvid + "&vid=" + vid + "&vinfo=1&tm=" + params[0]; 24 | vmsreq += "&enc=" + params[1]; 25 | vmsreq += "&qyid=" + uid + "&tn=" + Math.random() +"&um=1"; 26 | vmsreq += "&authkey=" + md5([params[0], tvid].join('')); 27 | 28 | return util.httpUtil.getHtml(vmsreq); 29 | }; 30 | 31 | var getIdsByHtml = function (html, url){ 32 | var match1 = html.match(/data-player-tvid="(.*?)"/) || url.match(/tvid=(.*?)&/ ), 33 | match2 = html.match(/data-player-videoid="(.*?)"/) || url.match(/vid=(.*?)&/); 34 | 35 | return [match1[1], match2[1]]; 36 | }; 37 | 38 | var getVRSXORCode = function (arg1, arg2){ 39 | return arg1 ^ ([103, 121, 72][arg2 % 3]); 40 | }; 41 | 42 | var getVrsEncodeCode = function (vlink){ 43 | var loc6 = 0, loc2 = "", 44 | loc3 = vlink.split("-"), loc4 = loc3.length; 45 | 46 | for(var i = loc4 - 1; i > -1; --i){ 47 | loc6 = getVRSXORCode(parseInt(loc3[loc4 - i - 1], 16), i); 48 | loc2 += String.fromCharCode(loc6); 49 | } 50 | 51 | return loc2.split('').reverse().join(''); 52 | }; 53 | 54 | var getVLnksByVMS = function (info){ 55 | if (info["data"]['vp']["tkl"] == ""){ 56 | // 有误 57 | } 58 | 59 | var bid = 0, vlnks; 60 | 61 | info["data"]["vp"]["tkl"][0]["vs"].forEach(function (el, i){ 62 | var elBid = parseInt(el["bid"], 10); 63 | 64 | if ((elBid <= 10) && (elBid >= bid)){ 65 | bid = elBid; 66 | 67 | vlnks = el["fs"]; 68 | 69 | if (el["fs"][0]["l"][0] != "/"){ 70 | if (getVrsEncodeCode(el["fs"][0]["l"]).substr(-3) == "mp4"){ 71 | vlnks = el["flvs"] 72 | } 73 | } 74 | } 75 | }); 76 | 77 | return vlnks; 78 | }; 79 | 80 | var getDispathKey = function (rid){ 81 | var url = "http://data.video.qiyi.com/t?tn=" + Math.random(); 82 | 83 | return util.httpUtil.getHtml(url).then(function (data){ 84 | var tp = ")(*&^flash@#$%a"; // swf 里面的处理 85 | var t = Math.floor((JSON.parse(data)["t"])/6e2); 86 | 87 | return md5(t + tp + rid); 88 | }); 89 | }; 90 | 91 | var analyseVMSCode = function(data, url, uid){ 92 | var info = JSON.parse(data), 93 | title = info["data"]["vi"]["vn"], 94 | urls, vLnks = getVLnksByVMS(info); 95 | 96 | urls = vLnks.map(function (el, i){ 97 | vlink = el["l"]; 98 | 99 | if (vlink[0] != "/"){ //编码过的 100 | vlink = getVrsEncodeCode(vlink); 101 | } 102 | 103 | return (function (vlink){ 104 | return getDispathKey(vlink.split("/").pop().split(".")[0]).then(function(data){ 105 | var baseUrlInfo, baseUrl, url; 106 | 107 | baseUrlInfo = info["data"]["vp"]["du"].split("/"); 108 | baseUrlInfo.splice(-1, 0 , data); 109 | baseUrl = baseUrlInfo.join("/"); 110 | 111 | url = baseUrl + vlink + '?su=' + uid + '&qyid=' + uuid.v4().replace(/-/g, ""); 112 | url += '&client=&z=&bt=&ct=&tn=' + (Math.floor(Math.random() * (20000 - 10000) + 10000)); 113 | 114 | return util.httpUtil.getHtml(url).then(function (data){ 115 | return { 116 | link : JSON.parse(data)["l"], 117 | size : el["b"] 118 | }; 119 | }); 120 | }); 121 | })(vlink); 122 | }); 123 | 124 | return Promise.all(urls).then(function(){ 125 | var size = 0, us = [], rets = {}; 126 | 127 | for (var i in arguments){ 128 | for(var j in arguments[i]){ 129 | size += arguments[i][j].size; 130 | us.push(arguments[i][j].link); 131 | } 132 | } 133 | 134 | rets = { 135 | size : size, 136 | urls : us, 137 | title : title 138 | }; 139 | 140 | return rets; 141 | }); 142 | }; 143 | 144 | var iqiyi = function (){}; 145 | 146 | iqiyi.prototype = { 147 | extract : function (url){ 148 | return util.httpUtil.getHtml(url).then(function (html){ 149 | var uid = uuid.v4().replace(/-/g, ""), 150 | ids = getIdsByHtml(html); 151 | 152 | return getVMS(ids[0], ids[1], uid).then(function (data){ 153 | return analyseVMSCode(data, url, uid).then(function(data){ 154 | data.site_postfix = postfix; 155 | return data; 156 | }); 157 | }); 158 | }, function (err){ 159 | console.info(err); 160 | }); 161 | } 162 | }; 163 | 164 | module.exports = iqiyi; 165 | -------------------------------------------------------------------------------- /extractor/letv.js: -------------------------------------------------------------------------------- 1 | var postfix = "mp4"; 2 | var uuid = require("uuid"); 3 | var md5 = require("md5"); 4 | var bigInt = require("big-integer"); 5 | 6 | var util = require("../util/util.js"); 7 | var letv = function (){}; 8 | 9 | var getHtmlByUrl = (function (){ 10 | var html; 11 | 12 | return function (url){ 13 | return new Promise(function (resolve, reject){ 14 | if (html){ 15 | resolve(html); 16 | } 17 | 18 | return util.httpUtil.getHtml(url).then(function (data){ 19 | resolve(data); 20 | }); 21 | }); 22 | }; 23 | })(); 24 | 25 | var getVid = function (url){ 26 | var reg1 = /http:\/\/www.letv.com\/ptv\/vplay\/(\d+).html/, 27 | reg2 = /http:\/\/www.letv.com\/ptv\/vplay\/(\d+).html/; 28 | 29 | return new Promise(function (resolve, reject){ 30 | if (url.match(reg1)){ 31 | resolve(url.match(reg1)[1]); 32 | } 33 | 34 | if (url.match(reg2)){ 35 | resolve(url.match(reg1)[2]); 36 | } 37 | 38 | return getHtmlByUrl(url).then(function(data){ 39 | resolve(data.match(/vid="(\d+)"/)[1]); 40 | }); 41 | }); 42 | } 43 | 44 | var getTitle = function (url){ 45 | return getHtmlByUrl(url).then(function (data){ 46 | return data.match(/name="irTitle" content="(.*?)"/)[1]; 47 | }) 48 | }; 49 | 50 | var calcTimeKey = function (t){ 51 | var ror = function (val, r_bits){ 52 | var opt1 = bigInt(val).and( 53 | bigInt(2).pow(32).minus(1) 54 | ).shiftRight( 55 | bigInt(r_bits).divmod(32)["remainder"] 56 | ); 57 | 58 | var opt2 = bigInt(val).shiftLeft( 59 | bigInt(32).minus( 60 | bigInt(r_bits.divmod(32)["remainder"]) 61 | ) 62 | ).and( 63 | bigInt(2).pow(32).minus(1) 64 | ); 65 | 66 | return opt1.or(opt2); 67 | }; 68 | 69 | return ror( 70 | ror(bigInt(t), bigInt(773625421).divmod(13)["remainder"]).xor(bigInt(773625421)), 71 | bigInt(773625421).divmod(17)["remainder"] 72 | ).toJSNumber(); 73 | }; 74 | 75 | var decode = function (data){ 76 | var version = data.slice(0, 5).toString().toLowerCase(); 77 | 78 | if (version == "vc_01"){ 79 | var loc2 = data.slice(5); 80 | 81 | var length = loc2.length; 82 | var loc4 = Array(2*length).fill(0); 83 | 84 | Array(length).fill(0).forEach(function (el, i){ 85 | loc4[2 * i] = bigInt(loc2[i]).shiftRight(4); 86 | loc4[2 * i + 1] = bigInt(loc2[i]).and(15); 87 | }); 88 | 89 | loc4 = loc4.map(function (el){ 90 | return (el.toJSNumber) ? el.toJSNumber() : el; 91 | }); 92 | 93 | var loc6 = loc4.slice(loc4.length - 11).concat(loc4.slice(0, loc4.length - 11)); 94 | 95 | var loc7 = Array(length).fill(0); 96 | 97 | loc7 = loc7.map(function (el, i){ 98 | return (bigInt(loc6[2 * i]).shiftLeft(4)).add(bigInt(loc6[2 * i + 1])); 99 | }); 100 | 101 | var loc = loc7.map(function (el){ 102 | el = (el.toJSNumber) ? el.toJSNumber() : el; 103 | return String.fromCharCode(el); 104 | }); 105 | 106 | return loc.join(''); 107 | } 108 | else { 109 | return data; 110 | } 111 | }; 112 | 113 | var getM3u8List = function (url){ 114 | return util.httpUtil.getHtml(url).then(function (data){ 115 | var info = JSON.parse(data); 116 | 117 | return util.httpUtil.getHtml(info["location"], undefined, false).then(function (m3u8){ 118 | var m3u8List = decode(m3u8).match(/http.*/g); 119 | 120 | return m3u8List; 121 | }); 122 | }); 123 | }; 124 | 125 | var _videoInfo = function (vid){ 126 | var url = ['http://api.letv.com/mms/out/video/playJson?id=', 127 | vid, '&platid=1&splatid=101&format=1&tkey=', 128 | calcTimeKey(Math.floor(+new Date()/1e3)), '&domain=www.letv.com'].join(''); 129 | 130 | return util.httpUtil.getHtml(url).then(function (data){ 131 | var info = JSON.parse(data); 132 | var supportsStreams = info["playurl"]["dispatch"]; 133 | var supportsStreamsKeys = Object.keys(supportsStreams); 134 | 135 | var selectStreamKey = supportsStreamsKeys.reduce(function (prev, current){ 136 | var testP = /p/; 137 | 138 | if (testP.test(prev) && !testP.test(current)){ 139 | return prev; 140 | } 141 | 142 | if (!testP.test(prev) && testP.test(current)){ 143 | return current; 144 | } 145 | 146 | if (testP.test(prev) && testP.test(current)){ 147 | var newPrev = parseInt(prev.replace(/p/g, ""), 10); 148 | var newCurrent = parseInt(current.replace(/p/g, ""), 10); 149 | 150 | return newPrev >= newCurrent ? prev : current; 151 | } 152 | 153 | return parseInt(prev, 10) >= parseInt(current, 10) ? prev : current; 154 | }, 0); 155 | 156 | var url = info["playurl"]["domain"][0] + info["playurl"]["dispatch"][selectStreamKey][0]; 157 | var ext = info["playurl"]["dispatch"][selectStreamKey][1].split('.')[-1]; 158 | url += ["&ctv=pc&m3v=1&termid=1&format=1&hwtype=un&ostype=Linux&tag=letv&sign=letv&expect=3&tn=", 159 | Math.random(), "&pay=0&iscpn=f9051&rateid=", selectStreamKey].join(''); 160 | 161 | return getM3u8List(url); 162 | }); 163 | }; 164 | 165 | var getVideoInfo = function (vid){ 166 | return _videoInfo(vid); 167 | }; 168 | 169 | letv.prototype = { 170 | extract : function (url){ 171 | return getVid(url).then(function (vid){ 172 | return getTitle(url).then(function (title){ 173 | return getVideoInfo(vid).then(function (data){ 174 | return { 175 | size : 'N/A', 176 | urls : data, 177 | title : title, 178 | site_postfix : postfix 179 | }; 180 | 181 | }) 182 | }); 183 | }, function (err){ 184 | console.info(err); 185 | }); 186 | } 187 | }; 188 | 189 | module.exports = letv; 190 | -------------------------------------------------------------------------------- /util/util.js: -------------------------------------------------------------------------------- 1 | var headers = { 2 | 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', 3 | 'Accept-Charset': 'UTF-8,*;q=0.5', 4 | 'Accept-Encoding': 'gzip,deflate,sdch', 5 | 'Accept-Language': 'en-US,en;q=0.8', 6 | 'User-Agent': 'Mozilla/5.0 (X11; Linux x86_64; rv:13.0) Gecko/20100101 Firefox/13.0' 7 | }; 8 | 9 | 10 | var fs = require("fs"); 11 | var path = require("path"); 12 | var request = require('request'); 13 | var urlparse = require('url').parse; 14 | var http = require('http'); 15 | var zlib = require('zlib'); 16 | var Download = require('download'); 17 | 18 | var httpUtil = { 19 | getHtml : function (url, userAgent, toString){ 20 | if (userAgent){ 21 | headers["User-Agent"] = userAgent; 22 | } 23 | 24 | if (toString === undefined){ 25 | toString = true; 26 | } 27 | 28 | var opt = { 29 | 'headers' : headers, 30 | 'url' : url 31 | }; 32 | 33 | return new Promise(function (resolve, reject){ 34 | var req = request.get(opt); 35 | 36 | req.on('response', function(res) { 37 | var chunks = []; 38 | 39 | res.on('data', function(chunk) { 40 | chunks.push(chunk); 41 | }); 42 | 43 | res.on('end', function() { 44 | var buffer = Buffer.concat(chunks); 45 | var encoding = res.headers['content-encoding']; 46 | 47 | if (encoding == 'gzip') { 48 | zlib.gunzip(buffer, function(err, decoded) { 49 | if (err){ 50 | reject(err); 51 | } 52 | 53 | var ret = toString ? decoded.toString() : decoded; 54 | resolve(ret); 55 | }); 56 | } 57 | else if (encoding == 'deflate') { 58 | zlib.inflate(buffer, function(err, decoded) { 59 | if (err){ 60 | reject(err); 61 | } 62 | var ret = toString ? decoded.toString() : decoded; 63 | resolve(ret); 64 | }); 65 | } 66 | else { 67 | var ret = toString ? buffer.toString() : buffer; 68 | resolve(ret); 69 | } 70 | }); 71 | }); 72 | req.on('error', function(err) { 73 | reject(err); 74 | }); 75 | }); 76 | }, 77 | 78 | getUrlSize : function(url){ 79 | var opt = { 80 | 'headers' : headers, 81 | 'url' : url 82 | }; 83 | 84 | return new Promise(function (resolve, reject){ 85 | var req = http.request(opt, function(res) { 86 | var resHeaders = res.headers; 87 | if (headers['transfer-encoding'] != 'chunked'){ 88 | size = parseInt(headers['content-length'], 10); 89 | } 90 | else{ 91 | size = 0; 92 | } 93 | 94 | resolve(size); 95 | }); 96 | }); 97 | 98 | } 99 | } 100 | 101 | var spaceUtil = { 102 | getSize : function (size){ 103 | return size + "Bytes"; 104 | } 105 | }; 106 | 107 | var fsUtil = { 108 | mkdir : function (dir){ 109 | while (fs.existsSync(dir)){ 110 | dir = [dir, +new Date()].join("_"); 111 | } 112 | 113 | fs.mkdirSync(dir); 114 | return dir; 115 | } 116 | 117 | }; 118 | 119 | var downloadUtil = { 120 | downloadAndSave : function (url, savefile) { 121 | return new Promise(function (resolve, reject){ 122 | // url = "http://42.81.80.162:80/178/6/77/letv-uts/14/ver_00_22-1009154802-avc-3012372-aac-128000-2583200-1017852967-0c7735268d4b38883aa086810e01cef4-1449630661353_mp4/ver_00_22_149_710_2_13924032_1032162300.ts?mltag=1&platid=1&splatid=101&playid=0&geo=CN-1-0-1&tag=letv&ch=&p1=&p2=&p3=&tss=ios&b=3152&bf=29&nlh=3072&path=&sign=letv&proxy=709953101,1711200372,3683272603&uuid=&ntm=1450188000&keyitem=GOw_33YJAAbXYE-cnQwpfLlv_b2zAkYctFVqe5bsXQpaGNn3T1-vhw..&its=0&nkey2=858cbe1db8fa85cbde01aec61751a3f6&uid=3659428872.rp&qos=4&enckit=&m3v=1&token=&vid=&liveid=&station=&app_name=&app_ver=&fcheck=0"; 123 | 124 | // var urlinfo = urlparse(url); 125 | 126 | // var options = { 127 | // method: 'GET', 128 | // host: urlinfo.host, 129 | // path: urlinfo.pathname, 130 | // headers: headers 131 | // }; 132 | 133 | // if(urlinfo.port) { 134 | // options.port = urlinfo.port; 135 | // } 136 | // if(urlinfo.search) { 137 | // options.path += urlinfo.search; 138 | // } 139 | 140 | // var req = http.request(options, function(res) { 141 | // var writestream = fs.createWriteStream(savefile); 142 | // writestream.on('close', function() { 143 | // resolve(res); 144 | // }); 145 | // res.pipe(writestream); 146 | // }); 147 | // req.end(); 148 | new Download({mode: '755'}) 149 | .get(url) 150 | .dest(path.dirname(savefile)) 151 | .rename(path.basename(savefile)) 152 | .run(function (){ 153 | resolve(); 154 | }); 155 | }); 156 | }, 157 | 158 | download : function (urls, downloadPostfix, workPath){ 159 | if (typeof workPath === "undefined"){ 160 | workPath = path.resolve(__dirname, '..', require("../config.json").download, (+new Date()) + ''); 161 | } 162 | 163 | workPath = util.fsUtil.mkdir(workPath); // 如有重名，会改变 164 | 165 | var downloaders = urls.map(function (el, i){ 166 | var file = path.resolve(workPath, '.', [i, ".", downloadPostfix].join('')); 167 | 168 | return (function (el){ 169 | return util.downloadUtil 170 | .downloadAndSave(el, file) 171 | .then(function (data){ 172 | return { 173 | path : workPath, 174 | idx : i, 175 | file : file 176 | }; 177 | }); 178 | })(el); 179 | }); 180 | 181 | return downloaders; 182 | } 183 | }; 184 | 185 | var util = {}; 186 | 187 | util.httpUtil = httpUtil; 188 | util.spaceUtil = spaceUtil; 189 | util.fsUtil = fsUtil; 190 | util.downloadUtil = downloadUtil; 191 | 192 | module.exports = util; --------------------------------------------------------------------------------