├── docs ├── design │ ├── readme.md │ ├── 调度.pdf │ ├── bee.png │ ├── 开关播.pdf │ ├── 新架构.pdf │ ├── 新架构1.pdf │ ├── 服务架构.pdf │ ├── 采集架构.pdf │ ├── cluster.png │ ├── 采集架构-2.pdf │ ├── new-design.png │ └── waiting-list.png ├── cock.md ├── diff.pdf ├── todo.pdf ├── V4架构图解.pdf ├── redis规范.jpg ├── 数据库设计规范.pdf ├── 数据消费系统.pdf ├── 平台数据 │ ├── all.png │ ├── huya.png │ ├── chushou.png │ ├── douyu.png │ ├── longzhu.png │ ├── panda.png │ ├── quanmin.png │ ├── zhanqi.png │ ├── 新数据流程.pdf │ ├── 重复ip.md │ ├── fentuan.xlsx │ ├── fentuan.numbers │ ├── 汇总.md │ ├── 数据汇总.txt │ └── 已释放.md ├── 斗鱼第三方开放平台API文档v2.2.pdf ├── 斗鱼弹幕服务器第三方接入协议v1.6.2.pdf ├── 直播宝2.0.md ├── redis-key.md ├── 视频流.md ├── 需求.txt ├── 数据采集需求.md ├── mysql索引策略和优化.md ├── 消息分类.md ├── huoshan.md ├── 战旗采集流程.md ├── pb接口文档.md ├── huya.md └── request-rule.md ├── architecture.pdf ├── architecture.png ├── test ├── test_ws.js ├── test_ts.js ├── test_rpc │ ├── client.js │ └── server.js ├── test_notice.js ├── test_stream.js ├── log_test.js ├── test_log.js ├── test_obj.js ├── xxx.d.ts ├── test_kugou.js ├── test_yy.js ├── test_longzhu.js ├── test_mysql.js ├── test_chushou.js ├── test_panda.js ├── test_gift.js ├── test_ink.js ├── test_huya.js ├── test_huyav2.js ├── demo.js ├── test_zhanqi.js ├── test_douyu.js ├── test_type.js ├── test_fanxing.js └── test_danmu.js ├── helper ├── sleep.js ├── device.js ├── pair.js ├── huoshan │ ├── python-scripts │ │ ├── complementation.py │ │ ├── cryptoAfter.py │ │ ├── unwifi.py │ │ └── tt_encrypt.py │ ├── requestOperation.js │ ├── encrypt.js │ ├── requestEncrypt.js │ └── device.js ├── useragent.js ├── cutbuf.js ├── qmSign.js ├── proto.js ├── http.js ├── giftManager.js ├── proxy.js ├── monitor.js └── gateway.json ├── danmu ├── test.js ├── YY.js ├── Conn.js ├── KugouH5.js ├── WSConn.js ├── Inke.js ├── 接收弹幕.md ├── Longzhu.js ├── index.js ├── Douyu.js ├── Chushou.js ├── inke.go ├── Huoshan.js ├── Panda.js ├── XYPanda.js └── Kugou.js ├── config ├── Logger.d.ts ├── redis.js ├── mysql.d.ts ├── log.js ├── http.js ├── config_local.js ├── config_test.js ├── config_prod.js ├── config.js ├── mysql.js ├── Logger.js ├── log_test.js └── log_prod.js ├── scripts ├── update_node.sh ├── install_python3.sh ├── all-list.sh └── all-danmu.sh ├── gather ├── sub │ ├── router.js │ ├── index.js │ ├── handler.js │ ├── check.js │ └── subs.js ├── nest.js ├── video-stream │ ├── index.js │ ├── QuanminStream.js │ ├── LongzhuStream.js │ ├── PandaStream.js │ ├── HuyaStream.js │ ├── FanxingStream.js │ ├── ZhanqiStream.js │ ├── DouyuStream.js │ └── VideoStream.js ├── lib │ ├── anchor.js │ ├── queue.js │ ├── broadcast.js │ ├── searcher.js │ ├── notice.js │ ├── util.js │ ├── recycle.js │ └── opportunity.js └── bee.js ├── index.js ├── README.md ├── LICENSE ├── .gitignore ├── package.json └── hosts.yaml /docs/design/readme.md: -------------------------------------------------------------------------------- 1 | # 架构设计图及源文件 -------------------------------------------------------------------------------- /docs/cock.md: -------------------------------------------------------------------------------- 1 | 可扩展性、强一致性、极强生存 性容忍磁盘，机器，机架甚至数据中心故障 2 | -------------------------------------------------------------------------------- /docs/diff.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Txiaozhe/gather/HEAD/docs/diff.pdf -------------------------------------------------------------------------------- /docs/todo.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Txiaozhe/gather/HEAD/docs/todo.pdf -------------------------------------------------------------------------------- /architecture.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Txiaozhe/gather/HEAD/architecture.pdf -------------------------------------------------------------------------------- /architecture.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Txiaozhe/gather/HEAD/architecture.png -------------------------------------------------------------------------------- /docs/V4架构图解.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Txiaozhe/gather/HEAD/docs/V4架构图解.pdf -------------------------------------------------------------------------------- /docs/redis规范.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Txiaozhe/gather/HEAD/docs/redis规范.jpg -------------------------------------------------------------------------------- /docs/数据库设计规范.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Txiaozhe/gather/HEAD/docs/数据库设计规范.pdf -------------------------------------------------------------------------------- /docs/数据消费系统.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Txiaozhe/gather/HEAD/docs/数据消费系统.pdf -------------------------------------------------------------------------------- /docs/design/调度.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Txiaozhe/gather/HEAD/docs/design/调度.pdf -------------------------------------------------------------------------------- /docs/平台数据/all.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Txiaozhe/gather/HEAD/docs/平台数据/all.png -------------------------------------------------------------------------------- /docs/平台数据/huya.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Txiaozhe/gather/HEAD/docs/平台数据/huya.png -------------------------------------------------------------------------------- /docs/design/bee.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Txiaozhe/gather/HEAD/docs/design/bee.png -------------------------------------------------------------------------------- /docs/design/开关播.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Txiaozhe/gather/HEAD/docs/design/开关播.pdf -------------------------------------------------------------------------------- /docs/design/新架构.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Txiaozhe/gather/HEAD/docs/design/新架构.pdf -------------------------------------------------------------------------------- /docs/design/新架构1.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Txiaozhe/gather/HEAD/docs/design/新架构1.pdf -------------------------------------------------------------------------------- /docs/design/服务架构.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Txiaozhe/gather/HEAD/docs/design/服务架构.pdf -------------------------------------------------------------------------------- /docs/design/采集架构.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Txiaozhe/gather/HEAD/docs/design/采集架构.pdf -------------------------------------------------------------------------------- /docs/平台数据/chushou.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Txiaozhe/gather/HEAD/docs/平台数据/chushou.png -------------------------------------------------------------------------------- /docs/平台数据/douyu.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Txiaozhe/gather/HEAD/docs/平台数据/douyu.png -------------------------------------------------------------------------------- /docs/平台数据/longzhu.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Txiaozhe/gather/HEAD/docs/平台数据/longzhu.png -------------------------------------------------------------------------------- /docs/平台数据/panda.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Txiaozhe/gather/HEAD/docs/平台数据/panda.png -------------------------------------------------------------------------------- /docs/平台数据/quanmin.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Txiaozhe/gather/HEAD/docs/平台数据/quanmin.png -------------------------------------------------------------------------------- /docs/平台数据/zhanqi.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Txiaozhe/gather/HEAD/docs/平台数据/zhanqi.png -------------------------------------------------------------------------------- /docs/平台数据/新数据流程.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Txiaozhe/gather/HEAD/docs/平台数据/新数据流程.pdf -------------------------------------------------------------------------------- /docs/平台数据/重复ip.md: -------------------------------------------------------------------------------- 1 | 重复ip 2 | 3 | 118.31.20.139 4 | 5 | 116.62.150.71 6 | 7 | 116.62.121.34 -------------------------------------------------------------------------------- /test/test_ws.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Creator: Tang Xiaoji 3 | * Time: 2018-05-09 4 | */ 5 | 6 | 7 | -------------------------------------------------------------------------------- /docs/design/cluster.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Txiaozhe/gather/HEAD/docs/design/cluster.png -------------------------------------------------------------------------------- /docs/design/采集架构-2.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Txiaozhe/gather/HEAD/docs/design/采集架构-2.pdf -------------------------------------------------------------------------------- /docs/平台数据/fentuan.xlsx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Txiaozhe/gather/HEAD/docs/平台数据/fentuan.xlsx -------------------------------------------------------------------------------- /docs/design/new-design.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Txiaozhe/gather/HEAD/docs/design/new-design.png -------------------------------------------------------------------------------- /docs/平台数据/fentuan.numbers: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Txiaozhe/gather/HEAD/docs/平台数据/fentuan.numbers -------------------------------------------------------------------------------- /docs/design/waiting-list.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Txiaozhe/gather/HEAD/docs/design/waiting-list.png -------------------------------------------------------------------------------- /docs/斗鱼第三方开放平台API文档v2.2.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Txiaozhe/gather/HEAD/docs/斗鱼第三方开放平台API文档v2.2.pdf -------------------------------------------------------------------------------- /docs/斗鱼弹幕服务器第三方接入协议v1.6.2.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Txiaozhe/gather/HEAD/docs/斗鱼弹幕服务器第三方接入协议v1.6.2.pdf -------------------------------------------------------------------------------- /test/test_ts.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Creator: Tang Xiaoji 3 | * Time: 2018-07-27 4 | */ 5 | 6 | 'use strict'; 7 | 8 | parse() 9 | -------------------------------------------------------------------------------- /docs/直播宝2.0.md: -------------------------------------------------------------------------------- 1 | 开关播添加标题 2 | 3 | 新开贵族、守护、周榜 采集 4 | 5 | 采单场数据、按场次记录？？ 6 | 7 | 最近一场？？定义 8 | 9 | 视频采集（开关播保证） 10 | 11 | -------------------------------------------------------------------------------- /helper/sleep.js: -------------------------------------------------------------------------------- 1 | module.exports = async function(interval){ 2 | return new Promise((resolve)=>{ 3 | setTimeout(resolve, interval); 4 | }); 5 | }; -------------------------------------------------------------------------------- /docs/redis-key.md: -------------------------------------------------------------------------------- 1 | 采集 2 | MYSQL- 3 | 4 | CATEGORY- 5 | 6 | TYPE- 7 | 8 | HASH:ANCHOR_INFO:HUYA_ 9 | 10 | ZHIBOBAO:MOBILE:CODE 11 | ZHIBOBAO:TOKEN:SECRET 12 | THIRD:OPENID:INFO 13 | -------------------------------------------------------------------------------- /docs/视频流.md: -------------------------------------------------------------------------------- 1 | start接口 plat rid url hid(直播记录标识) 2 | 3 | stop接口 4 | 5 | 6 | 7 | 维持开播的房间列表 && 注册用户 redis set 8 | 9 | 集合 key：SET:GATHER:STREAM:NEW 10 | value：plat::rid::url 11 | 12 | -------------------------------------------------------------------------------- /danmu/test.js: -------------------------------------------------------------------------------- 1 | const HuoshanApi = require('../helper/huoshan/api'); 2 | const a = require('./Huoshan'); 3 | 4 | (async () => { 5 | await a() 6 | await HuoshanApi.delay(2000); 7 | await a() 8 | })() -------------------------------------------------------------------------------- /config/Logger.d.ts: -------------------------------------------------------------------------------- 1 | declare class Logger { 2 | constructor(space: string); 3 | 4 | log(args: any) 5 | debug(arg: Object) 6 | error(title: string, plat: string, err: Error, info: String, print: boolean, report: boolean) 7 | } -------------------------------------------------------------------------------- /test/test_rpc/client.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Creator: Tang Xiaoji 3 | * Time: 2018-06-28 4 | */ 5 | 6 | 'use strict'; 7 | 8 | const rpc = require('../../gather/lib/rpc'); 9 | 10 | const client = new rpc.Client({}, 3000, '127.0.0.1'); 11 | -------------------------------------------------------------------------------- /test/test_notice.js: -------------------------------------------------------------------------------- 1 | const opp = require('../gather/lib/opportunity'); 2 | 3 | opp.getNoticeToCheck({ 4 | room_id: 123456, 5 | plat: 'quanmin', 6 | url: 'https://www.quanmin.tv/13359230' 7 | }).then(r => console.log(r)).catch(e => console.log(e)); 8 | -------------------------------------------------------------------------------- /test/test_stream.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Creator: Tang Xiaoji 3 | * Time: 2018-06-19 4 | */ 5 | 6 | 'use strict'; 7 | 8 | const video_stream = require('../video-stream'); 9 | 10 | const stream = video_stream['fanxing']; 11 | 12 | console.log(typeof stream['start']); 13 | -------------------------------------------------------------------------------- /config/redis.js: -------------------------------------------------------------------------------- 1 | const Redis = require('ioredis'); 2 | const config = require('./config').redis; 3 | const redis = new Redis({ 4 | port: config.port, 5 | host: config.host, 6 | password: config.auth, 7 | db: config.db 8 | }); 9 | 10 | module.exports = redis; 11 | -------------------------------------------------------------------------------- /test/log_test.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Creator: Tang Xiaoji 3 | * Time: 2017-12-26 4 | */ 5 | 6 | const Log4js = require('../config/log'); 7 | const logtail_chat = Log4js.getLogger('logtail_chat'); 8 | 9 | setInterval(() => { 10 | logtail_chat.info('jjjjjj'); 11 | }, 1000); 12 | -------------------------------------------------------------------------------- /config/mysql.d.ts: -------------------------------------------------------------------------------- 1 | /** 2 | * 3 | */ 4 | 5 | export declare function getConn(cb: (err: Error, conn: any) => {}) 6 | 7 | export declare function asyncGetConn(): Promise; 8 | 9 | export declare function asyncQuery(sql: string, params: [any], conn: PoolConnection): Promise; 10 | -------------------------------------------------------------------------------- /helper/device.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | PANDA: { 3 | pc_web: 1, 4 | ios: 2, 5 | android: 2 6 | }, 7 | LONGZHU: { 8 | 1: 1, // pc 9 | 2: 2, // android 10 | 3: 2 // ios 11 | }, 12 | QUANMIN: { 13 | PC: 1, 14 | android: 2, 15 | iOS: 2 16 | } 17 | }; 18 | -------------------------------------------------------------------------------- /test/test_rpc/server.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Creator: Tang Xiaoji 3 | * Time: 2018-06-28 4 | */ 5 | 6 | 'use strict'; 7 | 8 | const rpc = require('../../gather/lib/rpc'); 9 | 10 | const server = new rpc.Server({}, 3000, '127.0.0.1'); 11 | server.on('remote', (remote) => { 12 | console.log(remote) 13 | }); -------------------------------------------------------------------------------- /helper/pair.js: -------------------------------------------------------------------------------- 1 | module.exports = function pair(content, linesep, sep) { 2 | let kvs = {}; 3 | content = content.toString("utf-8").split(linesep); 4 | content.forEach(item => { 5 | item = item.split(sep); 6 | if (item.length == 2) { 7 | kvs[item[0]] = item[1]; 8 | } 9 | }); 10 | return kvs; 11 | }; 12 | -------------------------------------------------------------------------------- /scripts/update_node.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | export NVM_DIR="$HOME/.nvm" 3 | [ -s "$NVM_DIR/nvm.sh" ] && \. "$NVM_DIR/nvm.sh" # This loads nvm 4 | [ -s "$NVM_DIR/bash_completion" ] && \. "$NVM_DIR/bash_completion" # This loads nvm bash_completion 5 | 6 | nvm install 8.11.3 7 | nvm use 8.11.3 8 | nvm alias default 8.11.3 9 | npm install -g pm2 10 | -------------------------------------------------------------------------------- /helper/huoshan/python-scripts/complementation.py: -------------------------------------------------------------------------------- 1 | import sys 2 | 3 | def qiuyu(data): 4 | d = data.encode('utf-8') 5 | return (lambda b: b + (-len(b) % 16) * b'\0')(d) 6 | 7 | def main(): 8 | data = sys.argv[1] 9 | data = qiuyu(data) 10 | data = ''.join(map(chr, data)) 11 | print(data) 12 | 13 | if __name__ == '__main__': 14 | main() -------------------------------------------------------------------------------- /gather/sub/router.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Creator: Tang Xiaoji 3 | * Time: 2018-07-17 4 | */ 5 | 6 | 'use strict'; 7 | 8 | const Router = require('koa-router'); 9 | const router = new Router(); 10 | 11 | const hdl_sub = require('./handler')(); 12 | 13 | router.post('/sub', hdl_sub.sub); 14 | router.post('/unsub', hdl_sub.unSub); 15 | 16 | module.exports = router; 17 | -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | Douyu: require("./danmu/Douyu"), 3 | Panda: require("./danmu/Panda"), 4 | XYPanda: require("./danmu/XYPanda"), 5 | Zhanqi: require("./danmu/Zhanqi"), 6 | Longzhu: require("./danmu/Longzhu"), 7 | Chushou: require("./danmu/Chushou"), 8 | Huya: require("./danmu/Huya"), 9 | Huoshan: require("./danmu/Huoshan") 10 | }; 11 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Requires 2 | > node v8+ 3 | 4 | # Install 5 | > npm i 6 | 7 | # 简介 8 | * danmu: 弹幕采集客户端 9 | * gather: 调度及任务管理 10 | * helper: 工具 11 | * scripts: 执行脚本 12 | 13 | 14 | # 相关技术文章 15 | [直播大数据采集（一期）](https://txiaozhe.github.io/2019/12/15/live-data-fetch01/) 16 | 17 | [直播大数据采集（二期）](https://txiaozhe.github.io/2019/12/29/live-data-fetch02/) 18 | 19 | # 架构图 20 | ![](./architecture.png) 21 | -------------------------------------------------------------------------------- /config/log.js: -------------------------------------------------------------------------------- 1 | /** 2 | * 3 | */ 4 | 5 | const os = require('os'); 6 | 7 | let log; 8 | switch (os.platform()) { 9 | case 'darwin' : { 10 | log = require('./log_test'); 11 | break; 12 | } 13 | case 'linux' : { 14 | log = require('./log_prod'); 15 | break; 16 | } 17 | default: { 18 | log = require('./log_prod'); 19 | } 20 | } 21 | 22 | module.exports = log; 23 | -------------------------------------------------------------------------------- /test/test_log.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Creator: Tang Xiaoji 3 | * Time: 2018-06-25 4 | */ 5 | 6 | 'use strict'; 7 | 8 | // const Log4js = require('../config/log_test'); 9 | // const logt = Log4js.getLogger('logtail_data_native'); 10 | // logt.info({ddd:'dddddd'}); 11 | 12 | const Logger = require('../config/Logger'); 13 | const logtail = Logger.getLogtail(Logger.type.DataNative); 14 | 15 | Logger.write(logtail, {dde: 'ddd'}) -------------------------------------------------------------------------------- /test/test_obj.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Creator: Tang Xiaoji 3 | * Time: 2018-06-26 4 | */ 5 | 6 | 'use strict'; 7 | 8 | // const obj = { 9 | // n1: '222', 10 | // n2: 'ddd', 11 | // n3: 'fff' 12 | // }; 13 | // 14 | // console.log(obj) 15 | // delete obj.n1; 16 | // console.log(obj) 17 | 18 | const task = { 19 | status: 1 20 | } 21 | const s = task['status']; 22 | console.log(s) 23 | console.log([2, 3].includes(s)); -------------------------------------------------------------------------------- /helper/huoshan/python-scripts/cryptoAfter.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import gzip 3 | import zlib 4 | 5 | def cryptAfter(h): 6 | c = zlib.crc32(h.encode('utf-8')) 7 | g = bytes([31, 139, 8, 0, 0, 0, 0, 0, 0, 0]) + gzip.compress((h + ',' + str(c)).encode('utf-8'))[10:] 8 | return g 9 | 10 | def main(): 11 | res = cryptAfter(sys.argv[1]) 12 | res = "".join(map(chr, res)) 13 | print(res) 14 | 15 | if __name__ == '__main__': 16 | main() -------------------------------------------------------------------------------- /test/xxx.d.ts: -------------------------------------------------------------------------------- 1 | export function parse(connectionString: string): ConnectionOptions; 2 | 3 | export interface ConnectionOptions { 4 | host: string | null; 5 | password: string | null; 6 | user: string | null; 7 | port: number | null; 8 | database: string | null; 9 | client_encoding: string | null; 10 | ssl: boolean | null; 11 | 12 | application_name: string | null; 13 | fallback_application_name: string | null; 14 | } -------------------------------------------------------------------------------- /docs/需求.txt: -------------------------------------------------------------------------------- 1 | 1.scan：http消息获取; 2 | 1.在线主播房间列表 √ 3 | 2.人气 粉丝 × 4 | 3.平台主播key × 5 | 2.dispatch:调度服务 6 | 1.scan消息接收/与bee连接 消息传递 √ 7 | 2.房间状态管理,任务派发，负载均衡 √ 8 | 3.集群机器状态监控，动态伸缩 × 9 | 3.bee:采集机器 10 | 1.任务接收 √ 11 | 2.平台监听执行 √ 12 | 3.异常上报 √ 13 | 4.状态心跳 × 14 | 15 | 16 | 链接方式 17 | scan(http)->dispatch(tcp)<->bees； 18 | bee(log)->loghub 19 | bee(es client)->es 20 | loghub（投递）->odps; -------------------------------------------------------------------------------- /test/test_kugou.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Creator: Tang Xiaoji 3 | * Time: 2018-05-07 4 | */ 5 | 6 | const danmu = require('../danmu'); 7 | const fanxing = new danmu.DanmuClient(danmu.TYPE.Fanxing, 'http://fanxing.kugou.com/19191919'); 8 | 9 | fanxing.on('connect', () => { 10 | console.log('connect'); 11 | }); 12 | 13 | fanxing.on('data', (data) => { 14 | // console.log(data) 15 | }); 16 | 17 | fanxing.on('error', (err) => { 18 | console.log('err: ', err); 19 | }); 20 | -------------------------------------------------------------------------------- /test/test_yy.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Creator: Tang Xiaoji 3 | * Time: 2018-05-07 4 | */ 5 | 6 | const danmu = require('../danmu'); 7 | const huya = new danmu.DanmuClient(danmu.TYPE.YY, 'http://www.yy.com/25271902/25271902?tempId=16777217'); 8 | 9 | huya.on('connect', () => { 10 | console.log('connect'); 11 | }); 12 | 13 | huya.on('data', (data) => { 14 | // console.log(data) 15 | }); 16 | 17 | huya.on('error', (err) => { 18 | console.log('err: ', err); 19 | }); 20 | -------------------------------------------------------------------------------- /test/test_longzhu.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Creator: Tang Xiaoji 3 | * Time: 2018-05-07 4 | */ 5 | 6 | const danmu = require('../danmu'); 7 | const huya = new danmu.DanmuClient(danmu.TYPE.longzhu, 'http://star.longzhu.com/126244?from=challcontent'); 8 | 9 | huya.on('monitor_start', () => { 10 | console.log('connect'); 11 | }); 12 | 13 | huya.on('data', (data) => { 14 | console.log(data) 15 | }); 16 | 17 | huya.on('error', (err) => { 18 | console.log('err: ', err); 19 | }); 20 | 21 | -------------------------------------------------------------------------------- /helper/useragent.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | pc: 3 | "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/54.0.2840.71 Safari/537.36", 4 | mobile: 5 | "Mozilla/5.0 (iPhone; CPU iPhone OS 9_1 like Mac OS X) AppleWebKit/601.1.46 (KHTML, like Gecko) Version/9.0 Mobile/13B143 Safari/601.1", 6 | new_mobile: 'Mozilla/5.0 (Linux; Android 5.1.1; Nexus 6 Build/LYZ28E) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/63.0.3239.84 Mobile Safari/537.36' 7 | }; 8 | -------------------------------------------------------------------------------- /gather/nest.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Creator: Tang Xiaoji 3 | * Time: 2018-06-27 4 | */ 5 | 6 | 'use strict'; 7 | 8 | const Logger = require('../config/Logger'); 9 | const logger = new Logger('nest'); 10 | 11 | require('./sub/index'); // todo 打开监视 12 | 13 | require('./lib/dispather').initSearcher((plat, info) => { 14 | logger.log('initSearcher: ', plat, info); 15 | }); 16 | 17 | process.on('uncaughtException', (err) => { 18 | logger.error('nest start uncaughtException', 'nest', err); 19 | }); 20 | -------------------------------------------------------------------------------- /docs/数据采集需求.md: -------------------------------------------------------------------------------- 1 | 1、http://10.4.19.210:3001/v2/update 2 | 3 | ``` 4 | { 5 | plat: '' // 平台名称 6 | tasks: '' // 数据，需处理url，需加字段名：timestamp：扫描时间 7 | } 8 | ``` 9 | 10 | 2、两个es http://116.62.188.69:9100/ 配置：118.31.73.226:9200/ 11 | 12 | roomlist + 日期 都是 ‘global’ 13 | 14 | room 15 | 16 | 3、存日志（人气信息） 17 | 18 | /root/app/lives/ 、文件名：RoomLive 19 | 20 | 4、redis 持久化到mysql 21 | 22 | * 键： HASH:ROOM:INFO:%s:%s 23 | 24 | ​ 值：对应数据 25 | 26 | * 第一次开播时间 27 | 28 | first_time 29 | 30 | ​ 31 | 32 | -------------------------------------------------------------------------------- /test/test_mysql.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Creator: Tang Xiaoji 3 | * Time: 2018-01-09 4 | */ 5 | 6 | const config = require('../config/config'); 7 | const mysql = require('../config/mysql'); 8 | 9 | mysql.getConn((err, conn) => { 10 | conn.query('SELECT * FROM ?? LIMIT 10', [ 11 | config.tbl_name.db_fentuan_taskv2 12 | ], (err, result) => { 13 | if (err) { 14 | console.log(err); 15 | } else { 16 | console.log('获取数据成功: ', result); 17 | } 18 | conn.release(); 19 | }); 20 | }); 21 | -------------------------------------------------------------------------------- /docs/平台数据/汇总.md: -------------------------------------------------------------------------------- 1 | ```json 2 | { 3 | room_id: 1111111, // 房间id 4 | room_title: '滴水穿石', // 房间名称/房间简介/房间标签 5 | anchor_id: 0, // 主播id 6 | anchor_nick: 'djke', // 主播名/主播昵称 7 | plat: 'quanmin', 8 | url: 'http://quanmin.tv/123456', // 直播间地址 9 | online: 1000, // 人气值/在线人数 10 | follow: 1000, // 关注 11 | avatar: 'http://dewjndkjwedwe.jpg', // 主播头像 12 | cate_id: '5', 13 | cate_name: '英雄联盟', // 直播分类/游戏名 14 | city: -1, 15 | dateline: '2017-11-08', 16 | timestamp: 1131324536 17 | } 18 | ``` 19 | 20 | -------------------------------------------------------------------------------- /docs/mysql索引策略和优化.md: -------------------------------------------------------------------------------- 1 | mysql索引策略和优化 2 | 3 | ### 创建索引 4 | 5 | - 在经常查询而不经常增删改操作的字段加索引。 6 | - order by与group by后应直接使用字段，而且字段应该是索引字段。 7 | - 一个表上的索引不应该超过6个。 8 | - 索引字段的长度固定，且长度较短。 9 | - 索引字段重复不能过多，如果某个字段为主键，那么这个字段不用设为索引。 10 | - 在过滤性高的字段上加索引。 11 | 12 | ### 使用索引注意事项 13 | 14 | - 使用like关键字时，前置%会导致索引失效。 15 | - 使用null值会被自动从索引中排除，索引一般不会建立在有空值的列上。 16 | - 使用or关键字时，or左右字段如果存在一个没有索引，有索引字段也会失效。 17 | - 使用!=操作符时，将放弃使用索引。因为范围不确定，使用索引效率不高，会被引擎自动改为全表扫描。 18 | - 不要在索引字段进行运算。 19 | - 在使用复合索引时，最左前缀原则，查询时必须使用索引的第一个字段，否则索引失效；并且应尽量让字段顺序与索引顺序一致。 20 | - 避免隐式转换，定义的数据类型与传入的数据类型保持一致。 -------------------------------------------------------------------------------- /docs/消息分类.md: -------------------------------------------------------------------------------- 1 | ### longzhu 2 | 3 | type: gradeuserjoin：有等级的用户进房间 4 | 5 | commonjoin：普通用户进入 6 | 7 | vipjoin： 8 | 9 | onlineuserjoin： 10 | 11 | pkrank：敌我双方排行榜，pk动态信息 12 | 13 | chat： 14 | 15 | gift： 16 | 17 | hostjoin： 18 | 19 | heatValue: 心跳 20 | 21 | rollgame： 22 | 23 | rolling： 24 | 25 | wssrrank： 26 | 27 | 28 | 29 | ### quanmin 30 | 31 | AUTH: 1001 32 | SUB: 1002 33 | UNSUB: 1003 34 | HB: 1004 35 | BAR: 1005 36 | GIFT: 1006 37 | -------------------------------------------------------------------------------- /gather/video-stream/index.js: -------------------------------------------------------------------------------- 1 | const quanmin = require('./QuanminStream'); 2 | const fanxing = require('./FanxingStream'); 3 | const longzhu = require('./LongzhuStream'); 4 | const douyu = require('./DouyuStream'); 5 | const panda = require('./PandaStream'); 6 | // const chushou = require('./ChushouStream'); 7 | const zhanqi = require('./ZhanqiStream'); 8 | const huya = require('./HuyaStream'); 9 | 10 | module.exports = { 11 | quanmin, 12 | fanxing, 13 | longzhu, 14 | douyu, 15 | panda, 16 | // chushou, 17 | zhanqi, 18 | huya 19 | }; 20 | -------------------------------------------------------------------------------- /test/test_chushou.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Creator: Tang Xiaoji 3 | * Time: 2018-05-07 4 | */ 5 | 6 | const {DanmuClient, TYPE} = require('../danmu'); 7 | const chushou = new DanmuClient(TYPE.chushou, 'https://chushou.tv/room/7919686.htm'); 8 | 9 | chushou.on('monitor_start', () => { 10 | console.log('connect') 11 | }); 12 | 13 | chushou.on('data', (data) => { 14 | console.log(data) 15 | }); 16 | 17 | chushou.on('error', (err) => { 18 | console.log('error', err) 19 | }); 20 | 21 | chushou.on('close', () => { 22 | console.log('close') 23 | }); 24 | -------------------------------------------------------------------------------- /config/http.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Creator: Tang Xiaoji 3 | * Time: 2018-01-25 4 | */ 5 | 6 | const request = require('request-promise'); 7 | const RESPONSE_TIMEOUT = 15e3; 8 | const DEADLINE_TIMEOUT = 6e4; 9 | 10 | const Get = function(url) { 11 | return request({ 12 | method: 'GET', 13 | uri: url 14 | }); 15 | }; 16 | 17 | const Post = function(url, params) { 18 | return request({ 19 | method: 'POST', 20 | uri: url, 21 | gzip: true, 22 | forever: true, 23 | json: params 24 | }); 25 | }; 26 | 27 | module.exports = { 28 | Get, 29 | Post 30 | }; 31 | -------------------------------------------------------------------------------- /docs/平台数据/数据汇总.txt: -------------------------------------------------------------------------------- 1 | room_id: ele.room_id, // 房间id 2 | room_title: ele.room_name, // 房间名称 3 | anchor_id: ele.owner_uid, // 主播id 4 | anchor_nick: ele.nickname, // 主播昵称 5 | plat: plat.Douyu, // chushou/douyu/huya/longzhu/panda/pandaxy/quanmin/zhanqi 6 | url: `https://www.douyu.com/${ele.room_id}`, // 房间url，输入浏览器能直接到达直播间的地址 7 | online: ele.online, // 在线人数 8 | follow: fans, // 粉丝数/关注数 9 | avatar: this.getAvatarUrl(ele.room_id), // 头像，存入oss的url 10 | cate_id: ele.cate_id, // 分类id 11 | cate_name: ele.game_name, // 分类名称 12 | city: -1, // 主播所在城市 13 | dateline: 2017-12-11, // 当前时间 14 | timestamp: utils.getTS() // 当前时间戳 15 | -------------------------------------------------------------------------------- /test/test_panda.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Creator: Tang Xiaoji 3 | * Time: 2018-05-11 4 | */ 5 | 6 | const Panda = require('../danmu/Panda'); 7 | const moment = require('moment'); 8 | 9 | const panda = new Panda('https://www.panda.tv/2115408'); 10 | 11 | panda.on('connect', () => { 12 | console.log('connect') 13 | }); 14 | 15 | panda.on('data', (data) => { 16 | console.log(data[0].data) 17 | }); 18 | 19 | panda.on('error', (err) => { 20 | console.log('error', err) 21 | }); 22 | 23 | panda.on('close', () => { 24 | console.log('close'); 25 | console.log(moment().format('YYYY-MM-DD HH:mm:ss')); 26 | }); 27 | -------------------------------------------------------------------------------- /test/test_gift.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Creator: Tang Xiaoji 3 | * Time: 2018-07-05 4 | */ 5 | 6 | 'use strict'; 7 | 8 | // const Gift = require('../gather/lib/gift'); 9 | 10 | // const giftHelper = Gift.getInstance(); 11 | 12 | // giftHelper.initGift(8888888832032291, 'xypanda').then(r => console.log(r)).catch(e => console.log(e)); 13 | 14 | const fs = require('fs'); 15 | const util = require('../gather/lib/util'); 16 | util.getGiftList().then(r => { 17 | const gifts = JSON.parse(r.text).data; 18 | console.log(gifts) 19 | fs.writeFile('./gift.json', JSON.stringify(gifts.list), () => {}) 20 | }).catch(e => console.log(e)) -------------------------------------------------------------------------------- /helper/cutbuf.js: -------------------------------------------------------------------------------- 1 | module.exports = function(buffer, skip = 0) { 2 | let stack = 0; 3 | let contents = []; 4 | 5 | label: for (; buffer.length; ) { 6 | buffer = buffer.slice(skip); 7 | for (let i = 0, len = buffer.length; i < len; i++) { 8 | if (buffer[i] == 0x7b) { 9 | stack += 1; 10 | } else if (buffer[i] == 0x7d) { 11 | stack -= 1; 12 | } 13 | 14 | if (!stack) { 15 | contents.push(buffer.slice(0, i + 1).toString("utf-8")); 16 | buffer = buffer.slice(i + 1); 17 | continue label; 18 | } 19 | } 20 | } 21 | 22 | return contents; 23 | }; 24 | -------------------------------------------------------------------------------- /test/test_ink.js: -------------------------------------------------------------------------------- 1 | /** 2 | * 3 | */ 4 | 5 | const Inke = require('../danmu/Inke'); 6 | let inke = new Inke('http://www.inke.cn/live.html?uid=707750875&id=1534318348133212'); 7 | 8 | inke.on('connect', () => { 9 | console.log('connect2'); 10 | }); 11 | 12 | inke.on('data', (data) => { 13 | console.log(data) 14 | }); 15 | 16 | inke.on('error', (err) => { 17 | console.log('err: ', err); 18 | }); 19 | 20 | inke.on('initerror', (err) => { 21 | console.log('initerr: ', err); 22 | }); 23 | 24 | inke.on('close', (err) => { 25 | console.log('close: ', err); 26 | }); 27 | 28 | inke.on('destroy', (err) => { 29 | console.log('destroy: ', err); 30 | }); -------------------------------------------------------------------------------- /config/config_local.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | nest: { 3 | rpc_host: '127.0.0.1' 4 | }, 5 | mysql: { 6 | connectionLimit: 10, 7 | host: '192.168.18.240', 8 | user: 'root', 9 | port: 3310, 10 | password: '123456', 11 | database: 'db_fentuan', 12 | charset: 'utf8mb4' 13 | }, 14 | redis: { 15 | host: '192.168.19.101', 16 | port: 6379, 17 | db: 0 18 | }, 19 | elasticsearch: { 20 | host: '10.10.0.57:9200', 21 | maxSockets: 100, 22 | log: 'error' 23 | }, 24 | opp: { 25 | bee: { 26 | max_retry: 100, 27 | retry_delay: 1000 28 | }, 29 | search_task: 30 * 1000 30 | } 31 | }; 32 | -------------------------------------------------------------------------------- /gather/sub/index.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Creator: Tang Xiaoji 3 | * Time: 2018-07-16 4 | */ 5 | 6 | 'use strict'; 7 | 8 | const Koa = require('koa'); 9 | const bodyParser = require('koa-bodyparser'); 10 | const config = require('../../config/config'); 11 | const router = require('./router'); 12 | const app = new Koa(); 13 | const Logger = require('../../config/Logger'); 14 | const logger = new Logger('sub/http'); 15 | 16 | app 17 | .use(bodyParser()) 18 | .use(router.routes()) 19 | .use(router.allowedMethods()); 20 | 21 | app.listen(config.app.http_sub_port); 22 | logger.log('sub http server listen in: ', config.app.http_sub_port); 23 | 24 | require('./check'); 25 | -------------------------------------------------------------------------------- /test/test_huya.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Creator: Tang Xiaoji 3 | * Time: 2018-05-07 4 | */ 5 | 6 | const Huya = require('../danmu/Huya'); 7 | let huya = new Huya('http://www.huya.com/11710691'); 8 | 9 | huya.on('connect', () => { 10 | console.log('connect'); 11 | }); 12 | 13 | huya.on('data', (data) => { 14 | console.log(data) 15 | }); 16 | 17 | huya.on('error', (err) => { 18 | console.log('err: ', err); 19 | }); 20 | 21 | huya.on('initerror', (err) => { 22 | console.log('initerr: ', err); 23 | }); 24 | 25 | huya.on('close', (err) => { 26 | console.log('close: ', err); 27 | }); 28 | 29 | huya.on('destroy', (err) => { 30 | console.log('destroy: ', err); 31 | }); 32 | -------------------------------------------------------------------------------- /config/config_test.js: -------------------------------------------------------------------------------- 1 | module.exports = { 2 | nest: { 3 | rpc_host: '10.4.20.103' 4 | }, 5 | mysql: { 6 | connectionLimit: 10, 7 | host: 'rm-bp1tzi73g5y910h7h.mysql.rds.aliyuncs.com', 8 | user: 'fentuan', 9 | port: 3306, 10 | password: 'Fentuan2017', 11 | charset: 'utf8mb4' 12 | }, 13 | redis: { 14 | host: 'r-bp166c8a4faed514196.redis.rds.aliyuncs.com', 15 | port: 6379, 16 | auth: 'Fentuan2017', 17 | db: 0 18 | }, 19 | elasticsearch: { 20 | host: '10.10.0.57:9200', 21 | maxSockets: 100, 22 | log: 'error' 23 | }, 24 | opp: { 25 | bee: { 26 | max_retry: 100, 27 | retry_delay: 1000 28 | }, 29 | search_task: 30 * 1000 30 | } 31 | }; 32 | -------------------------------------------------------------------------------- /test/test_huyav2.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Creator: Tang Xiaoji 3 | * Time: 2018-07-20 4 | */ 5 | 6 | 'use strict'; 7 | 8 | 9 | // const Huya = require('../danmu/H'); 10 | // const client = new Huya('https://www.huya.com/kaerlol'); 11 | 12 | const danmu = require('../danmu'); 13 | const client = new danmu.DanmuClient(danmu.TYPE.huya, 'http://www.huya.com/baozha'); 14 | 15 | client.on('monitor_start', () => { 16 | console.log(`已连接huya 房间弹幕~`) 17 | }); 18 | 19 | client.on('data', data => { 20 | console.log(data) 21 | }); 22 | 23 | client.on('error', e => { 24 | console.log(e) 25 | }); 26 | 27 | client.on('close', () => { 28 | console.log('close') 29 | }); 30 | 31 | client.on('initerror', e => { 32 | console.log(e) 33 | }); 34 | -------------------------------------------------------------------------------- /gather/lib/anchor.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Creator: Tang Xiaoji 3 | * Time: 2018-02-28 4 | */ 5 | 6 | const request = require('request'); 7 | 8 | const fanxing = url => { 9 | return new Promise((resolve, reject) => { 10 | request.get(url, (err, res, body) => { 11 | if (err || res.statusCode !== 200) { 12 | resolve({info: '找不到直播间'}); 13 | } 14 | const reg = body && body.toString().match(/roomId: \'(\S*)\',/); 15 | const rid = reg ? +reg[1] : null; 16 | if (!rid || Number.isNaN(rid)) { 17 | resolve({info: '找不到直播间'}); 18 | } else { 19 | resolve({rid}); 20 | } 21 | }); 22 | }); 23 | }; 24 | 25 | module.exports = { 26 | fanxing 27 | }; 28 | 29 | // fanxing('http://fanxing.kugou.com/2467872').then(r => console.log(r)).catch(e => console.log(e)) 30 | -------------------------------------------------------------------------------- /config/config_prod.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Creator: Tang Xiaoji 3 | * Time: 2018-06-12 4 | */ 5 | 6 | 'use strict'; 7 | 8 | module.exports = { 9 | nest: { 10 | rpc_host: '10.4.20.103' 11 | }, 12 | mysql: { 13 | host: 'rm-bp1z6xst2m8uwjvge.mysql.rds.aliyuncs.com', 14 | port: 3306, 15 | connectionLimit: 10, 16 | user: 'fentuan', 17 | password: 'ALadHrb3bd7JORxq', 18 | charset: 'utf8mb4' 19 | }, 20 | redis: { 21 | host: 'r-bp166c8a4faed514196.redis.rds.aliyuncs.com', 22 | port: 6379, 23 | auth: 'Fentuan2017', 24 | db: 0 25 | }, 26 | elasticsearch: { 27 | host: '10.10.0.57:9200', 28 | maxSockets: 100, 29 | log: 'error' 30 | }, 31 | opp: { 32 | bee: { 33 | max_retry: 100, 34 | retry_delay: 1000 35 | }, 36 | search_task: 30 * 1000 37 | } 38 | }; 39 | -------------------------------------------------------------------------------- /config/config.js: -------------------------------------------------------------------------------- 1 | /** 2 | * 3 | */ 4 | 5 | const config_default = require('./config_default'); 6 | const config_prod = require('./config_prod'); 7 | const config_test = require('./config_test'); 8 | const config_local = require('./config_local'); 9 | const Logger = require('./Logger'); 10 | const logger = new Logger('launcher'); 11 | const os = require('os'); 12 | 13 | let config; 14 | 15 | if(os.platform() === 'darwin') { 16 | config = Object.assign(config_default, config_local); 17 | } else { 18 | const debug = !process.env.DEBUG_BEE || (process.env.DEBUG_BEE === 'true'); 19 | 20 | if(debug) { 21 | logger.log('测试环境...'); 22 | config = Object.assign(config_default, config_test); 23 | } else { 24 | logger.log('生产环境...'); 25 | config = Object.assign(config_default, config_prod); 26 | } 27 | } 28 | 29 | module.exports = config; 30 | -------------------------------------------------------------------------------- /helper/qmSign.js: -------------------------------------------------------------------------------- 1 | const crypto = require('crypto'); 2 | const appSecret= require('../config/config').quanmin.appSecret; 3 | 4 | module.exports = function (options) { 5 | // sign 6 | let headers = []; 7 | //1 参数按字典序 排序 8 | for (let item in options) { 9 | if (options.hasOwnProperty(item)) { 10 | headers.push(item); 11 | } 12 | } 13 | headers.sort((a, b) => a < b ? -1 : 1); 14 | //待加密字符串 15 | let canonicalizedQueryString = ""; 16 | //2 参数编码 17 | canonicalizedQueryString = headers.reduce((max, curr) => max + `&${curr}=${options[curr]}`, 18 | canonicalizedQueryString); 19 | //3 拼接appSecret 20 | canonicalizedQueryString +=`&appSecret=${appSecret}`; 21 | let hash = crypto.createHash('md5'); 22 | hash.update(canonicalizedQueryString.substr(1)); 23 | // logger.trace('options', options); 24 | return hash.digest().toString('hex').toUpperCase(); 25 | }; 26 | -------------------------------------------------------------------------------- /helper/huoshan/python-scripts/unwifi.py: -------------------------------------------------------------------------------- 1 | import random 2 | 3 | def uword(n): 4 | if n <= 0: 5 | return None 6 | elif n == 1: 7 | return '0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ'[random.randint(0, 61)] 8 | else: 9 | return ''.join([uword(1) for i in range(n)]) 10 | 11 | def ubyte(n): 12 | if n > 0: 13 | return ubyte(n - 1) + hex(random.randint(0, 0xff))[2:].zfill(2) 14 | return '' 15 | 16 | def uwifi(): 17 | wifibssid = ':'.join([ubyte(i) for i in [1, 1, 1, 1, 1, 1]]) 18 | i = None 19 | del i 20 | wifiip = '192.168.' + str(random.randint(0, 255)) + '.' + str(random.randint(0, 255)) 21 | wifimac = '02:00:00:00:00:00' 22 | wifissid = '"' + uword(random.randint(2, 8)) + '"' 23 | return locals() 24 | 25 | def main(): 26 | res = uwifi() 27 | print(res) 28 | 29 | if __name__ == '__main__': 30 | main() -------------------------------------------------------------------------------- /gather/bee.js: -------------------------------------------------------------------------------- 1 | /** 2 | * tangxiaoji 3 | */ 4 | 5 | const rpc = require('./lib/rpc'); 6 | const Logger = require('../config/Logger'); 7 | const logger = new Logger('beelauncher'); 8 | const config = require('../config/config'); 9 | 10 | logger.log('bee start', 'local: ', config.app.host, 'quanmin: ', config.quanmin.target); 11 | if(config.app.host === config.quanmin.target) { 12 | require('./lib/quanmin').check((err, count) => { 13 | logger.log(`inited quanmin ${count} tasks`); 14 | }); 15 | } else { 16 | const manager = require('./lib/bee'); 17 | manager.check((err, count) => { 18 | logger.log(`inited ${count} tasks`); 19 | new rpc.Client(manager); 20 | }); 21 | 22 | manager._beeCheck(count => logger.error('_beeCheck 任务自检', '_beeCheck', '', count)); 23 | } 24 | 25 | process.on('uncaughtException', (err) => { 26 | logger.error('bee start uncaughtException', 'bee', err); 27 | }); 28 | -------------------------------------------------------------------------------- /test/demo.js: -------------------------------------------------------------------------------- 1 | const { 2 | DanmuClient, 3 | TYPE 4 | } = require('../danmu'); 5 | const util = require('../gather/lib/util'); 6 | 7 | // node demo https://www.panda.tv/6666 8 | 9 | const url = process.argv[2]; 10 | const type = util.getType(url); 11 | if (type && TYPE.hasOwnProperty(type)) { 12 | const c = new DanmuClient(TYPE[type], url) 13 | .on('monitor_start', data => { 14 | console.log('start monitor:', data); 15 | }) 16 | .on('data', data => { 17 | if(data.type == 'gift'){ 18 | console.log('[D]', (new Date).toLocaleString(), data); 19 | } 20 | }) 21 | .on('error', e => { 22 | console.error('[E]', e && e.message); 23 | c.restart(); 24 | }) 25 | .on('close', e => { 26 | console.error('[E] close'); 27 | }) 28 | } else if (url) { 29 | console.error('[E] unsupport url:', url); 30 | } else { 31 | console.error('[E] missing url param'); 32 | } -------------------------------------------------------------------------------- /gather/video-stream/QuanminStream.js: -------------------------------------------------------------------------------- 1 | const VideoStream = require('./VideoStream'); 2 | const rp = require('request-promise'); 3 | const moment = require('moment'); 4 | 5 | class QuanminStream extends VideoStream { 6 | 7 | constructor (plat = 'quanmin') { 8 | super(plat); 9 | } 10 | 11 | async getStreamAddr (plat, roomId, url) { 12 | try { 13 | const res = await rp({ 14 | uri: `https://www.quanmin.tv/json/rooms/${roomId}/lineInfo.json`, 15 | qs: { 16 | _t: moment().format('YYYYMDHm') 17 | }, 18 | json: true 19 | }); 20 | const roomLine = res.room_lines[0]; 21 | let flvs = Object.values(roomLine.flv); 22 | flvs = flvs.filter(e => typeof e === 'object').sort((a, b) => a.quality - b.quality); 23 | return flvs[0].src; 24 | } catch (e) { 25 | console.log(`get plat: ${plat}, rid: ${roomId} stream addr error`); 26 | } 27 | } 28 | 29 | } 30 | 31 | module.exports = new QuanminStream(); -------------------------------------------------------------------------------- /scripts/install_python3.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | gcc -v 4 | 5 | if [[ $? == 0 ]] 6 | then 7 | python3 --version 8 | 9 | if [[ $? == 0 ]] 10 | then 11 | echo `ifconfig eth0 | grep inet | awk '{print $2}'` installed 12 | else 13 | 14 | wget -O /root/Python-3.6.5.tgz https://www.python.org/ftp/python/3.6.5/Python-3.6.5.tgz 15 | 16 | tar zxvf /root/Python-3.6.5.tgz -C /root/ && cd /root/Python-3.6.5 17 | 18 | ./configure --prefix=/usr/local/python3.6 19 | 20 | make && make install 21 | 22 | if [[ $? == 0 ]] 23 | then 24 | 25 | echo "export PATH=\$PATH:/usr/local/python3.6/bin" >> /root/.bashrc 26 | source /root/.bashrc 27 | 28 | echo `ifconfig eth0 | grep inet | awk '{print $2}'` success 29 | else 30 | echo `ifconfig eth0 | grep inet | awk '{print $2}'` makeFailed 31 | fi 32 | fi 33 | else 34 | echo `ifconfig eth0 | grep inet | awk '{print $2}'` needGcc 35 | fi 36 | -------------------------------------------------------------------------------- /gather/video-stream/LongzhuStream.js: -------------------------------------------------------------------------------- 1 | const VideoStream = require('./VideoStream'); 2 | const rp = require('request-promise'); 3 | 4 | class LongzhuStream extends VideoStream { 5 | 6 | constructor (plat = 'longzhu') { 7 | super(plat); 8 | } 9 | 10 | async getStreamAddr (plat, roomId, url) { 11 | try { 12 | let res = await rp({ 13 | uri: 'https://livestream.longzhu.com/live/getlivePlayurl', 14 | qs: { 15 | roomId, 16 | hostPullType: 2, 17 | isAdvanced: true, 18 | playUrlsType: 1, 19 | callback: 'jsonp3' 20 | } 21 | }); 22 | res = res.slice(7, -1); 23 | res = JSON.parse(res); 24 | let urls = res.playLines && res.playLines[0].urls; 25 | urls = urls.sort((a, b) => a.rateLevel - b.rateLevel); 26 | return urls[0].securityUrl; 27 | } catch (e) { 28 | console.log(`get plat: ${plat}, rid: ${roomId} stream addr error`); 29 | } 30 | } 31 | 32 | } 33 | 34 | module.exports = new LongzhuStream(); -------------------------------------------------------------------------------- /scripts/all-list.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # curl -XDELETE 'http://10.10.0.57:9200/roomlist-*/' 4 | curl -XPUT 'http://10.10.0.57:9200/_template/roomlist' -d ' 5 | { 6 | "template" : "roomlist-*", 7 | "mappings": { 8 | "_default_":{ 9 | "_all":{ "enabled":false } 10 | }, 11 | "global": { 12 | "dynamic":true, 13 | "properties": { 14 | "timestamp": { "type": "date" }, 15 | "plat": { "type": "string", "index": "not_analyzed" }, 16 | "online": { "type": "integer" }, 17 | "anchor_id": { "type": "string", "index": "not_analyzed" }, 18 | "cate_id": { "type": "string", "index": "not_analyzed" }, 19 | "url": { "type": "string", "index": "not_analyzed" }, 20 | "city": { "type": "string", "index": "not_analyzed" }, 21 | "follow": { "type": "integer" }, 22 | "online": { "type": "integer" }, 23 | "anchor_level": { "type": "integer" }, 24 | "room_id": { "type": "string", "index": "not_analyzed" } 25 | } 26 | } 27 | } 28 | } 29 | ' -------------------------------------------------------------------------------- /gather/lib/queue.js: -------------------------------------------------------------------------------- 1 | /** 2 | * 3 | * @type {number} 4 | */ 5 | 6 | const BUFFER_FLUSH_INTERVAL = 10 * 1000; 7 | 8 | class Queue { 9 | constructor(size, onFlush, recall = true) { 10 | this.size = size; 11 | this.onFlush = onFlush; 12 | this.timer = null; 13 | this.buffer = []; 14 | this.recall = recall; 15 | } 16 | 17 | push(...items) { 18 | if (items && items.length) { 19 | this.buffer.push(...items); 20 | } 21 | clearTimeout(this.timer); 22 | if (this.buffer.length >= this.size) { 23 | this.flush(); 24 | } else { 25 | if (this.recall) { 26 | this.timer = setTimeout(this.push.bind(this), BUFFER_FLUSH_INTERVAL); 27 | } 28 | } 29 | } 30 | 31 | /** 32 | * @param onFlush: (buf: [], err: () => {}) => {} 33 | */ 34 | flush(onFlush = this.onFlush) { 35 | const tmp = this.buffer; 36 | this.buffer = []; 37 | onFlush && onFlush(tmp, () => { 38 | this.buffer = this.buffer.concat(tmp); 39 | }); 40 | } 41 | } 42 | 43 | module.exports = Queue; 44 | -------------------------------------------------------------------------------- /gather/lib/broadcast.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Creator: Tang Xiaoji 3 | * Time: 2018-06-26 4 | */ 5 | 6 | 'use strict'; 7 | 8 | const moment = require('moment'); 9 | const Logger = require('../../config/Logger'); 10 | const broadcast_logtail = Logger.getLogtail(Logger.type.Broadcast); 11 | 12 | function start(info) { 13 | const now = moment().unix(); 14 | const now_format = moment().format('YYYY-MM-DD HH:mm:ss'); 15 | if(!info.start_time) info.start_time = now; 16 | if(!info.start_time_format) info.start_time_format = now_format; 17 | info.type = 'start'; 18 | 19 | delete info.status; 20 | Logger.write(broadcast_logtail, info); 21 | } 22 | 23 | function stop(info) { 24 | const now = moment().unix(); 25 | const now_format = moment().format('YYYY-MM-DD HH:mm:ss'); 26 | info.type = 'stop'; 27 | if(!info.stop_time) info.stop_time = now; 28 | if(!info.stop_time_format) info.stop_time_format = now_format; 29 | 30 | delete info.status; 31 | Logger.write(broadcast_logtail, info); 32 | } 33 | 34 | module.exports = { 35 | start, 36 | stop 37 | }; 38 | -------------------------------------------------------------------------------- /scripts/all-danmu.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | # curl -XDELETE 'http://10.10.0.57:9200/danmu-*/' 4 | curl -XPUT 'http://10.10.0.57:9200/_template/danmu' -d ' 5 | { 6 | "template" : "danmu-*", 7 | "mappings": { 8 | "_default_":{ 9 | "_all":{ "enabled":false } 10 | }, 11 | "global": { 12 | "dynamic":true, 13 | "properties": { 14 | "type": { "type": "string", "index": "not_analyzed" }, 15 | "plat": { "type": "string", "index": "not_analyzed" }, 16 | "timestamp": { "type": "date" }, 17 | "user_id": { "type": "string", "index": "not_analyzed" }, 18 | "user_level": { "type": "integer" }, 19 | "medal_level": { "type": "integer" }, 20 | "room_id": { "type": "string", "index": "not_analyzed" }, 21 | "cate_id": { "type": "string", "index": "not_analyzed" }, 22 | "device": { "type": "string", "index": "not_analyzed" }, 23 | "gift_id": { "type": "string", "index": "not_analyzed" }, 24 | "gift_count": { "type": "integer" }, 25 | "gift_cost": { "type": "double" } 26 | } 27 | } 28 | } 29 | } 30 | ' -------------------------------------------------------------------------------- /gather/video-stream/PandaStream.js: -------------------------------------------------------------------------------- 1 | const VideoStream = require('./VideoStream'); 2 | const rp = require('request-promise'); 3 | const QUALITY = { 4 | HIGH: '_mid', 5 | SUPER: '' 6 | }; 7 | 8 | class PandaStream extends VideoStream { 9 | 10 | constructor (plat = 'panda') { 11 | super(plat); 12 | } 13 | 14 | async getStreamAddr (plat, roomId, url) { 15 | try { 16 | const res = await rp({ 17 | uri: 'https://api.m.panda.tv/ajax_get_liveroom_baseinfo', 18 | qs: { 19 | roomid: roomId, 20 | slaveflag: 1, 21 | __version: '3.2.1.4923', 22 | __plat: 'ios', 23 | __channel: 'appstore' 24 | }, 25 | json: true 26 | }); 27 | const { plflag, room_key, sign, ts } = res.data.info.videoinfo; 28 | const pl = plflag.split('_')[1]; 29 | return `https://pl${pl}.live.panda.tv/live_panda/${room_key}${QUALITY.HIGH}.flv?sign=${sign}${ts}`; 30 | } catch (e) { 31 | console.log(`get plat: ${plat}, rid: ${roomId} stream addr error`); 32 | } 33 | } 34 | 35 | } 36 | 37 | module.exports = new PandaStream(); -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2019 Jim Tang 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /gather/sub/handler.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Creator: Tang Xiaoji 3 | * Time: 2018-07-17 4 | */ 5 | 6 | 'use strict'; 7 | 8 | const subs = require('./subs'); 9 | 10 | module.exports = () => { 11 | class SubHandler { 12 | async sub(ctx, next) { 13 | const {plat, rid} = ctx.request.body; 14 | if (!rid || !plat) { 15 | ctx.body = {code: 404, msg: '参数错误'}; 16 | await next(); 17 | } else { 18 | try { 19 | await subs.sub(plat, rid); 20 | ctx.body = {code: 0}; 21 | await next(); 22 | } catch (e) { 23 | ctx.body = {code: 400, msg: e}; 24 | await next(); 25 | } 26 | } 27 | } 28 | 29 | async unSub(ctx, next) { 30 | const {plat, rid} = ctx.request.body; 31 | if (!rid || !plat) { 32 | ctx.body = {code: 404, msg: '参数错误'}; 33 | await next(); 34 | } else { 35 | try { 36 | await subs.unSub(plat, rid); 37 | ctx.body = {code: 0}; 38 | await next(); 39 | } catch (e) { 40 | ctx.body = {code: 400, msg: e}; 41 | await next(); 42 | } 43 | } 44 | } 45 | } 46 | 47 | return new SubHandler(); 48 | }; 49 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Logs 2 | logs 3 | *.log 4 | npm-debug.log* 5 | yarn-debug.log* 6 | yarn-error.log* 7 | 8 | # Runtime data 9 | pids 10 | *.pid 11 | *.seed 12 | *.pid.lock 13 | 14 | # Directory for instrumented libs generated by jscoverage/JSCover 15 | lib-cov 16 | 17 | # Coverage directory used by tools like istanbul 18 | coverage 19 | 20 | # nyc test coverage 21 | .nyc_output 22 | 23 | # Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files) 24 | .grunt 25 | 26 | # Bower dependency directory (https://bower.io/) 27 | bower_components 28 | 29 | # node-waf configuration 30 | .lock-wscript 31 | 32 | # Compiled binary addons (https://nodejs.org/api/addons.html) 33 | build/Release 34 | 35 | # Dependency directories 36 | node_modules/ 37 | jspm_packages/ 38 | 39 | # TypeScript v1 declaration files 40 | typings/ 41 | 42 | # Optional npm cache directory 43 | .npm 44 | 45 | # Optional eslint cache 46 | .eslintcache 47 | 48 | # Optional REPL history 49 | .node_repl_history 50 | 51 | # Output of 'npm pack' 52 | *.tgz 53 | 54 | # Yarn Integrity file 55 | .yarn-integrity 56 | 57 | # dotenv environment variables file 58 | .env 59 | 60 | # next.js build output 61 | .next 62 | .DS_Store 63 | .vscode 64 | .idea 65 | -------------------------------------------------------------------------------- /docs/huoshan.md: -------------------------------------------------------------------------------- 1 | ### JSON FORMAT 2 | ``` 3 | { 4 | common：{ 5 | room_id, 6 | create_time, 7 | method, // 消息类型 8 | }， 9 | 10 | user_id, // 用户id 11 | user: { 12 | 13 | short_id, // 火山号 14 | gender, 15 | level 16 | 17 | }, 18 | ...others 19 | } 20 | ``` 21 | 22 | ### 消息类型 23 | 24 | #### ChatMessage(弹幕) 25 | ``` 26 | data.common.method: ChatMessage 27 | data.content: 弹幕内容 28 | ``` 29 | 30 | #### GiftMessage(礼物) 31 | ``` 32 | data.common.method: GiftMessage 33 | data.gift_id: 礼物id 34 | data.gift_id.combo_count: 礼物个数 35 | ``` 36 | 37 | #### DoodleGiftMessage(涂鸦礼物) 38 | ``` 39 | data.compose.points: [ // 涂鸦礼物数组，每个礼物一个 object， 涂鸦礼物可以由不同礼物即不同 礼物id 组成 40 | { 41 | x, // 涂鸦礼物屏幕显示 坐标 42 | y, // 涂鸦礼物屏幕显示 坐标 43 | id // 礼物id，数目为1 44 | } 45 | ] 46 | data.gift_id: 礼物id 47 | 没有礼物个数，涂鸦礼物一般为一个，由多个其他礼物构成 48 | ``` 49 | 50 | #### CommonLuckyMoneyMessage(手气红包) 51 | 一般为观众发，观众抢 52 | 53 | #### RoomNotifyMessage(其他直播间大礼物显示) 54 | 55 | #### RoomMessage(自己进入直播间提示信息) 56 | 57 | #### MemberMessage(观众进入直播间) 58 | 59 | #### DiggMessage(点击屏幕出现小动物等，没什么卵用) 60 | 61 | #### SunDailyRankMessage(主播个人排行榜信息) 62 | 63 | #### PushMessage (推送) 64 | 65 | #### UserStatsMessage(应该是没用) 66 | 67 | #### SocialMessage(没有辅助信息进行判断用途) 68 | -------------------------------------------------------------------------------- /config/mysql.js: -------------------------------------------------------------------------------- 1 | /** 2 | * tang xiaoji 3 | */ 4 | 5 | const mysql = require('mysql'); 6 | const config = require('./config'); 7 | const Logger = require('./Logger'); 8 | const logger = new Logger('mysql'); 9 | 10 | const pool = mysql.createPool(config.mysql); 11 | logger.log('mysql 已连接!'); 12 | 13 | exports.pool = pool; 14 | 15 | exports.getConn = (cb) => { 16 | pool.getConnection((err, conn) => { 17 | if (err) { 18 | cb(err); 19 | logger.error('mysql connect error', 'mysql', err); 20 | } else { 21 | cb(null, conn); 22 | } 23 | }); 24 | } 25 | 26 | exports.asyncGetConn = () => { 27 | return new Promise((resolve) => { 28 | pool.getConnection((err, conn) => { 29 | resolve([err, conn]); 30 | }); 31 | }); 32 | } 33 | 34 | exports.asyncQuery = (sql, params, conn) => { 35 | return new Promise((resolve) => { 36 | if (conn) { 37 | conn.query(sql, params, (err, res) => { 38 | resolve([err, res]); 39 | }); 40 | } else { 41 | pool.getConnection((err, conn) => { 42 | if (err) { 43 | resolve([err]); 44 | } else { 45 | conn.query(sql, params, (err, res) => { 46 | conn.release(); 47 | resolve([err, res]); 48 | }); 49 | } 50 | }); 51 | } 52 | }); 53 | } 54 | -------------------------------------------------------------------------------- /helper/proto.js: -------------------------------------------------------------------------------- 1 | const ProtoBuf = require('protobufjs'); 2 | 3 | /** 4 | * PB对象 5 | * @type {{}} 6 | */ 7 | const proto_root = {}; 8 | 9 | /** 10 | * PB封包 11 | * @param protoPath 12 | * @param msgType 13 | * @param body 14 | */ 15 | async function protoEncode(protoPath, msgType, body) { 16 | if (!proto_root[protoPath]) { 17 | proto_root[protoPath] = await ProtoBuf.load(protoPath); 18 | } 19 | let msgT = proto_root[protoPath].lookupType(msgType); 20 | let errMsg = msgT.verify(body); 21 | if (errMsg) { 22 | console.log('pb invalid body', body); 23 | throw new Error(errMsg); 24 | } 25 | let message = msgT.create(body); 26 | return msgT.encode(message).finish(); 27 | } 28 | 29 | /** 30 | * PB解包 31 | * @param protoPath 32 | * @param msgType 33 | * @param buffer 34 | */ 35 | async function protoDecode(protoPath, msgType, buffer) { 36 | if (!proto_root[protoPath]) { 37 | proto_root[protoPath] = await ProtoBuf.load(protoPath); 38 | } 39 | let msgT = proto_root[protoPath].lookupType(msgType); 40 | let message = msgT.decode(buffer); 41 | return msgT.toObject(message, { 42 | longs: String, 43 | enums: String, 44 | bytes: String 45 | // see ConversionOptions 46 | }); 47 | } 48 | 49 | module.exports = { 50 | encode: protoEncode, 51 | decode: protoDecode 52 | }; 53 | -------------------------------------------------------------------------------- /test/test_zhanqi.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Creator: Tang Xiaoji 3 | * Time: 2018-05-23 4 | */ 5 | 6 | const Zhanqi = require('../danmu/Zhanqi'); 7 | // let connected = 0; 8 | // let closed = 0; 9 | 10 | // let tasks = []; 11 | // for(let i = 0; i < 100; i++) { 12 | // tasks[i] = new Zhanqi('https://www.zhanqi.tv/873062370'); 13 | // console.log('listen', i); 14 | // tasks[i].on('connect', () => { 15 | // connected++; 16 | // // console.log(`connect ${i}`) 17 | // console.log('connected: ', connected, ' closed: ', closed); 18 | // }); 19 | 20 | // tasks[i].on('data', (data) => { 21 | // console.log(data) 22 | // }); 23 | 24 | // tasks[i].on('error', (err) => { 25 | // console.log(`error ${i}`, err) 26 | // }); 27 | 28 | // tasks[i].on('close', () => { 29 | // closed++; 30 | // // console.log(`close ${i}`) 31 | // console.log('connected: ', connected, ' closed: ', closed); 32 | // }); 33 | // } 34 | 35 | const zhanqi = new Zhanqi('https://www.zhanqi.tv/8888'); 36 | zhanqi.on('connect', () => { 37 | console.log('connected'); 38 | }); 39 | 40 | zhanqi.on('data', (data) => { 41 | console.log(data) 42 | }); 43 | 44 | zhanqi.on('error', (err) => { 45 | console.log(`error`, err) 46 | }); 47 | 48 | zhanqi.on('close', () => { 49 | console.log('connected: ', ' closed: '); 50 | }); 51 | -------------------------------------------------------------------------------- /test/test_douyu.js: -------------------------------------------------------------------------------- 1 | /** 2 | * 3 | * @type {module.Douyu|*} 4 | */ 5 | 6 | // const Douyu = require('../danmu/Douyu'); 7 | // let connected = 0; 8 | // let closed = 0; 9 | // 10 | // let tasks = []; 11 | // for(let i = 0; i < 600; i++) { 12 | // tasks[i] = new Douyu('http://www.douyu.com/4601529'); 13 | // 14 | // tasks[i].on('connect', () => { 15 | // connected++; 16 | // // console.log(`connect ${i}`) 17 | // console.log('connected: ', connected, ' closed: ', closed); 18 | // }); 19 | // 20 | // tasks[i].on('data', (data) => { 21 | // // console.log(data) 22 | // }); 23 | // 24 | // tasks[i].on('error', (err) => { 25 | // // console.log(`error ${i}`, err) 26 | // }); 27 | // 28 | // tasks[i].on('close', () => { 29 | // closed++; 30 | // // console.log(`close ${i}`) 31 | // console.log('connected: ', connected, ' closed: ', closed); 32 | // }); 33 | // } 34 | 35 | const {DanmuClient, TYPE} = require('../danmu'); 36 | const douyu = new DanmuClient(TYPE.douyu, 'https://www.douyu.com/1655193'); 37 | 38 | douyu.on('monitor_start', () => { 39 | console.log('connect') 40 | }); 41 | 42 | douyu.on('data', (data) => { 43 | console.log(data) 44 | }); 45 | 46 | douyu.on('error', (err) => { 47 | console.log('error', err) 48 | }); 49 | 50 | douyu.on('close', () => { 51 | console.log('close') 52 | }); 53 | -------------------------------------------------------------------------------- /danmu/YY.js: -------------------------------------------------------------------------------- 1 | const EventEmitter = require('events'); 2 | const YYHandler = require('../helper/yy-sdk'); 3 | const redis = require('../config/redis'); 4 | const config = require('../config/config'); 5 | 6 | module.exports = class YY extends EventEmitter { 7 | constructor (url = '', opts) { 8 | super(); 9 | this.url = url; 10 | 11 | this.init(); 12 | } 13 | 14 | async init () { 15 | try { 16 | await redis.lpush(config.LIST_GIFT_TASK_KEY(config.plat.yy, this.url)); 17 | 18 | this.socket = new YYHandler(); 19 | const [topSid, subSid] = this.parseSid(); 20 | this.socket.joinChannel(topSid, subSid); 21 | this.socket.agent().on('data', (data) => this.clientDataHandler(data)); 22 | this.socket.agent().on('gift', (data) => this.clientGiftHandler(data)); 23 | } catch (e) { 24 | this.emit('initerror', e); 25 | } 26 | } 27 | 28 | destroy () { 29 | this.socket.logout(); 30 | this.removeAllListeners(); 31 | } 32 | 33 | parseSid () { 34 | let [ , , , topSid, subSid] = this.url.split(/[/?]/); 35 | return [topSid, subSid]; 36 | } 37 | 38 | clientDataHandler (data) { 39 | data.type = 'chat'; 40 | this.emit('data', [].concat(data)); 41 | } 42 | 43 | clientGiftHandler (gift) { 44 | gift.type = 'gift'; 45 | this.emit('data', [].concat(gift)); 46 | } 47 | }; 48 | -------------------------------------------------------------------------------- /docs/战旗采集流程.md: -------------------------------------------------------------------------------- 1 | 请求 https://www.zhanqi.tv/873062370 2 | 3 | 匹配 regexp：/window.oPageConfig.oRoom\s*=\s*([\s\S]+?);\n/g; 4 | 5 | 获取 eyJsaXN0IjpbeyJpcCI6IjExMi4xMjQuMTkuMjAzIiwicG9ydCI6MTUwMTAsImlkIjo0NiwiY2hhdHJvb21faWQiOjQ2fSx7ImlwIjoiMTIwLjU1LjExNi4yMTkiLCJwb3J0IjoxNTAxMCwiaWQiOjk4LCJjaGF0cm9vbV9pZCI6OTh9LHsiaXAiOiI0Ny45Ni44Mi4xNDIiLCJwb3J0IjoxNTAxMCwiaWQiOjE0MywiY2hhdHJvb21faWQiOjE0M31dfQ== 6 | 7 | base64解析 8 | 9 | '{"list":[{"ip":"112.124.19.203","port":15010,"id":46,"chatroom_id":46},{"ip":"120.55.116.219","port":15010,"id":98,"chatroom_id":98},{"ip":"47.96.82.142","port":15010,"id":143,"chatroom_id":143}]}' 10 | 11 | 随机获取其中一个ip和port 12 | 13 | 14 | 15 | 请求房间信息roominfo： 16 | 17 | https://www.zhanqi.tv/api/public/room.viewer 18 | 19 | 20 | 21 | 使用上述获取的ip和port 连接tcp， 22 | 23 | 发送请求 (oroom为上述regexp匹配到的对象) 24 | 25 | {nickname: '', 26 | 27 | roomid: parseInt(this.oroom.id), 28 | 29 | gid: roomInfo.gid, 30 | 31 | sid: roomInfo.sid, 32 | 33 | ssid: roomInfo.sid, 34 | 35 | timestamp: roomInfo.timestamp, 36 | 37 | cmdid: 'loginreq', 38 | 39 | develop_date: '2015-06-07', 40 | 41 | fhost: 'zhanqi.tool', 42 | 43 | fx: 0, 44 | 45 | t: 0, 46 | 47 | thirdacount: '', 48 | 49 | uid: 0, 50 | 51 | ver: 2, 52 | 53 | vod: 0} 54 | 55 | 发送心跳包 [0xbb, 0xcc, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x59, 0x27] 56 | 57 | 58 | 59 | 编解码：小端法 60 | 61 | 62 | -------------------------------------------------------------------------------- /docs/平台数据/已释放.md: -------------------------------------------------------------------------------- 1 | 已释放 2 | 3 | 121.196.206.10 4 | 5 | 118.31.20.139 6 | 7 | 118.31.79.77 8 | 9 | 47.96.129.45 10 | 11 | 47.96.183.43 12 | 13 | 116.62.184.248 14 | 15 | 121.196.208.231 16 | 17 | 116.62.137.80 18 | 19 | 116.62.128.97 20 | 21 | 116.62.197.241 22 | 23 | 116.62.130.36 24 | 25 | 116.62.163.58 26 | 27 | 116.62.150.71 28 | 29 | --- 30 | 31 | 121.43.188.128 32 | 33 | 116.62.121.34 34 | 35 | 116.62.189.46 36 | 37 | 121.43.179.12 38 | 39 | 116.62.123.145 40 | 41 | 116.62.169.142 42 | 43 | 116.62.203.164 44 | 45 | 116.62.163.173 46 | 47 | 116.62.174.4 48 | 49 | 121.43.172.221 50 | 51 | 已重绑 52 | 53 | 10.4.20.38 54 | 55 | 10.4.20.22 56 | 57 | 。。。 58 | 59 | 10.4.20.36 60 | 61 | 10.4.20.21 62 | 63 | 10.4.20.16 64 | 65 | 重复ip 66 | 67 | 118.31.20.139 68 | 69 | 116.62.150.71 70 | 71 | 116.62.121.34 72 | 73 | 116.62.189.46 74 | 75 | 116.62.128.97 76 | 77 | 121.196.206.10 78 | 79 | 121.196.208.231 80 | 81 | 116.62.130.36 82 | 83 | 118.31.79.77 84 | 85 | 116.62.169.142 86 | 87 | 116.62.163.173 88 | 89 | 116.62.203.164 90 | 91 | 121.43.172.221 92 | 93 | 116.62.197.241 94 | 95 | 116.62.174.4 96 | 97 | 116.62.137.80 98 | 99 | 116.62.163.58 100 | 101 | 116.62.184.248 102 | 103 | 116.62.123.145 104 | 105 | 118.31.32.127 106 | 107 | 108 | 109 | 110 | 111 | Scan 112 | 113 | 116.62.136.151 -------------------------------------------------------------------------------- /gather/video-stream/HuyaStream.js: -------------------------------------------------------------------------------- 1 | const VideoStream = require('./VideoStream'); 2 | const rp = require('request-promise'); 3 | const QUALITY = { 4 | BLUE: 0, 5 | SUPER: 2000, 6 | HIGH: 1200, 7 | FLUENCY: 500 8 | }; 9 | 10 | class HuyaStream extends VideoStream { 11 | 12 | constructor (plat = 'huya') { 13 | super(plat); 14 | } 15 | 16 | async getStreamAddr (plat, roomId, url) { 17 | try { 18 | const { sStreamName, sFlvUrl, sFlvUrlSuffix, sFlvAntiCode } = await this.getParams(plat, roomId, url); 19 | return `${sFlvUrl}/${sStreamName}.${sFlvUrlSuffix}?${sFlvAntiCode}&ratio=${QUALITY.FLUENCY}`; 20 | } catch (e) { 21 | console.log(`get plat: ${plat}, rid: ${roomId} stream addr error`); 22 | } 23 | } 24 | 25 | async getParams (plat, roomId, url) { 26 | try { 27 | let res = await rp({ uri: url }); 28 | res = res.replace(/ |\r\n/g, ''); 29 | res = res.match(/"stream":(\S*)};window/)[1]; 30 | res = JSON.parse(res); 31 | const gameStreamInfoList = res.data[0].gameStreamInfoList; 32 | const gameStreamInfo = gameStreamInfoList[0]; 33 | const { sStreamName, sFlvUrl, sFlvUrlSuffix, sFlvAntiCode } = gameStreamInfo; 34 | return { sStreamName, sFlvUrl, sFlvUrlSuffix, sFlvAntiCode }; 35 | } catch (e) { 36 | console.log(`get plat: ${plat}, rid: ${roomId} query params error`); 37 | } 38 | } 39 | 40 | } 41 | 42 | module.exports = new HuyaStream(); -------------------------------------------------------------------------------- /gather/video-stream/FanxingStream.js: -------------------------------------------------------------------------------- 1 | const VideoStream = require('./VideoStream'); 2 | const rp = require('request-promise'); 3 | 4 | class FanxingStream extends VideoStream { 5 | 6 | constructor (plat = 'fanxing') { 7 | super(plat); 8 | 9 | this.uri = 'http://fx2.service.kugou.com/video/pc/live/pull/v1/streamaddr.jsonp' 10 | } 11 | 12 | spliceQuery (roomId) { 13 | let qs = { 14 | roomId, 15 | version: '1.0', 16 | platform: 7, 17 | streamType: '1-2-3', 18 | ch: 'fx', 19 | ua: 'fx-flash', 20 | kugouId: 1293910630, 21 | layout: 1, 22 | _: Math.random() 23 | }; 24 | let jsonpcallback = ''; 25 | jsonpcallback += this.uri; 26 | for (let [k, v] of Object.entries(qs)) { 27 | jsonpcallback += (k + v); 28 | } 29 | jsonpcallback = jsonpcallback.replace(/[:/.-]/g, ''); 30 | qs.jsonpcallback = jsonpcallback; 31 | return qs; 32 | } 33 | 34 | async getStreamAddr (plat, roomId, url) { 35 | try { 36 | const qs = this.spliceQuery(roomId); 37 | let res = await rp({ 38 | uri: this.uri, 39 | qs 40 | }); 41 | const str = res.slice(qs.jsonpcallback.length + 1, -1); 42 | res = JSON.parse(str); 43 | return res.data.httpflv[0]; 44 | } catch (e) { 45 | console.log(`get plat: ${plat}, rid: ${roomId} stream addr error`); 46 | } 47 | } 48 | 49 | } 50 | 51 | module.exports = new FanxingStream(); -------------------------------------------------------------------------------- /docs/pb接口文档.md: -------------------------------------------------------------------------------- 1 | * post /zhibobao.personal.account/Account/reqCode 2 | 3 | ```Json 4 | {mobile: '15158161535'} // 必须 5 | ``` 6 | 7 | * post /zhibobao.personal.account/Account/login 8 | 9 | ```Json 10 | { 11 | mobile: '15158161535', // 必须 12 | code: '123456', // 必须 13 | password: 'djkenwdew', 14 | type: '0', 15 | cacheId: 'nnnnn' 16 | } 17 | ``` 18 | 19 | * get /zhibobao.personal.account/Account/logout 20 | * post /zhibobao.personal.account/Account/pwlogin 21 | 22 | ```json 23 | { 24 | mobile: '151598161535', // 必须 25 | code: '111111', // 必须 26 | type: '0' 27 | } 28 | ``` 29 | 30 | * post /zhibobao.personal.account/Account/pwModify 31 | 32 | ```json 33 | { 34 | password: 'dewdwedw', // 必须 35 | mobile: '15158161535', // 必须 36 | code: '123457' // 必须 37 | } 38 | ``` 39 | 40 | * get /zhibobao.personal.account/Account/getInfo 41 | * post /zhibobao.personal.account/Account/modifyInfo 42 | 43 | ```json 44 | { 45 | nickname: 'hdehd', // 必须 46 | province: '浙江', // 必须 47 | city: '杭州', // 必须 48 | gender: 0(女)/1(男)/-1(未知), // 必须 49 | avatar: 'http://dedeknde.img' 50 | } 51 | ``` 52 | 53 | * post /zhibobao.personal.account/Account/checkOldMobile 54 | 55 | ```json 56 | { 57 | mobile: '151598161535', 58 | code: '123456' 59 | } 60 | ``` 61 | 62 | * post /zhibobao.personal.account/Account/modifyOldMobile 63 | 64 | ```json 65 | { 66 | mobile: '15158161535', 67 | code: '123456' 68 | } 69 | ``` 70 | 71 | 72 | 73 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "watcher", 3 | "version": "0.0.1", 4 | "description": "", 5 | "scripts": { 6 | "test": "./node_modules/mocha/bin/mocha test -t 5000" 7 | }, 8 | "author": "eroszhao", 9 | "license": "ISC", 10 | "dependencies": { 11 | "base-64": "^0.1.0", 12 | "bluebird": "^3.5.0", 13 | "buffer-crc32": "^0.2.13", 14 | "cheerio": "^1.0.0-rc.2", 15 | "child-process-promise": "^2.2.1", 16 | "compression": "^1.7.1", 17 | "debug": "^2.6.9", 18 | "elasticsearch": "^13.3.1", 19 | "express": "^4.15.5", 20 | "fast-xml-parser": "^3.11.1", 21 | "filereader": "^0.10.3", 22 | "ioredis": "^3.1.4", 23 | "ip": "^1.1.5", 24 | "json-socket": "^0.3.0", 25 | "koa": "^2.5.2", 26 | "koa-bodyparser": "^4.2.1", 27 | "koa-router": "^7.4.0", 28 | "lodash": "^4.17.4", 29 | "log4js": "^3.0.5", 30 | "md5": "^2.2.1", 31 | "moment": "^2.18.1", 32 | "msgpack-lite": "^0.1.26", 33 | "mysql": "^2.14.1", 34 | "protobufjs": "^6.8.0", 35 | "redis": "^2.8.0", 36 | "redis-parser": "^3.0.0", 37 | "request": "^2.83.0", 38 | "request-promise": "^4.2.2", 39 | "socks-proxy-agent": "^4.0.1", 40 | "superagent": "^3.6.0", 41 | "superagent-proxy": "^1.0.3", 42 | "tcharts.js": "0.0.4", 43 | "to-arraybuffer": "^1.0.1", 44 | "uuid": "^3.2.1", 45 | "voca": "^1.3.0", 46 | "websocket": "^1.0.24", 47 | "ws": "^5.2.2", 48 | "zlib": "^1.0.5" 49 | }, 50 | "devDependencies": { 51 | "mocha": "^3.5.3" 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /docs/huya.md: -------------------------------------------------------------------------------- 1 | 6501 礼物 2 | 1400 弹幕/系统消息 3 | 4 | ```javascript 5 | [ 6 | { cmd: 6501, 7 | data: 8 | { 9 | iItemType: 4, 10 | strPayId: '', 11 | iItemCount: 1, 12 | lPresenterUid: 367138632, 13 | lSenderUid: 1547367511, 14 | sPresenterNick: '卡尔', 15 | sSenderNick: '猪宝宝', 16 | sSendContent: '', 17 | iItemCountByGroup: 1, 18 | iItemGroup: 4, 19 | iSuperPupleLevel: 0, 20 | iComboScore: 4, 21 | iDisplayInfo: 3, 22 | iEffectType: 0, 23 | iSenderIcon: 'http://huyaimg.msstatic.com/avatar/1029/d2/de9fd910e70da4fb22eea0de9e8e28_180_135.jpg', 24 | iPresenterIcon: 'http://huyaimg.msstatic.com/avatar/1084/b7/896bc815db9560eabbcb4a227f62ba_180_135.jpg?1416387967', 25 | iTemplateType: 4, 26 | sExpand: '', 27 | bBusi: false, 28 | iColorEffectType: -1 29 | } 30 | } 31 | ] 32 | ``` 33 | 34 | ```javascript 35 | [ 36 | { 37 | cmd: 1400, 38 | data: 39 | { 40 | tUserInfo: [Object], 41 | lTid: 77259038, 42 | lSid: 2622305980, 43 | sContent: '叫声虎爷每人100💰💰💰', 44 | iShowMode: 0, 45 | tFormat: [Object], 46 | tBulletFormat: [Object], 47 | iTermType: 8, 48 | vDecorationPrefix: [Object], 49 | vDecorationSuffix: [Object], 50 | vAtSomeone: [Object], 51 | lPid: 367138632 52 | } 53 | } 54 | ] 55 | ``` -------------------------------------------------------------------------------- /helper/http.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Creator: Tang Xiaoji 3 | * Time: 2017-11-02 4 | */ 5 | 6 | const url = require('url'); 7 | const request = require('superagent'); 8 | require('superagent-proxy')(request); 9 | 10 | const rp = require('request-promise'); 11 | 12 | const RESPONSE_TIMEOUT = 15e3; 13 | const DEADLINE_TIMEOUT = 6e4; 14 | 15 | class HttpClient { 16 | constructor(response = RESPONSE_TIMEOUT, deadline = DEADLINE_TIMEOUT) { 17 | this.response = response; 18 | this.deadline = deadline; 19 | } 20 | 21 | Get(u, host, queryParams) { 22 | const srvUrl = url.parse(u); 23 | return request 24 | .get(u) 25 | .query(queryParams || {}) 26 | .set('host', host || srvUrl.hostname) 27 | .set('Content-Type', 'application/json') 28 | .set('Connection', 'keep-alive') 29 | .set('User-Agent', 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.115 Safari/537.36') 30 | .set('X-Requested-With', 'XMLHttpRequest') 31 | .timeout({ 32 | response: this.response, 33 | deadline: this.deadline 34 | }); 35 | } 36 | 37 | LongzhuGet(url) { 38 | return rp({ 39 | uri: url, 40 | headers: { 41 | 'Host': 'api.plu.cn', 42 | 'Content-Type': 'application/json', 43 | 'Connection': 'keep-alive', 44 | 'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/59.0.3071.115 Safari/537.36', 45 | 'X-Requested-With': 'XMLHttpRequest' 46 | }, 47 | }); 48 | } 49 | 50 | Post(url, params) { 51 | return rp({ 52 | method: 'POST', 53 | uri: url, 54 | gzip: true, 55 | forever: true, 56 | json: params 57 | }); 58 | } 59 | } 60 | 61 | module.exports = new HttpClient(); -------------------------------------------------------------------------------- /danmu/Conn.js: -------------------------------------------------------------------------------- 1 | const EventEmitter = require('events'); 2 | const Socket = require('net').Socket; 3 | 4 | const SOCKET_TIMEOUT = 3000; 5 | 6 | module.exports = class Conn extends EventEmitter { 7 | constructor() { 8 | super(); 9 | this.socket = new Socket(); 10 | this.socket.setNoDelay(true); 11 | this.socket.setTimeout(SOCKET_TIMEOUT); 12 | this.socket.on('timeout', this.timeoutHandler.bind(this)); 13 | this.socket.on('connect', this.connectHandler.bind(this)); 14 | this.socket.on('data', this.dataHandler.bind(this)); 15 | this.socket.on('close', this.closeHandler.bind(this)); 16 | this.socket.on('error', this.errorHandler.bind(this)); 17 | } 18 | 19 | get destroyed() { 20 | return this.socket.destroyed; 21 | } 22 | 23 | connect(port = '8081', host = '127.0.0.1') { 24 | this.socket.connect(port, host); 25 | } 26 | 27 | write(data, raw = false) { 28 | if (!raw && typeof this.encode == 'function') { 29 | data = this.encode(data); 30 | } 31 | this.socket.write(data); 32 | } 33 | 34 | destroy() { 35 | this.removeAllListeners(); 36 | this.socket.removeAllListeners(); 37 | this.socket.destroy(); 38 | this.emit('destroy'); 39 | } 40 | 41 | encode(data) { 42 | return data; 43 | } 44 | 45 | decode(buffer) { 46 | return buffer; 47 | } 48 | 49 | connectHandler() { 50 | this.emit('connect'); 51 | } 52 | 53 | timeoutHandler() { 54 | this.emit('timeout'); 55 | } 56 | 57 | dataHandler(buffer) { 58 | if (typeof this.encode == 'function') { 59 | buffer = this.decode(buffer); 60 | } 61 | this.emit('rawdata', buffer); 62 | } 63 | 64 | closeHandler() { 65 | this.emit('close'); 66 | } 67 | 68 | errorHandler(error) { 69 | this.emit('error', error); 70 | } 71 | }; 72 | -------------------------------------------------------------------------------- /test/test_type.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Creator: Tang Xiaoji 3 | * Time: 2018-05-21 4 | */ 5 | 6 | const Danmu = { 7 | Douyu: require('../danmu/Douyu'), 8 | Panda: require('../danmu/Panda'), 9 | XYPanda: require('../danmu/XYPanda'), 10 | Zhanqi: require('../danmu/Zhanqi'), 11 | Longzhu: require('../danmu/Longzhu'), 12 | Chushou: require('../danmu/Chushou'), 13 | Huya: require('../danmu/HuyaV2'), 14 | Kugou: require('../danmu/Kugou'), 15 | YY: require('../danmu/YY') 16 | }; 17 | 18 | let TYPE = {}; 19 | 20 | for (let key of Object.keys(Danmu)) { 21 | TYPE[key] = key; 22 | } 23 | 24 | const PATTERN = { 25 | douyu: /www\.douyu\.com\/\w+(?=\?|$)/i, 26 | panda: /www\.panda\.tv\/\w+(?=\?|$)/i, 27 | xypanda: /xingyan\.panda\.tv\/\w+(?=\?|$)/i, 28 | zhanqi: /www\.zhanqi\.tv\/\w+[\w\/]+(?=\?|$)/i, 29 | longzhu: /(star|y)\.longzhu\.com\/\w+(?=\?|$)/, 30 | chushou: /chushou\.tv\/room\/\w+\.htm(?=\?|$)/, 31 | huya: /www\.huya\.com\/\w+(?=\?|$)/, 32 | quanmin: /www\.quanmin\.tv\/(\d+|(v|star)\/\w+)(?=\?|$)/, 33 | fanxing: /fanxing\.kugou\.com\/\w+(?=\?|$)/i, 34 | yy: /www\.yy\.com\/\w+\/\w+\?tempId=\w+/i 35 | }; 36 | 37 | getType = url => { 38 | for (var type in PATTERN) { 39 | const fn = PATTERN[type]; 40 | if (fn instanceof RegExp && fn.test(url)) { 41 | return type; 42 | } else if (typeof fn === 'function' && fn(url)) { 43 | return type; 44 | } 45 | } 46 | }; 47 | 48 | valid = (plat, url) => { 49 | const type = getType(url); 50 | console.log(plat, type) 51 | return type && TYPE.hasOwnProperty(type) && plat === type.toLowerCase(); 52 | }; 53 | 54 | console.log(valid('yy', "http://www.yy.com/96397497/96397497?tempId=16777217")); 55 | console.log(valid('douyu', "http://www.douyu.com/12341")); 56 | console.log(valid('quanmin', "https://www.quanmin.tv/12116933")); 57 | -------------------------------------------------------------------------------- /test/test_fanxing.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Creator: Tang Xiaoji 3 | * Time: 2018-05-07 4 | */ 5 | 6 | const danmu = require('../danmu'); 7 | 8 | // let tasks = []; 9 | // let connected = 0; 10 | // let closed = 0; 11 | // 12 | // for(let i = 0; i < 1; i++) { 13 | // tasks[i] = new danmu.DanmuClient(danmu.TYPE.Fanxing, 'http://fanxing.kugou.com/1428462'); 14 | // 15 | // tasks[i].on('connect', () => { 16 | // console.log('connect'); 17 | // connected++; 18 | // console.log('connected: ', connected, ' closed: ', closed); 19 | // }); 20 | // 21 | // tasks[i].on('data', (data) => { 22 | // // console.log(data) 23 | // }); 24 | // 25 | // tasks[i].on('error', (err) => { 26 | // console.log('err: ', err); 27 | // closed++; 28 | // console.log('connected: ', connected, ' closed: ', closed); 29 | // }); 30 | // 31 | // tasks[i].on('close', () => { 32 | // console.log('close') 33 | // }); 34 | // } 35 | 36 | const fanxing = new danmu.DanmuClient(danmu.TYPE.fanxing, 'http://fanxing.kugou.com/2517805'); 37 | console.log(fanxing) 38 | 39 | fanxing.on('connect', () => { 40 | console.log('connect'); 41 | }); 42 | 43 | fanxing.on('data', (data) => { 44 | console.log(data) 45 | }); 46 | 47 | fanxing.on('error', (err) => { 48 | console.log('err: ', err); 49 | }); 50 | 51 | fanxing.on('close', () => { 52 | console.log('close') 53 | }); 54 | 55 | // const Kugou = require('../danmu/KugouH5'); 56 | // 57 | // const kugou = new Kugou('http://fanxing.kugou.com/1428462'); 58 | // 59 | // kugou.on('connect', () => { 60 | // console.log(`connect`) 61 | // }); 62 | // 63 | // kugou.on('data', (data) => { 64 | // console.log(data) 65 | // }); 66 | // 67 | // kugou.on('error', (err) => { 68 | // console.log(`error: `, err) 69 | // }); 70 | // 71 | // kugou.on('close', () => { 72 | // console.log('close') 73 | // }); -------------------------------------------------------------------------------- /gather/lib/searcher.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Creator: Tang Xiaoji 3 | * Time: 2018-06-23 4 | */ 5 | 6 | 'use strict'; 7 | 8 | const EventEmitter = require('events'); 9 | const config = require('../../config/config'); 10 | const redis = require('../../config/redis'); 11 | const moment = require('moment'); 12 | const Logger = require('../../config/Logger'); 13 | const logger = new Logger('searcher'); 14 | 15 | class Searcher extends EventEmitter { 16 | constructor() { 17 | super(); 18 | } 19 | 20 | async runTask(cb) { 21 | logger.log('开始检索任务...'); 22 | for (let plat in config.plat_effective) { 23 | if(config.plat_effective.hasOwnProperty(plat)) { 24 | try { 25 | const now = moment().unix(); 26 | const effect = now - config.plat_effective[plat]; 27 | 28 | const latest_update_time = await redis.get(config.TASK_LATEST_UPDATE_TIME_KEY(plat)); 29 | if (!latest_update_time || latest_update_time < effect) { 30 | // 超时未更新，设定任务失败 31 | this.emit('error', 'searcher', `超时未更新 latest: ${moment(Number(latest_update_time || 0) * 1000).format('YYYY-MM-DD HH:mm:ss')}`); 32 | } else { 33 | const start = await redis.zrangebyscore(config.SORTED_SET_TASK(plat), effect, '+inf'); 34 | const stop = await redis.zrangebyscore(config.SORTED_SET_TASK(plat), '-inf', effect); 35 | this.emit('start', plat, start); 36 | this.emit('stop', plat, stop); 37 | cb && cb(plat, `采集任务: ${start.length}, 关闭任务: ${stop.length}`); 38 | } 39 | } catch (e) { 40 | this.emit('error', 'searcher', e); 41 | continue; 42 | } 43 | } else { 44 | logger.log('没有这个平台', plat); 45 | } 46 | } 47 | 48 | setTimeout(() => this.runTask(cb), config.opp.search_task); 49 | } 50 | } 51 | 52 | module.exports = Searcher; 53 | -------------------------------------------------------------------------------- /danmu/KugouH5.js: -------------------------------------------------------------------------------- 1 | const Kugou = require('./Kugou'); 2 | const crypto = require('crypto'); 3 | const rp = require('request-promise'); 4 | const config = require('../config/config'); 5 | const Logger = require('../config/Logger'); 6 | const logger = new Logger('KugouH5/danmu'); 7 | 8 | module.exports = class KugouH5 extends Kugou { 9 | constructor (url = '', opts) { 10 | super(url, opts); 11 | } 12 | 13 | async getWs () { 14 | const formData = { 15 | roomId: this.roomId, 16 | kugouId: 0, 17 | token: 0, 18 | appid: 2815, 19 | channel: 2, 20 | platform: 207, 21 | times: +new Date() 22 | }; 23 | let res; 24 | try { 25 | formData.sign = this._generateSign(formData); 26 | res = await rp({ 27 | uri: 'http://mo.fanxing.kugou.com/mfx/h5/share/live/getSocketInfo', 28 | method: 'POST', 29 | formData 30 | }); 31 | res = res.match(/window.name='(\S*)';/)[1]; 32 | res = JSON.parse(res); 33 | const socketInfo = res.data.socketInfo; 34 | return socketInfo[0].ip + ':1314'; 35 | } catch (e) { 36 | logger.error('getws error', config.plat.fanxing, e); 37 | } 38 | } 39 | 40 | _generateSign (formData) { 41 | let values = Object.values(formData); 42 | values = values.sort(); 43 | values.sort(); 44 | const str = values.join('') + '#FX_md5*!'; 45 | return crypto.createHash('md5').update(str).digest('hex'); 46 | } 47 | 48 | login () { 49 | this.send({ 50 | appid: null, 51 | cmd: 201, 52 | kugouid: null, 53 | roomid: this.roomId 54 | }); 55 | } 56 | 57 | heartbeat () { 58 | this.scheduler = setInterval(() => { 59 | this.send('HEARTBEAT_REQUEST'); 60 | }, 5 * 1000); 61 | } 62 | 63 | _decode (data) { 64 | if (data.utf8Data === 'HEARTBEAT_RESPONSE') { 65 | return null; 66 | } 67 | return JSON.parse(data.utf8Data); 68 | } 69 | }; 70 | -------------------------------------------------------------------------------- /danmu/WSConn.js: -------------------------------------------------------------------------------- 1 | const EventEmitter = require('events'); 2 | const WebSocket = require('websocket').client; 3 | 4 | const SOCKET_TIMEOUT = 3000; 5 | 6 | module.exports = class WSConn extends EventEmitter { 7 | constructor() { 8 | super(); 9 | this.destroyed = false; 10 | this.connection = null; 11 | this.socket = new WebSocket(); 12 | this.socket.on('connect', this.connectHandler.bind(this)); 13 | this.socket.on('connectFailed', this.connectFaildHandler.bind(this)); 14 | } 15 | 16 | connect(url = '') { 17 | this.destroyed = false; 18 | this.socket.connect(url); 19 | } 20 | 21 | write(data, raw = false) { 22 | if (!raw && typeof this.encode == 'function') { 23 | data = this.encode(data); 24 | } 25 | this.socket.write(data); 26 | } 27 | 28 | destroy() { 29 | this.destroyed = true; 30 | if (this.connection) { 31 | this.connection.removeAllListeners(); 32 | this.connection.close(); 33 | } 34 | this.socket.removeAllListeners(); 35 | this.socket.abort(); 36 | this.emit('destroy'); 37 | this.removeAllListeners(); 38 | } 39 | 40 | encode(data) { 41 | return data; 42 | } 43 | 44 | decode(buffer) { 45 | return buffer; 46 | } 47 | 48 | connectHandler(connection) { 49 | connection.on('message', this.dataHandler.bind(this)); 50 | connection.on('close', this.closeHandler.bind(this)); 51 | connection.on('error', this.errorHandler.bind(this)); 52 | 53 | if (this.connection) { 54 | this.connection.removeAllListeners(); 55 | this.connection = null; 56 | } 57 | 58 | this.connection = connection; 59 | this.emit('connect'); 60 | } 61 | 62 | connectFaildHandler() { 63 | this.destroyed = true; 64 | this.emit('error', new Error('connect failded')); 65 | this.emit('close'); 66 | } 67 | 68 | dataHandler(buffer) { 69 | if (typeof this.decode == 'function') { 70 | buffer = this.decode(buffer); 71 | } 72 | this.emit('rawdata', buffer); 73 | } 74 | 75 | closeHandler() { 76 | this.destroyed = true; 77 | this.emit('close'); 78 | } 79 | 80 | errorHandler(error) { 81 | this.destroyed = true; 82 | this.emit('error', error); 83 | } 84 | }; 85 | -------------------------------------------------------------------------------- /danmu/Inke.js: -------------------------------------------------------------------------------- 1 | /** 2 | * tangxiaoji 2018-08-13 14:55:00 3 | */ 4 | 5 | const Conn = require('./Conn'); 6 | const zlib = require('zlib'); 7 | const InkDanmuAddr = "60.205.82.107"; 8 | const InkDanmuPort = 80; 9 | 10 | class Inke extends Conn { 11 | constructor(url = '') { 12 | super(); 13 | 14 | this.url = url; 15 | 16 | this.init(); 17 | } 18 | 19 | init() { 20 | this.on('connect', this.onConnectHandler); 21 | this.connect(InkDanmuPort, InkDanmuAddr); 22 | } 23 | 24 | onConnectHandler() { 25 | this.on('rawdata', this.clientDataHandler); 26 | console.log('connect1'); 27 | 28 | this.register(); 29 | } 30 | 31 | clientDataHandler(buf) { 32 | // this.emit('data', data); 33 | 34 | // console.log(buf); 35 | zlib.unzip(Buffer.from(buf), (err, res) => { 36 | if (err) { 37 | console.log(err); 38 | } else { 39 | // console.log(res); 40 | } 41 | }); 42 | } 43 | 44 | register() { 45 | const uid = "1A156FBB" 46 | // topic = "topic_88" 47 | // rid = "1533799576408588" 48 | // uid = "29F04AC0" 49 | const topic = "topic_49" 50 | const rid = "1533872684082049" 51 | const buffString = Buffer.from("090002034c04" + "0072" + uid + "000027100000002d", 'hex'); 52 | 53 | let buffer = Buffer.alloc(100, buffString); 54 | let offset = 0; 55 | 56 | offset = Buffer.byteLength(buffString); 57 | const arr = [0x0001, 0x0600, "domain", 0x0500, "group", 0x0800, topic, 0x1000, rid]; 58 | arr.forEach(a => { 59 | if (typeof a === 'string') { 60 | buffer.fill(Buffer.from(a), offset); 61 | offset += Buffer.byteLength(Buffer.from(a)); 62 | } else { 63 | buffer.fill(Buffer.from([a]), offset); 64 | offset += Buffer.byteLength(Buffer.from([a])); 65 | } 66 | console.log('offset: ', offset); 67 | console.log(buffer); 68 | }); 69 | this.write(buffer, true); 70 | } 71 | 72 | send() { 73 | 74 | } 75 | } 76 | 77 | module.exports = Inke 78 | -------------------------------------------------------------------------------- /gather/video-stream/ZhanqiStream.js: -------------------------------------------------------------------------------- 1 | const VideoStream = require('./VideoStream'); 2 | const rp = require('request-promise'); 3 | const QUALITY = { 4 | NORMAL: '_360p', 5 | HIGH: '_480p', 6 | SUPER: '_720p', 7 | HD: '' 8 | }; 9 | 10 | class ZhanqiStream extends VideoStream { 11 | 12 | constructor (plat = 'zhanqi') { 13 | super(plat); 14 | } 15 | 16 | async getStreamAddr (plat, roomId, url) { 17 | try { 18 | const [videoId, gid] = await Promise.all([ 19 | this.getVideoId(plat, roomId, url), 20 | this.getGid(plat, roomId) 21 | ]); 22 | const key = await this.getKey(plat, roomId, videoId, gid); 23 | return `https://alhdl-cdn.zhanqi.tv/zqlive/${videoId}${QUALITY.NORMAL}.flv?${key}&fhost=h5&platform=128` 24 | } catch (e) { 25 | console.log(`get plat: ${plat}, rid: ${roomId} stream addr error`); 26 | } 27 | } 28 | 29 | async getVideoId (plat, roomId, url) { 30 | try { 31 | const res = await rp({ uri: url }); 32 | const videoId = res.match(/"videoId":"(\S*)","chatStatus"/)[1]; 33 | return videoId; 34 | } catch (e) { 35 | console.log(`get plat: ${plat}, rid: ${roomId} video id error`); 36 | } 37 | } 38 | 39 | async getGid (plat, roomId) { 40 | try { 41 | const res = await rp({ 42 | uri: 'https://www.zhanqi.tv/api/public/room.viewer', 43 | qs: { 44 | uid: 401841505, 45 | _t: +new Date()/60000|0 46 | }, 47 | json: true 48 | }); 49 | return res.data.gid; 50 | } catch (e) { 51 | console.log(`get plat: ${plat}, rid: ${roomId} gid error`); 52 | } 53 | } 54 | 55 | async getKey (plat, roomId, videoId, gid) { 56 | try { 57 | const res = await rp({ 58 | uri: 'https://www.zhanqi.tv/api/public/burglar/chain', 59 | method: 'POST', 60 | headers: { 61 | 'Cookie': `gid=${gid}` 62 | }, 63 | formData: { 64 | stream: videoId + QUALITY.NORMAL + '.flv', 65 | cdnKey: 202, 66 | platform: 128 67 | }, 68 | json: true 69 | }); 70 | return res.data.key; 71 | } catch (e) { 72 | console.log(`get plat: ${plat}, rid: ${roomId} key error`); 73 | } 74 | } 75 | 76 | } 77 | 78 | module.exports = new ZhanqiStream(); -------------------------------------------------------------------------------- /config/Logger.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Creator: Tang Xiaoji 3 | * Time: 2018-01-23 4 | */ 5 | 6 | const moment = require('moment'); 7 | 8 | const TYPE = { 9 | Broadcast: 'logtail_broadcast', 10 | Alarm: 'logtail_alarm', 11 | DataNative: 'logtail_data_native' 12 | }; 13 | 14 | const LEVEL = { 15 | Debug: 'debug', 16 | Log: 'log', 17 | Info: 'info', 18 | Error: 'error', 19 | }; 20 | 21 | class Logger { 22 | constructor(space) { 23 | this.space = space; 24 | } 25 | 26 | log() { 27 | console.log(`\x1B[32m[INFO]\x1B[39m ${moment().format('YYYY-MM-DD HH:mm:ss')} ${this.space}: ${Array.prototype.slice.call(arguments).join(', ')}`); 28 | } 29 | 30 | debug(arg) { 31 | console.log(`\x1B[33m[DEBUG]\x1B[39m ${moment().format('YYYY-MM-DD HH:mm:ss')} ${this.space}: `, JSON.stringify(arg)); 32 | } 33 | 34 | /** 35 | * @param title 36 | * @param plat 37 | * @param err 38 | * @param info 39 | * @param print 是否打印在控制台 40 | * @param report 是否推loghub 41 | */ 42 | error(title, plat, err, info = '', print = false, report = true) { 43 | if(print) { 44 | if (err && err instanceof Error) { 45 | console.log(`\x1B[31m[ERROR]\x1B[39m ${moment().format('YYYY-MM-DD HH:mm:ss')} ${this.space}: ${plat} ${title}\n${err.stack}`) 46 | } else { 47 | console.log(`\x1B[31m[ERROR]\x1B[39m ${moment().format('YYYY-MM-DD HH:mm:ss')} ${this.space}: ${plat} ${title} ${err}`) 48 | } 49 | } 50 | 51 | if (report) { 52 | this.alarm(title, err, plat, info, LEVEL.Error); 53 | } 54 | } 55 | 56 | static getLogtail(type) { 57 | return require('../config/log').getLogger(type); 58 | } 59 | 60 | static write(logtail, info) { 61 | logtail.info(info); 62 | } 63 | 64 | alarm(title, err = '', plat = 'null', info = '', type = LEVEL.Log) { 65 | const payload = {}; 66 | payload.host = require('ip').address(); 67 | payload.title = title; 68 | payload.stack = err; 69 | payload.plat = plat; 70 | payload.type = type; 71 | payload.space = this.space; 72 | payload.info = info; 73 | if (err instanceof Error) { 74 | payload.stack = err.stack; 75 | } 76 | const logtail = Logger.getLogtail(TYPE.Alarm); 77 | Logger.write(logtail, payload); 78 | }; 79 | } 80 | 81 | Logger.type = TYPE; 82 | Logger.level = LEVEL; 83 | module.exports = Logger; 84 | -------------------------------------------------------------------------------- /gather/video-stream/DouyuStream.js: -------------------------------------------------------------------------------- 1 | const VideoStream = require('./VideoStream'); 2 | const rp = require('request-promise'); 3 | const crypto = require('crypto'); 4 | const QUALITY = { 5 | HIGH: 2, 6 | FLUENCY: 1, 7 | SUPER: 0 8 | }; 9 | 10 | class DouyuStream extends VideoStream { 11 | 12 | constructor (plat = 'douyu') { 13 | super(plat); 14 | } 15 | 16 | async getStreamAddr (plat, roomId, url) { 17 | try { 18 | const min = +new Date()/60000|0; 19 | const did = DouyuStream.createHash(); 20 | const res = await rp({ 21 | uri: `https://www.douyu.com/lapi/live/getPlay/${roomId}`, 22 | method: 'POST', 23 | form: { 24 | ver: '2017081401', 25 | cptl: '0002', 26 | sign: await this.getSign(roomId, min, did), 27 | rate: QUALITY.FLUENCY, 28 | cdn: '', 29 | tt: min, 30 | did 31 | }, 32 | json: true 33 | }); 34 | return res.data.rtmp_url + '/' + res.data.rtmp_live; 35 | } catch (e) { 36 | console.log(`get plat: ${plat}, rid: ${roomId} stream addr error`); 37 | } 38 | } 39 | 40 | async getSign (roomId, min, did) { 41 | try { 42 | const res = await rp({ 43 | uri: 'http://121.43.189.149:3103/sthvideo/getsign', 44 | method: 'POST', 45 | form: { 46 | roomid: roomId, 47 | min, 48 | did 49 | }, 50 | json: true 51 | }); 52 | return res.data.sign; 53 | } catch (e) { 54 | console.log(`douyu rid: ${roomId} get sign error`); 55 | } 56 | } 57 | 58 | /** 59 | * 生成指定长度随机字符串 60 | * @param length 61 | */ 62 | static randString (length) { 63 | const str = "0123456789abcdefghijklmnopqrstuvwyxz0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ"; 64 | const str_len = str.length; 65 | let rand = ''; 66 | for (let i = 0; i < length; i++) { 67 | rand = rand + str.charAt(Math.random() * str_len); 68 | } 69 | return rand; 70 | } 71 | 72 | /** 73 | * 生成随机摘要 74 | * @param str 75 | */ 76 | static createHash (str) { 77 | let hash = crypto.createHash('md5'); 78 | hash.update(DouyuStream.randString(20) + Date.now() + (str ? JSON.stringify(str) : ''), 'utf8'); 79 | return hash.digest('hex').toUpperCase(); 80 | } 81 | 82 | } 83 | 84 | module.exports = new DouyuStream(); -------------------------------------------------------------------------------- /danmu/接收弹幕.md: -------------------------------------------------------------------------------- 1 | 使用了GCDAsyncSocket，连接地址 60.205.82.107:80 2 | 3 | 请求连接的发送包生成方式如下 4 | 5 | 具体规则在IKSocketRoomConnection _subscribe:domain:group:topic:liveId:中 6 | 7 | ```objective-c 8 | NSMutableData *data = [[NSMutableData alloc] initWithCapacity:10]; 9 | 10 | UInt16 flag = 0x0001; 11 | 12 | [data appendBytes:&flag length:2]; //0100，append后00 01 顺序颠倒 13 | 14 | flag = 0x0600; 15 | 16 | [data appendBytes:&flag length:2];//01000006 17 | 18 | NSString *string = @"domain"; 19 | 20 | [data appendBytes:string.UTF8String length:string.length];//01000006 646f6d61 696e 21 | 22 | flag = 0x0500; 23 | 24 | [data appendBytes:&flag length:2];//01000006 646f6d61 696e0005 25 | 26 | string = @"group"; 27 | 28 | [data appendBytes:string.UTF8String length:string.length];//01000006 646f6d61 696e0005 67726f75 70 29 | 30 | flag = 0x0800; 31 | 32 | [data appendBytes:&flag length:2]; //data到这里的值是 01000006 646f6d61 696e0005 67726f75 700008 33 | 34 | string = @"topic_38";//38为房间号的最后两位 35 | 36 | [data appendBytes:string.UTF8String length:string.length]; 37 | //把string转换为16进制拼接在data后 38 | 39 | flag = 0x1000;//房间号长度为16，转换为16进制为0x0010, 40 | 41 | [data appendBytes:&flag length:2]; 42 | 43 | string = @"1528963660265338";//房间号 44 | 45 | [data appendBytes:string.UTF8String length:string.length]; 46 | 47 | NSString *queueID = @"0072";//请求的序号的16进制，可以填固定值 48 | 49 | NSString *uid = @"29fa5155";//uid的16进制 50 | 51 | NSString *buff = [NSString stringWithFormat:@"090002034c04%@%@000027100000002d",queueID,uid]; 52 | 53 | NSMutableData *buffer = [NSMutableData dataWithData:[self convertHexStrToData:buff]]; 54 | 55 | [buffer appendData:data]; 56 | 57 | 58 | ``` 59 | 60 | 其中"1528963660265357" 为房间的livid，"09000203 4c0400be 29fa5155 00002710 0000002d"为和账号绑定的Buffer 61 | 62 | 在IKTCPSocketPacket initWithCommandType:protoVersion:中生成 63 | 64 | 其中29fa5155 为uid的16进制 65 | 66 | 00be 是socket的序号的16进制 67 | 68 | 69 | 70 | 收到packet后，得到NSData类型，截取20位后的 71 | 72 | [data subdataWithRange:NSMakeRange(20, data.length-20)]; 73 | 74 | 得到的data前2个byte为0x1f,0x8b,符合GZIP类型，gunzippedData解压后得到NSData类型unzipData，用[NSJSONSerialization JSONObjectWithData:unzipData options:NSJSONReadingMutableContainers error:nil],得到的json类型，包含了弹幕的相关数据 75 | 76 | 77 | 78 | 房间号列表获取的方式是爬http://www.inke.cn/hotlive_list.html?page=1 79 | 80 | 这个网址，然后搜索live.html?uid= 后面比较长的那个id就是房间号 81 | 82 | 83 | 84 | -------------------------------------------------------------------------------- /danmu/Longzhu.js: -------------------------------------------------------------------------------- 1 | const WSConn = require('./WSConn'); 2 | const request = require('superagent'); 3 | const voca = require('voca'); 4 | const config = require('../config/config'); 5 | const Logger = require('../config/Logger'); 6 | const logger = new Logger('Longzhu/danmu'); 7 | 8 | const ROOMID_URL = 'http://star.longzhu.com/m/%s'; 9 | const WS_URL = 'ws://mbgows.plu.cn:8805/?room_id=%s&batch=1&group=0'; 10 | const REQ_TIMEOUT = 6000; 11 | 12 | module.exports = class Longzhu extends WSConn { 13 | constructor(url = '') { 14 | super(); 15 | this.roomId = -1; 16 | this.url = url; 17 | this.clientConnHandler = this.clientConnHandler.bind(this); 18 | this.clientDataHandler = this.clientDataHandler.bind(this); 19 | this.init(); 20 | } 21 | 22 | async init() { 23 | let roomId; 24 | try { 25 | roomId = await this.requestRoomId(); 26 | } catch (e) { 27 | logger.error('longzhu init', config.plat.longzhu, e); 28 | this.emit('initerror', e); 29 | return false; 30 | } 31 | 32 | if (roomId) { 33 | this.on('connect', this.clientConnHandler); 34 | this.connect(voca.sprintf(WS_URL, this.roomId)); 35 | } 36 | } 37 | 38 | async requestRoomId() { 39 | let parts = this.url.split('/'); 40 | let url = voca.sprintf(ROOMID_URL, parts[parts.length - 1]); 41 | let res = await request.get(url).timeout({ 42 | response: REQ_TIMEOUT 43 | }); 44 | 45 | if (res.status != 200 || !res.text) { 46 | throw new Error('request error'); 47 | } 48 | 49 | let body = res.text; 50 | let startIndex = body.indexOf('roomid: "') + 'roomid: "'.length; 51 | let temp = body.slice(startIndex); 52 | let endIndex = temp.indexOf('"'); 53 | if (!startIndex || !endIndex) { 54 | throw new Error('parse error'); 55 | } 56 | 57 | this.roomId = temp.slice(0, endIndex).trim(); 58 | return this.roomId; 59 | } 60 | 61 | decode(buffer) { 62 | try { 63 | buffer = JSON.parse(buffer); 64 | } catch (e) {} 65 | return buffer; 66 | } 67 | 68 | clientConnHandler() { 69 | this.on('rawdata', this.clientDataHandler); 70 | } 71 | 72 | clientDataHandler(data) { 73 | this.emit('data', [].concat(data)); 74 | } 75 | 76 | restart() { 77 | this.removeListener('connect', this.clientConnHandler); 78 | this.removeListener('rawdata', this.clientDataHandler); 79 | this.init(); 80 | } 81 | }; -------------------------------------------------------------------------------- /docs/request-rule.md: -------------------------------------------------------------------------------- 1 | # Request URL Rules 2 | 3 | ## URL 公共 Query 字段 4 | 5 | ### 业务 URL Query 字段 6 | 字段 | 意义 | 候选值和含义 7 | ---------------- | ----------- | ------------ 8 | product | 产品标识 | 直播宝: 1 9 | app | 应用标识 | Android: 1, iPhone: 2, PC: 3, 小程序: 4, PC Web: 5, Mobile Web: 6, Android pad: 7, iPhone pad: 8, Android TV: 9 10 | channel | 渠道标识 | zhibobao, xiaozaizhan, 等等 11 | ver_code | 版本号 | 18011211，32位以内的无符号整形数字 12 | ver_name | 版本名 | 字符串，点分隔数字，一共三位 13 | dev_id | 设备 ID | xxx-xxx-xxxx-xxxxx，字符串 14 | dev_os | 设备系统 | android_5_1, android_8_0, ios_11, win_7, win_8_1, chrome_63_0_3239_132, ie_8 15 | dev_brand | 设备品牌 | xiaomi, sony, iphone 16 | dev_model | 设备型号 | G8142 17 | dev_arch | CPU 型号 | armv7, x86，x64, amd64 18 | dev_conn | 网络连接 | 有线网络: 1, WiFi: 2, 2G: 3, 3G: 4, 4G: 5, 5G: 6 19 | my_uid | 用户 UID | 122222 20 | my_sid | 用户 SID | xxxxxxxxxxxxxxxxxxxxxxxxxxx 21 | my_token | 用户 token | xxxxxxxxxxxxxxxxxxxxxxxxxxx 22 | 23 | 24 | ### 校验 URL Query 字段 25 | 26 | 字段 | 意义 | 候选值和含义 27 | ---------------- | ----------------- | ------------ 28 | s_nonce | 随机字符串 | xxxxxxxxxxxxxxxxxxxxxxxxxxxx 29 | s_sign | 需要校验 | 类似于微信开发平台参数校验规则，值大概是: (sha1(arg1=111&arg2=222&arg3=333&arg4=444&s_nonce=xxxx&s_key=xxxxxxxxxxxx)).toLowerCase() 30 | s_raw | 不需要校验 | 值: 1, 给 Web 端和调试使用 31 | ``` 32 | 1. get: 33 | http://api.zhibobao.tv/app/update?product=1&app=3&channel=zbb&ver_code=18011211&ver_name=1.1.1&dev_id=xxxxxxxxxxxxxxxxxx&dev_os=win_7&dev_brand=sony&dev_model=G8142&dev_arch=arvmv7&dev_conn=WiFi&my_uid=12323&my_sid=121213123232342424242444&s_nonce=xxxxxx&s_sign=12222222222222222222222222222222222 34 | 35 | 2. get: 36 | http://api.zhibobao.tv/room/info/{roomid}?product=1&app=3&channel=zbb&ver_code=18011211&ver_name=1.1.1&dev_id=xxxxxxxxxxxxxxxxxx&dev_os=win_7&dev_brand=sony&dev_model=G8142&dev_arch=arvmv7&dev_conn=WiFi&my_uid=12323&my_sid=121213123232342424242444&s_nonce=xxxxxx&s_sign=12222222222222222222222222222222222 37 | 38 | 3. post: 39 | http://api.zhibobao.tv/user/auth/login?product=1&app=3&channel=zbb&ver_code=18011211&ver_name=1.1.1&dev_id=xxxxxxxxxxxxxxxxxx&dev_os=win_7&dev_brand=sony&dev_model=G8142&dev_arch=arvmv7&dev_conn=WiFi&my_uid=12323&my_sid=121213123232342424242444&s_nonce=xxxxxx&s_sign=12222222222222222222222222222222222 40 | 41 | post field: 42 | mobile=13923420000 43 | sms=123456 44 | 45 | ``` 46 | -------------------------------------------------------------------------------- /helper/giftManager.js: -------------------------------------------------------------------------------- 1 | // const redis = require('../gather/lib/redis'); 2 | const mysql = require('../config/mysql'); 3 | const config = require('../config/config'); 4 | const util = require('../gather/lib/util'); 5 | const request = require('superagent'); 6 | module.exports = { 7 | setGift: async (giftInfo) => { 8 | let field = ['gift_id', 'price', 'gift_cost', 'gift_name', 'plat', 9 | 'isBroadcast', 'token_type']; 10 | await mysql.queryAsync('INSERT INTO ?? (??) VALUES (?)', [ 11 | config.TABLE_NAME_GIFT, field, [giftInfo.gift_id, giftInfo.price || 0, giftInfo.gift_cost || 0, giftInfo.gift_name || '', giftInfo.plat, 12 | giftInfo.isBroadcast || 0, giftInfo.token_type || 0]]); 13 | return true; 14 | }, 15 | updateGift: async (giftInfo) => { 16 | if(!giftInfo.gift_name || !giftInfo.gift_id || !giftInfo.plat) { 17 | return false 18 | } 19 | 20 | await mysql.queryAsync('UPDATE ?? SET gift_name = ? WHERE gift_id = ? AND plat = ?', [ 21 | config.TABLE_NAME_GIFT, giftInfo.gift_name, giftInfo.gift_id, giftInfo.plat 22 | ]); 23 | return true; 24 | }, 25 | getGiftList: async (plat) => { 26 | return await mysql.queryAsync( 27 | 'SELECT * FROM ?? WHERE plat = ?', [ 28 | config.TABLE_NAME_GIFT, plat 29 | ]); 30 | }, 31 | getGift: async (plat, giftId) => { 32 | return await mysql.queryAsync( 33 | 'SELECT * FROM ?? WHERE plat = ? AND gift_id = ?', [ 34 | config.TABLE_NAME_GIFT, plat, giftId 35 | ]); 36 | }, 37 | requestGift: async (plat, roomId) => { 38 | switch (plat) { 39 | case 'Douyu': 40 | let dyRes = await request.get(`http://open.douyucdn.cn/api/RoomApi/room/${roomId}`); 41 | return dyRes.body.data.gift; 42 | break; 43 | case 'Panda': 44 | let pdRes = await request.get(`https://mall.gate.panda.tv/ajax_gift_gifts_get?roomid=${roomId}`); 45 | if(pdRes.body.errno===0){ 46 | return pdRes.body.data.items; 47 | }else { 48 | throw new Error(errno); 49 | } 50 | break; 51 | case 'XYPanda': 52 | let GIFT_URL = 'https://gift.xingyan.panda.tv/gifts?__plat=pc_web&hostid='; 53 | let url = `${GIFT_URL}${roomId}`; 54 | let xyRes = await request.get(url).timeout({ 55 | response: 6000 56 | }); 57 | if (!xyRes.text) { 58 | throw e; 59 | } 60 | try { 61 | let body = JSON.parse(xyRes.text); 62 | this.giftInfo = body.data; 63 | return body.data; 64 | } catch (e) { 65 | throw e; 66 | } 67 | break; 68 | default: 69 | console.log('[E]unknown plat', plat); 70 | util.alarm('[E]unknown plat', plat); 71 | } 72 | } 73 | }; -------------------------------------------------------------------------------- /danmu/index.js: -------------------------------------------------------------------------------- 1 | const EventEmitter = require('events'); 2 | const config = require('../config/config'); 3 | const Logger = require('../config/Logger'); 4 | const logger = new Logger('danmuindex'); 5 | const Danmu = { 6 | douyu: require('./Douyu'), 7 | panda: require('./Panda'), 8 | xypanda: require('./XYPanda'), 9 | zhanqi: require('./Zhanqi'), 10 | longzhu: require('./Longzhu'), 11 | chushou: require('./Chushou'), 12 | huya: require('./HuyaV2'), 13 | fanxing: require('./KugouH5'), 14 | yy: require('./YY'), 15 | huoshan: require('./Huoshan') 16 | }; 17 | 18 | let TYPE = {}; 19 | 20 | for (let key of Object.keys(Danmu)) { 21 | TYPE[key] = key; 22 | } 23 | 24 | class DanmuClient extends EventEmitter { 25 | constructor(type, url, opts) { 26 | super(); 27 | if (!config.plat.hasOwnProperty(type) || !url) { 28 | this.initErrorHandler('url error or plat error') 29 | } else { 30 | this.type = type; 31 | this.url = url; 32 | this.opts = opts; 33 | this.SocketClass = this.getClient(type, url); 34 | this.listen(); 35 | } 36 | } 37 | 38 | getClient(plat, url) { 39 | try { 40 | plat = config.plat[config.getRealType(url)] || plat; 41 | return Danmu[plat]; 42 | } catch (e) { 43 | logger.error('nothis danmu client', plat, e); 44 | } 45 | } 46 | 47 | listen() { 48 | if(!this.SocketClass) return; 49 | this.client = new this.SocketClass(this.url, this.opts); 50 | this.client.on('initerror', this.initErrorHandler.bind(this)); 51 | this.client.on('connect', this.connHandler.bind(this)); 52 | this.client.on('data', this.dataHandler.bind(this)); 53 | this.client.on('error', this.errorHandler.bind(this)); 54 | this.client.on('close', this.closeHandler.bind(this)); 55 | this.client.on('destroy', this.destroyHandler.bind(this)); 56 | } 57 | 58 | restart() { 59 | if (this.client) { 60 | this.client.restart(); 61 | } 62 | 63 | return !!this.client; 64 | } 65 | 66 | // todo： this.client null 67 | destroy() { 68 | this.removeAllListeners(); 69 | this.client && this.client.destroy(); 70 | } 71 | 72 | initErrorHandler(error) { 73 | this.emit('initerror', error); 74 | } 75 | 76 | connHandler() { 77 | this.emit('monitor_start'); 78 | } 79 | 80 | dataHandler(data) { 81 | this.emit('data', data); 82 | } 83 | 84 | errorHandler(error) { 85 | this.emit('error', error); 86 | } 87 | 88 | closeHandler(error) { 89 | this.emit('close', error); 90 | } 91 | 92 | destroyHandler() { 93 | this.emit('monitor_stop'); 94 | this.client = null; 95 | } 96 | } 97 | 98 | module.exports = { 99 | TYPE, 100 | DanmuClient 101 | }; 102 | -------------------------------------------------------------------------------- /helper/huoshan/requestOperation.js: -------------------------------------------------------------------------------- 1 | const rp = require('request-promise'); 2 | // const proxy = require('../proxy'); 3 | const superagent = require('superagent'); 4 | 5 | const Logger = require('../../config/Logger'); 6 | const logger = new Logger('huoshan/request'); 7 | 8 | class RequestOption { 9 | static async Get(url, query, headers, insecure = true, needParse = true) { 10 | let timeLong = 60000; 11 | let options = { 12 | // proxy: 'http://' + proxy.getIp(), 13 | uri: url, 14 | method: 'GET', 15 | headers, 16 | qs: query, 17 | insecure, 18 | simple: true, 19 | resolveWithFullResponse: true, 20 | json: true, 21 | timeout: timeLong 22 | } 23 | delete query['uuid']; 24 | return new Promise((resolve, reject) => { 25 | let timeout = setTimeout(() => { 26 | reject('force timeout'); 27 | }, timeLong); 28 | superagent 29 | .get(url) 30 | // .proxy('http://' + proxy.getIp()) 31 | .set(headers) 32 | .query(query) 33 | .then((data) => { 34 | clearTimeout(timeout); 35 | resolve({ 36 | body: needParse ? JSON.parse(data.res.text) : data.res.text 37 | }) 38 | }).catch((e) => { 39 | clearTimeout(timeout); 40 | reject(e); 41 | }) 42 | // rp(options) 43 | // .then((data) => { 44 | // resolve(data); 45 | // }).catch((e) => { 46 | // reject(e); 47 | // }).finally(() => { 48 | // clearTimeout(timeout) 49 | // }); 50 | }); 51 | 52 | } 53 | 54 | static async Post(url, query = {}, data, headers = {}, insecure = true) { 55 | 56 | if(data && !Buffer.isBuffer(data)) { 57 | let _data = []; 58 | for(let [k, v] of Object.entries(data)) { 59 | _data.push(`${k}=${v}`); 60 | } 61 | 62 | data = _data.join('&'); 63 | }; 64 | 65 | let options = { 66 | // proxy: 'http://' + proxy.getIp(), 67 | url: url, 68 | method: 'POST', 69 | headers, 70 | qs: query, 71 | body: data, 72 | insecure, 73 | simple: true, 74 | resolveWithFullResponse: true, 75 | }; 76 | // if(data && !Buffer.isBuffer(data) && typeof(data) === 'object') options.json = true; 77 | // if(!url === 'https://i.snssdk.com/ies/antispam/upload_device_info/') options.qs = query; 78 | return await rp(options); 79 | } 80 | } 81 | module.exports = RequestOption -------------------------------------------------------------------------------- /gather/sub/check.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Creator: Tang Xiaoji 3 | * Time: 2018-07-17 4 | */ 5 | 6 | 'use strict'; 7 | 8 | const mysql = require('../../config/mysql'); 9 | const config = require('../../config/config'); 10 | const Logger = require('../../config/Logger'); 11 | const logger = new Logger('sub/check'); 12 | const subs = require('./subs'); 13 | 14 | class Check { 15 | constructor() { 16 | this.check(); 17 | } 18 | 19 | async check() { 20 | const [err, conn] = await mysql.asyncGetConn(); 21 | for (let plat in config.plat) { 22 | if (config.plat.hasOwnProperty(plat)) { 23 | if (err) { 24 | logger.error('check getconn error', plat, err); 25 | } else { 26 | const [err, res] = await mysql.asyncQuery('SELECT room_id FROM ?? WHERE platform_name = ?', [ 27 | config.tbl_name.db_user_all_room_bind, plat 28 | ], conn); 29 | 30 | if (err) { 31 | logger.error('search binds from mysql error', plat, err); 32 | } else { 33 | try { 34 | const subslist = await subs.getSubList(plat) 35 | const news = {}, old = {}; 36 | res.forEach(r => news[r.room_id] = 1); 37 | subslist.forEach(s => old[s] = 1); 38 | await this.subHandler(plat, news, old); 39 | } catch (e) { 40 | logger.error('search subslist from redis error', plat, e); 41 | } 42 | } 43 | } 44 | } 45 | } 46 | 47 | conn.release(); 48 | setTimeout(this.check.bind(this), config.timeout.check_sub_list); // 10mim 49 | } 50 | 51 | async subHandler(plat, news, old) { 52 | const plan = { 53 | sub: [], 54 | unSub: [], 55 | update: [] 56 | }; 57 | for (let new_rid in news) { 58 | if (news.hasOwnProperty(new_rid)) { 59 | if (!old.hasOwnProperty(new_rid)) { 60 | plan.sub.push(new_rid); 61 | } else { 62 | plan.update.push(new_rid); 63 | } 64 | } 65 | } 66 | 67 | for (let old_rid in old) { 68 | if (old.hasOwnProperty(old_rid)) { 69 | if (!news.hasOwnProperty(old_rid)) { 70 | plan.unSub.push(old_rid); 71 | } 72 | } 73 | } 74 | 75 | await this.todo(plat, plan); 76 | } 77 | 78 | async todo(plat, plan) { 79 | for (let method in plan) { 80 | if (plan.hasOwnProperty(method)) { 81 | const list = plan[method]; 82 | if (list.length && typeof subs[method] === 'function') { 83 | for (let rid of list) { 84 | try { 85 | await (subs[method](plat, rid)); 86 | } catch (e) { 87 | logger.error(`check ${method}`, plat, e); 88 | } 89 | } 90 | logger.error(`长期订阅`, plat, method, list.length); 91 | } 92 | } 93 | } 94 | } 95 | } 96 | 97 | module.exports = new Check(); 98 | -------------------------------------------------------------------------------- /helper/huoshan/encrypt.js: -------------------------------------------------------------------------------- 1 | const exec = require('child-process-promise').exec; 2 | const crypto = require('crypto'); 3 | 4 | const Logger = require('../../config/Logger'); 5 | const logger = new Logger('huoshan/encrypt'); 6 | 7 | const tt_encrypt = async (data) => { 8 | data = JSON.stringify(data); 9 | let res = await exec(`PYTHONIOENCODING=utf-8 python3 ${__dirname}/python-scripts/tt_encrypt.py ${JSON.stringify(data)}`); 10 | // logger.log('error: ', res.stderr); 11 | // logger.log('out: ', res.stdout.toString()); 12 | data = res.stdout; 13 | data = data.split('\n'); 14 | data = data.slice(0, data.length - 1); 15 | data = data.join('\n') 16 | return new Buffer(data.toString(), 'binary'); 17 | } 18 | 19 | const unwifi = async () => { 20 | try{ 21 | let res = await exec(`PYTHONIOENCODING=utf-8 python3 ${__dirname}/python-scripts/unwifi.py`); 22 | if(res.stderr) throw new Error(res.stderr); 23 | res = `${res.stdout.split('\n')[0]}`; 24 | res = res.replace(/"/g, '\\"'); 25 | res = res.replace(/'/g, '"'); 26 | res = JSON.parse(res); 27 | return res; 28 | } catch (e) { 29 | logger.log('unwifi e: ', e); 30 | return {}; 31 | } 32 | } 33 | 34 | const dataFormat = async (data) => { 35 | data = JSON.stringify(data, null, 4); 36 | data = JSON.stringify(data, null, 4); 37 | let res = await exec(`PYTHONIOENCODING=utf-8 python3 ${__dirname}/python-scripts/complementation.py ${data}`); 38 | data = res.stdout; 39 | data = data.split('\n')[0]; 40 | return data; 41 | } 42 | 43 | const cipheriv = async ( 44 | data, 45 | algorithm = 'aes-128-cfb', 46 | key = 'eagleye_9fd&fwfl', 47 | iv = '\x00\x01\x02\x03\x04\x05\x06\x07\x08\x09\x0a\x0b\x0c\x0d\x0e\x0f', 48 | input = 'binary', 49 | output = 'hex' 50 | ) => { 51 | data = await dataFormat(data); 52 | buf = data 53 | key = new Buffer(key, 'binary'); 54 | iv = new Buffer(iv, 'binary'); 55 | let encrypted = ''; 56 | const cip = crypto.createCipheriv(algorithm, key, iv); 57 | cip.setAutoPadding(true); 58 | encrypted += cip.update(buf, input, output); 59 | encrypted += cip.final(output); 60 | logger.log('encrypted length ====>', encrypted.length) 61 | return encrypted; 62 | } 63 | 64 | const cryptoAfter = async (data) => { 65 | data = data + ''; 66 | let res = await exec(`PYTHONIOENCODING=utf-8 python3 ${__dirname}/python-scripts/cryptoAfter.py ${data}`); 67 | data = res.stdout; 68 | data = data.split('\n'); 69 | 70 | data = data.slice(0, data.length - 1); 71 | data = data.join('\n') 72 | return new Buffer(data, 'binary'); 73 | } 74 | 75 | const j_a = async (data) => { 76 | let encrypted = await cipheriv(data); 77 | let res = await cryptoAfter(encrypted); 78 | return res 79 | } 80 | 81 | module.exports = { 82 | tt_encrypt, 83 | unwifi, 84 | dataFormat, 85 | cryptoAfter, 86 | cipheriv, 87 | j_a 88 | } 89 | 90 | // (async () => { 91 | // await j_a({a: 111}); 92 | // })() -------------------------------------------------------------------------------- /danmu/Douyu.js: -------------------------------------------------------------------------------- 1 | const Conn = require('./Conn'); 2 | const SOCKET_HOST = 'openbarrage.douyutv.com'; 3 | const SOCKET_PORT = 8601; 4 | const KEEPALIVE_INTERVAL = 30000; 5 | 6 | module.exports = class Douyu extends Conn { 7 | constructor(url = '') { 8 | super(); 9 | this.url = url; 10 | this.roomId = (() => { 11 | let parts = this.url.split('/'); 12 | return parts[parts.length - 1]; 13 | })(); 14 | this.buffer = Buffer.alloc(0); 15 | this.isLoginReg = false; 16 | this.keepaliveTimer = null; 17 | this.clientConnHandler = this.clientConnHandler.bind(this); 18 | this.clientDataHandler = this.clientDataHandler.bind(this); 19 | this.init(); 20 | } 21 | 22 | async init() { 23 | this.on('connect', this.clientConnHandler); 24 | this.connect(SOCKET_PORT, SOCKET_HOST); 25 | } 26 | 27 | destroy() { 28 | clearTimeout(this.keepaliveTimer); 29 | super.destroy(); 30 | } 31 | 32 | encode(data) { 33 | let length = Buffer.byteLength(data); 34 | let buffer = Buffer.alloc(8 + 4 + length + 1); 35 | buffer.writeInt32LE(9 + length); 36 | buffer.writeInt32LE(9 + length, 4); 37 | buffer.writeInt32LE(689, 8); 38 | buffer.write(data, 12); 39 | return buffer; 40 | } 41 | 42 | decode(buffer) { 43 | let contents = []; 44 | this.buffer = Buffer.concat([this.buffer, buffer]); 45 | 46 | try { 47 | for (; ;) { 48 | let totalLen = this.buffer.length; 49 | let contentLen = this.buffer.readInt32LE(0); 50 | let segLen = 4 + contentLen; 51 | if (totalLen < segLen) { 52 | break; 53 | } 54 | 55 | let kvs = {}; 56 | let content = this.buffer 57 | .slice(12, 12 + contentLen - 8 - 1) 58 | .toString('utf-8'); 59 | 60 | content.split('/').forEach(item => { 61 | let pairs = item.split('@='); 62 | if (pairs.length == 2) { 63 | kvs[pairs[0]] = pairs[1]; 64 | } 65 | }); 66 | 67 | contents.push(kvs); 68 | this.buffer = this.buffer.slice(segLen); 69 | } 70 | } catch (e) { 71 | } 72 | 73 | return contents; 74 | } 75 | 76 | async keepalive() { 77 | this.keepaliveTimer = setTimeout(() => { 78 | this.write(`type@=keeplive/tick@=${this.roomId}/`); 79 | this.keepalive(); 80 | }, KEEPALIVE_INTERVAL); 81 | } 82 | 83 | clientConnHandler() { 84 | this.on('rawdata', this.clientDataHandler); 85 | this.write(`type@=loginreq/roomid@=${this.roomId}/`); 86 | } 87 | 88 | async clientDataHandler(data) { 89 | if (!this.isLoginReg) { 90 | this.isLoginReg = true; 91 | this.keepalive(); 92 | this.write(`type@=joingroup/rid@=${this.roomId}/gid@=-9999/`); 93 | return; 94 | } 95 | this.emit('data', data); 96 | } 97 | 98 | restart() { 99 | this.isLoginReg = false; 100 | this.buffer = Buffer.alloc(0); 101 | clearTimeout(this.keepaliveTimer); 102 | this.removeListener('connect', this.clientConnHandler); 103 | this.removeListener('rawdata', this.clientDataHandler); 104 | this.init(); 105 | } 106 | }; -------------------------------------------------------------------------------- /gather/lib/notice.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Creator: Tang Xiaoji 3 | * Time: 2018-03-07 4 | */ 5 | 6 | const http = require('../../config/http'); 7 | const anchor = require('./anchor'); 8 | const rp = require('request-promise'); 9 | 10 | const Plat = { 11 | douyu: async (url) => { 12 | const info = await http.Get(url); 13 | return info.toString().replace(/\r\n/g, '').replace(/\s/g, '').match(/直播公告(\S*)class="columnrec"/, )[1].match(/column-cotent">(\S*)<\/p>/)[1] || ''; 14 | }, 15 | 16 | huya: async (url) => { 17 | const info = await http.Get(url); 18 | return info.toString().replace(/\r\n/g, '').replace(/\s/g, '').match(/J_roomNoticeText">公告:(\S*)room-player-wrap/)[1].match(/(\S*)<\/span>/)[1] || ''; 19 | }, 20 | 21 | longzhu: async (url) => { 22 | const info = await http.Get(url); 23 | return info.toString().replace(/\r\n/g, '').replace(/\s/g, '').match(/Desc":"(\S*)","Type"/)[1] || ''; 24 | }, 25 | 26 | panda: async (url) => { 27 | if(url.match(/xingyan/)) { 28 | const info = await http.Get(url); 29 | const utf8 = info.toString().replace(/\r\n/g, '').replace(/\s/g, '').match(/signature":"(\S*)","is_anchor/)[1] || ''; 30 | return unescape(utf8.replace(/\\u/g, '%u')); 31 | } 32 | const info = await http.Get(url); 33 | const utf8 = info.toString().replace(/\r\n/g, '').replace(/\s/g, '').match(/bulletin":"(\S*)","details/)[1] || ''; 34 | return unescape(utf8.replace(/\\u/g, '%u')); 35 | }, 36 | 37 | quanmin: async (url) => { 38 | const info = await http.Get(url); 39 | const r1 = info.toString().replace(/\r\n/g, '').replace(/\s/g, '').match(/"announcement":"(\S*)","play_at":"\d{4}-\d{2}-\d{4}/)[1] || ''; 40 | if (r1) { 41 | return r1.match(/(\S*)","play_at":"\d{4}-\d{2}-\d{4}/)[1] || '' 42 | } 43 | 44 | return '' 45 | }, 46 | 47 | zhanqi: async (url) => { 48 | const info = await http.Get(url); 49 | return info.toString().replace(/\r\n/g, '').replace(/\s/g, '').match(/js-room-notice-item">(\S*)js-room-notice-item"/)[1].match(/(\S*)<\/p>/)[1] || ''; 50 | }, 51 | 52 | fanxing: async (url) => { 53 | const res = await anchor.fanxing(url); 54 | if (res.info) { 55 | return ''; 56 | } 57 | const roomInfoUrl = `http://visitor.fanxing.kugou.com/VServices/RoomService.RoomService.getInfo/${res.rid}/`; 58 | const info = await rp({ 59 | uri: roomInfoUrl, 60 | headers: { 61 | 'Host': 'visitor.fanxing.kugou.com', 62 | 'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_13_3) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.7 Safari/537.36', 63 | 'Referer': `http://fanxing.kugou.com/${res.rid}`, 64 | } 65 | }); 66 | let data = info.match(/\((.+)\)/)[1]; 67 | data = JSON.parse(data); 68 | return data.data && data.data.publicMesg || ''; 69 | } 70 | }; 71 | 72 | const notice = module.exports = async (plat, url) => { 73 | if(!plat || !Plat[plat]) { 74 | return ''; 75 | } 76 | 77 | try { 78 | return await Plat[plat](url); 79 | } catch (e) { 80 | return '' 81 | } 82 | }; 83 | 84 | // notice('quanmin', 'https://www.quanmin.tv/13359230').then(r => console.log(r)).catch(e => console.log(e)); 85 | -------------------------------------------------------------------------------- /helper/huoshan/requestEncrypt.js: -------------------------------------------------------------------------------- 1 | const net = require('net'); 2 | 3 | const Logger = require('../../config/Logger'); 4 | const logger = new Logger('huoshan/tcp'); 5 | 6 | class RequestEncrypt { 7 | constructor() { 8 | this.serverUrl = '60.190.228.34'; 9 | this.serverPort = '1113'; 10 | this.online = false; 11 | this.client = null; 12 | this.mas = null; 13 | this.masFlag = false; 14 | this.connect(); 15 | // this.masFlagIntervel(); 16 | logger.log('chushihua') 17 | } 18 | 19 | // masFlagIntervel(timeLong = 2000) { 20 | // setInterval(() => { 21 | // this.masFlag = false 22 | // }, timeLong) 23 | // } 24 | 25 | connect() { 26 | let client = net.connect(this.serverPort, this.serverUrl, () => { 27 | logger.log('连接 服务端 '); 28 | this.online = true; 29 | this.client = client; 30 | }); 31 | 32 | client.on('data', this.clientDataHandler.bind(this)); 33 | 34 | client.on('close', () => { 35 | logger.log('客户端收到关闭信号'); 36 | this.remove(); 37 | }) 38 | 39 | client.on('error', (e) => { 40 | logger.log('错误信息: ', e); 41 | this.remove(); 42 | if(this.client) this.client.end(); 43 | }) 44 | client.write('ping\n'); 45 | } 46 | 47 | remove() { 48 | this.online = false; 49 | this.client = null; 50 | this.masFlag = false; 51 | } 52 | 53 | end() { 54 | logger.log('强制关闭'); 55 | this.client.end('强制关闭'); 56 | } 57 | 58 | clientDataHandler(data) { 59 | data = data.toString().replace(/[\r\n]/g, ''); 60 | if(data !== 'pong') { 61 | // logger.log('data ====> ', data); 62 | // logger.log('=================') 63 | let reg = /mas/g; 64 | let masCount = data.match(reg).length; 65 | if(masCount !== 1) { 66 | data = data.split('}'); 67 | if(data.length > 1) { 68 | data = data[0] + '}' 69 | } else { 70 | logger.log('获取加密参数失败', data, ' end'); 71 | return; 72 | } 73 | } 74 | // logger.log('data ====> ', data); 75 | data = JSON.parse(data); 76 | this.mas = data; 77 | if(!this.masFlag) { 78 | // this.masFlag = true; 79 | } 80 | } 81 | 82 | } 83 | 84 | send(ts, url, params, device_id) { 85 | // logger.log('in send --> ', ts, url, device_id) 86 | if(!this.online) this.connect(); 87 | if(!this.client) return false; 88 | // if(this.masFlag) return this.mas; 89 | let input = { 90 | 'ts': ts, 91 | 'url': url, 92 | 'params': params, 93 | 'device_id': device_id 94 | } 95 | input = new Buffer(JSON.stringify(input) + '\n', 'binary'); 96 | this.client.write(input); 97 | } 98 | } 99 | // requestE = new RequestEncrypt(); 100 | module.exports = new RequestEncrypt(); -------------------------------------------------------------------------------- /helper/proxy.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Creator: Tang Xiaoji 3 | * Time: 2018-02-08 4 | */ 5 | 6 | const _ = require('lodash'); 7 | const request = require("request"); 8 | 9 | const order = 'a7e109386c74ca83843f1bff64a1a52b'; 10 | const apiURL = 'http://api.ip.data5u.com/dynamic/get.html?order=' + order + '&sep=3'; 11 | 12 | const MAX_PROXY_LIST_LENGTH = 5; 13 | 14 | let Proxy_List = []; 15 | let running = false; 16 | function getProxyList() { 17 | const options = { 18 | method: 'GET', 19 | url: apiURL, 20 | gzip: true, 21 | encoding: null, 22 | headers: {}, 23 | }; 24 | 25 | request(options, function (error, response, body) { 26 | try { 27 | const res = Buffer.from(body).toString(); 28 | if (error || !res.match(/\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}:\d{1,5}/g)) { 29 | 30 | } else { 31 | running = true; 32 | Proxy_List = Proxy_List.concat(_.pull(res.split('\n'), '')); 33 | } 34 | } catch (e) { 35 | console.log('get proxy list error: ', e) 36 | } 37 | }); 38 | 39 | if(Proxy_List.length > MAX_PROXY_LIST_LENGTH) { 40 | Proxy_List = Proxy_List.slice(Proxy_List.length - MAX_PROXY_LIST_LENGTH); 41 | } 42 | 43 | setTimeout(getProxyList, 5000); 44 | } 45 | 46 | if(!running) { 47 | // getProxyList(); TODO: 48 | } 49 | 50 | function getIp(cb) { 51 | if(cb) { 52 | cb(Proxy_List[_.random(0, Proxy_List.length - 1)]); 53 | return; 54 | } 55 | 56 | if(Proxy_List.length) { 57 | return Proxy_List[_.random(0, Proxy_List.length - 1)]; 58 | } else { 59 | return ''; 60 | } 61 | } 62 | 63 | let ip = ''; 64 | function getRealIp(cb) { 65 | const options = { 66 | method: 'GET', 67 | url: apiURL, 68 | gzip: true, 69 | encoding: null, 70 | headers: {}, 71 | }; 72 | 73 | request(options, function (error, response, body) { 74 | if(error) { 75 | ip = getIp(); 76 | cb(ip); 77 | } else { 78 | const res = Buffer.from(body).toString(); 79 | if(!res.match(/\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}:\d{1,5}/g)) { 80 | ip = getIp(); 81 | cb(ip); 82 | } else { 83 | ip = res.split('\n')[0]; 84 | cb(ip); 85 | } 86 | } 87 | }); 88 | } 89 | 90 | function asyncGetRealIp() { 91 | const options = { 92 | method: 'GET', 93 | url: apiURL, 94 | gzip: true, 95 | encoding: null, 96 | headers: {}, 97 | }; 98 | 99 | return new Promise((resolve) => { 100 | request(options, function (error, r, body) { 101 | if(error) { 102 | ip = getIp(); 103 | resolve(ip); 104 | } else { 105 | const res = Buffer.from(body).toString(); 106 | if(!res.match(/\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}:\d{1,5}/g)) { 107 | ip = getIp(); 108 | resolve(ip); 109 | } else { 110 | ip = res.split('\n')[0]; 111 | resolve(ip); 112 | } 113 | } 114 | }); 115 | }) 116 | } 117 | 118 | function proxyRequest(opt, force_timeout) { 119 | return new Promise((resolve, reject) => { 120 | const timeout = setTimeout(() => { 121 | reject('force timeout'); 122 | }, force_timeout); 123 | 124 | request(opt, (err, response, body) => { 125 | if (err) { 126 | clearTimeout(timeout); 127 | reject(err); 128 | } else { 129 | clearTimeout(timeout); 130 | resolve(body); 131 | } 132 | }) 133 | }) 134 | } 135 | 136 | module.exports = { 137 | getIp, 138 | getRealIp, 139 | asyncGetRealIp, 140 | proxyRequest 141 | }; 142 | -------------------------------------------------------------------------------- /danmu/Chushou.js: -------------------------------------------------------------------------------- 1 | const EventEmitter = require('events'); 2 | const request = require('superagent'); 3 | const _ = require('lodash'); 4 | const config = require('../config/config'); 5 | const redis = require('../config/redis'); 6 | 7 | const JSONP_CALLBACK = 'jQuery110209616188693157892_'; 8 | const CHAT_URL = 'https://chat.chushou.tv/chat/get.htm'; 9 | const RETRY_TIMEOUT = 30 * 1000; 10 | const REQ_TIMEOUT = 6000; 11 | const MAX_RETYR = 5; 12 | 13 | module.exports = class Chushou extends EventEmitter { 14 | constructor(url = '') { 15 | super(); 16 | this.url = url; 17 | this.timer = null; 18 | this.roomId = -1; 19 | this.breakpoint = ''; 20 | this.cid = -1; 21 | this.retryCount = 0; 22 | this.agent = request.agent(); 23 | this.init(); 24 | } 25 | 26 | async init() { 27 | try { 28 | const roomStatus = await this.requestRoomStatus(); 29 | if (roomStatus) { 30 | this.emit('connect'); 31 | this.connect(); 32 | } else { 33 | this.emit('initerror', 'unable to get chushou room info'); 34 | } 35 | } catch (e) { 36 | this.emit('initerror', e); 37 | } 38 | } 39 | 40 | async requestRoomStatus() { 41 | try { 42 | let parts = this.url.split('/'); 43 | let roomId = parts[parts.length - 1]; 44 | if (!roomId || roomId.lastIndexOf('.') === -1) { 45 | this.emit('initerror', new Error('url error')); 46 | return false; 47 | } 48 | 49 | roomId = roomId.slice(0, roomId.lastIndexOf('.')); 50 | await redis.lpush(config.LIST_GIFT_TASK_KEY(config.plat.chushou), roomId); 51 | 52 | this.roomId = roomId; 53 | return true; 54 | } catch (e) { 55 | throw e; 56 | } 57 | } 58 | 59 | connect() { 60 | clearTimeout(this.timer); 61 | this.timer = setTimeout(async () => { 62 | try { 63 | await this.clientDataHandler(); 64 | } catch (e) { 65 | if (this.retryCount++ > MAX_RETYR) { 66 | this.emit('error', e); 67 | return; 68 | } 69 | } 70 | this.connect(); 71 | }, RETRY_TIMEOUT); 72 | } 73 | 74 | async clientDataHandler() { 75 | if (!this.agent) { 76 | return; 77 | } 78 | 79 | let callback = `${JSONP_CALLBACK}${+new Date()}`; 80 | let res = await this.agent 81 | .get(CHAT_URL) 82 | .timeout(REQ_TIMEOUT) 83 | .query({ 84 | style: 2, 85 | roomId: this.roomId, 86 | breakpoint: this.breakpoint, 87 | _: Date.now(), 88 | callback: callback 89 | }) 90 | .set('Referer', this.url) 91 | .set('Pragma', 'no-cache') 92 | .set('Cache-Control', 'no-cache'); 93 | 94 | if (res.status != 200 || !res.text) { 95 | throw new Error('[E] request [%s] error', CHAT_URL, res.status, res.text); 96 | } 97 | 98 | let body = res.text; 99 | if (body.indexOf(callback) > -1) { 100 | body = body.replace(callback, '').replace(/^\(|\);?$/g, ''); 101 | let data = JSON.parse(body).data; 102 | this.breakpoint = body.breakpoint; 103 | 104 | let items = data.items; 105 | let index = _.findIndex(items, { 106 | id: this.cid 107 | }); 108 | if (index > -1) { 109 | items = items.splice(index + 1); 110 | } 111 | 112 | if (items && items.length) { 113 | this.cid = items[items.length - 1].id; 114 | this.emit('data', items); 115 | } 116 | } 117 | } 118 | 119 | async restart() { 120 | this.breakpoint = ''; 121 | this.agent = request.agent(); 122 | this.init(); 123 | } 124 | 125 | destroy() { 126 | clearTimeout(this.timer); 127 | this.cid = -1; 128 | this.breakpoint = ''; 129 | this.agent = null; 130 | this.emit('destroy'); 131 | this.removeAllListeners(); 132 | } 133 | }; -------------------------------------------------------------------------------- /gather/sub/subs.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Creator: Tang Xiaoji 3 | * Time: 2018-07-16 4 | */ 5 | 6 | 'use strict'; 7 | 8 | const config = require('../../config/config'); 9 | const redis = require('../../config/redis'); 10 | const mysql = require('../../config/mysql'); 11 | const moment = require('moment'); 12 | const Logger = require('../../config/Logger'); 13 | const logger = new Logger('nest/sub/subs'); 14 | 15 | const subStatus = { 16 | sub: 1, 17 | notSub: 2 18 | } 19 | 20 | class Subs { 21 | async sub(plat, rid) { 22 | if (!config.plat.hasOwnProperty(plat)) { 23 | throw new Error('plat error'); 24 | } 25 | 26 | this.updateTaskSubStatus(plat, rid, subStatus.sub); 27 | try { 28 | return new Promise((resolve, reject) => { 29 | redis.multi([ 30 | ['set', config.STRING_SUB_TASK_KEY(plat, rid), moment().format('YYYY-MM-DD HH:mm:ss'), 'EX', 12 * 60 * 60], 31 | ['sadd', config.SET_SUB_TASK_KEY(plat), rid] 32 | ]).exec((err, res) => { 33 | if (err) { 34 | reject(err); 35 | } else { 36 | resolve(res); 37 | } 38 | }); 39 | }); 40 | } catch (e) { 41 | throw e; 42 | } 43 | } 44 | 45 | async update(plat, rid) { 46 | if (!config.plat.hasOwnProperty(plat)) { 47 | throw new Error('plat error'); 48 | } 49 | 50 | this.updateTaskSubStatus(plat, rid, subStatus.sub); 51 | try { 52 | return await redis.set(config.STRING_SUB_TASK_KEY(plat, rid), moment().format('YYYY-MM-DD HH:mm:ss'), 'EX', 12 * 60 * 60); 53 | } catch (e) { 54 | throw e; 55 | } 56 | } 57 | 58 | async isSub(plat, rid) { 59 | if (!config.plat.hasOwnProperty(plat)) { 60 | throw new Error('plat error'); 61 | } 62 | 63 | try { 64 | const isMember = await redis.sismember(config.SET_SUB_TASK_KEY(plat), rid); 65 | if (isMember === 1) { 66 | const info = await redis.get(config.STRING_SUB_TASK_KEY(plat, rid)); 67 | if (info) { 68 | return true; 69 | } else { 70 | await redis.srem(config.SET_SUB_TASK_KEY(plat), rid); 71 | return false; 72 | } 73 | } else { 74 | return false; 75 | } 76 | } catch (e) { 77 | throw e; 78 | } 79 | } 80 | 81 | async unSub(plat, rid) { 82 | if (!config.plat.hasOwnProperty(plat)) { 83 | throw new Error('plat error'); 84 | } 85 | 86 | this.updateTaskSubStatus(plat, rid, subStatus.notSub); 87 | try { 88 | return new Promise((resolve, reject) => { 89 | redis.multi([ 90 | ['del', config.STRING_SUB_TASK_KEY(plat, rid)], 91 | ['srem', config.SET_SUB_TASK_KEY(plat), rid] 92 | ]).exec((err, res) => { 93 | if (err) { 94 | reject(err); 95 | } else { 96 | resolve(res); 97 | } 98 | }); 99 | }); 100 | } catch (e) { 101 | throw e; 102 | } 103 | } 104 | 105 | async getSubList(plat) { 106 | if (!config.plat.hasOwnProperty(plat)) { 107 | throw new Error('plat error'); 108 | } 109 | 110 | try { 111 | return await redis.smembers(config.SET_SUB_TASK_KEY(plat)); 112 | } catch (e) { 113 | throw e; 114 | } 115 | } 116 | 117 | async updateTaskSubStatus(plat, rid, status) { 118 | const [err, conn] = await mysql.asyncGetConn(); 119 | if (err) { 120 | logger.error('sub getconn error', plat, err, rid); 121 | } else { 122 | const [err] = await mysql.asyncQuery('UPDATE ?? SET sub = ? WHERE plat = ? AND room_id = ? LIMIT 1', [ 123 | config.tbl_name.db_fentuan_taskv2, status, plat, rid 124 | ], conn); 125 | 126 | if (err) { 127 | logger.error('sub update sub status error', plat, err, rid); 128 | } 129 | } 130 | 131 | conn.release(); 132 | } 133 | } 134 | 135 | module.exports = new Subs(); 136 | -------------------------------------------------------------------------------- /helper/huoshan/device.js: -------------------------------------------------------------------------------- 1 | const Logger = require('../../config/Logger'); 2 | const logger = new Logger('huoshan/device'); 3 | 4 | const zfill = (str, size) => { 5 | while(str.length < size) { 6 | str = '0' + str; 7 | } 8 | return str; 9 | } 10 | 11 | const ubyte = (n) => { 12 | if(n > 0) { 13 | let start = 0; 14 | let end = parseInt('ff', 16); 15 | let randomNum = Math.floor(Math.random() * (end - start + 1) + start); 16 | randomNum = randomNum.toString(16); 17 | return ubyte(n - 1) + zfill(randomNum, 2); 18 | } 19 | return ''; 20 | } 21 | 22 | const uint = (n) => { 23 | let start = Math.pow(10, n - 1); 24 | let end = Math.pow(10, n) - 1; 25 | let res = Math.floor(Math.random() * (end - start + 1) + start); 26 | return res.toString(); 27 | } 28 | 29 | const ui = (i) => { 30 | return i[Math.floor(Math.random() * (i.length - 1 + 1))]; 31 | } 32 | const getDeviceInfo = () => { 33 | let [cpu, cpu_abi, density_dpi, description, device_brand, device_manufacturer, device_model, fingerprint, mem, os_api, os_version, resolution, rom, rom_version, sdtotal] = ui([ 34 | ['abi: armeabi-v7anProcessor\t: ARMv7 Processor rev 1 (v7l)\nBogoMIPS\t: 38.40\n\nBogoMIPS\t: 38.40\n\nBogoMIPS\t: 38.40\n\nBogoMIPS\t: 38.40\n\nFeatures\t: swp half thumb fastmult vfp edsp neon vfpv3 tls vfpv4 idiva idivt \nCPU implementer\t: 0x51\nCPU architecture: 7\nCPU variant\t: 0x2\nCPU part\t: 0x06f\nCPU revision\t: 1\n\nHardware\t: Qualcomm MSM8974PRO-AC\nRevision\t: 0000\nSerial\t\t: 0000000000000000\n', 'armeabi-v7a', 480, 'cancro-user 6.0.1 MMB29M 7.11.30 release-keys', 'Xiaomi', 'Xiaomi', 'MI 4LTE', 'Xiaomi\/cancro_wc_lte\/cancro:6.0.1\/MMB29M\/7.11.30:user\/release-keys', 1938935808, 23, '6.0.1', '1080*1920', 'MIUI-7.11.30', 'miui_V9_7.11.30', 13369495552], 35 | ]); 36 | 37 | let applist = ui([ 38 | [], 39 | ]).concat(['com.baidu.BaiduMap', 'com.eg.android.AlipayGphone', 'com.sina.weibo', 'com.snssdk.api', 'com.ss.android.ugc.live', 'com.ss.android.ugc.aweme', 'com.taobao.taobao', 'com.tencent.mm', 'com.tencent.mobileqq', 'com.tencent.weishi']).sort() 40 | 41 | let idfa = ubyte(8); 42 | let imei = uint(15); 43 | let mac = [1, 1, 1, 1, 1, 1].map(i => ubyte(i).toUpperCase()).join(':'); 44 | let serial_number = uint(8); 45 | 46 | let display_density = {120: 'ldpi', 240: 'hdpi', 320: 'xhdpi'}; 47 | 48 | const deviceInfo = { 49 | cellid: 0, 50 | device_platform: 'android', 51 | file: '', 52 | h: 'ffff0000-ffff1000 r-xp 00000000 00:00 0 [vectors]\n', 53 | host: 0, 54 | language: 'zh', 55 | lock: 5000, 56 | os: 'Android', 57 | provider: '', 58 | region: 'CN', 59 | root: 0, 60 | sim: 1, 61 | sim_serial_number: [], 62 | temperature: 1, 63 | timezone: 8, 64 | type: 0, 65 | tz_name: 'Asia\/Shanghai', 66 | tz_offset: 28800, 67 | usb: 1, 68 | vpn: 0, 69 | 70 | clientudid: [4, 2, 2, 2, 6].map(i => ubyte(i)).join('-'), 71 | idfa, 72 | imei, 73 | mac, 74 | serial_number, 75 | 76 | cpu, cpu_abi, density_dpi, description, device_brand, device_manufacturer, device_model, fingerprint, mem, os_api, os_version, resolution, rom, rom_version, sdtotal, 77 | applist, 78 | apkcount: applist.length, 79 | 80 | brand: device_brand + ' ' + device_model, 81 | build_serial: serial_number, 82 | device_type: device_model, 83 | display: resolution.replace('*', ','), 84 | display_density: display_density[density_dpi] || 'mdpi', 85 | dpi: density_dpi, 86 | mc: mac, 87 | openudid: idfa, 88 | udid: imei, 89 | uuid: imei, 90 | } 91 | return deviceInfo; 92 | } 93 | 94 | const getDeviceId = () => { 95 | 96 | } 97 | 98 | 99 | module.exports = { 100 | deviceInfo: getDeviceInfo 101 | } -------------------------------------------------------------------------------- /danmu/inke.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "bytes" 5 | "compress/gzip" 6 | "encoding/binary" 7 | "encoding/hex" 8 | "fmt" 9 | "io" 10 | "io/ioutil" 11 | "net" 12 | "os" 13 | "time" 14 | ) 15 | 16 | // 数据包的类型 17 | const ( 18 | HEART_BEAT_PACKET = 0x00 19 | REPORT_PACKET = 0x01 20 | ) 21 | 22 | // InkDanmuServer 弹幕服务器地址 23 | const InkDanmuServer = "60.205.82.107:80" 24 | 25 | // 客户端对象 26 | type TcpClient struct { 27 | connection *net.TCPConn 28 | hawkServer *net.TCPAddr 29 | stopChan chan struct{} 30 | } 31 | 32 | func main() { 33 | // 拿到服务器地址信息 34 | hawkServer, err := net.ResolveTCPAddr("tcp", InkDanmuServer) 35 | if err != nil { 36 | fmt.Printf("hawk server [%s] resolve error: [%s]", InkDanmuServer, err.Error()) 37 | os.Exit(1) 38 | } 39 | // 连接服务器 40 | connection, err := net.DialTCP("tcp", nil, hawkServer) 41 | if err != nil { 42 | fmt.Printf("connect to hawk server error: [%s]", err.Error()) 43 | os.Exit(1) 44 | } 45 | 46 | sender(connection) 47 | 48 | client := &TcpClient{ 49 | connection: connection, 50 | hawkServer: hawkServer, 51 | stopChan: make(chan struct{}), 52 | } 53 | // 启动接收 54 | go client.receivePackets() 55 | 56 | // 发送心跳的goroutine 57 | go func() { 58 | heartBeatTick := time.Tick(2 * time.Second) 59 | for { 60 | select { 61 | case <-heartBeatTick: 62 | client.sendHeartPacket() 63 | case <-client.stopChan: 64 | return 65 | } 66 | } 67 | }() 68 | 69 | // 等待退出 70 | <-client.stopChan 71 | } 72 | 73 | func handleData(d []byte) { 74 | jsonStr, _ := GzipDecode(d) 75 | // fmt.Printf("payload length:%0x header %0x json %v \n", len(nBuff), buffer[:20], string(jsonStr)) 76 | 77 | fmt.Println("json", string(jsonStr)) 78 | } 79 | 80 | // 接收数据包 81 | func (client *TcpClient) receivePackets() { 82 | defer close(client.stopChan) 83 | 84 | for { 85 | buffer := make([]byte, 2048) 86 | if _, err := io.ReadFull(client.connection, buffer[:20]); err != nil { 87 | fmt.Println("read header", err) 88 | return 89 | } 90 | 91 | length := binary.BigEndian.Uint16(buffer[18:20]) 92 | 93 | if _, err := io.ReadFull(client.connection, buffer[20:20+length]); err != nil { 94 | fmt.Println("read header", err) 95 | return 96 | } 97 | 98 | handleData(buffer[20 : 20+length]) 99 | } 100 | } 101 | 102 | // 发送数据包 103 | func (client *TcpClient) sendReportPacket() { 104 | sender(client.connection) 105 | } 106 | 107 | // 发送心跳包，与发送数据包一样 108 | func (client *TcpClient) sendHeartPacket() { 109 | sender(client.connection) 110 | fmt.Println("Send heartbeat data success!") 111 | } 112 | 113 | func sender(conn net.Conn) { 114 | uid := "1A156FBB" 115 | // topic := "topic_88" 116 | // rid := "1533799576408588" 117 | // uid := "29F04AC0" 118 | topic := "topic_49" 119 | rid := "1533872684082049" 120 | buffString := "090002034c04" + "0072" + uid + "000027100000002d" 121 | buff, _ := hex.DecodeString(buffString) 122 | 123 | var buffer bytes.Buffer 124 | buffer.Write(buff) 125 | fmt.Println(buffer) 126 | binary.Write(&buffer, binary.LittleEndian, uint16(1)) 127 | fmt.Println(buffer) 128 | binary.Write(&buffer, binary.LittleEndian, uint16(0x600)) 129 | fmt.Println(buffer) 130 | buffer.Write([]byte("domain")) 131 | fmt.Println(buffer) 132 | binary.Write(&buffer, binary.LittleEndian, uint16(0x500)) 133 | fmt.Println(buffer) 134 | 135 | buffer.Write([]byte("group")) 136 | fmt.Println(buffer) 137 | binary.Write(&buffer, binary.LittleEndian, uint16(0x800)) 138 | fmt.Println(buffer) 139 | buffer.Write([]byte(topic)) 140 | fmt.Println(buffer) 141 | binary.Write(&buffer, binary.LittleEndian, uint16(0x1000)) 142 | fmt.Println(buffer) 143 | buffer.Write([]byte(rid)) 144 | fmt.Println(buffer) 145 | conn.Write(buffer.Bytes()) 146 | } 147 | 148 | func GzipDecode(in []byte) (out []byte, err error) { 149 | reader, err := gzip.NewReader(bytes.NewReader(in)) 150 | if err != nil { 151 | return nil, err 152 | } 153 | 154 | out, err = ioutil.ReadAll(reader) 155 | reader.Close() 156 | return 157 | } 158 | -------------------------------------------------------------------------------- /gather/lib/util.js: -------------------------------------------------------------------------------- 1 | /** 2 | * tang xiaoji 3 | */ 4 | const request = require('superagent'); 5 | const STRING = 'abcdefghijklmnopqrstuvwxyz0123456789'; 6 | const sign = require('../../helper/qmSign'); 7 | const crc32 = require('buffer-crc32'); 8 | const _ = require('lodash'); 9 | const config = require('../../config/config'); 10 | 11 | exports.getIndexName = prefix => { 12 | const d = new Date(); 13 | return [ 14 | prefix, 15 | d.getFullYear(), 16 | ('0' + (d.getMonth() + 1)).slice(-2), 17 | ('0' + d.getDate()).slice(-2) 18 | ].join(''); 19 | }; 20 | 21 | exports.fib = n => { 22 | let stack = []; 23 | if (n < 3) { 24 | return 1; 25 | } 26 | stack.push(1); 27 | stack.push(1); 28 | let i = 3; 29 | while (i < n) { 30 | let tmp1 = stack.pop(); 31 | let tmp2 = stack.pop(); 32 | stack.push(tmp2); 33 | stack.push(tmp1); 34 | stack.push(tmp1 + tmp2); 35 | i++; 36 | } 37 | return stack.pop() + stack.pop(); 38 | }; 39 | 40 | exports.getToken = async () => { 41 | let n = 8; 42 | let nonceStr = ''; 43 | while (n--) { 44 | nonceStr += STRING.charAt(_.random(0, 35)); 45 | } 46 | 47 | const options = { 48 | appId: 'qmfeb3ecafda826617', 49 | cid: '39', 50 | nonceStr: nonceStr 51 | }; 52 | 53 | options.sign = sign(options); 54 | 55 | let url = Object.keys(options) 56 | .reduce((res, curr) => res + `&${curr}=${options[curr]}`, ''); 57 | url = 'http://open.quanmin.tv/app/get/token?' + url.substr(1); 58 | 59 | return await request.get(url); 60 | }; 61 | 62 | exports.getGiftList = async () => { 63 | let n = 8; 64 | let nonceStr = ''; 65 | while (n--) { 66 | nonceStr += STRING.charAt(_.random(0, 35)); 67 | } 68 | 69 | const options = { 70 | appId: 'qmfeb3ecafda826617', 71 | cid: '39', 72 | nonceStr: nonceStr 73 | }; 74 | 75 | options.sign = sign(options); 76 | 77 | let url = Object.keys(options) 78 | .reduce((res, curr) => res + `&${curr}=${options[curr]}`, ''); 79 | url = 'http://open.quanmin.tv/public/gift/list?' + url.substr(1); 80 | 81 | return await request.get(url); 82 | }; 83 | 84 | exports.encode = (data) => { 85 | let {service, content} = data; 86 | if (!content) { 87 | content = Buffer.from([]); 88 | } 89 | let header = Buffer.alloc(24); 90 | //length 91 | header.writeInt32LE(20 + content.length, 0); 92 | //sequence int随机值 93 | header.writeInt32LE(_.random(1, Math.pow(2, 31) - 1), 4); 94 | //type 95 | header.writeInt32LE(service || 0, 8); 96 | //ver 97 | header.writeInt32LE(2, 12); 98 | //crc 99 | header.writeUInt32LE(crc32.unsigned(content || 0), 16); 100 | //reserve 101 | header.writeUInt32LE(0, 20); 102 | return Buffer.concat([header, content], 24 + content.length); 103 | }; 104 | 105 | exports.getIndexName = (prefix) => { 106 | const d = new Date(); 107 | return [ 108 | prefix, 109 | d.getFullYear(), 110 | ('0' + (d.getMonth() + 1)).slice(-2), 111 | ('0' + d.getDate()).slice(-2) 112 | ].join(''); 113 | }; 114 | 115 | exports.trimNativeDataInfo = (room, info) => { 116 | const keys = [ 117 | 'anchor_id', 'anchor_nick', 'cate_id', 118 | 'cate_name', 'client_ip', 'room_title', 119 | 'info', 'plat', 'room_id', 'url', 120 | 'room_title' 121 | ]; 122 | 123 | let log = {}; 124 | keys.forEach(key => { 125 | switch (key) { 126 | case 'info': { 127 | log['info'] = JSON.stringify(info); 128 | break; 129 | } 130 | case 'client_ip': { 131 | log['client_ip'] = config.app.host; 132 | break; 133 | } 134 | case 'plat': { 135 | if (room.plat === config.plat.xypanda) { 136 | log['plat'] = config.plat.panda; 137 | break; 138 | } 139 | } 140 | default: { 141 | if(room.hasOwnProperty(key)) { 142 | log[key] = room[key] 143 | } else { 144 | log[key] = '' 145 | } 146 | } 147 | } 148 | }); 149 | 150 | return log; 151 | }; 152 | 153 | exports.sleep = (time = 60 * 1000) => { 154 | return new Promise((resolve) => { 155 | setTimeout(resolve, time); 156 | }); 157 | } 158 | -------------------------------------------------------------------------------- /hosts.yaml: -------------------------------------------------------------------------------- 1 | [gitlab_ansible] 2 | fentuan-gitlab_ansible-0_1 ansible_ssh_host=10.8.0.1 3 | 4 | [elasticsearch] 5 | fentuan-elasticsearch-0_56 ansible_ssh_host=10.10.0.56 6 | fentuan-elasticsearch-0_55 ansible_ssh_host=10.10.0.55 7 | 8 | [bee] 9 | fentuan-bee-19_231 ansible_ssh_host=10.4.19.231 10 | fentuan-bee-20_3 ansible_ssh_host=10.4.20.3 11 | fentuan-bee-19_253 ansible_ssh_host=10.4.19.253 12 | fentuan-bee-19_255 ansible_ssh_host=10.4.19.255 13 | fentuan-bee-20_7 ansible_ssh_host=10.4.20.7 14 | fentuan-bee-19_205 ansible_ssh_host=10.4.19.205 15 | fentuan-bee-19_221 ansible_ssh_host=10.4.19.221 16 | fentuan-bee-19_203 ansible_ssh_host=10.4.19.203 17 | fentuan-bee-19_227 ansible_ssh_host=10.4.19.227 18 | fentuan-bee-20_6 ansible_ssh_host=10.4.20.6 19 | fentuan-bee-20_4 ansible_ssh_host=10.4.20.4 20 | fentuan-bee-19_198 ansible_ssh_host=10.4.19.198 21 | fentuan-bee-19_206 ansible_ssh_host=10.4.19.206 22 | fentuan-bee-19_222 ansible_ssh_host=10.4.19.222 23 | fentuan-bee-19_228 ansible_ssh_host=10.4.19.228 24 | fentuan-bee-20_2 ansible_ssh_host=10.4.20.2 25 | fentuan-bee-19_235 ansible_ssh_host=10.4.19.235 26 | fentuan-bee-19_233 ansible_ssh_host=10.4.19.233 27 | fentuan-bee-19_250 ansible_ssh_host=10.4.19.250 28 | fentuan-bee-20_5 ansible_ssh_host=10.4.20.5 29 | fentuan-bee-19_234 ansible_ssh_host=10.4.19.234 30 | fentuan-bee-20_0 ansible_ssh_host=10.4.20.0 31 | fentuan-bee-20_1 ansible_ssh_host=10.4.20.1 32 | fentuan-bee-19_200 ansible_ssh_host=10.4.19.200 33 | fentuan-bee-19_202 ansible_ssh_host=10.4.19.202 34 | fentuan-bee-19_199 ansible_ssh_host=10.4.19.199 35 | fentuan-bee-19_226 ansible_ssh_host=10.4.19.226 36 | fentuan-bee-19_218 ansible_ssh_host=10.4.19.218 37 | fentuan-bee-20_8 ansible_ssh_host=10.4.20.8 38 | fentuan-bee-19_201 ansible_ssh_host=10.4.19.201 39 | fentuan-bee-19_251 ansible_ssh_host=10.4.19.251 40 | fentuan-bee-19_225 ansible_ssh_host=10.4.19.225 41 | fentuan-bee-19_204 ansible_ssh_host=10.4.19.204 42 | fentuan-bee-19_224 ansible_ssh_host=10.4.19.224 43 | fentuan-bee-19_223 ansible_ssh_host=10.4.19.223 44 | fentuan-bee-19_229 ansible_ssh_host=10.4.19.229 45 | fentuan-bee-19_252 ansible_ssh_host=10.4.19.252 46 | fentuan-bee-19_217 ansible_ssh_host=10.4.19.217 47 | fentuan-bee-19_219 ansible_ssh_host=10.4.19.219 48 | fentuan-bee-19_220 ansible_ssh_host=10.4.19.220 49 | fentuan-bee-19_236 ansible_ssh_host=10.4.19.236 50 | fentuan-bee-19_232 ansible_ssh_host=10.4.19.232 51 | fentuan-bee-19_230 ansible_ssh_host=10.4.19.230 52 | fentuan-bee-19_254 ansible_ssh_host=10.4.19.254 53 | 54 | [newbee] 55 | fentuan-bee-20_38 ansible_ssh_host=10.4.20.38 56 | fentuan-bee-20_22 ansible_ssh_host=10.4.20.22 57 | fentuan-bee-20_35 ansible_ssh_host=10.4.20.35 58 | fentuan-bee-20_20 ansible_ssh_host=10.4.20.20 59 | fentuan-bee-20_24 ansible_ssh_host=10.4.20.24 60 | fentuan-bee-20_23 ansible_ssh_host=10.4.20.23 61 | fentuan-bee-20_33 ansible_ssh_host=10.4.20.33 62 | fentuan-bee-20_36 ansible_ssh_host=10.4.20.36 63 | fentuan-bee-20_21 ansible_ssh_host=10.4.20.21 64 | fentuan-bee-20_16 ansible_ssh_host=10.4.20.16 65 | fentuan-bee-20_17 ansible_ssh_host=10.4.20.17 66 | fentuan-bee-20_18 ansible_ssh_host=10.4.20.18 67 | fentuan-bee-20_32 ansible_ssh_host=10.4.20.32 68 | fentuan-bee-20_39 ansible_ssh_host=10.4.20.39 69 | fentuan-bee-20_26 ansible_ssh_host=10.4.20.26 70 | fentuan-bee-20_30 ansible_ssh_host=10.4.20.30 71 | fentuan-bee-20_28 ansible_ssh_host=10.4.20.28 72 | fentuan-bee-20_34 ansible_ssh_host=10.4.20.34 73 | fentuan-bee-20_31 ansible_ssh_host=10.4.20.31 74 | fentuan-bee-20_19 ansible_ssh_host=10.4.20.19 75 | fentuan-bee-20_29 ansible_ssh_host=10.4.20.29 76 | fentuan-bee-20_37 ansible_ssh_host=10.4.20.37 77 | fentuan-bee-20_25 ansible_ssh_host=10.4.20.25 78 | fentuan-bee-20_27 ansible_ssh_host=10.4.20.27 79 | fentuan-bee-20_14 ansible_ssh_host=10.4.20.14 80 | fentuan-bee-20_10 ansible_ssh_host=10.4.20.10 81 | fentuan-bee-20_13 ansible_ssh_host=10.4.20.13 82 | fentuan-bee-20_11 ansible_ssh_host=10.4.20.11 83 | fentuan-bee-20_12 ansible_ssh_host=10.4.20.12 84 | fentuan-bee-20_9 ansible_ssh_host=10.4.20.9 85 | fentuan-bee-19_206 ansible_ssh_host=10.4.19.206 86 | fentuan-bee-19_201 ansible_ssh_host=10.4.19.201 87 | fentuan-bee-19_203 ansible_ssh_host=10.4.19.203 88 | fentuan-bee-19_202 ansible_ssh_host=10.4.19.202 89 | fentuan-bee-19_204 ansible_ssh_host=10.4.19.204 90 | fentuan-bee-19_205 ansible_ssh_host=10.4.19.205 91 | fentuan-bee-19_200 ansible_ssh_host=10.4.19.200 92 | fentuan-bee-19_199 ansible_ssh_host=10.4.19.199 93 | fentuan-bee-19_198 ansible_ssh_host=10.4.19.198 94 | -------------------------------------------------------------------------------- /test/test_danmu.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Creator: Tang Xiaoji 3 | * Time: 2018-01-16 4 | */ 5 | 6 | const moment = require('moment'); 7 | const Socket = require('net').Socket; 8 | const socket = new Socket(); 9 | 10 | const Log4js = require('../config/log'); 11 | 12 | const logtail_chat = Log4js.getLogger('logtail_chat'); 13 | const logtail_gift = Log4js.getLogger('logtail_gift'); 14 | 15 | socket.on('error', err => { 16 | console.log(err); 17 | }); 18 | 19 | socket.on('timeout', () => { 20 | console.log('timeout'); 21 | }); 22 | 23 | socket.on('connect', () => { 24 | console.log('connect'); 25 | }); 26 | 27 | socket.on('data', (buf) => { 28 | let msg = {}; 29 | let buffer = Buffer.from(buf); 30 | let offset = 0; 31 | msg.length = buffer.readInt32LE(offset); 32 | offset += 4; 33 | msg.sequence = buffer.readInt32LE(offset); 34 | offset += 4; 35 | msg.type = buffer.readInt32LE(offset); 36 | offset += 4; 37 | msg.ver = buffer.readInt32LE(offset); 38 | offset += 4; 39 | msg.crc = buffer.readInt32LE(offset); 40 | offset += 4; 41 | msg.reserve = buffer.readInt32LE(offset); 42 | offset += 4; 43 | msg.body = msgPack.decode(buffer.slice(offset, buffer.length)); 44 | 45 | switch (msg.type) { 46 | case 1001: { 47 | send(1002, { 48 | owid: [] 49 | }); 50 | offset = 0; 51 | keepalive(); 52 | break; 53 | } 54 | case 1002: { 55 | offset = 0; 56 | break; 57 | } 58 | case 1004: { 59 | console.log('心跳...'); 60 | offset = 0; 61 | break; 62 | } 63 | case 1005: 64 | logtail_chat.info(_trimChat(msg.body)); 65 | offset = 0; 66 | break; 67 | case 1006: 68 | offset = 0; 69 | break; 70 | } 71 | 72 | buffer = null; 73 | }); 74 | 75 | const msgPack = require('msgpack-lite'); 76 | const crc32 = require('buffer-crc32'); 77 | 78 | const request = require('superagent'); 79 | const sign = require('../helper/qmSign'); 80 | const STRING = 'abcdefghijklmnopqrstuvwxyz0123456789'; 81 | const _ = require('lodash'); 82 | let n = 8; 83 | let nonceStr = ''; 84 | while (n--) { 85 | nonceStr += STRING.charAt(_.random(0, 35)); 86 | } 87 | 88 | const options = { 89 | appId: 'qmfeb3ecafda826617', 90 | cid: '39', 91 | nonceStr: nonceStr 92 | }; 93 | 94 | options.sign = sign(options); 95 | 96 | let url = Object.keys(options) 97 | .reduce((res, curr) => res + `&${curr}=${options[curr]}`, ''); 98 | url = 'http://open.quanmin.tv/app/get/token?' + url.substr(1); 99 | 100 | console.log(url); 101 | 102 | request.get(url).then((res) => { 103 | const token = res.body.data.token; 104 | 105 | socket.connect('8700', '101.132.85.192'); 106 | 107 | send(1001, { 108 | appId: options.appId, 109 | token: token 110 | }); 111 | }).catch(err => { 112 | console.log(err); 113 | }); 114 | 115 | function send(service, data) { 116 | let content; 117 | if (data) { 118 | content = msgPack.encode(data); 119 | } 120 | let tmp = encode({ 121 | service: service, 122 | content: content 123 | }); 124 | socket.write(tmp); 125 | } 126 | 127 | function encode(data) { 128 | let {service, content} = data; 129 | if (!content) { 130 | content = Buffer.from([]); 131 | } 132 | let header = Buffer.alloc(24); 133 | //length 134 | header.writeInt32LE(20 + content.length, 0); 135 | //sequence int随机值 136 | header.writeInt32LE(_.random(1, Math.pow(2, 31) - 1), 4); 137 | //type 138 | header.writeInt32LE(service || 0, 8); 139 | //ver 140 | header.writeInt32LE(2, 12); 141 | //crc 142 | header.writeUInt32LE(crc32.unsigned(content || 0), 16); 143 | //reserve 144 | header.writeUInt32LE(0, 20); 145 | return Buffer.concat([header, content], 24 + content.length); 146 | } 147 | 148 | function keepalive() { 149 | setTimeout(() => { 150 | send(1004); 151 | keepalive(); 152 | }, 10 * 1000); 153 | } 154 | 155 | function _trimChat(msg) { 156 | return JSON.stringify({ 157 | cate_id: -1, 158 | cate_name: '', 159 | client_ip: '10.4.20.15', 160 | datetime: moment().format('YYYY-MM-DD HH-mm-ss'), 161 | device: 1, 162 | first_time: -1, 163 | medal_level: msg.roomAttr.guard, 164 | plat: 'quanmin', 165 | room_id: -1, 166 | room_nick: '', 167 | timestamp: moment().unix(), 168 | txt: msg.txt, 169 | type: 'chat', 170 | user_id: msg.user.uid, 171 | user_level: msg.user.level, 172 | user_nick: msg.user.nickname 173 | }); 174 | } 175 | -------------------------------------------------------------------------------- /helper/huoshan/python-scripts/tt_encrypt.py: -------------------------------------------------------------------------------- 1 | #!/usr/local/bin/python3 2 | import json 3 | import gzip 4 | import zlib 5 | import sys 6 | 7 | # data encrypt 8 | def tt_encrypt( 9 | data, 10 | key=b'!*ss!_defaul%t54k&ey', 11 | sbox0=b'\x63\x7c\x77\x7b\xf2\x6b\x6f\xc5\x30\x01\x67\x2b\xfe\xd7\xab\x76\xca\x82\xc9\x7d\xfa\x59\x47\xf0\xad\xd4\xa2\xaf\x9c\xa4\x72\xc0\xb7\xfd\x93\x26\x36\x3f\xf7\xcc\x34\xa5\xe5\xf1\x71\xd8\x31\x15\x04\xc7\x23\xc3\x18\x96\x05\x9a\x07\x12\x80\xe2\xeb\x27\xb2\x75\x09\x83\x2c\x1a\x1b\x6e\x5a\xa0\x52\x3b\xd6\xb3\x29\xe3\x2f\x84\x53\xd1\x00\xed\x20\xfc\xb1\x5b\x6a\xcb\xbe\x39\x4a\x4c\x58\xcf\xd0\xef\xaa\xfb\x43\x4d\x33\x85\x45\xf9\x02\x7f\x50\x3c\x9f\xa8\x51\xa3\x40\x8f\x92\x9d\x38\xf5\xbc\xb6\xda\x21\x10\xff\xf3\xd2\xcd\x0c\x13\xec\x5f\x97\x44\x17\xc4\xa7\x7e\x3d\x64\x5d\x19\x73\x60\x81\x4f\xdc\x22\x2a\x90\x88\x46\xee\xb8\x14\xde\x5e\x0b\xdb\xe0\x32\x3a\x0a\x49\x06\x24\x5c\xc2\xd3\xac\x62\x91\x95\xe4\x79\xe7\xc8\x37\x6d\x8d\xd5\x4e\xa9\x6c\x56\xf4\xea\x65\x7a\xae\x08\xba\x78\x25\x2e\x1c\xa6\xb4\xc6\xe8\xdd\x74\x1f\x4b\xbd\x8b\x8a\x70\x3e\xb5\x66\x48\x03\xf6\x0e\x61\x35\x57\xb9\x86\xc1\x1d\x9e\xe1\xf8\x98\x11\x69\xd9\x8e\x94\x9b\x1e\x87\xe9\xce\x55\x28\xdf\x8c\xa1\x89\x0d\xbf\xe6\x42\x68\x41\x99\x2d\x0f\xb0\x54\xbb\x16', 12 | sbox1=b'\x52\x09\x6a\xd5\x30\x36\xa5\x38\xbf\x40\xa3\x9e\x81\xf3\xd7\xfb\x7c\xe3\x39\x82\x9b\x2f\xff\x87\x34\x8e\x43\x44\xc4\xde\xe9\xcb\x54\x7b\x94\x32\xa6\xc2\x23\x3d\xee\x4c\x95\x0b\x42\xfa\xc3\x4e\x08\x2e\xa1\x66\x28\xd9\x24\xb2\x76\x5b\xa2\x49\x6d\x8b\xd1\x25\x72\xf8\xf6\x64\x86\x68\x98\x16\xd4\xa4\x5c\xcc\x5d\x65\xb6\x92\x6c\x70\x48\x50\xfd\xed\xb9\xda\x5e\x15\x46\x57\xa7\x8d\x9d\x84\x90\xd8\xab\x00\x8c\xbc\xd3\x0a\xf7\xe4\x58\x05\xb8\xb3\x45\x06\xd0\x2c\x1e\x8f\xca\x3f\x0f\x02\xc1\xaf\xbd\x03\x01\x13\x8a\x6b\x3a\x91\x11\x41\x4f\x67\xdc\xea\x97\xf2\xcf\xce\xf0\xb4\xe6\x73\x96\xac\x74\x22\xe7\xad\x35\x85\xe2\xf9\x37\xe8\x1c\x75\xdf\x6e\x47\xf1\x1a\x71\x1d\x29\xc5\x89\x6f\xb7\x62\x0e\xaa\x18\xbe\x1b\xfc\x56\x3e\x4b\xc6\xd2\x79\x20\x9a\xdb\xc0\xfe\x78\xcd\x5a\xf4\x1f\xdd\xa8\x33\x88\x07\xc7\x31\xb1\x12\x10\x59\x27\x80\xec\x5f\x60\x51\x7f\xa9\x19\xb5\x4a\x0d\x2d\xe5\x7a\x9f\x93\xc9\x9c\xef\xa0\xe0\x3b\x4d\xae\x2a\xf5\xb0\xc8\xeb\xbb\x3c\x83\x53\x99\x61\x17\x2b\x04\x7e\xba\x77\xd6\x26\xe1\x69\x14\x63\x55\x21\x0c\x7d' 13 | ): 14 | # data = '{"_gen_time":1532598343824,"header":{"access":"wifi","aid":1112,"appkey":"56ea65c067e58eea7e000c63","display_name":"火山小视频","not_request_sender":0,"package":"com.ss.android.ugc.live","release_build":"dfa0993_20180620","sdk_version":"2.5.4.0","sig_hash":"aea615ab910015038f73c47e45d21466","build_serial":"49176505","clientudid":"3fdfa56d-e8ec-2a5d-431e-a2257c67a86a","cpu_abi":"armeabi-v7a","density_dpi":480,"device_brand":"Xiaomi","device_manufacturer":"Xiaomi","device_model":"MI 4LTE","display_density":"mdpi","language":"zh","mc":"A2:6A:A0:A6:72:EE","openudid":"53d4e91578487185","os":"Android","os_api":23,"os_version":"6.0.1","region":"CN","resolution":"1080x1920","rom":"MIUI-7.11.30","rom_version":"miui_V9_7.11.30","serial_number":"49176505","sim_serial_number":[],"timezone":8,"tz_name":"Asia/Shanghai","tz_offset":28800,"udid":"502025975825073","app_version":"4.2.0.3","channel":"xiaomi","manifest_version_code":4203,"update_version_code":4203,"version_code":4203},"magic_tag":"ss_app_log"}' 15 | def ROR(bytes, start, end, byte): 16 | r = bytes[start:end] 17 | b = len(r) - byte 18 | r = r[b:] + r[:b] 19 | return bytes[0:start] + r + bytes[end:len(bytes)] 20 | if not isinstance(key, bytearray): 21 | key = bytearray(key) 22 | while True: 23 | if len(key) < 16: 24 | key.append(sbox1[key[len(key) - 1]]) 25 | else: 26 | key = key[0:16].translate(sbox0) 27 | break 28 | input = gzip.compress(data.encode('utf-8', 'surrogateescape')) 29 | output = bytearray(input) 30 | headers = [31, 139, 8, 0, 0, 0, 0, 0, 0, 0] 31 | for i in range(len(headers)): 32 | output[i] = headers[i] 33 | fill = (-len(input)) & 15 34 | output.extend(bytearray(fill)) 35 | output = output.translate(sbox0) 36 | for i in range(int(len(output) / 16)): 37 | i *= 16 38 | output = ROR(output, i + 4, i + 8, 3) 39 | output = ROR(output, i + 8, i + 12, 2) 40 | output = ROR(output, i + 12, i + 16, 1) 41 | for j in range(16): 42 | output[i + j] ^= key[j] 43 | return bytes([116, 99, 2, fill]) + bytes(output) 44 | 45 | def main(): 46 | result = tt_encrypt(sys.argv[1]) 47 | result = "".join(map(chr, result)) # 转成str进行传输 此方案不用考虑编码 48 | print(result) 49 | sys.stdout.flush() 50 | # return result 51 | 52 | if __name__ == '__main__': 53 | main() 54 | 55 | 56 | -------------------------------------------------------------------------------- /helper/monitor.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Creator: Tang Xiaoji 3 | * Time: 2018-06-20 4 | */ 5 | 6 | 'use strict'; 7 | 8 | const Logger = require('../config/Logger'); 9 | const logger = new Logger('cpu/mem/pkg monitor'); 10 | const {exec} = require('child_process'); 11 | const os = require('os'); 12 | const config = require('../config/config'); 13 | const mysql = require('../config/mysql'); 14 | 15 | function getCpuUsed() { 16 | return new Promise((resolve) => { 17 | try { 18 | const plat = os.platform(); 19 | if (plat !== 'linux') { 20 | resolve({ 21 | used: '0%' 22 | }); 23 | } else { 24 | exec('top -bn 1 -i -c', (error, stdout, stderr) => { 25 | if (error) { 26 | logger.error('getCpuUsed', 'monitor', error); 27 | resolve({ 28 | used: '-' 29 | }); 30 | } else if (stderr) { 31 | logger.error('getCpuUsed', 'monitor', stderr); 32 | resolve({ 33 | used: '-' 34 | }); 35 | } else { 36 | try { 37 | const idle = (100 - parseFloat(stdout.match(/ (\S*)%id,/)[1])).toFixed(1); 38 | resolve({ 39 | used: `${idle}%` 40 | }); 41 | } catch (e) { 42 | resolve({ 43 | used: '-' 44 | }); 45 | } 46 | } 47 | }); 48 | } 49 | } catch (e) { 50 | resolve({ 51 | used: '-' 52 | }); 53 | } 54 | }); 55 | } 56 | 57 | function getMemoryUsed() { 58 | return new Promise((resolve) => { 59 | try { 60 | const plat = os.platform(); 61 | if (plat !== 'linux') { 62 | resolve({ 63 | used: '0%' 64 | }); 65 | } else { 66 | exec('awk \'{if($1 ~ /MemTotal/ || $1 ~ /MemFree/ || $1 ~ /Buffers/ || $1 ~ /Cached/) print $1" "$2}\' /proc/meminfo', (error, stdout, stderr) => { 67 | if (error) { 68 | logger.error('getMemoryUsed', 'monitor', error); 69 | resolve({ 70 | used: '-' 71 | }); 72 | } else if (stderr) { 73 | logger.error('getMemoryUsed', 'monitor', stderr); 74 | resolve({ 75 | used: '-' 76 | }); 77 | } else { 78 | try { 79 | const total = parseInt(stdout.match(/MemTotal: (\S*)/)[1]); 80 | const free = parseInt(stdout.match(/MemFree: (\S*)/)[1]); 81 | const buffer = parseInt(stdout.match(/Buffers: (\S*)/)[1]); 82 | const cache = parseInt(stdout.match(/Cached: (\S*)/)[1]); 83 | resolve({ 84 | used: `${(((total - (free + buffer + cache)) / total) * 100).toFixed(1)}%` 85 | }); 86 | } catch (e) { 87 | resolve({ 88 | used: '-' 89 | }); 90 | } 91 | } 92 | }); 93 | } 94 | } catch (e) { 95 | resolve({ 96 | used: '-' 97 | }); 98 | } 99 | }); 100 | } 101 | 102 | async function updateSystemInfo() { 103 | try { 104 | const cpuInfo = await getCpuUsed(); 105 | const memInfo = await getMemoryUsed(); 106 | 107 | mysql.getConn((err, conn) => { 108 | if (err) { 109 | logger.error('检查系统性能 getconn error', 'monitor', err); 110 | setTimeout(updateSystemInfo, config.timeout.update_system_info); 111 | } else { 112 | conn.query('INSERT INTO ?? ( ?? ) VALUES ( ? ) ON DUPLICATE KEY UPDATE cpu = ?, memory = ?, pkgs = ?', [ 113 | config.tbl_name.db_fentuan_monitor_v2, [ 114 | 'cpu', 'memory', 'pkgs', 'client_ip' 115 | ], [ 116 | cpuInfo.used, memInfo.used, 0, config.app.host 117 | ], cpuInfo.used, memInfo.used, 0 118 | ], (e) => { 119 | conn.release(); 120 | setTimeout(updateSystemInfo, config.timeout.update_system_info); 121 | if (e) { 122 | logger.error('检查系统性能 error', 'monitor', e); 123 | } 124 | }); 125 | } 126 | }) 127 | } catch (e) { 128 | logger.error('检查系统性能 cpu/mem error', 'monitor', e); 129 | setTimeout(updateSystemInfo, config.timeout.update_system_info); 130 | } 131 | } 132 | 133 | process.on('uncaughtException', (err) => { 134 | logger.error('bee start uncaughtException', 'monitor', err); 135 | }); 136 | 137 | module.exports = { 138 | getCpuUsed, 139 | getMemoryUsed, 140 | updateSystemInfo 141 | }; 142 | 143 | // getCpuUsed().then(r => console.log('cpu: ', r)).catch(e => console.log(e)); 144 | // getMemoryUsed().then(r => console.log('mem: ', r)).catch(e => console.log(e)); 145 | -------------------------------------------------------------------------------- /danmu/Huoshan.js: -------------------------------------------------------------------------------- 1 | const EventEmitter = require('events'); 2 | const _ = require('lodash'); 3 | const config = require('../config/config'); 4 | // const redis = require('../config/redis'); 5 | const HuoshanApi = require('../helper/huoshan/api'); 6 | 7 | const RETRY_TIMEOUT = 20 * 1000; 8 | const REQ_TIMEOUT = 6000; 9 | const MAX_RETYR = 5; 10 | const LIVE_ROOM_ID_RETRY = 30 * 60 * 1000; // 频繁检测 需要代理 然而采集需要大量代理 11 | 12 | module.exports = class Huoshan extends EventEmitter { 13 | constructor(url = '', opts) { 14 | super(); 15 | this.url = url; 16 | this.roomId = -1; 17 | this.anchorId = opts ? opts.anchor_id : ''; 18 | this.retryCount = 0; 19 | this.cursor = 0; 20 | this.liveRoomTimer = null; 21 | this.destroyFlag = false; 22 | this.init(); 23 | } 24 | 25 | async init() { 26 | try { 27 | let count = 0; 28 | while (!Object.keys(HuoshanApi.query).length && count < 10) { 29 | count++; 30 | await HuoshanApi.delay(1000) 31 | } 32 | if(count >= 10) { 33 | this.emit('initerror', new Error('获取设备信息失败')); 34 | return false; 35 | } 36 | // 尽量避免同时大量房间初始化 (同时大量初始化, 拿不到房间号) 37 | await HuoshanApi.delay(Math.floor(Math.random() * 120) * 1000); 38 | 39 | const roomStatus = await this.requestRoomStatus(); 40 | const enterStatus = true; // await this.enterRoom(); 41 | if (roomStatus && enterStatus) { 42 | if(!this.destroyFlag) { 43 | 44 | this.emit('connect'); 45 | this.connect(); 46 | 47 | // 重复获取 live room id, 防止主播短时间内频繁开关播 -> 直播间号随之改变, 带来采集不到信息的情况 48 | if(this.liveRoomTimer) clearInterval(this.liveRoomTimer); 49 | this.liveRoomTimer = setInterval(this.getLiveRoomId.bind(this), LIVE_ROOM_ID_RETRY); 50 | } 51 | } else { 52 | this.emit('initerror', 'unable to get chushou room info'); 53 | } 54 | } catch (e) { 55 | this.emit('initerror', e); 56 | } 57 | } 58 | 59 | async getLiveRoomId () { 60 | try { 61 | let roomId = await HuoshanApi.getLiveRoomIdByAnchorId(this.anchorId); 62 | if(!roomId) return false; 63 | 64 | if(this.roomId !== roomId) { 65 | this.roomId = roomId; 66 | this.cursor = 0; 67 | }; 68 | return true; 69 | } catch (e) { 70 | return false; 71 | } 72 | } 73 | 74 | async requestRoomStatus() { 75 | try { 76 | let roomId = await this.getLiveRoomId(); 77 | if (!roomId) { 78 | this.emit('initerror', new Error('huoshan huoshanId error')); 79 | return false; 80 | } 81 | // await redis.lpush(config.LIST_GIFT_TASK_KEY(config.plat.huoshan), roomId); 82 | return true; 83 | } catch (e) { 84 | throw e; 85 | } 86 | } 87 | 88 | connect() { 89 | clearTimeout(this.timer); 90 | this.timer = setTimeout(async () => { 91 | try { 92 | await this.clientDataHandler(); 93 | } catch (e) { 94 | if (this.retryCount++ > MAX_RETYR) { 95 | this.emit('error', e); 96 | return; 97 | } 98 | } 99 | this.connect(); 100 | }, RETRY_TIMEOUT); 101 | } 102 | 103 | async clientDataHandler() { 104 | 105 | let res = await HuoshanApi.fetchMessagePolling(this.roomId, this.cursor); 106 | 107 | if (res.status_code !== 0) { 108 | throw new Error('[E] request [%s] error', this.roomId, res.status_code, res); 109 | } 110 | let items = []; 111 | res.data.forEach(_tmp => { 112 | items.push(_tmp); 113 | }) 114 | this.cursor = res.extra.cursor || 0; 115 | 116 | if (items && items.length) { 117 | this.emit('data', items); 118 | } 119 | return; 120 | } 121 | 122 | async enterRoom() { 123 | try { 124 | let res = await HuoshanApi.enterRoom(this.roomId); 125 | if (res.status_code === 0) return true; 126 | return false; 127 | } catch (e) { 128 | return false; 129 | } 130 | } 131 | 132 | async restart() { 133 | this.cursor = 0; 134 | this.init(); 135 | } 136 | 137 | destroy() { 138 | clearTimeout(this.timer); 139 | if(this.liveRoomTimer) clearInterval(this.liveRoomTimer); 140 | this.destroyFlag = true; 141 | this.cursor = 0; 142 | this.emit('destroy'); 143 | this.removeAllListeners(); 144 | } 145 | }; 146 | 147 | // const huoshan = new Huoshan('', {anchor_id: 100720770940}) 148 | // module.exports = async () => { 149 | // await HuoshanApi.delay(2000) 150 | // let res = await HuoshanApi.giftList(); 151 | // res.data.forEach(_tmp => { 152 | // console.log(_tmp.id, ' -- ', _tmp.name || _tmp.describe, ' -- ', _tmp.diamond_count) 153 | // }) 154 | // console.log('res ===> ', res.data.length) 155 | // } -------------------------------------------------------------------------------- /helper/gateway.json: -------------------------------------------------------------------------------- 1 | { 2 | "303215245": { 3 | "protocol": "Gateway.Shared.Notify", 4 | "msgType": "linkerProtocol.SharedNotify" 5 | }, 6 | "479781885": { 7 | "protocol": "Gateway.Gift.Notify", 8 | "msgType": "linkerProtocol.GiftNotify" 9 | }, 10 | "508840654": { 11 | "protocol": "Gateway.RoomJoin.Resp", 12 | "msgType": "linkerProtocol.RoomJoinResp" 13 | }, 14 | "570090790": { 15 | "protocol": "Gateway.Link.Notify", 16 | "msgType": "linkerProtocol.LinkNotify" 17 | }, 18 | "586698102": { 19 | "protocol": "Gateway.Manager.Notify", 20 | "msgType": "linkerProtocol.ManagerNotify" 21 | }, 22 | "617685563": { 23 | "protocol": "Gateway.Zan.Up", 24 | "msgType": "linkerProtocol.ZanUp" 25 | }, 26 | "669161096": { 27 | "protocol": "Gateway.Chat.Notify", 28 | "msgType": "linkerProtocol.ChatNotify" 29 | }, 30 | "676536667": { 31 | "protocol": "Gateway.RoomJoin.Notify", 32 | "msgType": "linkerProtocol.RoomJoinNotify" 33 | }, 34 | "697088987": { 35 | "protocol": "Gateway.OfficialLiveStatus.Notify", 36 | "msgType": "linkerProtocol.OfficialLiveStatusNotify" 37 | }, 38 | "-194078186": { 39 | "protocol": "Gateway.RoomUpdate.Notify", 40 | "msgType": "linkerProtocol.RoomUpdateNotify" 41 | }, 42 | "825420722": { 43 | "protocol": "Gateway.UserAttr.Notify", 44 | "msgType": "linkerProtocol.UserAttrNotify" 45 | }, 46 | "934367122": { 47 | "protocol": "Gateway.AnchorReady.Notify", 48 | "msgType": "linkerProtocol.AnchorReadyNotify" 49 | }, 50 | "1009213983": { 51 | "protocol": "Gateway.Link.Apply", 52 | "msgType": "linkerProtocol.LinkApply" 53 | }, 54 | "1150328117": { 55 | "protocol": "Gateway.GlobalMsg.Notify", 56 | "msgType": "linkerProtocol.GlobalMsg" 57 | }, 58 | "1301742417": { 59 | "protocol": "Gateway.RedEnvelope.Notify", 60 | "msgType": "linkerProtocol.RedEnvelopeNotify" 61 | }, 62 | "1452789148": { 63 | "protocol": "Gateway.Link.Cancel", 64 | "msgType": "linkerProtocol.LinkCancel" 65 | }, 66 | "1629484155": { 67 | "protocol": "Gateway.Login.Req", 68 | "msgType": "linkerProtocol.LoginReq" 69 | }, 70 | "1793833185": { 71 | "protocol": "Gateway.Link.Broadcast", 72 | "msgType": "linkerProtocol.LinkBroadcast" 73 | }, 74 | "1893073187": { 75 | "protocol": "Gateway.Chat.Up", 76 | "msgType": "linkerProtocol.ChatUp" 77 | }, 78 | "1952773763": { 79 | "protocol": "Gateway.Followed.Notify", 80 | "msgType": "linkerProtocol.FollowedNotify" 81 | }, 82 | "2005355379": { 83 | "protocol": "Gateway.Login.Resp", 84 | "msgType": "linkerProtocol.LoginResp" 85 | }, 86 | "-1260603416": { 87 | "protocol": "Gateway.RoomJoin.Req", 88 | "msgType": "linkerProtocol.RoomJoinReq" 89 | }, 90 | "-83894905": { 91 | "protocol": "Gateway.RoomJoin.Up", 92 | "msgType": "linkerProtocol.RoomJoinNotifyUp" 93 | }, 94 | "-742394014": { 95 | "protocol": "Gateway.RoomLeave.Up", 96 | "msgType": "linkerProtocol.RoomLeaveReq" 97 | }, 98 | "-35313019": { 99 | "protocol": "Gateway.Gift.Up", 100 | "msgType": "linkerProtocol.GiftUp" 101 | }, 102 | "-1425862037": { 103 | "protocol": "Gateway.Zan.Notify", 104 | "msgType": "linkerProtocol.ZanNotify" 105 | }, 106 | "-2118180727": { 107 | "protocol": "Gateway.Guard.Notify", 108 | "msgType": "linkerProtocol.RoomGuardNotify" 109 | }, 110 | "-358042289": { 111 | "protocol": "Gateway.AnchorLiveStatus.Notify", 112 | "msgType": "linkerProtocol.AnchorLiveStatusNotify" 113 | }, 114 | "-686543790": { 115 | "protocol": "Gateway.UserLevelUp.Notify", 116 | "msgType": "linkerProtocol.UserLevelUpNotify" 117 | }, 118 | "-2059302971": { 119 | "protocol": "Gateway.Kickout.Notify", 120 | "msgType": "linkerProtocol.KickoutNotify" 121 | }, 122 | "-1844875324": { 123 | "protocol": "Gateway.Link.Close", 124 | "msgType": "linkerProtocol.LinkClose" 125 | }, 126 | "-1281235494": { 127 | "protocol": "Gateway.Link.Accept", 128 | "msgType": "linkerProtocol.LinkAccept" 129 | }, 130 | "-225158577": { 131 | "protocol": "Gateway.Banner.Notify", 132 | "msgType": "linkerProtocol.BannerNotify" 133 | }, 134 | "-2068322255": { 135 | "protocol": "Gateway.OfficalRoomGift.Send", 136 | "msgType": "linkerProtocol.OfficialLiveLikeUp" 137 | }, 138 | "-2020231711": { 139 | "protocol": "Gateway.OfficialLiveLive.Notify", 140 | "msgType": "linkerProtocol.OfficialLiveLiveNotify" 141 | }, 142 | "-397601416": { 143 | "protocol": "Gateway.OfficialLiveNavigation.Change", 144 | "msgType": "linkerProtocol.OfficialLiveNavigationChange" 145 | }, 146 | "-1438216223": { 147 | "protocol": "Gateway.AnchorIndicate.Notify", 148 | "msgType": "linkerProtocol.AnchorIndicateNotify" 149 | } 150 | } 151 | -------------------------------------------------------------------------------- /gather/lib/recycle.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Creator: Tang Xiaoji 3 | * Time: 2018-06-27 4 | */ 5 | 6 | 'use strict'; 7 | 8 | const config = require('../../config/config'); 9 | const EventEmitter = require('events'); 10 | const Logger = require('../../config/Logger'); 11 | const logger = new Logger('recycle'); 12 | const mysql = require('../../config/mysql'); 13 | 14 | const EVENTS = { 15 | Reopen: 'recycle.reopen' 16 | }; 17 | 18 | class Recycle extends EventEmitter { 19 | constructor() { 20 | super(); 21 | 22 | setInterval(this.scan.bind(this), config.timeout.recycle_fail_task); 23 | } 24 | 25 | // 搜索所有失败的任务进行回收 26 | scan() { 27 | const plan = { 28 | reopen: [], // status: FAILED 29 | }; 30 | for (let plat in config.plat) { 31 | if (config.plat.hasOwnProperty(plat) && !config.blockList.includes(plat)) { 32 | mysql.getConn((err, conn) => { 33 | if (err) { 34 | logger.error('scan getconn error', plat, err); 35 | } else { 36 | conn.query('SELECT room_id, client_ip FROM ?? WHERE plat = ? AND status = ?', [ 37 | config.tbl_name.db_fentuan_taskv2, plat, config.status.FAILED 38 | ], (err, rooms) => { 39 | conn.release(); 40 | if(err) { 41 | logger.error('scan error', plat, err); 42 | } else { 43 | if (rooms && rooms.length) { 44 | rooms.forEach(room => { 45 | plan.reopen.push(room); 46 | }); 47 | 48 | this._todo(plat, plan); 49 | logger.error('recyclescan', plat, `已回收：${plat} ${rooms.length}`); 50 | } else { 51 | logger.error('recyclescan', plat, '没有需要回收的任务'); 52 | } 53 | } 54 | }); 55 | } 56 | }); 57 | } 58 | } 59 | } 60 | 61 | _todo(plat, plan) { 62 | for (let type in plan) { 63 | const method = this[type]; 64 | if (typeof method === 'function') { 65 | (method.bind(this))(plat, plan[type]); 66 | } 67 | } 68 | } 69 | 70 | reopen(plat, rooms) { 71 | this.emit(EVENTS.Reopen, plat, rooms); 72 | } 73 | 74 | // 回收掉线的机器上的任务 75 | recycleFailNodeTask(ip) { 76 | const list = { 77 | del: [], 78 | failed: [] 79 | }; 80 | mysql.getConn((err, conn) => { 81 | if (err) { 82 | logger.error('recycleFailNodeTask getconn0 error', ip, err); 83 | } else { 84 | conn.query('SELECT url, status, sub FROM ?? WHERE client_ip = ?', [ 85 | config.tbl_name.db_fentuan_taskv2, ip 86 | ], (err, results) => { 87 | conn.release(); 88 | if (err) { 89 | logger.error('recycleFailNodeTask select error', ip, err); 90 | } else { 91 | results.forEach(task => { 92 | if (task.sub == 1) { 93 | // 长期订阅的房间，无论开播与否都当做失败房间来进行回收 94 | list.failed.push(task.url); 95 | } else { 96 | switch (task.status) { 97 | case config.status.CLOSED: { 98 | // 删除关闭的任务 99 | list.del.push(task.url); 100 | break; 101 | } 102 | default: { 103 | list.failed.push(task.url); 104 | } 105 | } 106 | } 107 | }); 108 | 109 | for (let type in list) { 110 | const method = this[type]; 111 | if (typeof method === 'function') { 112 | method(list[type]); 113 | } 114 | } 115 | } 116 | }); 117 | } 118 | }) 119 | } 120 | 121 | del(urls) { 122 | urls.forEach(url => { 123 | mysql.getConn((err, conn) => { 124 | if (err) { 125 | logger.error('recycleFailNodeTask getconn1 error', config.getRealType(url), err); 126 | } else { 127 | conn.query('DELETE FROM ?? WHERE url = ? AND sub = ?', [ 128 | config.tbl_name.db_fentuan_taskv2, url, 2 129 | ], (err) => { 130 | conn.release(); 131 | if (err) { 132 | logger.error('recycleFailNodeTask del error', config.getRealType(url), err, url); 133 | } 134 | }); 135 | } 136 | }); 137 | }); 138 | } 139 | 140 | failed(urls) { 141 | urls.forEach(url => { 142 | mysql.getConn((err, conn) => { 143 | if (err) { 144 | logger.error('recycleFailNodeTask getconn2 error', config.getRealType(url), err); 145 | } else { 146 | conn.query('UPDATE ?? SET status = ? WHERE url = ?', [ 147 | config.tbl_name.db_fentuan_taskv2, config.status.FAILED, url 148 | ], (err) => { 149 | conn.release(); 150 | if (err) { 151 | logger.error('recycleFailNodeTask update error', config.getRealType(url), err, url); 152 | } else { 153 | logger.error('机器掉线，回收任务', config.getRealType(url), url); 154 | } 155 | }); 156 | } 157 | }); 158 | }); 159 | } 160 | } 161 | 162 | Recycle.events = EVENTS; 163 | module.exports = Recycle; 164 | -------------------------------------------------------------------------------- /gather/video-stream/VideoStream.js: -------------------------------------------------------------------------------- 1 | const redis = require('../../config/redis'); 2 | const ACTION = { 3 | STOP: 0, 4 | START: 1, 5 | UPDATE: 2 6 | }; 7 | const MAX_SLEEP = 10; 8 | const MAX_QUEUE_LEN = 10000; 9 | 10 | module.exports = class VideoStream { 11 | 12 | constructor (plat) { 13 | this.plat = plat; 14 | this.check(); 15 | } 16 | 17 | async updateLiving (plat, roomId) { 18 | return redis.sadd('SET:GATHER:STREAM:LIVING', `${plat}:${roomId}`); 19 | } 20 | 21 | async living () { 22 | const living = await redis.smembers('SET:GATHER:STREAM:LIVING'); 23 | let arr = []; 24 | for (let e of living) { 25 | let [plat, roomId] = e.split(':'); 26 | arr.push([plat, roomId]); 27 | } 28 | return arr; 29 | } 30 | 31 | async isLiving (plat, roomId) { 32 | return redis.sismember('SET:GATHER:STREAM:LIVING', `${plat}:${roomId}`); 33 | } 34 | 35 | async removeLiving (plat, roomId) { 36 | return redis.srem('SET:GATHER:STREAM:LIVING', `${plat}:${roomId}`); 37 | } 38 | 39 | async incrSleeping (plat, roomId) { 40 | return redis.hincrby('HASH:GATHER:STREAM:SLEEPING', `${plat}:${roomId}`, 1); 41 | } 42 | 43 | async sleeping () { 44 | return redis.hgetall('HASH:GATHER:STREAM:SLEEPING'); 45 | } 46 | 47 | async dead () { 48 | const sleeping = await this.sleeping(); 49 | let arr = []; 50 | for (let [k, v] of Object.entries(sleeping)) { 51 | if (v > MAX_SLEEP) { 52 | let [plat, roomId] = k.split(':'); 53 | arr.push([plat, roomId]); 54 | } 55 | } 56 | return arr; 57 | } 58 | 59 | async removeDead (arr) { 60 | let fields = []; 61 | for (let [plat, roomId] of arr) { 62 | fields.push(`${plat}:${roomId}`); 63 | } 64 | return redis.hdel('HASH:GATHER:STREAM:SLEEPING', fields); 65 | } 66 | 67 | async setHid (plat, roomId, url, hid) { 68 | return redis.hset('HASH:GATHER:STREAM:HID', `${plat}:${roomId}`, `${url}::${hid}`); 69 | } 70 | 71 | async getHid (plat, roomId) { 72 | const res = await redis.hget('HASH:GATHER:STREAM:HID', `${plat}:${roomId}`); 73 | return res ? res.split('::') : []; 74 | } 75 | 76 | async delHid (plat, roomId) { 77 | return redis.hdel('HASH:GATHER:STREAM:HID', `${plat}:${roomId}`); 78 | } 79 | 80 | async start (plat, roomId, url, hid) { 81 | await this.setHid(plat, roomId, url, hid); 82 | await this.updateLiving(plat, roomId); 83 | const streamAddr = await this.getStreamAddr(plat, roomId, url); 84 | await this.sendSign(hid, plat, roomId, ACTION.START, streamAddr); 85 | } 86 | 87 | async update (plat, roomId) { 88 | const [url, hid] = await this.getHid(plat, roomId); 89 | const streamAddr = await this.getStreamAddr(plat, roomId, url); 90 | if (streamAddr) { 91 | await this.sendSign(hid, plat, roomId, ACTION.UPDATE, streamAddr); 92 | } 93 | } 94 | 95 | async stop (plat, roomId, url, hid) { 96 | if (!hid) { 97 | [, hid] = await this.getHid(plat, roomId); 98 | } 99 | await this.delHid(plat, roomId); 100 | await this.removeLiving(plat, roomId); 101 | await this.sendSign(hid, plat, roomId, ACTION.STOP); 102 | } 103 | 104 | /** 105 | * 106 | * @param {*} hid 107 | * @param {*} plat 108 | * @param {*} roomId 109 | * @param {*} action 开关播信号，1：开播，0：关播，2：更新 110 | * @param {*} streamAddr 111 | * @param {*} opts 扩展字段 112 | */ 113 | async sendSign (hid, plat, roomId, action, streamAddr = '', opts = '') { 114 | const value = `${hid}::${plat}::${roomId}::${action}::${streamAddr}::${opts}`; 115 | if (action !== ACTION.UPDATE) { 116 | console.log('send sign: ', value); 117 | } 118 | return redis.lpush('LIST:GATHER:STREAM', value); 119 | } 120 | 121 | async getSignQueueLen () { 122 | return redis.llen('LIST:GATHER:STREAM'); 123 | } 124 | 125 | async getStreamAddr (plat, roomId, url) { 126 | return ''; 127 | } 128 | 129 | async newest (plat) { 130 | let arr = []; 131 | const roomList = await redis.smembers(`SET:GATHER:STREAM:NEW:${plat.toUpperCase()}`); 132 | for (let e of roomList) { 133 | let [plat, roomId] = e.split(':'); 134 | arr.push([plat, roomId]); 135 | } 136 | return arr; 137 | } 138 | 139 | async inNewest (plat, roomId) { 140 | return redis.sismember(`SET:GATHER:STREAM:NEW:${plat.toUpperCase()}`, `${plat}:${roomId}`); 141 | } 142 | 143 | check () { 144 | setInterval(async () => { 145 | console.log(`${this.plat} run video stream schedule !`); 146 | const living = await this.living(); 147 | for (let [plat, roomId] of living) { 148 | if (plat !== this.plat) { 149 | continue; 150 | } 151 | const inNewest = await this.inNewest(plat, roomId); 152 | if (inNewest) { 153 | const len = await this.getSignQueueLen(); 154 | if (len < MAX_QUEUE_LEN) { 155 | await this.update(plat, roomId); 156 | } 157 | } else { 158 | await this.incrSleeping(plat, roomId); 159 | } 160 | } 161 | 162 | const dead = await this.dead(); 163 | for (let [plat, roomId] of dead) { 164 | const isLiving = await this.isLiving(plat, roomId); 165 | if (isLiving) { 166 | await this.stop(plat, roomId); 167 | } 168 | } 169 | if (dead.length) { 170 | await this.removeDead(dead); 171 | } 172 | }, 60 * 1000); 173 | } 174 | 175 | } -------------------------------------------------------------------------------- /config/log_test.js: -------------------------------------------------------------------------------- 1 | const Log4js = require('log4js'); 2 | const moment = require('moment'); 3 | Log4js.addLayout('json', function () { 4 | return function (logEvent) { 5 | logEvent.datetime = moment().format('YYYY-MM-DD HH:mm:ss'); 6 | for (let key in logEvent.data[0]) { 7 | if (logEvent.data[0].hasOwnProperty(key)) { 8 | logEvent[key] = logEvent.data[0][key]; 9 | } 10 | } 11 | logEvent.timestamp = (moment().unix()).toString(); 12 | delete logEvent.startTime; 13 | delete logEvent.pid; 14 | delete logEvent.data; 15 | delete logEvent.categoryName; 16 | delete logEvent.level; 17 | delete logEvent.context; 18 | return JSON.stringify(logEvent) 19 | } 20 | }); 21 | 22 | Log4js.configure({ 23 | appenders: { 24 | out: { 25 | type: 'stdout' 26 | }, 27 | json_broadcast: { 28 | type: "file", 29 | layout: { 30 | type: 'json' 31 | }, 32 | filename: "./broadcast.log", 33 | maxLogSize: 512 * 1024 * 1024, 34 | backups: 4 35 | }, 36 | json_alarm: { 37 | type: "file", 38 | layout: { 39 | type: 'json' 40 | }, 41 | filename: "./alarm.log", 42 | maxLogSize: 512 * 1024 * 1024, 43 | backups: 4 44 | }, 45 | json_data_native: { 46 | type: "file", 47 | layout: { 48 | type: 'json' 49 | }, 50 | filename: `./data-native.log`, 51 | maxLogSize: 512 * 1024 * 1024, 52 | backups: 4 53 | }, 54 | json_data_native_douyu: { 55 | type: "file", 56 | layout: { 57 | type: 'json' 58 | }, 59 | filename: `./data-native-douyu.log`, 60 | maxLogSize: 512 * 1024 * 1024, 61 | backups: 4 62 | }, 63 | json_data_native_huya: { 64 | type: "file", 65 | layout: { 66 | type: 'json' 67 | }, 68 | filename: `./data-native-huya.log`, 69 | maxLogSize: 512 * 1024 * 1024, 70 | backups: 4 71 | }, 72 | json_data_native_longzhu: { 73 | type: "file", 74 | layout: { 75 | type: 'json' 76 | }, 77 | filename: `./data-native-longzhu.log`, 78 | maxLogSize: 512 * 1024 * 1024, 79 | backups: 4 80 | }, 81 | json_data_native_quanmin: { 82 | type: "file", 83 | layout: { 84 | type: 'json' 85 | }, 86 | filename: `./data-native-quanmin.log`, 87 | maxLogSize: 512 * 1024 * 1024, 88 | backups: 4 89 | }, 90 | json_data_native_zhanqi: { 91 | type: "file", 92 | layout: { 93 | type: 'json' 94 | }, 95 | filename: `./data-native-zhanqi.log`, 96 | maxLogSize: 512 * 1024 * 1024, 97 | backups: 4 98 | }, 99 | json_data_native_chushou: { 100 | type: "file", 101 | layout: { 102 | type: 'json' 103 | }, 104 | filename: `./data-native-chushou.log`, 105 | maxLogSize: 512 * 1024 * 1024, 106 | backups: 4 107 | }, 108 | json_data_native_panda: { 109 | type: "file", 110 | layout: { 111 | type: 'json' 112 | }, 113 | filename: `./data-native-panda.log`, 114 | maxLogSize: 512 * 1024 * 1024, 115 | backups: 4 116 | }, 117 | json_data_native_fanxing: { 118 | type: "file", 119 | layout: { 120 | type: 'json' 121 | }, 122 | filename: `./data-native-fanxing.log`, 123 | maxLogSize: 512 * 1024 * 1024, 124 | backups: 4 125 | }, 126 | json_data_native_huoshan: { 127 | type: "file", 128 | layout: { 129 | type: 'json' 130 | }, 131 | filename: `./data-native-huoshan.log`, 132 | maxLogSize: 512 * 1024 * 1024, 133 | backups: 4 134 | }, 135 | json_data_native_yy: { 136 | type: "file", 137 | layout: { 138 | type: 'json' 139 | }, 140 | filename: `./data-native-yy.log`, 141 | maxLogSize: 512 * 1024 * 1024, 142 | backups: 4 143 | } 144 | }, 145 | categories: { 146 | default: { 147 | appenders: ['out'], 148 | level: 'info' 149 | }, 150 | logtail_broadcast: { 151 | appenders: ['json_broadcast'], 152 | level: 'info' 153 | }, 154 | logtail_alarm: { 155 | appenders: ['json_alarm'], 156 | level: 'info' 157 | }, 158 | logtail_data_native: { 159 | appenders: ['json_data_native'], 160 | level: 'info' 161 | }, 162 | logtail_data_native_douyu: { 163 | appenders: ['json_data_native_douyu'], 164 | level: 'info' 165 | }, 166 | logtail_data_native_huya: { 167 | appenders: ['json_data_native_huya'], 168 | level: 'info' 169 | }, 170 | logtail_data_native_longzhu: { 171 | appenders: ['json_data_native_longzhu'], 172 | level: 'info' 173 | }, 174 | logtail_data_native_zhanqi: { 175 | appenders: ['json_data_native_zhanqi'], 176 | level: 'info' 177 | }, 178 | logtail_data_native_quanmin: { 179 | appenders: ['json_data_native_quanmin'], 180 | level: 'info' 181 | }, 182 | logtail_data_native_chushou: { 183 | appenders: ['json_data_native_chushou'], 184 | level: 'info' 185 | }, 186 | logtail_data_native_fanxing: { 187 | appenders: ['json_data_native_fanxing'], 188 | level: 'info' 189 | }, 190 | logtail_data_native_huoshan: { 191 | appenders: ['json_data_native_huoshan'], 192 | level: 'info' 193 | }, 194 | logtail_data_native_yy: { 195 | appenders: ['json_data_native_yy'], 196 | level: 'info' 197 | }, 198 | logtail_data_native_panda: { 199 | appenders: ['json_data_native_panda'], 200 | level: 'info' 201 | }, 202 | logtail_data_native_xypanda: { 203 | appenders: ['json_data_native_panda'], 204 | level: 'info' 205 | } 206 | } 207 | }); 208 | 209 | module.exports = Log4js; 210 | -------------------------------------------------------------------------------- /gather/lib/opportunity.js: -------------------------------------------------------------------------------- 1 | /** 2 | * Creator: Tang Xiaoji 3 | * Time: 2018-07-27 4 | */ 5 | 6 | 'use strict'; 7 | 8 | const config = require('../../config/config'); 9 | const video_stream = require('../video-stream'); 10 | const redis = require('../../config/redis'); 11 | const mysql = require('../../config/mysql'); 12 | const moment = require('moment'); 13 | const notice = require('./notice'); 14 | 15 | class Opportunity { 16 | constructor(debug = false) { 17 | this.isDebug = debug; 18 | } 19 | 20 | async updateStartInfo (plat, rid, start_time = moment().unix(), stop_time = null) { 21 | if (this.isDebug) { 22 | return; 23 | } 24 | 25 | return new Promise((resolve, reject) => { 26 | if (!config.plat.hasOwnProperty(plat)) { 27 | reject('plat error'); 28 | } else { 29 | if (plat === 'xypanda') { 30 | plat = 'panda'; 31 | } 32 | mysql.pool.query('INSERT INTO ?? ( ?? ) VALUES ( ? ) ON DUPLICATE KEY UPDATE start_time = ?, stop_time = ?;', [ 33 | config.tbl_name.db_fentuan_anchor_start_time, [ 34 | 'plat', 'room_id', 'room_nick', 'start_time', 'stop_time' 35 | ], [plat, rid, '', start_time, stop_time], 36 | start_time, stop_time 37 | ], (err, res) => { 38 | if (err) { 39 | reject(err); 40 | } else { 41 | resolve(res); 42 | } 43 | }); 44 | } 45 | }); 46 | } 47 | 48 | async updateHistoryStartInfo(plat, rid, hid, start_time = moment().unix(), stop_time = null) { 49 | if (this.isDebug) { 50 | return; 51 | } 52 | 53 | return new Promise((resolve, reject) => { 54 | if (!config.plat.hasOwnProperty(plat)) { 55 | reject('plat error'); 56 | } else { 57 | if (plat === 'xypanda') { 58 | plat = 'panda'; 59 | } 60 | mysql.pool.query('INSERT INTO ?? ( ?? ) VALUES ( ? ) ON DUPLICATE KEY UPDATE start_time = ?, stop_time = ?;', [ 61 | config.tbl_name.db_fentuan_anchor_start_time_history, [ 62 | 'plat', 'room_id', 'room_nick', 'start_time', 'stop_time', 'hid' 63 | ], [plat, rid, '', start_time, stop_time, hid], 64 | start_time, stop_time 65 | ], (err, res) => { 66 | if (err) { 67 | reject(err); 68 | } else { 69 | resolve(res); 70 | } 71 | }); 72 | } 73 | }); 74 | } 75 | 76 | async updateLiveStatus(rid, status, platform) { 77 | if (this.isDebug) { 78 | return; 79 | } 80 | 81 | return new Promise((resolve, reject) => { 82 | mysql.pool.query('UPDATE ?? SET status = ? WHERE platform = ? AND rid = ? LIMIT 1', [ 83 | config.tbl_name.db_rpt_fentuan_anchor_info_explore, status, platform, rid 84 | ], (err, res) => { 85 | if (err) { 86 | reject(err); 87 | } else { 88 | resolve(res); 89 | } 90 | }); 91 | }); 92 | } 93 | 94 | /** 95 | * @param type stop/start 96 | * @param plat quanmin/huya/douyu... 97 | * @param rid 98 | * @param url 99 | * @param hid 100 | */ 101 | async checkRegisterUser(type, plat, rid, url, hid) { 102 | if (this.isDebug) { 103 | return; 104 | } 105 | 106 | return new Promise((resolve, reject) => { 107 | if (plat === 'xypanda') { 108 | plat = 'panda'; 109 | } 110 | mysql.pool.query('SELECT room_id, plat FROM ?? WHERE plat = ? AND room_id = ? AND bind = ?', [ 111 | config.tbl_name.db_user_user_room_bind, plat, rid, 1 112 | ], (err, res) => { 113 | if (err) { 114 | reject(err); 115 | } else { 116 | const stream = video_stream[plat]; 117 | if (res && res.length && stream && typeof stream[type] === 'function') { 118 | stream[type](plat, rid, url, hid).then(r => { 119 | resolve(r); 120 | }).catch(e => { 121 | reject(e); 122 | }); 123 | } 124 | } 125 | }); 126 | }); 127 | } 128 | 129 | async updateRoomList(plat, tasks) { 130 | if (this.isDebug) { 131 | return; 132 | } 133 | 134 | let newest = []; 135 | for (let task of tasks) { 136 | newest.push(`${plat}:${task.room_id}`); 137 | } 138 | return await redis.sadd(config.SET_GATHER_STREAM_NEW(plat), newest); 139 | } 140 | 141 | async removeRoom(plat, rid) { 142 | if (this.isDebug) { 143 | return; 144 | } 145 | 146 | return await redis.srem(config.SET_GATHER_STREAM_NEW(plat), `${plat}:${rid}`); 147 | } 148 | 149 | async pushLiveReport(rid, platform, hid, startTime, stopTime) { 150 | if (this.isDebug) { 151 | return; 152 | } 153 | 154 | return await redis.lpush(config.LIST_ANCHOR_LIVE_REPORT(), JSON.stringify({ 155 | rid: rid, 156 | platform: platform, 157 | hid: hid, 158 | startTime: startTime, 159 | stopTime: stopTime 160 | })); 161 | } 162 | 163 | // 获取公告并校验，只在开播时调用 164 | async getNoticeToCheck(roominfo) { 165 | const noti = await notice(roominfo.plat, roominfo.url); 166 | const isMatch = /((\D)\d{5,}(\D))|(\d{5,}(\D))|((\D)\d{5,})/g.test(noti); 167 | 168 | if (isMatch) { 169 | const [err, res] = await mysql.asyncQuery('INSERT INTO ?? ( ?? ) VALUES ( ? ) ON DUPLICATE KEY UPDATE notice = ?, isCheck = ?', [ 170 | config.tbl_name.db_fentuan_anchor_att, [ 171 | 'rid', 'platform', 'notice', 'isCheck', 'cover' 172 | ], [ 173 | roominfo.room_id, config.plat_no[roominfo.plat], noti, 1, '' 174 | ], noti, 1 175 | ]); 176 | 177 | if (err) { 178 | throw err; 179 | } else { 180 | return res; 181 | } 182 | } else { 183 | return '' 184 | } 185 | } 186 | } 187 | 188 | module.exports = new Opportunity(); 189 | -------------------------------------------------------------------------------- /config/log_prod.js: -------------------------------------------------------------------------------- 1 | const Log4js = require('log4js'); 2 | const moment = require('moment'); 3 | Log4js.addLayout('json', function () { 4 | return function (logEvent) { 5 | logEvent.datetime = moment().format('YYYY-MM-DD HH:mm:ss'); 6 | for (let key in logEvent.data[0]) { 7 | if (logEvent.data[0].hasOwnProperty(key)) { 8 | logEvent[key] = logEvent.data[0][key]; 9 | } 10 | } 11 | logEvent.timestamp = (moment().unix()).toString(); 12 | delete logEvent.startTime; 13 | delete logEvent.pid; 14 | delete logEvent.data; 15 | delete logEvent.categoryName; 16 | delete logEvent.level; 17 | delete logEvent.context; 18 | return JSON.stringify(logEvent) 19 | } 20 | }); 21 | 22 | Log4js.configure({ 23 | appenders: { 24 | out: { 25 | type: 'stdout' 26 | }, 27 | json_broadcast: { 28 | type: "file", 29 | layout: { 30 | type: 'json' 31 | }, 32 | filename: "/root/logs/logtail/broadcast.log", 33 | maxLogSize: 512 * 1024 * 1024, 34 | backups: 4 35 | }, 36 | json_alarm: { 37 | type: "file", 38 | layout: { 39 | type: 'json' 40 | }, 41 | filename: "/root/logs/logtail/alarm.log", 42 | maxLogSize: 512 * 1024 * 1024, 43 | backups: 4 44 | }, 45 | json_data_native: { 46 | type: "file", 47 | layout: { 48 | type: 'json' 49 | }, 50 | filename: `/root/logs/logtail/data-native.log`, 51 | maxLogSize: 512 * 1024 * 1024, 52 | backups: 4 53 | }, 54 | json_data_native_douyu: { 55 | type: "file", 56 | layout: { 57 | type: 'json' 58 | }, 59 | filename: `/root/logs/logtail/data-native-douyu.log`, 60 | maxLogSize: 512 * 1024 * 1024, 61 | backups: 4 62 | }, 63 | json_data_native_huya: { 64 | type: "file", 65 | layout: { 66 | type: 'json' 67 | }, 68 | filename: `/root/logs/logtail/data-native-huya.log`, 69 | maxLogSize: 512 * 1024 * 1024, 70 | backups: 4 71 | }, 72 | json_data_native_longzhu: { 73 | type: "file", 74 | layout: { 75 | type: 'json' 76 | }, 77 | filename: `/root/logs/logtail/data-native-longzhu.log`, 78 | maxLogSize: 512 * 1024 * 1024, 79 | backups: 4 80 | }, 81 | json_data_native_quanmin: { 82 | type: "file", 83 | layout: { 84 | type: 'json' 85 | }, 86 | filename: `/root/logs/logtail/data-native-quanmin.log`, 87 | maxLogSize: 512 * 1024 * 1024, 88 | backups: 4 89 | }, 90 | json_data_native_zhanqi: { 91 | type: "file", 92 | layout: { 93 | type: 'json' 94 | }, 95 | filename: `/root/logs/logtail/data-native-zhanqi.log`, 96 | maxLogSize: 512 * 1024 * 1024, 97 | backups: 4 98 | }, 99 | json_data_native_chushou: { 100 | type: "file", 101 | layout: { 102 | type: 'json' 103 | }, 104 | filename: `/root/logs/logtail/data-native-chushou.log`, 105 | maxLogSize: 512 * 1024 * 1024, 106 | backups: 4 107 | }, 108 | json_data_native_panda: { 109 | type: "file", 110 | layout: { 111 | type: 'json' 112 | }, 113 | filename: `/root/logs/logtail/data-native-panda.log`, 114 | maxLogSize: 512 * 1024 * 1024, 115 | backups: 4 116 | }, 117 | json_data_native_fanxing: { 118 | type: "file", 119 | layout: { 120 | type: 'json' 121 | }, 122 | filename: `/root/logs/logtail/data-native-fanxing.log`, 123 | maxLogSize: 512 * 1024 * 1024, 124 | backups: 4 125 | }, 126 | json_data_native_huoshan: { 127 | type: "file", 128 | layout: { 129 | type: 'json' 130 | }, 131 | filename: `/root/logs/logtail/data-native-huoshan.log`, 132 | maxLogSize: 512 * 1024 * 1024, 133 | backups: 4 134 | }, 135 | json_data_native_yy: { 136 | type: "file", 137 | layout: { 138 | type: 'json' 139 | }, 140 | filename: `/root/logs/logtail/data-native-yy.log`, 141 | maxLogSize: 512 * 1024 * 1024, 142 | backups: 4 143 | } 144 | }, 145 | categories: { 146 | default: { 147 | appenders: ['out'], 148 | level: 'info' 149 | }, 150 | logtail_broadcast: { 151 | appenders: ['json_broadcast'], 152 | level: 'info' 153 | }, 154 | logtail_alarm: { 155 | appenders: ['json_alarm'], 156 | level: 'info' 157 | }, 158 | logtail_data_native: { 159 | appenders: ['json_data_native'], 160 | level: 'info' 161 | }, 162 | logtail_data_native_douyu: { 163 | appenders: ['json_data_native_douyu'], 164 | level: 'info' 165 | }, 166 | logtail_data_native_huya: { 167 | appenders: ['json_data_native_huya'], 168 | level: 'info' 169 | }, 170 | logtail_data_native_longzhu: { 171 | appenders: ['json_data_native_longzhu'], 172 | level: 'info' 173 | }, 174 | logtail_data_native_zhanqi: { 175 | appenders: ['json_data_native_zhanqi'], 176 | level: 'info' 177 | }, 178 | logtail_data_native_quanmin: { 179 | appenders: ['json_data_native_quanmin'], 180 | level: 'info' 181 | }, 182 | logtail_data_native_chushou: { 183 | appenders: ['json_data_native_chushou'], 184 | level: 'info' 185 | }, 186 | logtail_data_native_fanxing: { 187 | appenders: ['json_data_native_fanxing'], 188 | level: 'info' 189 | }, 190 | logtail_data_native_huoshan: { 191 | appenders: ['json_data_native_huoshan'], 192 | level: 'info' 193 | }, 194 | logtail_data_native_yy: { 195 | appenders: ['json_data_native_yy'], 196 | level: 'info' 197 | }, 198 | logtail_data_native_panda: { 199 | appenders: ['json_data_native_panda'], 200 | level: 'info' 201 | }, 202 | logtail_data_native_xypanda: { 203 | appenders: ['json_data_native_panda'], 204 | level: 'info' 205 | } 206 | } 207 | }); 208 | 209 | module.exports = Log4js; 210 | -------------------------------------------------------------------------------- /danmu/Panda.js: -------------------------------------------------------------------------------- 1 | const Conn = require('./Conn'); 2 | const voca = require('voca'); 3 | const request = require('superagent'); 4 | const _ = require('lodash'); 5 | const pair = require('../helper/pair'); 6 | const cutbuf = require('../helper/cutbuf'); 7 | 8 | const ADDRESS_URL = 'https://api.homer.panda.tv/chatroom/getinfo?rid=0&roomid=%s&retry=0&__version=3.2.1.4923&__plat=web'; 9 | const KEEPALIVE_INTERVAL = 30000; 10 | const REQ_TIMEOUT = 6000; 11 | const PROTOCOL = { 12 | LOGIN_SUCCESS: 393222, 13 | HEARTBEAT: 393217, 14 | RECVMSG: 393219 15 | }; 16 | 17 | module.exports = class Panda extends Conn { 18 | constructor(url = '') { 19 | super(); 20 | this.url = url; 21 | this.roomId = -1; 22 | this.serverInfo = null; 23 | this.buffer = Buffer.alloc(0); 24 | this.isLoginReg = false; 25 | this.keepaliveTimer = null; 26 | this.clientConnHandler = this.clientConnHandler.bind(this); 27 | this.clientDataHandler = this.clientDataHandler.bind(this); 28 | this.init(); 29 | } 30 | 31 | async init() { 32 | let serverAddress; 33 | try { 34 | serverAddress = await this.requestServerAddress(); 35 | } catch (e) { 36 | this.emit('initerror', e); 37 | return false; 38 | } 39 | 40 | if (serverAddress) { 41 | this.on('connect', this.clientConnHandler); 42 | serverAddress = serverAddress.split(':'); 43 | this.connect(serverAddress[1], serverAddress[0]); 44 | } 45 | } 46 | 47 | async requestServerAddress() { 48 | let parts = this.url.split('/'); 49 | let len = parts.length; 50 | this.roomId = parts[len - 1] || parts[len - 2]; 51 | 52 | let url = voca.sprintf(ADDRESS_URL, this.roomId); 53 | let res = await request.get(url).timeout({ 54 | response: REQ_TIMEOUT 55 | }); 56 | let body = res.text; 57 | try { 58 | body = JSON.parse(body); 59 | } catch (e) {} 60 | 61 | if (!body || body.errno || !body.data) { 62 | throw new Error('request error'); 63 | } 64 | 65 | this.serverInfo = body.data; 66 | let serverAddress = body.data.chat_addr_list; 67 | let rand = Math.floor(Math.random() * serverAddress.length); 68 | return serverAddress[rand]; 69 | } 70 | 71 | destroy() { 72 | clearTimeout(this.keepaliveTimer); 73 | super.destroy(); 74 | } 75 | 76 | encode(data) { 77 | let length = Buffer.byteLength(data); 78 | let buffer = Buffer.alloc(5 + 1 + length + 4); 79 | buffer.fill(Buffer.from([0x00, 0x06, 0x00, 0x02, 0x00]), 0); 80 | buffer.writeUInt8(length, 5); 81 | buffer.write(data, 6); 82 | buffer.fill(Buffer.from([0x00, 0x06, 0x00, 0x00]), 6 + length); 83 | return buffer; 84 | } 85 | 86 | decode(buffer) { 87 | let contents = []; 88 | this.buffer = Buffer.concat([this.buffer, buffer]); 89 | 90 | try { 91 | let count = 0; 92 | label: for (;;) { 93 | let protocol = this.buffer.readInt32BE(0); 94 | switch (protocol) { 95 | case PROTOCOL.LOGIN_SUCCESS: { 96 | let len = this.buffer.readInt16BE(4); 97 | let content = this.buffer.slice(4 + 2, 4 + 2 + len); 98 | if (len > content.length) { 99 | break label; 100 | } 101 | 102 | let kvs = pair(content, '\n', ':'); 103 | contents.push({ 104 | protocol: protocol, 105 | data: kvs 106 | }); 107 | this.buffer = this.buffer.slice(4 + 2 + len); 108 | break; 109 | } 110 | case PROTOCOL.HEARTBEAT: { 111 | contents.push({ 112 | protocol: protocol, 113 | data: {} 114 | }); 115 | this.buffer = this.buffer.slice(4); 116 | break; 117 | } 118 | case PROTOCOL.RECVMSG: { 119 | let len = this.buffer.readInt16BE(4); 120 | let content = this.buffer.slice(4 + 2, 4 + 2 + len); 121 | 122 | if (len > content.length) { 123 | break label; 124 | } 125 | 126 | let extMsg = pair(content, '\n', ':'); 127 | let extLen = 4 + 2 + len; 128 | let itemLen = this.buffer.readInt32BE(extLen); 129 | content = this.buffer.slice(extLen + 4, extLen + 4 + itemLen); 130 | if (itemLen > content.length) { 131 | break label; 132 | } 133 | 134 | content = cutbuf(content, 16); 135 | for (let item of content) { 136 | content = JSON.parse(item); 137 | contents.push({ 138 | protocol: protocol, 139 | data: content, 140 | ext: extMsg 141 | }); 142 | } 143 | 144 | this.buffer = this.buffer.slice(extLen + 4 + itemLen); 145 | break; 146 | } 147 | } 148 | } 149 | } catch (e) {} 150 | 151 | return contents; 152 | } 153 | 154 | async keepalive() { 155 | this.keepaliveTimer = setTimeout(() => { 156 | this.write(Buffer.from([0x00, 0x06, 0x00, 0x00]), true); 157 | this.keepalive(); 158 | }, KEEPALIVE_INTERVAL); 159 | } 160 | 161 | clientConnHandler() { 162 | let { rid, appid, ts, sign, authType } = this.serverInfo; 163 | let data = [ 164 | ['u', `${rid}@${appid}`], 165 | ['k', 1], 166 | ['t', 300], 167 | ['ts', ts], 168 | ['sign', sign], 169 | ['authtype', authType] 170 | ]; 171 | 172 | data = data.map(item => { 173 | return `${item[0]}:${item[1]}`; 174 | }); 175 | 176 | this.on('rawdata', this.clientDataHandler); 177 | this.write(data.join('\n')); 178 | } 179 | 180 | async clientDataHandler(data) { 181 | if (!this.isLoginReg) { 182 | let index = _.findIndex(data, { 183 | protocol: PROTOCOL.HEARTBEAT 184 | }); 185 | if (index > -1) { 186 | this.isLoginReg = true; 187 | this.keepalive(); 188 | } 189 | } 190 | this.emit('data', data); 191 | } 192 | 193 | restart() { 194 | this.isLoginReg = false; 195 | this.buffer = Buffer.alloc(0); 196 | clearTimeout(this.keepaliveTimer); 197 | this.removeListener('connect', this.clientConnHandler); 198 | this.removeListener('rawdata', this.clientDataHandler); 199 | this.init(); 200 | } 201 | }; -------------------------------------------------------------------------------- /danmu/XYPanda.js: -------------------------------------------------------------------------------- 1 | const Conn = require('./Conn'); 2 | const voca = require('voca'); 3 | const request = require('superagent'); 4 | const _ = require('lodash'); 5 | const md5 = require('md5'); 6 | 7 | const SOCKETSERVER_URL = '%s//online.panda.tv/dispatch'; 8 | const KEEPALIVE_INTERVAL = 30000; 9 | const REQ_TIMEOUT = 6000; 10 | const PROTOCOL = { 11 | REG: 2147483649, 12 | REG_SUCCESS: 2147483648 13 | }; 14 | 15 | module.exports = class XYPanda extends Conn { 16 | constructor(url = '') { 17 | super(); 18 | this.url = url; 19 | this.roomId = null; 20 | this.serverInfo = null; 21 | this.isLoginReg = false; 22 | this.keepaliveTimer = null; 23 | this.buffer = Buffer.alloc(0); 24 | this.guid = this.genGuid(); 25 | this.clientConnHandler = this.clientConnHandler.bind(this); 26 | this.clientDataHandler = this.clientDataHandler.bind(this); 27 | this.restartHandler = this.restart.bind(this); 28 | this.init(); 29 | } 30 | 31 | async init() { 32 | let serverAddress; 33 | try { 34 | serverAddress = await this.requestServerAddress(); 35 | 36 | } catch (e) { 37 | this.emit('initerror', e); 38 | return false; 39 | } 40 | 41 | if (serverAddress) { 42 | this.on('connect', this.clientConnHandler); 43 | this.connect(serverAddress[1], serverAddress[0]); 44 | } 45 | } 46 | 47 | async requestServerAddress() { 48 | let parts = this.url.split('/'); 49 | let guid = this.guid; 50 | let cluster = 'v3'; 51 | let plat = 'pc_web'; 52 | let xid = parts[parts.length - 1]; 53 | this.roomId = xid; 54 | this.timestamp = Math.round(new Date().getTime() / 1000); 55 | this.sign = md5(`uzY@H/C!N^G9K:EY${guid}${cluster}${this.timestamp}`); 56 | 57 | let url = voca.sprintf(SOCKETSERVER_URL, parts[0]); 58 | let res = await request 59 | .get(url) 60 | .timeout({ 61 | response: REQ_TIMEOUT 62 | }) 63 | .query({ 64 | guid: guid, 65 | time: this.timestamp, 66 | cluster: cluster, 67 | plat: plat, 68 | xid: xid, 69 | sign: this.sign 70 | }); 71 | 72 | let body = res.body; 73 | if (!body || body.error) { 74 | throw new Error('request error'); 75 | } 76 | 77 | this.serverInfo = body; 78 | return [body.addr, body.port]; 79 | } 80 | 81 | destroy() { 82 | clearTimeout(this.keepaliveTimer); 83 | super.destroy(); 84 | } 85 | 86 | encode(data) { 87 | let { cmd, buffer } = data; 88 | let headerBuf = Buffer.alloc(4 + 8 + 4 + 4 + 4); 89 | headerBuf.writeUInt8(3, 0); 90 | headerBuf.writeUInt8(0, 1); 91 | headerBuf.writeUInt8(0, 2); 92 | headerBuf.writeUInt8(0, 3); 93 | headerBuf.writeDoubleBE(Math.random(), 4); 94 | headerBuf.writeUInt32BE(cmd, 12); 95 | headerBuf.writeUInt32BE(0, 16); 96 | headerBuf.writeUInt32BE(buffer.length, 20); 97 | return Buffer.concat([headerBuf, buffer]); 98 | } 99 | 100 | decode(buffer) { 101 | let contents = []; 102 | this.buffer = Buffer.concat([this.buffer, buffer]); 103 | if (this.buffer.length < 24) { 104 | return contents; 105 | } 106 | 107 | try { 108 | for (;;) { 109 | let pad = this.buffer.readUInt32BE(0); 110 | let rand = this.buffer.readDoubleBE(4); 111 | let cmd = this.buffer.readUInt32BE(12); 112 | let length = this.buffer.readUInt32BE(20); 113 | let data = this.buffer.slice(24, 24 + length).toString('utf-8'); 114 | if (cmd != PROTOCOL.REG_SUCCESS) { 115 | data = JSON.parse(data); 116 | } 117 | 118 | contents.push({ 119 | rand: rand, 120 | cmd: cmd, 121 | data: data 122 | }); 123 | this.buffer = this.buffer.slice(24 + length); 124 | } 125 | } catch (e) {} 126 | 127 | return contents; 128 | } 129 | 130 | // u can wait the `close` event to restart 131 | // but we register again to protect client after 30s 132 | async keepalive() { 133 | // 熊猫星颜不需要keepalive 134 | return; 135 | clearTimeout(this.keepaliveTimer); 136 | this.keepaliveTimer = setTimeout(() => { 137 | this.reg2server(); 138 | this.keepalive(); 139 | }, KEEPALIVE_INTERVAL); 140 | } 141 | 142 | reg2server() { 143 | let guidBuf = this.str2Buffer(this.guid); 144 | let rndBuf = this.str2Buffer(this.serverInfo.rnd); 145 | let buf = Buffer.alloc(36); 146 | buf.fill(guidBuf.slice(0, 16), 0, 16); 147 | buf.fill(rndBuf.slice(0, 20), 16, 36); 148 | this.write({ 149 | cmd: PROTOCOL.REG, 150 | buffer: buf 151 | }); 152 | } 153 | 154 | clientConnHandler() { 155 | this.on('rawdata', this.clientDataHandler); 156 | this.reg2server(); 157 | } 158 | 159 | async clientDataHandler(data) { 160 | if (!this.isLoginReg) { 161 | let index = _.findIndex(data, { 162 | cmd: PROTOCOL.REG_SUCCESS 163 | }); 164 | if (index > -1) { 165 | this.isLoginReg = true; 166 | this.keepalive(); 167 | } 168 | } 169 | let d = [].concat(data).filter(i => { 170 | return i && i.data && i.data.to == this.roomId; 171 | }); 172 | if(d.length){ 173 | // console.log('DATA',JSON.stringify(d)); 174 | this.emit('data', d); 175 | } 176 | } 177 | 178 | restart(error) { 179 | this.isLoginReg = false; 180 | this.guid = this.genGuid(); 181 | this.buffer = Buffer.alloc(0); 182 | clearTimeout(this.keepaliveTimer); 183 | this.removeListener('connect', this.clientConnHandler); 184 | this.removeListener('rawdata', this.clientDataHandler); 185 | this.init(); 186 | } 187 | 188 | genGuid() { 189 | let rand = parseInt(Math.random() * 1000000000).toString(16); 190 | let timestamp = new Date().getTime().toString(16); 191 | let thex = ('0000000000000000' + timestamp).substr(-16); 192 | let rhex = ('000000000000' + rand).substr(-12); 193 | return '7773' + thex + rhex; 194 | } 195 | 196 | str2Buffer(str) { 197 | let length = str.length; 198 | str = str.replace(/^0x|\s|:/gm, ''); 199 | if ((length & 1) == 1) { 200 | str = '0' + str; 201 | } 202 | 203 | let index = 0; 204 | let bufArr = []; 205 | while (index < length) { 206 | bufArr[index / 2] = parseInt(str.substr(index, 2), 16); 207 | index += 2; 208 | } 209 | 210 | return Buffer.from(bufArr); 211 | } 212 | }; -------------------------------------------------------------------------------- /danmu/Kugou.js: -------------------------------------------------------------------------------- 1 | const WSConn = require('./WSConn'); 2 | const rp = require('request-promise'); 3 | const redis = require('../config/redis'); 4 | const config = require('../config/config'); 5 | const Logger = require('../config/Logger'); 6 | const logger = new Logger('Fanxing/danmu'); 7 | const Gift = require('../gather/lib/gift'); 8 | const giftHelper = Gift.getInstance(); 9 | 10 | module.exports = class Kugou extends WSConn { 11 | 12 | constructor (url = '', opts) { 13 | super(); 14 | this.url = url; 15 | this.roomId = this.parseRoomId(url); 16 | 17 | this.roomInfo = opts || {}; 18 | 19 | this.init(); 20 | } 21 | 22 | async init () { 23 | const giftInfo = await this.getGift(); 24 | for(let key in giftInfo) { 25 | if (giftInfo.hasOwnProperty(key)) { 26 | giftHelper.push({ 27 | plat: config.plat.fanxing, 28 | gift_id: giftInfo[key].giftId, 29 | gift_cost: giftInfo[key].price, 30 | isBroadcast: 0, 31 | token_type: 0, 32 | gift_name: giftInfo[key].giftName, 33 | gift_img: '' 34 | }); 35 | } 36 | } 37 | try { 38 | const ws = await this.getWs(); 39 | this.on('connect', this.clientConnHandler); 40 | this.connect(`ws://${ws}`); 41 | } catch (e) { 42 | this.emit('initerror', e); 43 | } 44 | } 45 | 46 | parseRoomId (url = '') { 47 | return url.split('/').pop(); 48 | } 49 | 50 | async _getGiftCache () { 51 | const gift = await redis.get('DANMU:KUGOU:GIFT'); 52 | if (gift) { 53 | return JSON.parse(gift); 54 | } 55 | } 56 | 57 | async _updateGiftCache (gift) { 58 | return await redis.set('DANMU:KUGOU:GIFT', JSON.stringify(gift), 'EX', 60 * 60 * 24); 59 | } 60 | 61 | async getGift () { 62 | const opts = { 63 | uri: 'http://act.fanxing.kugou.com/api/YueZhan/getGiftList', 64 | qs: { 65 | args: [154], 66 | jsoncallback: 'jsonpcallback_httpactfanxingkugoucomapiphpactYueZhangetGiftListargs154' 67 | }, 68 | headers: { 69 | 'User-Agent': encodeURIComponent('酷狗直播 3.95.2 rv:3.95.0.1 (iPhone; iPhone OS 9.3.2; zh_CN)') 70 | } 71 | }; 72 | try { 73 | const gift = await this._getGiftCache(); 74 | if (gift) { 75 | return gift; 76 | } 77 | let res = await rp(opts); 78 | res = res.match(/\((.+)\)/)[1]; 79 | res = JSON.parse(res); 80 | const giftList = res.data.giftList || []; 81 | let obj = {}; 82 | for (let e of giftList) { 83 | for (let gift of e.giftTypeList) { 84 | obj[gift.giftId] = gift; 85 | obj[gift.giftName] = gift; 86 | } 87 | } 88 | if (giftList.length) { 89 | this._updateGiftCache(obj); 90 | } 91 | return obj; 92 | } catch (e) { 93 | console.error('Kugou get gift list err: ', e.toString()); 94 | return {}; 95 | } 96 | } 97 | 98 | async _getWsCache () { 99 | const addrs = await redis.get('DANMU:KUGOU:WS'); 100 | if (addrs) { 101 | return JSON.parse(addrs); 102 | } 103 | } 104 | 105 | async _updateWsCache (addrs) { 106 | return await redis.set('DANMU:KUGOU:WS', JSON.stringify(addrs), 'EX', 60 * 60 * 24); 107 | } 108 | 109 | async getWs () { 110 | const opts = { 111 | uri: 'https://fx2.service.kugou.com/socket_scheduler/pc/v2/address.jsonp', 112 | qs: { 113 | jsonpcallback: 'jsonpcallback_httpsfx2servicekugoucomsocket_schedulerpcv2addressjsonp', 114 | _p: 0, 115 | _v: '7.0.0', 116 | pv: 20171111, 117 | rid: this.roomId, 118 | cid: 100, 119 | at: 101, 120 | _: +new Date() 121 | }, 122 | headers: { 123 | 'User-Agent': encodeURIComponent('酷狗直播 3.95.2 rv:3.95.0.1 (iPhone; iPhone OS 9.3.2; zh_CN)') 124 | } 125 | }; 126 | try { 127 | let addrs = await this._getWsCache(); 128 | if (addrs) { 129 | return addrs[Math.floor((Math.random() * addrs.length))].host; 130 | } 131 | let res = await rp(opts); 132 | res = res.match(/\((.+)\)/)[1]; 133 | res = JSON.parse(res); 134 | addrs = res.data.addrs || []; 135 | if (addrs.length) { 136 | this._updateWsCache(addrs); 137 | } 138 | return addrs[Math.floor((Math.random() * addrs.length))].host; 139 | } catch (e) { 140 | console.error('can not get ws address: ', e); 141 | const addrsBackup = [ 142 | '119.146.204.143:1315', 143 | '119.146.204.169:1315', 144 | '119.146.204.167:1315' 145 | ]; 146 | console.warn('select a backup address !!!'); 147 | return addrsBackup[Math.floor((Math.random() * addrsBackup.length))]; 148 | } 149 | } 150 | 151 | // ws创建成功，登录并发送心跳，同时接收数据 152 | clientConnHandler () { 153 | this.on('rawdata', this.clientDataHandler); 154 | this.on('close', this.clientCloseHandler); 155 | this.login(); 156 | this.heartbeat(); 157 | } 158 | 159 | clientDataHandler (data) { 160 | data = this._decode(data); 161 | if (data) { 162 | this.emit('data', [].concat(data)); 163 | } 164 | } 165 | 166 | clientCloseHandler () { 167 | clearInterval(this.scheduler); 168 | } 169 | 170 | destroy () { 171 | clearInterval(this.scheduler); 172 | super.destroy(); 173 | } 174 | 175 | login () { 176 | this.send({ 177 | appid: 1010, 178 | clientid: 100, 179 | cmd: 201, 180 | kugouid: 0, 181 | referer: 0, 182 | roomId: this.roomId, 183 | soctoken: '', 184 | token: '', 185 | v: 20171111 186 | }); 187 | } 188 | 189 | // 每5秒心跳，保持连接 190 | heartbeat () { 191 | this.scheduler = setInterval(() => { 192 | this.send('H'); 193 | }, 5 * 1000); 194 | } 195 | 196 | offset (data) { 197 | this.send({ 198 | cmd: 211, 199 | kugouId: 1293910630, 200 | msgId: data.msgId, 201 | offset: data.offset, 202 | roomId: data.content.roomid || this.roomId, 203 | rpt: 0 204 | }); 205 | } 206 | 207 | send (data) { 208 | this.connection.send(this._encode(data)); 209 | } 210 | 211 | _encode (data) { 212 | if (typeof data === 'object') { 213 | return JSON.stringify(data); 214 | } 215 | return data; 216 | } 217 | 218 | _decode (data) { 219 | // 心跳消息 220 | if (data.utf8Data === 'H') { 221 | return null; 222 | } 223 | const parsedData = JSON.parse(data.utf8Data); 224 | if (parsedData.offset) { 225 | this.offset(parsedData); 226 | } 227 | return parsedData; 228 | } 229 | 230 | } --------------------------------------------------------------------------------