├── .gitignore ├── .project ├── .pydevproject ├── README.md ├── __init__.py ├── app ├── __init__.py ├── common │ ├── __init__.py │ └── function.py ├── config │ ├── __init__.py │ └── config.py ├── db │ ├── DBmain.py │ └── __init__.py ├── douyu │ ├── __init__.py │ ├── html_downloader.py │ ├── html_parser.py │ ├── liveDataByRoomId.py │ ├── liveDb.py │ ├── main.py │ └── url_manager.py └── server │ ├── TextMsgHandle.py │ ├── __init__.py │ ├── douyuServer.py │ ├── main.py │ ├── subscribe.py │ └── userDb.py ├── doc ├── index.rst ├── show.png ├── spider.sql ├── sub.png ├── top.jpg └── word.png ├── main.py └── send.py /.gitignore: -------------------------------------------------------------------------------- 1 | wxpy.pkl 2 | wxpy_puid.pkl 3 | friend.txt 4 | app/__pycache__ 5 | app/*/__pycache__ 6 | autoSend 7 | send_* 8 | venv 9 | .idea 10 | -------------------------------------------------------------------------------- /.project: -------------------------------------------------------------------------------- 1 | 2 | 3 | wxPython 4 | 5 | 6 | 7 | 8 | 9 | org.python.pydev.PyDevBuilder 10 | 11 | 12 | 13 | 14 | 15 | org.python.pydev.pythonNature 16 | 17 | 18 | -------------------------------------------------------------------------------- /.pydevproject: -------------------------------------------------------------------------------- 1 | 2 | 3 | Default 4 | python 2.7 5 | 6 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # 使用wxpy爬取需求信息发送微信 2 | 3 | ## 技术栈： 4 | 5 | python3.6 + pip3 + wxpy + wechat_sender + urllib + bs4 +pymysql 6 | 7 | ## 下载 (运行`python3.6 `) 8 | ##### 如果安装报错，请用pip安装指定模块，对于的wxpy的学习，请github自行搜索 9 | 10 | git clone https://github.com/Topthinking/wxPython.git 11 | 12 | cd wxPython 13 | 14 | python main.py 15 | 16 | ## 2017年6月25日 17 | ``` 18 | 初次提交,本地数据进行斗鱼主播的直播情况查询 19 | ``` 20 | ## 2017年6月27日 21 | ``` 22 | 1.使用pymysql来动态捕获查询数据 23 | 2.同时扩展了库与库之间的关系，待优化... 24 | ``` 25 | ## 2017年6月28日 26 | ``` 27 | 1.数据的增删改查 28 | 2.回复格式为 dy:[名称]:[房间号]:[别名] 即可完成添加或者修改， 29 | 例如回复: dy:yyf:58428:rua , 30 | 结果就是更新rua别名，那么就可以回复rua获取房间号58428的直播情况 31 | 可以回复多个别名，他们以英文的逗号隔开，比如 dy:yyf:58428:rua,胖头鱼 32 | 3.优化sql语句对数据库的直接操作，防止sql注入 33 | 4.同时处理了输入格式错误返回的信息 34 | ``` 35 | ## 2017年6月30日 36 | ``` 37 | 1.加入订阅功能，具体回复："我的订阅" 查看详情 38 | 2.放出数据库表结构，在doc目录下 39 | ``` 40 | ## 说明 41 | 42 | > 本项目主要学习Python爬虫，配合微信发送爬取信息，使得学习不会那么枯燥 43 | 44 | > 如果觉得不错的话，您可以点右上角 "Star" 支持一下 谢谢！ ^_^ 45 | 46 | > 如有问题请直接在 Issues 中提，或者您发现问题并有非常好的解决方案，欢迎 PR 👍 47 | 48 | ### 扫描二维码，验证信息输入'top' 49 | #### 工作时间可用(10:00 - 19:00) 50 | ![](https://github.com/Topthinking/wxPython/blob/master/doc/top.jpg) 51 | 52 | # 效果截图 53 | 54 | 55 | 56 | # 词汇截图 57 | -------------------------------------------------------------------------------- /__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Topthinking/wxPython/d64f107e6b05a8238f9643d4c3f992c4c4d87357/__init__.py -------------------------------------------------------------------------------- /app/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Topthinking/wxPython/d64f107e6b05a8238f9643d4c3f992c4c4d87357/app/__init__.py -------------------------------------------------------------------------------- /app/common/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Topthinking/wxPython/d64f107e6b05a8238f9643d4c3f992c4c4d87357/app/common/__init__.py -------------------------------------------------------------------------------- /app/common/function.py: -------------------------------------------------------------------------------- 1 | # coding:utf-8 2 | class CommonFn(object): 3 | def __init__(self): 4 | pass 5 | 6 | # 打印对象所有集合 7 | @staticmethod 8 | def prn_obj(obj): 9 | print('\n'.join(['%s:%s' % item for item in obj.__dict__.items()])) 10 | -------------------------------------------------------------------------------- /app/config/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Topthinking/wxPython/d64f107e6b05a8238f9643d4c3f992c4c4d87357/app/config/__init__.py -------------------------------------------------------------------------------- /app/config/config.py: -------------------------------------------------------------------------------- 1 | # coding:utf-8 2 | import pymysql 3 | 4 | 5 | class Config(object): 6 | 7 | @staticmethod 8 | def database(): 9 | return dict( 10 | host="127.0.0.1", 11 | port=3306, 12 | user="root", 13 | passwd="root", 14 | db="spider", 15 | charset="utf8", 16 | cursorclass=pymysql.cursors.DictCursor 17 | ) 18 | -------------------------------------------------------------------------------- /app/db/DBmain.py: -------------------------------------------------------------------------------- 1 | # coding:utf-8 2 | import pymysql 3 | from app.config import config 4 | 5 | 6 | class DBModel(object): 7 | def __init__(self): 8 | self.conf = config.Config() 9 | 10 | def _connect(self): 11 | self.conn = pymysql.connect(**self.conf.database()); 12 | 13 | def exeAllMySQL(self, sql, data=None): 14 | self._connect() 15 | try: 16 | with self.conn.cursor() as cursor: 17 | # 执行sql语句，进行查询 18 | if data is None: 19 | cursor.execute(sql) 20 | else: 21 | cursor.execute(sql, data) 22 | # 获取查询结果 23 | result = cursor.fetchall() 24 | # 没有设置默认自动提交，需要主动提交，以保存所执行的语句 25 | self.conn.commit() 26 | finally: 27 | self.conn.close() 28 | 29 | return result 30 | 31 | def exeOneMySQL(self, sql, data=None): 32 | self._connect() 33 | try: 34 | with self.conn.cursor() as cursor: 35 | # 执行sql语句，进行查询 36 | if data is None: 37 | cursor.execute(sql) 38 | else: 39 | cursor.execute(sql, data) 40 | # 获取查询结果 41 | result = cursor.fetchone() 42 | # 没有设置默认自动提交，需要主动提交，以保存所执行的语句 43 | self.conn.commit() 44 | finally: 45 | self.conn.close() 46 | 47 | return result 48 | 49 | def insertMySQL(self, sql, data): 50 | self._connect() 51 | try: 52 | with self.conn.cursor() as cursor: 53 | # 执行sql语句，进行查询 54 | cursor.execute(sql, (data)) 55 | 56 | # 没有设置默认自动提交，需要主动提交，以保存所执行的语句 57 | self.conn.commit() 58 | finally: 59 | self.conn.close() 60 | 61 | def updateMySQL(self, sql, data): 62 | self._connect() 63 | try: 64 | with self.conn.cursor() as cursor: 65 | # 执行sql语句，进行查询 66 | cursor.execute(sql, (data)) 67 | 68 | # 没有设置默认自动提交，需要主动提交，以保存所执行的语句 69 | self.conn.commit() 70 | finally: 71 | self.conn.close() 72 | 73 | def insertMySQLById(self, sql, data): 74 | self._connect() 75 | try: 76 | with self.conn.cursor() as cursor: 77 | # 执行sql语句，进行查询 78 | cursor.execute(sql, data) 79 | sql = " SELECT LAST_INSERT_ID() as id " 80 | cursor.execute(sql) 81 | result = cursor.fetchone() 82 | # 没有设置默认自动提交，需要主动提交，以保存所执行的语句 83 | self.conn.commit() 84 | finally: 85 | self.conn.close() 86 | return result 87 | -------------------------------------------------------------------------------- /app/db/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Topthinking/wxPython/d64f107e6b05a8238f9643d4c3f992c4c4d87357/app/db/__init__.py -------------------------------------------------------------------------------- /app/douyu/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Topthinking/wxPython/d64f107e6b05a8238f9643d4c3f992c4c4d87357/app/douyu/__init__.py -------------------------------------------------------------------------------- /app/douyu/html_downloader.py: -------------------------------------------------------------------------------- 1 | # coding:utf-8 2 | import urllib.request 3 | 4 | 5 | class HtmlDownloader(object): 6 | 7 | @staticmethod 8 | def download(url): 9 | print(url) 10 | 11 | if url is None: 12 | return None 13 | 14 | headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:23.0) Gecko/20100101 Firefox/23.0'} 15 | 16 | req = urllib.request.Request(url=url, headers=headers) 17 | 18 | response = urllib.request.urlopen(req) 19 | 20 | if response.getcode() != 200: 21 | return None 22 | 23 | return response.read().decode() 24 | -------------------------------------------------------------------------------- /app/douyu/html_parser.py: -------------------------------------------------------------------------------- 1 | # coding:utf-8 2 | from bs4 import BeautifulSoup 3 | import json 4 | 5 | 6 | class HtmlParser(object): 7 | def __init__(self): 8 | self.liveRoom = [] 9 | self.attentionRoom = [] 10 | 11 | def _get_live_room(self, soup): 12 | 13 | lists = soup.find_all("li") 14 | 15 | if lists is None or len(lists) == 0: 16 | return 17 | 18 | cur_rooms = [] 19 | 20 | for _list in lists: 21 | rInfo = _list.find("a") 22 | if "data-rid" not in rInfo.attrs: 23 | continue 24 | rid = rInfo.attrs["data-rid"] 25 | rName = rInfo.attrs["title"] 26 | 27 | uNameInfo = rInfo.find("span", class_="dy-name") 28 | if uNameInfo is None: 29 | rUace = '-未识别-' 30 | else: 31 | rUace = uNameInfo.get_text() 32 | 33 | rNumber = rInfo.find("span", class_="dy-num") 34 | if rNumber is None: 35 | rNum = 0 36 | else: 37 | rNum = rNumber.get_text() 38 | 39 | tmpRoom = { 40 | "roomId": rid, 41 | "roomName": rName, 42 | "nickName": rUace, 43 | "roomNum": rNum 44 | } 45 | 46 | self.liveRoom.append(tmpRoom) 47 | 48 | cur_rooms.append(tmpRoom) 49 | 50 | return cur_rooms 51 | 52 | def getLiveRoom(self): 53 | return self.liveRoom 54 | 55 | def clearAttentionRoom(self): 56 | self.attentionRoom = [] 57 | 58 | def getAttentionRoom(self): 59 | return self.attentionRoom 60 | 61 | def parse(self, html_cont): 62 | 63 | soup = BeautifulSoup(html_cont, 'html.parser', from_encoding="utf-8") 64 | 65 | return self._get_live_room(soup) 66 | 67 | def jsonParse(self, content): 68 | 69 | roomsData = json.loads(content["$ROOM.showData"]) 70 | 71 | print(roomsData["child_cate"]["url"]) 72 | 73 | return 74 | 75 | # 已开播 76 | if roomInfo["show_status"] == 1: 77 | self.attentionRoom.append({ 78 | "roomId": str(roomInfo['room_id']), 79 | "roomName": roomInfo['room_name'], 80 | "nickName": roomInfo['owner_name'], 81 | "roomNum": roomInfo['levelInfo']['upgrade_exp'] 82 | }) 83 | 84 | @staticmethod 85 | def liveDataByRoomIdParse(content): 86 | 87 | roomInfo = json.loads(content["$ROOM"]) 88 | 89 | roomshowData = json.loads(content["$ROOM.showData"]) 90 | 91 | if "child_cate" not in roomshowData: 92 | search_url = roomshowData["game"]["url"] 93 | else: 94 | search_url = roomshowData["child_cate"]["url"] 95 | 96 | return { 97 | "roomId": str(roomInfo['room_id']), 98 | "roomName": roomInfo['room_name'], 99 | "nickName": roomInfo['owner_name'], 100 | "show_status": roomInfo["show_status"], 101 | "search_url": search_url 102 | } 103 | -------------------------------------------------------------------------------- /app/douyu/liveDataByRoomId.py: -------------------------------------------------------------------------------- 1 | # coding:utf-8 2 | # 根据roomID获取直播详情 3 | 4 | import json 5 | 6 | from future.types.newbytes import unicode 7 | 8 | from app.douyu import url_manager, html_downloader, html_parser 9 | 10 | 11 | class LiveDataByRoomId(object): 12 | def __init__(self): 13 | self.urls = url_manager.UrlManager() 14 | self.downloader = html_downloader.HtmlDownloader() 15 | self.parser = html_parser.HtmlParser() 16 | 17 | # 根据roomID号获取基本参数 18 | def attention(self, roomId): 19 | 20 | result = '' 21 | 22 | try: 23 | url = "http://www.douyu.com/ztCache/WebM/room/" + str(roomId) 24 | 25 | html_cont = self.downloader.download(url) 26 | 27 | content = json.loads(html_cont) 28 | 29 | if not content: 30 | result = None 31 | 32 | data = self.parser.liveDataByRoomIdParse(content) 33 | 34 | # 正在直播，需要获取直播人气值 35 | if data["show_status"] == 1: 36 | liveData = self.crawAttention(roomId, data["search_url"]) 37 | data["roomNum"] = liveData["roomNum"] 38 | else: 39 | data["roomNum"] = 0 40 | 41 | result = data 42 | 43 | except Exception as e: 44 | print('craw failed:%s' % (e)) 45 | 46 | return result 47 | 48 | # 根据分类的url获取具体的人气值 49 | 50 | def crawAttention(self, roomId, child_cate_url): 51 | 52 | findNumber = True 53 | count = 1 54 | 55 | result = '' 56 | 57 | while findNumber: 58 | try: 59 | url = "https://www.douyu.com" + str(child_cate_url) + "?page=" + str(count) + "&isAjax=1" 60 | 61 | html_cont = self.downloader.download(url) 62 | 63 | roomData = self.parser.parse(html_cont) 64 | 65 | for roomD in roomData: 66 | if roomD["roomId"] == str(roomId): 67 | findNumber = False 68 | result = roomD 69 | 70 | count = count + 1 71 | 72 | except Exception as e: 73 | print('craw failed:%s' % e) 74 | 75 | return result 76 | 77 | # 根据roomId判定房间是否存在 78 | 79 | def isExistRoom(self, roomId): 80 | url = "http://www.douyu.com/ztCache/WebM/room/" + str(roomId) 81 | 82 | html_cont = self.downloader.download(url) 83 | 84 | if unicode(html_cont, "utf-8") == '': 85 | return False 86 | 87 | content = json.loads(html_cont) 88 | 89 | if not content: 90 | result = False 91 | 92 | roomInfo = json.loads(content["$ROOM"]) 93 | 94 | if not roomInfo['room_name']: 95 | return False 96 | else: 97 | return True 98 | -------------------------------------------------------------------------------- /app/douyu/liveDb.py: -------------------------------------------------------------------------------- 1 | # coding:utf-8 2 | from app.db import DBmain 3 | 4 | 5 | class LiveDb(object): 6 | 7 | def __init__(self): 8 | self.db = DBmain.DBModel() 9 | 10 | def searchLiveState(self, param): 11 | sql = " SELECT * FROM dyLiveRoom WHERE alias LIKE %s " 12 | return self.db.exeAllMySQL(sql, param) 13 | 14 | def addLiveState(self, param): 15 | sql = "INSERT INTO dyLiveRoom (name,roomID,alias) values(%s,%s,%s)" 16 | return self.db.insertMySQL(sql, param) 17 | 18 | def isExistLive(self, param): 19 | sql = " SELECT * FROM dyLiveRoom WHERE roomId = %s "; 20 | return self.db.exeOneMySQL(sql, param) 21 | 22 | def updateLiveInfo(self, param): 23 | sql = " UPDATE dyLiveRoom set name=%s,alias=%s WHERE roomId = %s " 24 | return self.db.updateMySQL(sql, param) 25 | -------------------------------------------------------------------------------- /app/douyu/main.py: -------------------------------------------------------------------------------- 1 | # coding:utf-8 2 | import threading 3 | 4 | from wechat_sender import Sender 5 | 6 | from app.douyu import url_manager, html_downloader, html_parser 7 | import json 8 | from future.backports.misc import count 9 | 10 | 11 | class SpilerLiveMain(object): 12 | def __init__(self): 13 | self.urls = url_manager.UrlManager() 14 | self.downloader = html_downloader.HtmlDownloader() 15 | self.parser = html_parser.HtmlParser() 16 | pass 17 | 18 | def craw(self, page=10): 19 | count = 1 20 | 21 | while count < page: 22 | try: 23 | url = "https://www.douyu.com/directory/all?page=" + str(count) + "&isAjax=1" 24 | 25 | html_cont = self.downloader.download(url) 26 | 27 | self.parser.parse(html_cont) 28 | 29 | count = count + 1 30 | 31 | except Exception as e: 32 | print('craw failed:%s' % (e)) 33 | 34 | return self.parser.getLiveRoom() 35 | 36 | def attention(self, rooms=[]): 37 | if len(rooms) == 0: 38 | return 39 | 40 | for room in rooms: 41 | try: 42 | url = "http://www.douyu.com/ztCache/WebM/room/" + str(room) 43 | 44 | html_cont = self.downloader.download(url) 45 | 46 | content = json.loads(html_cont) 47 | 48 | if not content: 49 | continue 50 | 51 | self.parser.jsonParse(content) 52 | 53 | except Exception as e: 54 | print('craw failed:%s' % (e)) 55 | 56 | return self.parser.getAttentionRoom() 57 | 58 | def crawAttention(self, rooms=[]): 59 | 60 | if len(rooms) == 0: 61 | return 62 | count = 1 63 | new_rooms = [] 64 | 65 | while len(rooms): 66 | try: 67 | url = "https://www.douyu.com/directory/all?page=" + str(count) + "&isAjax=1" 68 | 69 | html_cont = self.downloader.download(url) 70 | 71 | roomData = self.parser.parse(html_cont) 72 | 73 | for roomD in roomData: 74 | if roomD["roomId"] in rooms: 75 | rooms.remove(roomD["roomId"]) 76 | new_rooms.append(roomD) 77 | 78 | count = count + 1 79 | 80 | except Exception as e: 81 | print('craw failed:%s' % e) 82 | 83 | return new_rooms 84 | 85 | def clearAttention(self): 86 | self.parser.clearAttentionRoom() 87 | -------------------------------------------------------------------------------- /app/douyu/url_manager.py: -------------------------------------------------------------------------------- 1 | # coding:utf-8 2 | 3 | class UrlManager(object): 4 | def __init__(self): 5 | self.url = '' 6 | 7 | def has_new_url(self): 8 | return self.url != '' 9 | 10 | def get_new_url(self): 11 | if self.url == '': 12 | return None 13 | 14 | return self.url 15 | 16 | def add_new_url(self, url): 17 | self.url = url 18 | -------------------------------------------------------------------------------- /app/server/TextMsgHandle.py: -------------------------------------------------------------------------------- 1 | # coding:utf-8 2 | # 消息处理，文本消息 3 | from app.common import function 4 | from app.server import userDb, subscribe 5 | 6 | import sys 7 | 8 | 9 | class TextMsg(object): 10 | def __init__(self): 11 | self.common = function.CommonFn() 12 | self.userdbSer = userDb.User() 13 | pass 14 | 15 | def start(self, msg): 16 | # 判断用户的订阅的哪些信息 17 | # self.common.prn_obj(msg) 18 | 19 | msg.chat.send('已接收数据，正在处理中...') 20 | 21 | param = (msg.raw["FromUserName"],) 22 | data = self.userdbSer.getUserInfoByUserName(param) 23 | 24 | userId = data["id"] 25 | 26 | if msg.text == "我的订阅": 27 | 28 | self._mySubList(userId, msg) 29 | 30 | else: 31 | # 判定是否是请求订阅，格式：订阅:[斗鱼直播] 32 | sub = msg.text.split(":") 33 | 34 | if len(sub) == 2: 35 | if sub[0] == "订阅": 36 | # 订阅功能 37 | self._addSub(sub, userId, msg) 38 | elif sub[0] == "取消订阅": 39 | # 取消订阅功能 40 | self._cancelSub(sub, userId, msg) 41 | else: 42 | # 信息请求 43 | # 获取订阅信息 44 | param = (data["id"],) 45 | subData = self.userdbSer.getUserSubInfo(param) 46 | 47 | if len(subData) == 0: 48 | info = self._replay_subInfo() 49 | msg.chat.send("您需要订阅\n" + info) 50 | else: 51 | # 前往订阅类，处理订阅对应的信息请求 52 | subscribe.SubscribeHandle(subData, msg) 53 | 54 | def _addSub(self, sub, userId, msg): 55 | subName = sub[1] 56 | param = (subName, subName) 57 | subInfo = self.userdbSer.getSubInfo(param) 58 | if subInfo is None: 59 | info = self._replay_subInfo() 60 | msg.chat.send("订阅名称：【" + subName + "】系统还未提供\n" + info); 61 | else: 62 | # 判定有没有订阅该项目 63 | param = (userId, subInfo["id"]) 64 | isSub = self.userdbSer.isSubUserInfo(param) 65 | 66 | if isSub is None: 67 | # 添加订阅 68 | # uid,subsribe_id,status 69 | param = (userId, subInfo["id"], 1) 70 | self.userdbSer.InsertUserSubInfo(param) 71 | msg.chat.send("订阅：【" + subInfo["name"] + "】成功，可以使用其功能") 72 | else: 73 | # 更新添加 74 | if str(isSub["status"]) == "1": 75 | msg.chat.send("您已经订阅：【" + subInfo["name"] + "】") 76 | else: 77 | param = (1, userId, subInfo["id"]) 78 | self.userdbSer.updateUserSubInfo(param) 79 | msg.chat.send("订阅：【" + subInfo["name"] + "】成功，可以使用其功能") 80 | 81 | def _cancelSub(self, sub, userId, msg): 82 | subName = sub[1] 83 | param = (subName, subName) 84 | subInfo = self.userdbSer.getSubInfo(param) 85 | if subInfo is None: 86 | info = self._replay_subInfo() 87 | msg.chat.send("订阅名称：【" + subName + "】系统还未提供\n" + info); 88 | else: 89 | # 判定有没有订阅该项目 90 | param = (userId, subInfo["id"]) 91 | isSub = self.userdbSer.isSubUserInfo(param) 92 | 93 | if isSub is None: 94 | info = self._replay_subInfo() 95 | msg.chat.send("您还未订阅：【" + subInfo["name"] + "】\n" + info) 96 | else: 97 | # 取消订阅 98 | # uid,subsribe_id,status 99 | param = (0, userId, subInfo["id"]) 100 | self.userdbSer.updateUserSubInfo(param) 101 | msg.chat.send("取消订阅：【" + subInfo["name"] + "】成功") 102 | 103 | def _mySubList(self, userId, msg): 104 | param = (userId,) 105 | subData = self.userdbSer.getUserSubInfo(param) 106 | 107 | if len(subData) == 0: 108 | info = self._replay_subInfo() 109 | msg.chat.send("您还未进行订阅\n" + info) 110 | else: 111 | # 列出我的订阅 112 | info = '[咖啡]我的订阅列表：\n' 113 | count = 1 114 | for sub in subData: 115 | info = info + str(count) + "、订阅号：" + str(sub["id"]) + "；订阅名：" + str(sub["name"]) + "\n" 116 | count = count + 1 117 | 118 | sysInfo = self._replay_subInfo() 119 | 120 | msg.chat.send(info + sysInfo) 121 | pass 122 | 123 | def _replay_subInfo(self): 124 | # 获取目前可以订阅的功能 125 | subs = self.userdbSer.getSubList() 126 | 127 | info = '[拳头]当前系统可用订阅类型：\n' 128 | count = 1 129 | for sub in subs: 130 | info = info + str(count) + "、订阅号：" + str(sub["id"]) + "；订阅名：" + str(sub["name"]) + "\n" 131 | count = count + 1 132 | 133 | info = info + "\n您可以回复: 订阅:[订阅号,订阅名] \n[玫瑰]例如: 订阅:" + str(subs[0]["name"]) 134 | info = info + "\n[嘘]把订阅改为取消订阅，其余不变，即可完成取消 " 135 | info = info + "\n回复：我的订阅 即可查询当前订阅内容\n[爱心]谢谢使用[爱心]" 136 | return info 137 | -------------------------------------------------------------------------------- /app/server/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Topthinking/wxPython/d64f107e6b05a8238f9643d4c3f992c4c4d87357/app/server/__init__.py -------------------------------------------------------------------------------- /app/server/douyuServer.py: -------------------------------------------------------------------------------- 1 | # coding:utf-8 2 | # 处理斗鱼的数据接口 3 | from app.douyu import liveDataByRoomId, liveDb 4 | 5 | 6 | class DouyuServer(object): 7 | 8 | def __init__(self, msg): 9 | # 连接数据库 10 | self.douyuLiveDb = liveDb.LiveDb() 11 | self.liveSer = liveDataByRoomId.LiveDataByRoomId() 12 | self.msg = msg 13 | 14 | def liveDataByRoomId(self): 15 | 16 | newMsg = self.msg.text.split(":") 17 | 18 | sendMsg = self.msg.chat 19 | 20 | # 查询 21 | if len(newMsg) == 0 or newMsg[0] != "dy": 22 | 23 | param = ("%" + self.msg.text + "%",) 24 | 25 | lists = self.douyuLiveDb.searchLiveState(param) 26 | 27 | if len(lists) == 0: 28 | sendMsg.send( 29 | "您的查询超过系统爬取的范围\n可以回复格式为\n dy:[名称]:[房间号]:[别名1,别名2,别名3]\n即可完成添加或者修改，目前针对斗鱼直播数据\n例如： " 30 | "dy:yyf:58428:rua,胖头鱼") 31 | 32 | searchLive = False 33 | 34 | for _list in lists: 35 | liveData = self.liveSer.attention(_list["roomId"]) 36 | 37 | self.msg.chat.send(self._livereplayInfo(liveData)) 38 | 39 | searchLive = True 40 | 41 | if searchLive: 42 | sendMsg.send("数据发送完毕！") 43 | # 插入 44 | elif len(newMsg) != 0 and newMsg[0] == "dy": 45 | 46 | if len(newMsg) != 4: 47 | sendMsg.send("错误格式的数据") 48 | 49 | # name,roomId,alias 50 | # dy:top:12345:a,b,v,d 51 | # newMsg[1],newMsg[2],newMsg[3] 52 | 53 | # 先判断房间号在斗鱼是否存在 54 | if not self.liveSer.isExistRoom(newMsg[2]): 55 | sendMsg.send("房间号:【" + newMsg[2] + "】在斗鱼不存在，无法添加") 56 | 57 | param = (newMsg[2],) 58 | 59 | data = self.douyuLiveDb.isExistLive(param) 60 | 61 | if data is None: 62 | 63 | # 添加数据 64 | param = (newMsg[1], newMsg[2], newMsg[3]) 65 | 66 | self.douyuLiveDb.addLiveState(param) 67 | 68 | sendMsg.send("数据添加成功") 69 | 70 | else: 71 | 72 | # 更新数据 73 | alias = data["alias"].split(",") 74 | 75 | newAlias = newMsg[3].split(",") 76 | 77 | for newAlia in newAlias: 78 | if newAlia not in alias: 79 | alias.append(newAlia) 80 | 81 | alias = ",".join(alias) 82 | 83 | # 更新房间信息 84 | param = (newMsg[1], alias, newMsg[2]) 85 | 86 | self.douyuLiveDb.updateLiveInfo(param) 87 | 88 | sendMsg.send("房间号:【" + newMsg[2] + "】数据更新成功！") 89 | 90 | @staticmethod 91 | def _livereplayInfo(liveData): 92 | 93 | # 正在直播 94 | if liveData['show_status'] == 1: 95 | info = "房间名称:【" + liveData['roomName'] + "】已开播" \ 96 | "\n房间号：" + liveData['roomId'] + \ 97 | "\n主播昵称：" + liveData['nickName'] + \ 98 | "\n人气值：" + liveData['roomNum'] + \ 99 | "\n访问：https://www.douyu.com/" + liveData['roomId'] + "/" 100 | 101 | else: 102 | info = "房间名称:【" + liveData['roomName'] + "】未开播" \ 103 | "\n房间号：" + liveData['roomId'] + \ 104 | "\n主播昵称：" + liveData['nickName'] + \ 105 | "\n访问：https://www.douyu.com/" + liveData['roomId'] + "/" 106 | 107 | return info 108 | -------------------------------------------------------------------------------- /app/server/main.py: -------------------------------------------------------------------------------- 1 | """ 2 | 初始化机器人，以及响应好友消息请求 3 | """ 4 | # coding:utf-8 5 | import datetime 6 | import time 7 | 8 | from wechat_sender.listener import listen 9 | from wxpy.api.bot import Bot 10 | from app.common import function 11 | from app.server import userDb, TextMsgHandle 12 | ''' 13 | 微信个人服务器类 14 | ''' 15 | 16 | 17 | class ServerController(object): 18 | def __init__(self, init=False): 19 | # 初始化机器人 20 | self.bot = Bot(init) 21 | self.bot.enable_puid("wxpy_puid.pkl") 22 | self.common = function.CommonFn() 23 | self.userdbser = userDb.User() 24 | self.textmsg = TextMsgHandle.TextMsg() 25 | 26 | # 存储所有朋友 27 | self.friends = [] 28 | 29 | # 获取朋友列表 30 | self._get_friend() 31 | 32 | def _get_friend(self): 33 | friends = self.bot.friends() 34 | # 遍历所有好友，进行存储和更新好友信息 35 | friendpuid = set() 36 | 37 | for friend in friends: 38 | # 判断重复的puid 39 | if friend.puid in friendpuid: 40 | continue 41 | 42 | friendpuid.add(friend.puid) 43 | tmp_friends = self.bot.friends().search(puid=friend.puid) 44 | 45 | for tmp_friend in tmp_friends: 46 | 47 | # 机器人自己不加入 48 | if tmp_friend.raw["UserName"] == self.bot.self.raw["UserName"]: 49 | continue 50 | 51 | # 刷新当前数据库好友信息 52 | self._actionUserInfo(tmp_friend) 53 | 54 | def start(self): 55 | print("【" + self.bot.self.raw["NickName"] + "】登录成功") 56 | listen(self.bot, self.friends) 57 | self.bot.join() 58 | 59 | def replay(self): 60 | @self.bot.register() 61 | def print_others(msg): 62 | print(msg) 63 | 64 | @self.bot.register(self.friends) 65 | def reply_my_friend(msg): 66 | # 添加好友的提示 67 | if msg.type == "Note": 68 | return '' 69 | 70 | if msg.type != "Text": 71 | return "暂时支持文本格式的" 72 | 73 | # 发送的文本信息 74 | if msg.type == "Text": 75 | self.textmsg.start(msg) 76 | 77 | @self.bot.register(msg_types="Friends") 78 | def auto_accept_friends(msg): 79 | if "top" in msg.text.lower(): 80 | new_friend = msg.card.accept() 81 | 82 | # 对添加的用户进行保存 83 | self._actionUserInfo(new_friend) 84 | 85 | new_friend.send( 86 | '您好，已经接受好友请求了\n访问 https://github.com/Topthinking/wxPython 查看更多' 87 | ) 88 | 89 | def _actionUserInfo(self, friend): 90 | """ 91 | 添加用户 92 | nick_name,user_name,puid,add_time 93 | 更新用户 94 | nick_name,user_name,puid,update_time,id 95 | """ 96 | curTime = int(time.mktime(datetime.datetime.now().timetuple())) 97 | # 判定该添加的朋友之前是否存在备注 98 | if friend.raw["RemarkName"] == '': 99 | # 添加新的朋友到数据库 100 | param = (friend.raw["NickName"], friend.raw["UserName"], 101 | friend.puid, curTime) 102 | remarkName = self.userdbser.insertUserGetInertId(param) 103 | self.bot.core.set_alias( 104 | userName=friend.raw["UserName"], alias=remarkName) 105 | else: 106 | # 更新 107 | # 1.先查询是否存在该用户 存在就更新 否则就添加 108 | param = (friend.raw["RemarkName"], ) 109 | info = self.userdbser.isExistUser(param) 110 | 111 | if info is None: 112 | # 添加新的朋友到数据库 113 | param = (friend.raw["NickName"], friend.raw["UserName"], 114 | friend.puid, curTime) 115 | remarkName = self.userdbser.insertUserGetInertId(param) 116 | self.bot.core.set_alias( 117 | userName=friend.raw["UserName"], alias=remarkName) 118 | else: 119 | param = (friend.raw["NickName"], friend.raw["UserName"], 120 | friend.puid, curTime, friend.raw["RemarkName"]) 121 | self.userdbser.updateUserInfo(param) 122 | 123 | self.friends.append(friend) 124 | -------------------------------------------------------------------------------- /app/server/subscribe.py: -------------------------------------------------------------------------------- 1 | # coding:utf-8 2 | 3 | from app.server import douyuServer 4 | 5 | 6 | class SubscribeHandle(object): 7 | def __init__(self, data, msg): 8 | self.data = data 9 | self.msg = msg 10 | self._diffSub() 11 | pass 12 | 13 | def _diffSub(self): 14 | res = False 15 | for sub in self.data: 16 | if sub["name"] == "斗鱼直播": 17 | # 连接斗鱼数据查询 18 | douyuSer = douyuServer.DouyuServer(self.msg) 19 | 20 | # 根据roomID号爬取斗鱼直播情况 21 | douyuSer.liveDataByRoomId() 22 | res = True 23 | 24 | if not res: 25 | self.msg.chat.send("待开发") 26 | -------------------------------------------------------------------------------- /app/server/userDb.py: -------------------------------------------------------------------------------- 1 | # coding:utf-8 2 | from app.db import DBmain 3 | 4 | 5 | class User(object): 6 | 7 | def __init__(self): 8 | self.db = DBmain.DBModel() 9 | pass 10 | 11 | def isExistUser(self, param): 12 | sql = ' SELECT * FROM user WHERE id = %s '; 13 | return self.db.exeOneMySQL(sql, param) 14 | 15 | def insertUserGetInertId(self, param): 16 | sql = " INSERT INTO user (nick_name,user_name,puid,add_time) values(%s,%s,%s,%s) " 17 | data = self.db.insertMySQLById(sql, param) 18 | return data["id"] 19 | 20 | def updateUserInfo(self, param): 21 | sql = " UPDATE user set nick_name=%s,user_name=%s,puid=%s,update_time=%s WHERE id=%s " 22 | return self.db.updateMySQL(sql, param) 23 | 24 | def getUserInfoByUserName(self, param): 25 | sql = " SELECT * FROM user WHERE user_name = %s "; 26 | return self.db.exeOneMySQL(sql, param) 27 | 28 | def getToalUserInfo(self): 29 | sql = " SELECT * FROM user "; 30 | return self.db.exeAllMySQL(sql) 31 | 32 | def getUserSubInfo(self, param): 33 | sql = " SELECT s.name,s.id FROM user_sub as us \ 34 | INNER JOIN subscribe as s \ 35 | ON us.subscribe_id = s.id \ 36 | WHERE us.uid = %s and us.status=1 " 37 | return self.db.exeAllMySQL(sql, param) 38 | 39 | def getSubInfo(self, param): 40 | sql = " SELECT * FROM subscribe WHERE name = %s or id = %s " 41 | return self.db.exeOneMySQL(sql, param) 42 | 43 | def InsertUserSubInfo(self, param): 44 | sql = " INSERT INTO user_sub (uid,subscribe_id,status) values(%s,%s,%s) " 45 | self.db.insertMySQL(sql, param) 46 | 47 | def updateUserSubInfo(self, param): 48 | sql = " UPDATE user_sub SET status = %s WHERE uid = %s and subscribe_id = %s " 49 | self.db.updateMySQL(sql, param) 50 | 51 | def isSubUserInfo(self, param): 52 | sql = " SELECT * FROM user_sub WHERE uid = %s and subscribe_id = %s " 53 | return self.db.exeOneMySQL(sql, param) 54 | 55 | def getSubList(self): 56 | sql = " SELECT * FROM subscribe WHERE status=1 " 57 | return self.db.exeAllMySQL(sql) 58 | -------------------------------------------------------------------------------- /doc/index.rst: -------------------------------------------------------------------------------- 1 | 使用wxpy爬取需求信息发送微信 2 | ============================== 3 | 4 | 技术栈： 5 | 6 | python3.6 + pip3 + wxpy + wechat_sender + urllib + bs4 +pymysql 7 | 8 | 下载 (运行`python3.6 `) 9 | 如果安装报错，请用pip安装指定模块，对于的wxpy的学习，请github自行搜索 10 | 11 | git clone https://github.com/Topthinking/wxPython.git 12 | 13 | cd wxPython 14 | 15 | python main.py 16 | 17 | 2017年6月25日 18 | 19 | * 初次提交,本地数据进行斗鱼主播的直播情况查询 20 | 21 | 2017年6月27日 22 | 23 | * 1.使用pymysql来动态捕获查询数据 24 | * 2.同时扩展了库与库之间的关系，待优化... 25 | 26 | 2017年6月28日 27 | 28 | * 1.数据的增删改查 29 | * 2.回复格式为 dy:[名称]:[房间号]:[别名] 即可完成添加或者修改， 30 | * 例如回复: dy:yyf:58428:rua , 31 | * 结果就是更新rua别名，那么就可以回复rua获取房间号58428的直播情况 32 | * 可以回复多个别名，他们以英文的逗号隔开，比如 dy:yyf:58428:rua,胖头鱼 33 | * 3.优化sql语句对数据库的直接操作，防止sql注入 34 | * 4.同时处理了输入格式错误返回的信息 35 | 36 | 2017年6月30日 37 | 38 | * 1.加入订阅功能，具体回复："我的订阅" 查看详情 39 | * 2.放出数据库表结构，在doc目录下 40 | 41 | 说明 42 | 43 | * 本项目主要学习Python爬虫，配合微信发送爬取信息，使得学习不会那么枯燥 44 | 45 | * 如果觉得不错的话，您可以点右上角 "Star" 支持一下 谢谢！ ^_^ 46 | 47 | * 如有问题请直接在 Issues 中提，或者您发现问题并有非常好的解决方案，欢迎 PR 👍 48 | 49 | ** 扫描二维码，验证信息输入'top' ** 50 | 51 | .. image:: top.jpg 52 | 53 | 效果截图 54 | 55 | .. image:: show.png 56 | 57 | .. image:: sub.png 58 | 59 | 词汇截图 60 | 61 | .. image:: word.png 62 | -------------------------------------------------------------------------------- /doc/show.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Topthinking/wxPython/d64f107e6b05a8238f9643d4c3f992c4c4d87357/doc/show.png -------------------------------------------------------------------------------- /doc/spider.sql: -------------------------------------------------------------------------------- 1 | /* 2 | Navicat MySQL Data Transfer 3 | 4 | Source Server : top 5 | Source Server Version : 50556 6 | Source Database : spider 7 | 8 | Target Server Type : MYSQL 9 | Target Server Version : 50556 10 | File Encoding : 65001 11 | 12 | Date: 2017-06-30 14:34:39 13 | */ 14 | 15 | SET FOREIGN_KEY_CHECKS=0; 16 | 17 | -- ---------------------------- 18 | -- Table structure for `dyLiveRoom` 19 | -- ---------------------------- 20 | DROP TABLE IF EXISTS `dyLiveRoom`; 21 | CREATE TABLE `dyLiveRoom` ( 22 | `id` int(10) unsigned NOT NULL AUTO_INCREMENT, 23 | `name` varchar(100) DEFAULT NULL, 24 | `roomId` int(11) DEFAULT NULL, 25 | `alias` varchar(3000) DEFAULT NULL, 26 | PRIMARY KEY (`id`), 27 | UNIQUE KEY `roomId` (`roomId`) USING HASH, 28 | KEY `alias` (`alias`(255)) USING BTREE 29 | ) ENGINE=InnoDB AUTO_INCREMENT=37 DEFAULT CHARSET=utf8; 30 | 31 | -- ---------------------------- 32 | -- Records of dyLiveRoom 33 | -- ---------------------------- 34 | INSERT INTO `dyLiveRoom` VALUES ('1', 'yyf', '58428', '鱼鱼风,歪歪爱抚,yyf,yyfyyf,胖头鱼,y,小僵尸,rua,huh,,yyfyyfyyf,ruarua,66666,200斤,姜计就计'); 35 | INSERT INTO `dyLiveRoom` VALUES ('2', '8', '64609', '8,绿帽8,下面八,八师傅,下面师傅,下面,八路军'); 36 | INSERT INTO `dyLiveRoom` VALUES ('3', 'pis', '532152', 'p,绿帽p,小树人,报警,绿色保护着妮,pis'); 37 | INSERT INTO `dyLiveRoom` VALUES ('4', '单车', '339610', '毒奶,车长老,单车武士'); 38 | INSERT INTO `dyLiveRoom` VALUES ('5', '威海', '491416', '威海大叔'); 39 | INSERT INTO `dyLiveRoom` VALUES ('7', '820', '507882', '乌鲁鲁,820,566,八老板,小乌贼,噜噜噜,乌露露，呜呜呜，露露,倚天'); 40 | INSERT INTO `dyLiveRoom` VALUES ('25', 'zsmj', '52876', '方丈,马甲,zsmj'); 41 | INSERT INTO `dyLiveRoom` VALUES ('26', '这是哪位啊', '980511', '美女'); 42 | INSERT INTO `dyLiveRoom` VALUES ('27', '绅士', '45662', '破嘴,魔兽'); 43 | INSERT INTO `dyLiveRoom` VALUES ('28', '红叶动漫', '326183', '红叶动漫'); 44 | INSERT INTO `dyLiveRoom` VALUES ('29', '随机', '664795', '随机1'); 45 | INSERT INTO `dyLiveRoom` VALUES ('30', '随机', '66475', '随机2'); 46 | INSERT INTO `dyLiveRoom` VALUES ('31', '天使焦', '97376', '猪皇,猪,天使焦,天使宝宝,猪仔'); 47 | INSERT INTO `dyLiveRoom` VALUES ('32', '岁月', '58674', '疯了,碧霞祠,阿西吧,我都醉了'); 48 | INSERT INTO `dyLiveRoom` VALUES ('33', '岁月', '58624', 'logout,开机,关机'); 49 | INSERT INTO `dyLiveRoom` VALUES ('34', '卡卡', '55353', '卡露露,卡露露，卡卡，卡师傅,噜噜噜噜噜,良智'); 50 | INSERT INTO `dyLiveRoom` VALUES ('35', '冷冷', '20360', '喝冷水，冷冷，冷了个冷'); 51 | INSERT INTO `dyLiveRoom` VALUES ('36', '主播', '36548', 'ruyrty,bhbj'); 52 | 53 | -- ---------------------------- 54 | -- Table structure for `subscribe` 55 | -- ---------------------------- 56 | DROP TABLE IF EXISTS `subscribe`; 57 | CREATE TABLE `subscribe` ( 58 | `id` int(10) unsigned NOT NULL AUTO_INCREMENT, 59 | `name` varchar(100) DEFAULT NULL, 60 | `status` tinyint(4) NOT NULL DEFAULT '1', 61 | `desc` varchar(3000) DEFAULT NULL, 62 | PRIMARY KEY (`id`) 63 | ) ENGINE=InnoDB AUTO_INCREMENT=2 DEFAULT CHARSET=utf8; 64 | 65 | -- ---------------------------- 66 | -- Records of subscribe 67 | -- ---------------------------- 68 | INSERT INTO `subscribe` VALUES ('1', '斗鱼直播', '1', null); 69 | 70 | -- ---------------------------- 71 | -- Table structure for `user` 72 | -- ---------------------------- 73 | DROP TABLE IF EXISTS `user`; 74 | CREATE TABLE `user` ( 75 | `id` int(10) unsigned NOT NULL AUTO_INCREMENT, 76 | `nick_name` varchar(300) DEFAULT NULL, 77 | `user_name` varchar(3000) DEFAULT NULL, 78 | `puid` varchar(30) DEFAULT NULL, 79 | `add_time` int(11) DEFAULT NULL, 80 | `update_time` int(11) DEFAULT NULL, 81 | PRIMARY KEY (`id`) 82 | ) ENGINE=InnoDB AUTO_INCREMENT=128 DEFAULT CHARSET=utf8; 83 | 84 | -- ---------------------------- 85 | -- Table structure for `user_sub` 86 | -- ---------------------------- 87 | DROP TABLE IF EXISTS `user_sub`; 88 | CREATE TABLE `user_sub` ( 89 | `id` int(10) unsigned NOT NULL AUTO_INCREMENT, 90 | `uid` int(11) DEFAULT NULL, 91 | `subscribe_id` int(11) DEFAULT NULL, 92 | `status` tinyint(4) NOT NULL DEFAULT '1', 93 | PRIMARY KEY (`id`) 94 | ) ENGINE=InnoDB AUTO_INCREMENT=4 DEFAULT CHARSET=utf8; -------------------------------------------------------------------------------- /doc/sub.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Topthinking/wxPython/d64f107e6b05a8238f9643d4c3f992c4c4d87357/doc/sub.png -------------------------------------------------------------------------------- /doc/top.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Topthinking/wxPython/d64f107e6b05a8238f9643d4c3f992c4c4d87357/doc/top.jpg -------------------------------------------------------------------------------- /doc/word.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Topthinking/wxPython/d64f107e6b05a8238f9643d4c3f992c4c4d87357/doc/word.png -------------------------------------------------------------------------------- /main.py: -------------------------------------------------------------------------------- 1 | # coding=utf-8 2 | """ 3 | 主入口文件 4 | """ 5 | from app.server.main import ServerController 6 | 7 | if __name__ == '__main__': 8 | wx_server = ServerController(True) 9 | wx_server.replay() 10 | wx_server.start() 11 | -------------------------------------------------------------------------------- /send.py: -------------------------------------------------------------------------------- 1 | from app.server.send_update_notify import VerUpdateSend 2 | 3 | if __name__ == '__main__': 4 | wx_send = VerUpdateSend(True) 5 | wx_send.sendMsg("欢迎使用top机器人,更多信息访问https://github.com/Topthinking/wxPython/") 6 | --------------------------------------------------------------------------------