├── LICENSE ├── README.md ├── data_scrapy ├── history_stock_quotes_scrapy.py └── stock_basic_scrapy.py ├── data_viewer ├── __pycache__ │ └── rightview.cpython-35.pyc ├── echarts.min.js ├── main.py ├── rightview.py └── template.html ├── requirements.txt └── utils ├── __pycache__ └── myDict.cpython-35.pyc └── myDict.py /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2020 Python3Spiders 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # StockSpider 2 | 股票数据爬虫+分析+可视化框架 3 | -------------------------------------------------------------------------------- /data_scrapy/history_stock_quotes_scrapy.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # author: inspurer(月小水长) 3 | # pc_type lenovo 4 | # create_time: 2019/12/1 23:07 5 | # file_name: history_stock_quotes_scrapy.py 6 | # github https://github.com/inspurer 7 | # qq邮箱 2391527690@qq.com 8 | # 微信公众号 月小水长(ID: inspurer) 9 | 10 | import requests 11 | 12 | from lxml import etree 13 | 14 | from stock_analysis.utils import myDict 15 | 16 | import pymongo 17 | 18 | client = pymongo.MongoClient(host="localhost",port=27017) 19 | db = client['stock'] 20 | table_basic = db['basic'] 21 | 22 | basis = table_basic.find() 23 | print(basis.count()) 24 | stock_basis = myDict.AllowKeyRepeatDict() 25 | for basic in basis: 26 | stock_basis.add(key=basic['name'],value=basic['code']) 27 | 28 | stock_name = '中信证券' 29 | stock_code = stock_basis.query(key=stock_name)[0] 30 | table_quotes = db[stock_name+'_'+stock_code] 31 | 32 | 33 | headers = { 34 | 'Referer': 'http://quotes.money.163.com/trade/lsjysj_600030.html?year=2019&season=4', 35 | 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36' 36 | } 37 | 38 | base_url = 'http://quotes.money.163.com/trade/lsjysj_{code}.html' 39 | # 2019 年 40 | year = 2019 41 | season = 1 42 | # 四季度 43 | for season in range(1,5): 44 | params = { 45 | 'year':year, 46 | 'season':season 47 | } 48 | response = requests.get(url=base_url.format(code=stock_code),headers=headers,params=params) 49 | 50 | html = etree.HTML(response.text) 51 | for tr in html.xpath('//table[@class="table_bg001 border_box limit_sale"]/tr'): 52 | tds = tr.xpath('.//td/text()') 53 | date = tds[0].replace('-','') 54 | open,high,low,close = tds[1],tds[2],tds[3],tds[4] 55 | change,change_pencentage = tds[5],tds[6] 56 | deal_amount,deal_money = tds[7].replace(',',''),tds[8].replace(',','') # 成交量，成交金额（万元） 57 | print(date,open,high,low,close,change,change_pencentage,deal_amount,deal_money) 58 | data = { 59 | 'date': int(date), 60 | 'open': open, 61 | 'high': high, 62 | 'low': low, 63 | 'close': close, 64 | 'change': change, 65 | 'change_pencentage': change_pencentage, 66 | 'deal_amount': deal_amount, 67 | 'deal_money': deal_money 68 | } 69 | # 上海和深圳可能有两个同名的股票 70 | # 上海股市也可能有同名股票，但是code不一样 71 | table_quotes.update_one({'date': date}, {'$set': data}, True) -------------------------------------------------------------------------------- /data_scrapy/stock_basic_scrapy.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # author: inspurer(月小水长) 3 | # pc_type lenovo 4 | # create_time: 2019/12/1 19:24 5 | # file_name: stock_basic_scrapy.py 6 | # github https://github.com/inspurer 7 | # qq邮箱 2391527690@qq.com 8 | # 微信公众号 月小水长(ID: inspurer) 9 | 10 | import requests 11 | 12 | import pymongo 13 | 14 | 15 | client = pymongo.MongoClient(host="localhost",port=27017) 16 | db = client['stock'] 17 | table_basic = db['basic'] 18 | table_quotes = db['quotes'] 19 | 20 | from lxml import etree 21 | headers = { 22 | 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.142 Safari/537.36' 23 | } 24 | 25 | response = requests.get(url='http://quote.eastmoney.com/stock_list.html',headers=headers) 26 | 27 | html = etree.HTML(response.text.encode(response.encoding).decode('gbk')) 28 | 29 | location = '上海' 30 | 31 | print(len(html.xpath('//div[@id="quotesearch"]/ul/li/a/text()'))) 32 | 33 | for stock in html.xpath('//div[@id="quotesearch"]/ul/li/a/text()'): 34 | left = stock.index('(') 35 | name = stock[0:left] 36 | code = stock[left+1:-1] 37 | if name=='凌云B股': 38 | location = '深圳' 39 | print(name,code) 40 | data = { 41 | 'location':location, 42 | 'name':name, 43 | 'code':code 44 | } 45 | # 上海和深圳可能有两个同名的股票 46 | # 上海股市也可能有同名股票，但是code不一样 47 | res = table_basic.update_one({'code':code}, {'$set': data}, True) 48 | # if res.modified_count>0: 49 | # print(name,code) 50 | 51 | -------------------------------------------------------------------------------- /data_viewer/__pycache__/rightview.cpython-35.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Python3Spiders/StockSpider/46edf2f7d85142107aa0f35a34ae964affb51e6e/data_viewer/__pycache__/rightview.cpython-35.pyc -------------------------------------------------------------------------------- /data_viewer/main.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # author: inspurer(月小水长) 3 | # pc_type lenovo 4 | # create_time: 2019/12/2 15:22 5 | # file_name: gui.py 6 | # github https://github.com/inspurer 7 | # qq邮箱 2391527690@qq.com 8 | # 微信公众号 月小水长(ID: inspurer) 9 | 10 | from PyQt5.QtCore import QUrl, Qt 11 | from PyQt5.QtWebEngineWidgets import QWebEngineView 12 | from PyQt5.QtWidgets import QApplication, QHBoxLayout, QWidget, \ 13 | QAction,QMainWindow,QVBoxLayout,QPushButton 14 | from PyQt5.QtGui import QIcon 15 | from pyecharts import Bar, Pie, Line, Overlap,Kline 16 | from pyecharts_javascripthon.api import TRANSLATOR 17 | 18 | from stock_analysis.utils import myDict 19 | from rightview import RightTableView 20 | 21 | import pymongo 22 | 23 | 24 | class Visualization(QMainWindow): 25 | 26 | themes = ['light', 'dark'] 27 | themesTip = ['切换为深色主题','切换为浅色主题'] 28 | 29 | def __init__(self): 30 | super().__init__() 31 | 32 | self.leftTopView = None 33 | self.leftTopEcharts = False 34 | 35 | self.BottomView = None 36 | self.BottomEcharts = False 37 | self.initDataSet() 38 | self.initUi() 39 | self.loadUrl() 40 | 41 | def initUi(self): 42 | 43 | self.statusBar().showMessage('加载中...') 44 | 45 | self.setGeometry(100, 60, 600, 400) 46 | self.setWindowTitle('关注微信公众号：月小水长') 47 | self.setWindowIcon(QIcon('logo.jpg')) 48 | 49 | self.themeSetAct = QAction('更换图表主题(&T)', self) 50 | self.themeSetAct.setShortcut('Ctrl+T') 51 | # 默认浅色主题 52 | self.themeIndex = 0 53 | self.themeSetAct.setStatusTip(Visualization.themesTip[self.themeIndex]) 54 | self.themeSetAct.triggered.connect(self.changeTheme) 55 | 56 | menubar = self.menuBar() 57 | setMenu = menubar.addMenu('设置(&S)') 58 | setMenu.addAction(self.themeSetAct) 59 | 60 | 61 | self.widget = QWidget() 62 | self.setCentralWidget(self.widget) 63 | 64 | # 添加web view 65 | self.leftTopView = QWebEngineView() 66 | self.leftTopView.setContextMenuPolicy(Qt.NoContextMenu) 67 | 68 | self.BottomView = QWebEngineView() 69 | self.BottomView.setContextMenuPolicy(Qt.NoContextMenu) 70 | 71 | 72 | 73 | h1box = QHBoxLayout() 74 | h1box.addWidget(self.leftTopView) 75 | h1box.addWidget(RightTableView()) 76 | h1box.setStretch(0,1) 77 | h1box.setStretch(1,1) 78 | 79 | 80 | h2box = QHBoxLayout() 81 | # v2box = QVBoxLayout() 82 | h2box.addWidget(self.BottomView) 83 | 84 | 85 | vbox = QVBoxLayout() 86 | vbox.addLayout(h1box) 87 | vbox.addLayout(h2box) 88 | vbox.setStretch(0,1) 89 | vbox.setStretch(1,1) 90 | 91 | 92 | self.widget.setLayout(vbox) 93 | 94 | def resizeEvent(self, *args, **kwargs): 95 | w, h = self.width(), self.height() 96 | 97 | 98 | def changeTheme(self): 99 | self.themeIndex = (self.themeIndex+1)%2 100 | self.themeSetAct.setStatusTip(Visualization.themesTip[self.themeIndex]) 101 | 102 | if not self.BottomView: 103 | return 104 | options = self.getOptions(type='K') 105 | 106 | self.BottomView.page().runJavaScript( 107 | ''' 108 | myChart.dispose(); 109 | var myChart = echarts.init(document.getElementById('container'), '{theme}', {{renderer: 'canvas'}}); 110 | myChart.clear(); 111 | window.onresize = function(){{ 112 | myChart.resize(); 113 | }} 114 | var option = eval({options}); 115 | myChart.setOption(option); 116 | '''.format(theme=Visualization.themes[self.themeIndex],options=options) 117 | ) 118 | 119 | if not self.leftTopView: 120 | return 121 | options = self.getOptions(type='Pie') 122 | 123 | self.leftTopView.page().runJavaScript( 124 | ''' 125 | myChart.dispose(); 126 | var myChart = echarts.init(document.getElementById('container'), '{theme}', {{renderer: 'canvas'}}); 127 | myChart.clear(); 128 | window.onresize = function(){{ 129 | myChart.resize(); 130 | }} 131 | var option = eval({options}); 132 | myChart.setOption(option); 133 | '''.format(theme=Visualization.themes[self.themeIndex], options=options) 134 | ) 135 | 136 | def loadUrl(self): 137 | url = QUrl("file:///template.html") 138 | 139 | self.leftTopView.load(url) 140 | self.leftTopView.loadFinished.connect(self.setOptions) 141 | 142 | self.BottomView.load(url) 143 | self.BottomView.loadFinished.connect(self.setOptions) 144 | self.statusBar().showMessage('准备就绪') 145 | 146 | def setOptions(self): 147 | if not self.BottomView: 148 | return 149 | if not self.BottomEcharts: 150 | # 初始化echarts 151 | self.BottomView.page().runJavaScript( 152 | ''' 153 | var myChart = echarts.init(document.getElementById('container'), 'light', {renderer: 'canvas'}); 154 | window.onresize = function(){{ 155 | myChart.resize(); 156 | }} 157 | ''' 158 | ) 159 | self.BottomEcharts = True 160 | 161 | options = self.getOptions(type='K') 162 | 163 | self.BottomView.page().runJavaScript( 164 | ''' 165 | var option = eval({}); 166 | myChart.setOption(option); 167 | '''.format(options) 168 | ) 169 | 170 | if not self.leftTopView: 171 | return 172 | if not self.leftTopEcharts: 173 | # 初始化echarts 174 | self.leftTopView.page().runJavaScript( 175 | ''' 176 | var myChart = echarts.init(document.getElementById('container'), 'light', {renderer: 'canvas'}); 177 | window.onresize = function(){{ 178 | myChart.resize(); 179 | }} 180 | ''' 181 | ) 182 | self.leftTopEcharts = True 183 | 184 | options = self.getOptions(type='Pie') 185 | 186 | self.leftTopView.page().runJavaScript( 187 | ''' 188 | var option = eval({}); 189 | myChart.setOption(option); 190 | '''.format(options) 191 | ) 192 | 193 | def getOptions(self,type): 194 | if type==None or type=='K': 195 | return self.createKlines() 196 | elif type=='Pie': 197 | return self.create_pie(v=[3000,600,5000]) 198 | 199 | 200 | def createKlines(self): 201 | overlap = Overlap() 202 | for quote in self.quote_data: 203 | line = Line(quote['title']) 204 | print(quote) 205 | line.add('open',quote['date'],quote['open'],is_smooth=True) 206 | line.add('close',quote['date'],quote['close'],is_smooth=True) 207 | line.add('high',quote['date'],quote['high'],is_smooth=True) 208 | line.add('low',quote['date'],quote['low'],is_smooth=True) 209 | 210 | overlap.add(line) 211 | 212 | snippet = TRANSLATOR.translate(overlap.options) 213 | options = snippet.as_snippet() 214 | return options 215 | 216 | def create_pie(self, v): 217 | pie = Pie() 218 | pie.add("昨日行情",['涨','平','跌'], v, is_label_show=True) 219 | snippet = TRANSLATOR.translate(pie.options) 220 | options = snippet.as_snippet() 221 | return options 222 | 223 | 224 | def initDataSet(self): 225 | client = pymongo.MongoClient(host="localhost", port=27017) 226 | db = client['stock'] 227 | table_basic = db['basic'] 228 | 229 | self.stock_basis = myDict.AllowKeyRepeatDict() 230 | for basic in table_basic.find(): 231 | self.stock_basis.add(key=basic['name'], value=basic['code']) 232 | 233 | self.quote_data = [] 234 | 235 | stock_name = '中信证券' 236 | stock_code = self.stock_basis.query(key=stock_name)[0] 237 | title = stock_name + '_' + stock_code 238 | table_quote = db[title] 239 | queryDateRange = [20190301,20191130] 240 | date = [] 241 | open = [] 242 | close = [] 243 | high = [] 244 | low = [] 245 | # 为什么不能用 in 而要用 gte lte 246 | for quote in table_quote.find({'date':{'$gte':queryDateRange[0],'$lte':queryDateRange[1]}}).sort('date',pymongo.ASCENDING): 247 | date.append(str(quote['date'])) 248 | open.append(float(quote['open'])) 249 | close.append(float(quote['close'])) 250 | high.append(float(quote['high'])) 251 | low.append(float(quote['low'])) 252 | 253 | 254 | self.quote_data.append({ 255 | 'title':title, 256 | 'date':date, 257 | 'open':open, 258 | 'close': close, 259 | 'high': high, 260 | 'low': low 261 | 262 | }) 263 | self.statusBar().showMessage('数据加载完成') 264 | print('数据加载完成') 265 | 266 | 267 | 268 | 269 | if __name__ == '__main__': 270 | import sys 271 | 272 | app = QApplication(sys.argv) 273 | app.setStyle('fusion') 274 | form = Visualization() 275 | form.show() 276 | sys.exit(app.exec_()) -------------------------------------------------------------------------------- /data_viewer/rightview.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # author: inspurer(月小水长) 3 | # pc_type lenovo 4 | # create_time: 2019/12/18 21:54 5 | # file_name: rightview.py 6 | # github https://github.com/inspurer 7 | # qq邮箱 2391527690@qq.com 8 | # 微信公众号 月小水长(ID: inspurer) 9 | 10 | import sys 11 | from PyQt5.QtWidgets import QApplication,QWidget,QVBoxLayout,QTabWidget,QLabel,QTableWidget,QAbstractItemView,QTableWidgetItem 12 | from PyQt5.QtCore import Qt 13 | 14 | 15 | class RightTableView(QWidget): 16 | def __init__(self): 17 | super().__init__() 18 | 19 | self.mainLayout = QVBoxLayout() 20 | 21 | tabWidgets = QTabWidget() 22 | 23 | label = QLabel("前一日涨幅排名前十的股票详细信息") 24 | tabWidgets.addTab(label, "涨幅排名") 25 | label = QLabel("前一日成交量排名前十的股票详细信息") 26 | tabWidgets.addTab(label, "成交量排名") 27 | 28 | tabWidgets.currentChanged['int'].connect(self.tabClicked) # 绑定标签点击时的信号与槽函数 29 | 30 | self.mainLayout.addWidget(tabWidgets) 31 | 32 | self.tableView = QTableWidget() 33 | self.table = QTableWidget(self) 34 | self.table.setColumnCount(6) 35 | self.table.setSelectionBehavior(QAbstractItemView.SelectRows) # 设置表格的选取方式是行选取 36 | self.table.setSelectionMode(QAbstractItemView.SingleSelection) # 设置选取方式为单个选取 37 | self.table.setHorizontalHeaderLabels(["股票代码", "开盘", "收盘",'最高','最低','成交量']) # 设置行表头 38 | 39 | self.mainLayout.addWidget(self.table) 40 | self.mainLayout.setStretch(0,1) 41 | self.mainLayout.setStretch(1,12) 42 | self.setLayout(self.mainLayout) 43 | 44 | self.updateView() 45 | 46 | def updateView(self): 47 | self.table.insertRow(0) 48 | 49 | stock_code = QTableWidgetItem("1001") 50 | stock_code.setFlags(Qt.ItemIsSelectable | Qt.ItemIsEnabled) # 设置物件的状态为只可被选择（未设置可编辑） 51 | stock_code.setTextAlignment(Qt.AlignCenter) 52 | 53 | stock_open = QTableWidgetItem("10.20") # 我们要求它可以修改，所以使用默认的状态即可 54 | stock_open.setFlags(Qt.ItemIsSelectable | Qt.ItemIsEnabled) # 设置物件的状态为只可被选择 55 | stock_open.setTextAlignment(Qt.AlignCenter) 56 | 57 | stock_close = QTableWidgetItem("10.20") # 我们要求它可以修改，所以使用默认的状态即可 58 | stock_close.setFlags(Qt.ItemIsSelectable | Qt.ItemIsEnabled) # 设置物件的状态为只可被选择 59 | stock_close.setTextAlignment(Qt.AlignCenter) 60 | 61 | 62 | stock_high = QTableWidgetItem("10.20") # 我们要求它可以修改，所以使用默认的状态即可 63 | stock_high.setFlags(Qt.ItemIsSelectable | Qt.ItemIsEnabled) # 设置物件的状态为只可被选择 64 | stock_high.setTextAlignment(Qt.AlignCenter) 65 | 66 | 67 | stock_low = QTableWidgetItem("10.20") # 我们要求它可以修改，所以使用默认的状态即可 68 | stock_low.setFlags(Qt.ItemIsSelectable | Qt.ItemIsEnabled) # 设置物件的状态为只可被选择 69 | stock_low.setTextAlignment(Qt.AlignCenter) 70 | 71 | 72 | stock_dealNum = QTableWidgetItem("10.20") # 我们要求它可以修改，所以使用默认的状态即可 73 | stock_dealNum.setFlags(Qt.ItemIsSelectable | Qt.ItemIsEnabled) # 设置物件的状态为只可被选择 74 | stock_dealNum.setTextAlignment(Qt.AlignCenter) 75 | 76 | 77 | self.table.setItem(0, 0, stock_code) 78 | self.table.setItem(0, 1, stock_open) 79 | self.table.setItem(0, 2, stock_close) 80 | self.table.setItem(0, 3, stock_high) 81 | self.table.setItem(0, 4, stock_low) 82 | self.table.setItem(0, 5, stock_dealNum) 83 | 84 | 85 | 86 | def tabClicked(self,index): 87 | print(index) 88 | 89 | 90 | if __name__ == '__main__': 91 | app = QApplication(sys.argv) 92 | mainWin = RightTableView() 93 | mainWin.show() 94 | sys.exit(app.exec_()) 95 | -------------------------------------------------------------------------------- /data_viewer/template.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | Echarts 6 | 7 | 8 |
9 | 10 | 11 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | requests==2.22.0 2 | lxml==4.2.3 3 | pymongo==3.7.1 4 | PyQt5==5.13.1 -------------------------------------------------------------------------------- /utils/__pycache__/myDict.cpython-35.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Python3Spiders/StockSpider/46edf2f7d85142107aa0f35a34ae964affb51e6e/utils/__pycache__/myDict.cpython-35.pyc -------------------------------------------------------------------------------- /utils/myDict.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # author: inspurer(月小水长) 3 | # pc_type lenovo 4 | # create_time: 2019/12/2 12:25 5 | # file_name: myDict.py 6 | # github https://github.com/inspurer 7 | # qq邮箱 2391527690@qq.com 8 | # 微信公众号 月小水长(ID: inspurer) 9 | 10 | class AllowKeyRepeatDict(): 11 | ''' 12 | 自定义允许键重复的字典 13 | 其本质是一个列表，列表元素为字典，核心思想是把键重复的item分散到不同字典 14 | 封装后列表对外操作像字典 15 | ''' 16 | def __init__(self): 17 | self.dictList = [] 18 | 19 | def add(self,key,value): 20 | length = len(self.dictList) 21 | i = 0 22 | while i0: 43 | if self.dictList[length-1]=={}: 44 | del self.dictList[length-1] 45 | length -= 1 46 | 47 | 48 | def query(self,key): 49 | ''' 50 | :param key: 查询的健 51 | :return: 由于允许键重复，返回形式是一个列表 52 | ''' 53 | result = [] 54 | length = len(self.dictList) 55 | for i in range(length): 56 | response = self.dictList[i].get(key,None) 57 | if not response: 58 | return result 59 | result.append(response) 60 | return result 61 | 62 | def __str__(self): 63 | ''' 64 | :return: 打印整个字典 65 | ''' 66 | resStr = '' 67 | 68 | length = len(self.dictList) 69 | 70 | if length==0: 71 | return '该字典为空' 72 | 73 | for i in range(length): 74 | for k,v in self.dictList[i].items(): 75 | aItem = 'key:{:<8}value:{:<8}\n'.format(k,v) 76 | resStr += aItem 77 | 78 | return resStr 79 | 80 | 81 | 82 | --------------------------------------------------------------------------------