├── cronjob ├── image └── README │ ├── 1688050852474.png │ └── 1688050972640.png ├── requirements.txt ├── 一键运行.bat ├── docker-compose-ecs.yml ├── README.md ├── Dockerfile ├── Dcokerfile_crontab ├── docker-compose-local.yml ├── update.py ├── plot.py ├── update_ddb_tables.dos ├── app.py ├── mirror_v4.dos └── v4.ipynb /cronjob: -------------------------------------------------------------------------------- 1 | 0 17 * * * root python /app/script.py >> /var/log/cron.log 2>&1 2 | -------------------------------------------------------------------------------- /image/README/1688050852474.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/flizzywine/kline-mirror/HEAD/image/README/1688050852474.png -------------------------------------------------------------------------------- /image/README/1688050972640.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/flizzywine/kline-mirror/HEAD/image/README/1688050972640.png -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | streamlit 2 | tushare 3 | akshare 4 | bs4 5 | streamlit-aggrid 6 | streamlit-option-menu 7 | plotly 8 | -------------------------------------------------------------------------------- /一键运行.bat: -------------------------------------------------------------------------------- 1 | start D:\kline\dolphindb\DolphinDB_Win64_V1.30.21.2\server\dolphindb.exe 2 | start streamlit run app.py 3 | 4 | 5 | 6 | 7 | -------------------------------------------------------------------------------- /docker-compose-ecs.yml: -------------------------------------------------------------------------------- 1 | version: '3.3' 2 | services: 3 | streamlit: 4 | image: kline_mirror 5 | ports: 6 | - "8501:8501" 7 | volumes: 8 | - /home/ecs-assist-user:/app 9 | command: ["streamlit", "run", "app.py"] 10 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Kline Mirror 2 | 3 | 4 | Kline Mirror 是一个 中国股票市场的复盘软件。 具有以下功能。 5 | 6 | 1. 查阅历史交易日期的股票K线。 7 | 8 | 2. 限定概念题材，筛选股票。 9 | 10 | 3. 跟踪概念题材的涨幅，而不是单个股票的涨幅。 11 | 12 | 4. 多种不同的筛选方式，对股票进行筛选，如：已经大涨，已经大跌等。 13 | 14 | 15 | ## 大盘情况 16 | 17 | 18 | ![1688050852474](image/README/1688050852474.png) 19 | 20 | ## 个股情况 21 | 22 | 23 | ![1688050972640](image/README/1688050972640.png) 24 | -------------------------------------------------------------------------------- /Dockerfile: -------------------------------------------------------------------------------- 1 | 2 | # Use an official Python runtime as the base image 3 | # FROM python:3.9-slim-buster 一会调试完成再用 4 | FROM python:3.9-slim-buster 5 | # Set the working directory inside the container 6 | WORKDIR /app 7 | 8 | # 把应用的依赖包 全部下载下来 9 | COPY requirements.txt . 10 | 11 | RUN pip config set global.index-url https://mirrors.aliyun.com/pypi/simple/ \ 12 | && pip install -r requirements.txt 13 | 14 | EXPOSE 8501 15 | -------------------------------------------------------------------------------- /Dcokerfile_crontab: -------------------------------------------------------------------------------- 1 | FROM python:3.9-slim 2 | 3 | 4 | WORKDIR /app 5 | 6 | # 将Python脚本复制到容器中 7 | COPY update.py /app/update.py 8 | 9 | COPY requirements.txt /app/requirements.txt 10 | 11 | # 安装所需的依赖项 12 | RUN pip install -r requirements.txt 13 | 14 | COPY cronjob /etc/cron.d/cronjob 15 | 16 | # 赋予Cron文件执行权限 17 | RUN chmod 0644 /etc/cron.d/cronjob 18 | 19 | # 应用新的Cron任务 20 | RUN crontab /etc/cron.d/cronjob 21 | 22 | CMD ["python", "update.py"] 23 | 24 | 25 | 26 | 27 | 28 | -------------------------------------------------------------------------------- /docker-compose-local.yml: -------------------------------------------------------------------------------- 1 | version: '3.3' 2 | services: 3 | dolphindb: 4 | image: dolphindb/dolphindb:v2.00.9.8 5 | ports: 6 | - "8848:8848" 7 | volumes: 8 | - /Users/cf/Documents/kline/data:/data/ddb/server/data 9 | command: ["tail", "-f", "/dev/null"] 10 | 11 | streamlit: 12 | image: kline_mirror 13 | ports: 14 | - "8501:8501" 15 | volumes: 16 | - /Users/cf/Documents/kline/kline_mirror/:/app 17 | depends_on: 18 | - dolphindb 19 | command: ["streamlit", "run", "app.py"] 20 | -------------------------------------------------------------------------------- /update.py: -------------------------------------------------------------------------------- 1 | import pandas as pd 2 | import akshare as ak 3 | import datetime 4 | import tushare as ts 5 | import dolphindb 6 | import numpy as np 7 | import time 8 | import requests 9 | def get_conn(): 10 | db = dolphindb.session() 11 | db.connect("localhost", 8848, "admin", "123456") 12 | return db 13 | 14 | conn = get_conn() 15 | 16 | # 连接tushare 17 | token = "f1ce53736e3b6777425d3df97c05e7460a55534db8ece60114c9e2a3" 18 | ts.set_token(token) 19 | pro = ts.pro_api() 20 | 21 | 22 | # 更新daily_adj 23 | def update_daily_adj(date): 24 | # 先下载 daily 数据 25 | # 判断是否已经有date的数据 26 | conn.run("daily_adj = loadTable('dfs://stock', `daily_adj)") 27 | if conn.run(f"select 1 from daily_adj where data_dt={date} limit 1").shape[0] == 0: 28 | print(f"已经存在{date}的daily_adj数据") 29 | else: 30 | # 下载数据 31 | df = pro.daily(start_date=date.replace(".",""), end_date=date.replace(".",""), adj='hfq') 32 | if df.shape[0] == 0: 33 | print(f"{date}没有数据") 34 | return 35 | df = df.query("not ts_code.str.contains('.BJ')", engine='python') 36 | # 把chg>20的刷成21, 把chg<-20的刷成-21 37 | df.pct_chg[df.pct_chg > 21] = 21 38 | df.pct_chg[df.pct_chg < -21] = -21 39 | # to daily_adj 40 | new_order = ['trade_date', 'ts_code', 'pct_chg', 'open','high', 'low', 'close', 'amount'] 41 | df = df.reindex(columns=new_order) 42 | df.rename(columns={'trade_date':'data_dt', 'amount':'amt', "pct_chg":"chg"}, inplace=True) 43 | 44 | # 日期从string转成datetime,然后上传. 45 | df['data_dt'] = pd.to_datetime(df['data_dt'], format='%Y%m%d').values.astype('datetime64[D]') 46 | df['ts_code'] = df['ts_code'].astype("object") 47 | conn.upload({'daily_adj_tmp':df}) 48 | conn.run("tableInsert(daily_adj,(select * from daily_adj_tmp))") 49 | 50 | print(f"更新完成{date}的daily_adj数据") 51 | 52 | # 批量更新daily_adj 53 | def batch_update_daily_adj(): 54 | conn.run("daily_adj = loadTable('dfs://stock', `daily_adj)") 55 | start_date = np.datetime_as_string(conn.run("select max(data_dt) from daily_adj").values[0][0], unit='D').replace("-",".") 56 | today = datetime.datetime.today().strftime("%Y.%m.%d") 57 | end_date = conn.run(f"transFreq({today}, 'SSE')").astype("str").replace("-",".") 58 | print(start_date,"---", end_date) 59 | download_dates = conn.run(f"getMarketCalendar('SSE', {start_date}, {end_date})").tolist() 60 | 61 | for date in download_dates: 62 | date = date.strftime("%Y.%m.%d") 63 | print(date) 64 | while True: 65 | try: 66 | update_daily_adj(date) 67 | break 68 | except requests.exceptions.ConnectTimeout: 69 | print("出错了, 重试") 70 | time.sleep(5) 71 | continue 72 | print("更新完成批量daily_adj") 73 | 74 | 75 | 76 | # # 更新股票列表 77 | def update_basic(): 78 | df = pro.stock_basic() 79 | df = df[(df['market'] != '北交所') & (df['market'] != 'CDR')] # 过滤北交所和CDR. 80 | df['ts_code'] = df['ts_code'].astype("object") 81 | df['symbol'] = df['ts_code'].str.slice(0,6) 82 | conn.upload({'basic_tmp':df}) 83 | conn.run(""" 84 | db = database("dfs://stock"); 85 | if (existsTable("dfs://stock", 'basic')) db.dropTable('basic'); 86 | basic = db.createTable(basic_tmp, 'basic'); 87 | basic.tableInsert(select * from basic_tmp);""") 88 | print("更新完成basic") 89 | # df.to_sql("basic",sdb._instance, index=False, if_exists="replace", method='multi') 90 | # 要新增一个symbol 字段, 用来存储股票代码, 例如: 000001 91 | # 每次都删除重新创建一个. 92 | 93 | 94 | 95 | # 更新概念与题材列表 96 | # 每天更新,新增的概念. 每月进行一次全量的更新 97 | 98 | # 增量更新概念 99 | # 清洗概念,对于不想要的概念,进行清洗. 100 | # 对于没有概念的股票,进行清洗.排除. 101 | 102 | 103 | ## 获取最新的概念列表 104 | def update_concept_list(): 105 | concept_list_df = ak.stock_board_concept_name_ths() 106 | concept_list_df['日期'] = pd.to_datetime(concept_list_df['日期'], format="%Y.%m.%d") 107 | conn.upload({"concept_list_df": concept_list_df}) 108 | first_update_concept_flag = conn.run(" not existsTable('dfs://stock', 'concept_list_old')") 109 | # 是否是第一次更新概念 110 | if first_update_concept_flag: 111 | # 如果是第一次更新概念, 则全量更新 112 | # // 日期, 概念名称, 成分股数量, 网址, 代码 113 | conn.run(""" 114 | db = database("dfs://stock"); 115 | concept_list_old = db.createTable(concept_list_df, 'concept_list_old'); 116 | concept_list_old.append!(select * from concept_list_df); 117 | truncate('dfs://stock' , `concept_list_old); 118 | concept_list_new = db.createTable(concept_list_df, 'concept_list_new'); 119 | concept_list_new.append!(select * from concept_list_df); 120 | """) 121 | else: 122 | # 比对old new ,找出成分股数量不同的概念, 这些概念就是新增的概念. 123 | # // 日期, 概念名称, 成分股数量, 网址, 代码 124 | conn.run(""" 125 | db = database("dfs://stock"); 126 | concept_list_new = loadTable("dfs://stock", 'concept_list_new'); 127 | concept_list_old = loadTable("dfs://stock", 'concept_list_old'); 128 | 129 | truncate('dfs://stock' , `concept_list_old); // 清空old 130 | concept_list_old.append!(select * from concept_list_new); // 原来 new 的值赋给 old 131 | truncate('dfs://stock' , `concept_list_new);// 清空new 132 | concept_list_new.append!(select * from concept_list_df); 133 | """) 134 | print("更新完成concept_list") 135 | 136 | def update_concept_cons(concept): 137 | conn.run("db = database('dfs://stock');") 138 | if conn.run(f"existsTable('dfs://stock', 'concept_cons')"): 139 | conn.run("concept_cons = loadTable('dfs://stock', 'concept_cons');") 140 | 141 | # if conn.run(f"select * from concept_cons where 概念名称 = '{concept}'").shape[0] > 0: 142 | # print("已经存在", concept) 143 | # return 144 | while True: 145 | try: 146 | print("开始下载", concept) 147 | df = ak.stock_board_concept_cons_ths(symbol=concept) 148 | df['概念名称'] = concept 149 | df = df[['代码', '名称', '概念名称']] 150 | conn.upload({"concept_cons_df": df}) 151 | if not conn.run(f"existsTable('dfs://stock', 'concept_cons')"): 152 | conn.run(""" 153 | t2 = select b.ts_code, c.* from basic b 154 | left join concept_cons_df c on b.symbol = c.代码 155 | concept_cons = db.createTable(t2, 'concept_cons'); 156 | concept_cons.append!(select * from t2); 157 | """) 158 | else: 159 | conn.run("concept_cons = loadTable('dfs://stock', 'concept_cons');") 160 | conn.run(f""" 161 | delete from concept_cons where 概念名称 = '{concept}'; 162 | t2 = select b.ts_code, c.* from basic b 163 | left join concept_cons_df c on b.symbol = c.代码 164 | concept_cons.append!(select * from t2); 165 | """) 166 | print("下载完成", concept) 167 | break 168 | except (requests.exceptions.ConnectTimeout,ValueError): 169 | print("出错了, 重试") 170 | time.sleep(5) 171 | continue 172 | 173 | 174 | def batch_update_concept_cons(): 175 | 176 | # 找到在 new中存在, 在old中不存在的概念 177 | # // 日期, 概念名称, 成分股数量, 网址, 代码 178 | conn.run("concept_list_new = loadTable('dfs://stock', 'concept_list_new');") 179 | conn.run("concept_list_old = loadTable('dfs://stock', 'concept_list_old');") 180 | concept_list = conn.run("""select 概念名称 as concept from concept_list_new new 181 | where not exists(select 1 from concept_list_old old 182 | where new.概念名称 = old.概念名称) 183 | or new.概念名称 in (select 概念名称 from concept_list_old old 184 | where new.概念名称 = old.概念名称 and (new.成分股数量 <> old.成分股数量 or 185 | new.日期 <> old.日期)) 186 | order by 日期 desc""")['concept'].values 187 | # concept_list = conn.run("""select 概念名称 as concept from concept_list_new 188 | # order by 日期 desc""")['concept'].values 189 | 190 | for concept in concept_list: 191 | update_concept_cons(concept) 192 | print("更新完成concept_cons") 193 | 194 | 195 | 196 | def update_ddb_tables(): 197 | # 更新market_index 198 | # 更新stock_ma 199 | # 更新stock_ind 200 | # 更新concept_ind 201 | # 更新concept_cons 202 | conn.runFile("update_ddb_tables.dos") 203 | # 冷启动 204 | # conn.run(""" 205 | # start_date = 2022.01.01 206 | # end_date = 2022.12.31 207 | # init_market_index(start_date, end_date) 208 | # init_stock_ma(start_date, end_date) 209 | # init_stock_ind(start_date, end_date) 210 | # //init_concept_ind(start_date, end_date) // concept_ind 需要单独启动一遍, 因为换concept_cons了 211 | # """) 212 | 213 | # 热更新 214 | conn.run(""" 215 | update_market_index() 216 | update_stock_ma() 217 | update_stock_ind() 218 | update_concept_ind() 219 | """) 220 | 221 | print("更新完成ddb_tables") 222 | 223 | 224 | def update_all(): 225 | update_basic() 226 | update_concept_list() 227 | batch_update_concept_cons() 228 | batch_update_daily_adj() 229 | update_ddb_tables() 230 | 231 | -------------------------------------------------------------------------------- /plot.py: -------------------------------------------------------------------------------- 1 | import streamlit as st 2 | import pandas as pd 3 | 4 | from st_aggrid import AgGrid, GridOptionsBuilder 5 | from st_aggrid.shared import GridUpdateMode 6 | from plotly.subplots import make_subplots 7 | import plotly.express as px 8 | import plotly.graph_objects as go 9 | import datetime 10 | 11 | import dolphindb 12 | 13 | # connect to database server 14 | @st.cache_resource 15 | def get_conn(): 16 | db = dolphindb.session() 17 | db.connect("localhost", 8848, "admin", "123456") 18 | return db 19 | 20 | # @st.cache_resource 21 | # def get_conn(): 22 | # return sqlite3.connect("stock.db", check_same_thread=False) 23 | 24 | conn = get_conn() 25 | 26 | 27 | config = {"editSelection":False, "displayModeBar": False, 28 | "responsive":True, "editable":False, 29 | "showAxisDragHandles":False, "showAxisRangeEntryBoxes":False} 30 | 31 | @st.cache_data 32 | def MA(S,N): #求序列的N日简单移动平均值，返回序列 33 | return pd.Series(S).rolling(N).mean().values 34 | 35 | 36 | def plot_kline_fig(df, cur_date, height=500): 37 | """需要满足, df 中有 columns: data_dt, chg, open, close, high, low, amt""" 38 | dates = df['data_dt'] 39 | 40 | cur_date = pd.to_datetime(cur_date, format="%Y.%m.%d") 41 | open = df['open'] 42 | high = df['high'] 43 | low = df['low'] 44 | close = df['close'] 45 | MA10 = MA(close, 10) 46 | MA5 = MA(close, 5) 47 | MA20 = MA(close, 20) 48 | amount = df['amt'] 49 | pct_chg = df['chg'] 50 | amount_color = df[['open','close']].apply(lambda x :'red' if x['close']>=x['open'] else 'green', axis=1) 51 | hover_text = list(map(lambda pct: f"涨跌幅:{pct:.2f}", pct_chg)) 52 | 53 | fig = make_subplots(rows=5, cols=1, 54 | specs=[[{"rowspan": 4}], 55 | [{}], 56 | [{}], 57 | [{}], 58 | [{}], 59 | ]) 60 | 61 | # 均线MA10 62 | line_ma = go.Scatter(x=dates, y=MA10, line=dict(color='#D2B48C',dash='solid', width=1),name='MA10', hoverinfo='skip') 63 | fig.add_trace(line_ma, row=1, col=1) 64 | 65 | # 均线MA5 66 | line_ma2 = go.Scatter(x=dates, y=MA5, line=dict(color='#FFD700',dash='solid', width=1),name='MA20', hoverinfo='skip') 67 | fig.add_trace(line_ma2, row=1, col=1) 68 | # 均线MA20 69 | line_ma20 = go.Scatter(x=dates, y=MA20, line=dict(color='#DDA0DD',dash='solid', width=1),name='MA20', hoverinfo='skip') 70 | fig.add_trace(line_ma20, row=1, col=1) 71 | 72 | # 蜡烛图 73 | candle_stick = go.Candlestick(x=dates, open=open, high=high, low=low, close=close, 74 | hovertext = hover_text, 75 | increasing=dict(line=dict(color="#FF0000")), 76 | decreasing=dict(line=dict(color="#00FF00")), 77 | name='' 78 | ) 79 | fig.add_trace(candle_stick, row=1, col=1) 80 | 81 | # 成交量图 82 | amount_bar = go.Bar(x=dates, y=amount, marker_color=amount_color,name='成交量') 83 | fig.add_trace(amount_bar, row=5, col=1) 84 | fig.update_yaxes(range=[amount.min(), amount.max()], row=5, col=1) 85 | fig.update_xaxes(dict(type="category")) 86 | 87 | # 垂直标记点线. 88 | max_high = high.max() 89 | min_low = low.min() 90 | line = go.Scatter(x=[cur_date, cur_date], y=[min_low, max_high],line=dict(color="gray", dash='dot'),hoverinfo='skip') 91 | fig.add_trace(line, row=1, col=1) 92 | 93 | # 成交量垂直标记线 94 | max_amount = amount.max() 95 | min_amount = 0 96 | line2 = go.Scatter(x=[cur_date, cur_date],y=[min_amount,max_amount],line=dict(color="gray", dash='dot'),hoverinfo='skip') 97 | fig.add_trace(line2,row=5,col=1) 98 | 99 | fig.update_layout( 100 | xaxis=dict(showgrid=False, visible=False,type = "category"), 101 | yaxis=dict(showgrid=False) 102 | ) 103 | fig.update_layout(xaxis_rangeslider_visible=False) 104 | fig.update_layout(showlegend=False) 105 | fig.update_layout(xaxis_visible=False) 106 | fig.update_layout(yaxis_title=None, xaxis_title=None) 107 | fig.update_layout(height=height) 108 | fig.update_xaxes(tickfont=dict(size=1)) 109 | fig.update_layout(autosize=True) 110 | st.plotly_chart(fig, use_container_width=False, config=config) 111 | 112 | 113 | def read_kline_df(data_dt, ts_code, type="股票"): 114 | if type == "股票": 115 | script = f""" 116 | daily_adj = loadTable("dfs://stock", "daily_adj") 117 | cur_date = {data_dt} 118 | start_date = temporalAdd(cur_date, -20, "SSE") // 交易日偏移 119 | end_date = temporalAdd(cur_date, 20, "SSE") // 交易日偏移 120 | select data_dt, ts_code, chg, open, high, low, close, amt/100000 as amt 121 | from daily_adj where data_dt between start_date:end_date and ts_code = `{ts_code} 122 | """ 123 | 124 | elif type == "综合指数": 125 | script = f""" 126 | market_index = loadTable("dfs://stock", "market_index") 127 | cur_date = {data_dt} 128 | start_date = temporalAdd(cur_date, -20, "SSE") // 交易日偏移 129 | end_date = temporalAdd(cur_date, 20, "SSE") // 交易日偏移 130 | select data_dt, cum_open as open, cum_high as high, 131 | cum_low as low, cum_close as close, amt, chg as chg 132 | from market_index where 133 | data_dt between start_date:end_date order by data_dt asc 134 | """ 135 | 136 | df = conn.run(script) 137 | return df 138 | 139 | 140 | 141 | def plot_grid_fig(df: pd.DataFrame, type="股票" , height: int =500): 142 | options = GridOptionsBuilder.from_dataframe( 143 | df 144 | ) 145 | options.configure_side_bar(filters_panel=False, columns_panel=False) 146 | options.configure_selection("single") 147 | update_mode = GridUpdateMode.SELECTION_CHANGED 148 | selection = AgGrid( 149 | df, 150 | gridOptions=options.build(), 151 | update_mode=update_mode, 152 | theme='material', 153 | height=height 154 | ) 155 | if type == "股票": 156 | if selection and selection['selected_rows']: 157 | selected_stock = selection['selected_rows'][0] 158 | 159 | if 'ts_code' in selected_stock: 160 | ts_code = selected_stock['ts_code'] 161 | st.session_state["ts_code"] = ts_code 162 | 163 | 164 | elif type == "题材": 165 | if selection and selection['selected_rows']: 166 | selected_concept = selection['selected_rows'][0] 167 | 168 | if 'concept' in selected_concept: 169 | concept = selected_concept['concept'] 170 | st.session_state["concept"] = concept 171 | 172 | 173 | def plot_concept_fig(df: pd.DataFrame, y_lable , cur_date , height: int =500): 174 | 175 | fig = px.line(df, x="data_dt", y=y_lable, color='concept',markers=True) 176 | max_amount = df[y_lable].max() 177 | min_amount = df[y_lable].min() 178 | # cur_date 转成 日期格式 179 | cur_date = cur_date.replace(".","-") 180 | line2 = go.Scatter(x=[cur_date, cur_date],y=[min_amount,max_amount],line=dict(color="gray", dash='dot'),hoverinfo='skip',mode='lines+markers') 181 | fig.add_trace(line2) 182 | fig.update_layout(xaxis_rangeslider_visible=False) 183 | fig.update_layout(showlegend=True) 184 | fig.update_layout(xaxis_visible=False) 185 | fig.update_layout(yaxis_title=None, xaxis_title=None) 186 | fig.update_layout(height=height) 187 | fig.update_xaxes(tickfont=dict(size=1)) 188 | fig.update_layout(autosize=True) 189 | st.plotly_chart(fig, use_container_width=False, config=config) 190 | 191 | 192 | @st.cache_data 193 | def read_concept_df(data_dt, type, concept=None): 194 | conn.run(f"cur_date = {data_dt};t5 = temporalAdd(cur_date, -5, 'SSE') ;t_5 = temporalAdd(cur_date, 5, 'SSE');") 195 | conn.run("concept_ind = loadTable('dfs://stock', 'concept_ind')") 196 | if type == "中期大涨的概念": 197 | script = f""" 198 | select * from concept_ind where data_dt between t5:t_5 and rn_dl_desc < 10 199 | order by data_dt asc 200 | """ 201 | y_label = "dl" 202 | 203 | elif type == "中期大跌的概念": 204 | script = f""" 205 | select * from concept_ind where data_dt between t5:t_5 and rn_dl_asc < 10 206 | order by data_dt asc 207 | """ 208 | y_label = "dl" 209 | 210 | elif type == "当天大涨的概念": 211 | script = f""" 212 | select * from concept_ind where data_dt between t5:t_5 and rn_chg_desc < 10 213 | order by data_dt asc 214 | """ 215 | y_label = "chg" 216 | 217 | 218 | if type == "当天大跌的概念": 219 | script = f""" 220 | select * from concept_ind where data_dt between t5:t_5 and rn_chg_asc < 10 221 | order by data_dt asc 222 | """ 223 | y_label = "chg" 224 | 225 | elif type == "有潜力的概念": 226 | script = f""" 227 | select data_dt, concept, ql from concept_ind 228 | where data_dt between t5:t_5 229 | and rn_ql_desc < 10 order by data_dt asc 230 | """ 231 | y_label = "ql" 232 | 233 | elif type == "概念K线" and concept is not None: 234 | script = f""" 235 | select data_dt, concept, dl from concept_ind 236 | where data_dt between t5:t_5 and concept like '%{concept}%' 237 | order by data_dt asc 238 | """ 239 | y_label = "dl" 240 | 241 | df = conn.run(script) 242 | return df, y_label 243 | 244 | 245 | 246 | 247 | 248 | 249 | -------------------------------------------------------------------------------- /update_ddb_tables.dos: -------------------------------------------------------------------------------- 1 | 2 | def write_table(tab_name, t) { 3 | db = database("dfs://stock") 4 | if (existsTable("dfs://stock", tab_name)) db.dropTable(tab_name) 5 | pt = db.createPartitionedTable(t, tab_name, `data_dt) 6 | pt.append!(select * from t) 7 | } 8 | 9 | def init_market_index(start_date, end_date) { 10 | db = database("dfs://stock") 11 | pt = db.loadTable(`daily_adj) 12 | t2 = select data_dt, ts_code, open\close as open_pct, 13 | high\close as high_pct, low\close as low_pct, ratios(close) as close_pct, 14 | amt\100000 as amt 15 | from pt where data_dt between start_date:end_date context by ts_code 16 | 17 | t3 = select data_dt, avg(open_pct) as open_pct, avg(high_pct) as high_pct, 18 | avg(low_pct) as low_pct, avg(close_pct) as close_pct, sum(amt) as amt from t2 group by data_dt 19 | 20 | t4 = select data_dt, cumprod(close_pct) as cum_close, amt , 100*(close_pct-1) as chg , 21 | open_pct, high_pct, low_pct, close_pct from t3 22 | update t4 set 23 | cum_open = cum_close*open_pct 24 | from t4 25 | update t4 set cum_high = max(cum_close*(1+ (high_pct-1)\3), max(cum_open, cum_close)), 26 | cum_low = min(cum_close*(1+ (low_pct-1)\3), min(cum_close, cum_open)) 27 | from t4 28 | t5 = select data_dt, cum_open,cum_low,cum_high,cum_close, amt, chg from t4 29 | 30 | tab_name = `market_index 31 | if (existsTable("dfs://stock", tab_name)) db.dropTable(tab_name) 32 | pt2 = db.createPartitionedTable(t5, tab_name, `data_dt) 33 | pt2.append!(select * from t5) 34 | } 35 | // market_index 在新增的时候,需要做什么呢?? 36 | 37 | def update_market_index() { 38 | db = database("dfs://stock") 39 | daily_adj = db.loadTable(`daily_adj) 40 | market_index = db.loadTable(`market_index) 41 | pre_date = (select max(data_dt) from market_index)['max_data_dt'][0] 42 | start_date = temporalAdd(select max(data_dt) from market_index, 1, 'SSE')['max_data_dt'][0] 43 | write_start_date = (select max(data_dt) from market_index)['max_data_dt'][0] 44 | end_date = (select max(data_dt) from daily_adj)['max_data_dt'][0] 45 | 46 | 47 | t2 = select data_dt, ts_code, open\close as open_pct, 48 | high\close as high_pct, low\close as low_pct, ratios(close) as close_pct, 49 | amt\100000 as amt 50 | from daily_adj where data_dt between start_date:end_date context by ts_code 51 | 52 | t3 = select data_dt, avg(open_pct) as open_pct, avg(high_pct) as high_pct, 53 | avg(low_pct) as low_pct, avg(close_pct) as close_pct, sum(amt) as amt from t2 group by data_dt 54 | 55 | t4 = select data_dt, cumprod(close_pct) as cum_close, amt , 100*(close_pct-1) as chg , 56 | open_pct, high_pct, low_pct, close_pct from t3 57 | update t4 set 58 | cum_open = cum_close*open_pct 59 | from t4 60 | update t4 set cum_high = max(cum_close*(1+ (high_pct-1)\3), max(cum_open, cum_close)), 61 | cum_low = min(cum_close*(1+ (low_pct-1)\3), min(cum_close, cum_open)) 62 | from t4 63 | 64 | prev_cum_open = (select cum_open from market_index where data_dt = pre_date)['cum_open'][0] 65 | 66 | prev_cum_close = (select cum_close from market_index where data_dt = pre_date)['cum_close'][0] 67 | prev_cum_high = (select cum_high from market_index where data_dt = pre_date)['cum_high'][0] 68 | prev_cum_low = (select cum_low from market_index where data_dt = pre_date)['cum_low'][0] 69 | t5 = select data_dt, cum_open*prev_cum_open as cum_open, 70 | cum_low*prev_cum_low as cum_low, 71 | cum_high*prev_cum_high as cum_high, 72 | cum_close*prev_cum_close as cum_close, 73 | amt, chg from t4 74 | 75 | market_index.append!(select * from t5 where data_dt > write_start_date) 76 | 77 | // 下面这句话用于验证 日期是不是完整. 78 | // set(getMarketCalendar("SSE", pre_date, end_date)) - set((select distinct data_dt from market_index where data_dt >= pre_date)['data_dt']) 79 | } 80 | 81 | // 如果要更新,就要 计算当天的昨天的差距, 然后, 把整个流程重走一遍. 82 | 83 | 84 | def init_stock_ma(start_date, end_date) { 85 | db = database("dfs://stock") 86 | daily_adj = db.loadTable(`daily_adj) 87 | tab_name = `stock_ma 88 | t1 = select data_dt, ts_code, chg, close, high, low, open, amt/100000 as amt, mavg(close, 3) as ma3, mavg(close, 5) as ma5, 89 | mavg(close, 10) as ma10, mavg(close, 20) as ma20, mavg(close, 60) as ma60 90 | from daily_adj where data_dt between start_date:end_date context by ts_code 91 | if (existsTable("dfs://stock", tab_name)) db.dropTable(tab_name) 92 | stock_ma = db.createPartitionedTable(t1, `stock_ma, `data_dt) 93 | stock_ma.append!(select * from t1) 94 | } 95 | 96 | def update_stock_ma() { 97 | db = database("dfs://stock") 98 | daily_adj = db.loadTable(`daily_adj) 99 | stock_ma = db.loadTable(`stock_ma) 100 | start_date = (temporalAdd(select max(data_dt) from stock_ma, -60, 'SSE'))['max_data_dt'][0] 101 | write_start_date = (select max(data_dt) from stock_ma)['max_data_dt'][0] 102 | end_date = (select max(data_dt) from daily_adj)['max_data_dt'][0] 103 | 104 | t1 = select data_dt, ts_code, chg, close, high, low, open, amt/100000 as amt, mavg(close, 3) as ma3, mavg(close, 5) as ma5, 105 | mavg(close, 10) as ma10, mavg(close, 20) as ma20, mavg(close, 60) as ma60 106 | from daily_adj where data_dt between start_date:end_date context by ts_code 107 | stock_ma.append!(select * from t1 where data_dt > write_start_date) 108 | 109 | // select * from stock_ma where data_dt > write_start_date limit 10 110 | } 111 | 112 | // 在计算 stock_ma的时候, 如果是追加, 不能只追加当天, 必须把限制日期, 放松到60天以前,不然是没数据的. 113 | // 计算 stock_ind 的时候, 日期要放到 20天前, 然后只取当天的追加进去. 114 | // market_index 115 | 116 | 117 | def init_stock_ind(start_date, end_date) { 118 | tab_name = "stock_ind" 119 | db = database("dfs://stock") 120 | pt = db.loadTable(`stock_ma) 121 | t1 = select data_dt, ts_code, amt, chg, 122 | (close-ma20)/ma20*sqrt(amt+5) as dl, 123 | 100*close/move(close, 5) as zf5, 124 | 100*move(close,5)/move(close, 10) as zf5_10, 125 | abs(ma5-ma20)/ma20 + abs(ma5-ma10)/ma10 as bl, 126 | abs(ma5-ma10)/ma10 as bl_mid 127 | from pt where data_dt between start_date:end_date 128 | context by ts_code 129 | 130 | update t1 set bl_amt = bl * sqrt(amt) from t1 131 | update t1 set ql = mmax(chg, 7) * abs(1\bl_amt) from t1 132 | 133 | update t1 set rn_dl_desc = rank(dl, ascending=false) from t1 context by data_dt 134 | update t1 set rn_dl_asc = rank(dl, ascending=true) from t1 context by data_dt 135 | update t1 set rn_zf5_desc = rank(zf5, ascending=false) from t1 context by data_dt 136 | update t1 set rn_zf5_asc = rank(zf5, ascending=true) from t1 context by data_dt 137 | update t1 set rn_bl_amt_asc = rank(bl_amt, ascending=true) from t1 context by data_dt 138 | update t1 set rn_bl_mid_asc = rank(bl_mid, ascending=true) from t1 context by data_dt 139 | update t1 set rn_ql_desc = rank(ql, ascending=false) from t1 context by data_dt 140 | write_table(tab_name, t1) 141 | } 142 | 143 | def update_stock_ind() { 144 | tab_name = "stock_ind" 145 | db = database("dfs://stock") 146 | stock_ma = db.loadTable(`stock_ma) 147 | stock_ind = db.loadTable(`stock_ind) 148 | start_date = (temporalAdd(select max(data_dt) from stock_ind, -20, 'SSE'))['max_data_dt'][0] 149 | write_start_date = (select max(data_dt) from stock_ind)['max_data_dt'][0] 150 | end_date = (select max(data_dt) from stock_ma)['max_data_dt'][0] 151 | t1 = select data_dt, ts_code, amt, chg, 152 | (close-ma20)/ma20*sqrt(amt+5) as dl, 153 | 100*close/move(close, 5) as zf5, 154 | 100*move(close,5)/move(close, 10) as zf5_10, 155 | abs(ma5-ma20)/ma20 + abs(ma5-ma10)/ma10 as bl, 156 | abs(ma5-ma10)/ma10 as bl_mid 157 | from stock_ma where data_dt between start_date:end_date 158 | context by ts_code 159 | 160 | update t1 set bl_amt = bl * sqrt(amt) from t1 161 | update t1 set ql = mmax(chg, 7) * abs(1\bl_amt) from t1 162 | 163 | update t1 set rn_dl_desc = rank(dl, ascending=false) from t1 context by data_dt 164 | update t1 set rn_dl_asc = rank(dl, ascending=true) from t1 context by data_dt 165 | update t1 set rn_zf5_desc = rank(zf5, ascending=false) from t1 context by data_dt 166 | update t1 set rn_zf5_asc = rank(zf5, ascending=true) from t1 context by data_dt 167 | update t1 set rn_bl_amt_asc = rank(bl_amt, ascending=true) from t1 context by data_dt 168 | update t1 set rn_bl_mid_asc = rank(bl_mid, ascending=true) from t1 context by data_dt 169 | update t1 set rn_ql_desc = rank(ql, ascending=false) from t1 context by data_dt 170 | 171 | stock_ind.append!(select * from t1 where data_dt > write_start_date) 172 | 173 | } 174 | 175 | def init_concept_ind(start_date, end_date) { 176 | tab_name = "concept_ind" 177 | db = database("dfs://stock") 178 | stock_ind = db.loadTable(`stock_ind) 179 | concept_cons = db.loadTable("concept_cons") // TODO 180 | 181 | // start_date = 2022.01.01 182 | // end_date = 2022.12.31 183 | 184 | t1 = select a.data_dt, b.概念名称 as concept, 185 | sqrt(count(*)) as cnt_sqrt, 186 | avg(signum(dl)*pow(dl ,2)) as dl, 187 | avg(signum(chg)*pow(chg,2)*(amt+5)) as chg, 188 | avg(signum(ql)*pow(ql,2)) as ql 189 | from concept_cons b 190 | left join (select * from stock_ind where data_dt between start_date:end_date) a 191 | on b.ts_code = a.ts_code 192 | group by b.概念名称 as concept, a.data_dt 193 | 194 | 195 | update t1 set dl = dl * cnt_sqrt from t1 196 | update t1 set chg = chg * cnt_sqrt from t1 197 | update t1 set ql = ql * cnt_sqrt from t1 198 | 199 | update t1 set rn_dl_desc = rank(dl, ascending=false) from t1 context by data_dt 200 | update t1 set rn_dl_asc = rank(dl, ascending=true) from t1 context by data_dt 201 | update t1 set rn_chg_desc = rank(chg, ascending=false) from t1 context by data_dt 202 | update t1 set rn_chg_asc = rank(chg, ascending=true) from t1 context by data_dt 203 | update t1 set rn_ql_desc = rank(ql, ascending=false) from t1 context by data_dt 204 | update t1 set rn_ql_asc = rank(ql, ascending=true) from t1 context by data_dt 205 | write_table(tab_name, t1) 206 | 207 | } 208 | 209 | 210 | 211 | 212 | def update_concept_ind() { 213 | tab_name = "concept_ind" 214 | db = database("dfs://stock") 215 | stock_ind = db.loadTable(`stock_ind) 216 | concept_cons = db.loadTable("concept_cons") // TODO 217 | concept_ind = db.loadTable(`concept_ind) 218 | start_date = (temporalAdd(select max(data_dt) from concept_ind, 1, 'SSE'))['max_data_dt'][0] 219 | write_start_date = (select max(data_dt) from concept_ind)['max_data_dt'][0] 220 | end_date = (select max(data_dt) from stock_ind)['max_data_dt'][0] 221 | 222 | t1 = select a.data_dt, b.概念名称 as concept, 223 | sqrt(count(*)) as cnt_sqrt, 224 | avg(signum(dl)*pow(dl ,2)) as dl, 225 | avg(signum(chg)*pow(chg,2)*(amt+5)) as chg, 226 | avg(signum(ql)*pow(ql,2)) as ql 227 | from concept_cons b 228 | left join (select * from stock_ind where data_dt between start_date:end_date) a 229 | on b.ts_code = a.ts_code 230 | group by b.概念名称 as concept, a.data_dt 231 | 232 | update t1 set dl = dl * cnt_sqrt from t1 233 | update t1 set chg = chg * cnt_sqrt from t1 234 | update t1 set ql = ql * cnt_sqrt from t1 235 | 236 | update t1 set rn_dl_desc = rank(dl, ascending=false) from t1 context by data_dt 237 | update t1 set rn_dl_asc = rank(dl, ascending=true) from t1 context by data_dt 238 | update t1 set rn_chg_desc = rank(chg, ascending=false) from t1 context by data_dt 239 | update t1 set rn_chg_asc = rank(chg, ascending=true) from t1 context by data_dt 240 | update t1 set rn_ql_desc = rank(ql, ascending=false) from t1 context by data_dt 241 | update t1 set rn_ql_asc = rank(ql, ascending=true) from t1 context by data_dt 242 | 243 | concept_ind.append!(select * from t1 where data_dt > write_start_date) 244 | } 245 | 246 | -------------------------------------------------------------------------------- /app.py: -------------------------------------------------------------------------------- 1 | # Kline Mirror V4.0 2 | # 第一版, 实现基本的可视化功能, 但过于冗杂, 且没有各类指标. 3 | # 第二版, 引入了dolphindb, 做到了真正的时间序列分析,并优化了选股操作. 4 | # 第三版, 用数学定义了大涨大跌. 热门题材. 5 | # 第四版, 以概念题材为核心观察对象, 数据库完全切换到dolphindb 6 | 7 | import streamlit as st 8 | import pandas as pd 9 | import sqlite3 10 | import datetime 11 | import numpy as np 12 | 13 | from streamlit_option_menu import option_menu 14 | 15 | st.set_page_config(layout="wide", page_title="Kline Mirror V4.0", page_icon=":shark:") 16 | 17 | # from utils import * 18 | from plot import * 19 | from update import * 20 | 21 | 22 | @st.cache_resource 23 | def get_conn(): 24 | db = dolphindb.session() 25 | db.connect("localhost", 8848, "admin", "123456") 26 | return db 27 | 28 | conn = get_conn() 29 | 30 | # 默认的配置 ,来自 session_state表, field in ('tag', 'concept', 'date', 'ts_code') 31 | def init_default_config(): 32 | script = """ 33 | if (not existsTable("dfs://stock", `session_state)) { 34 | schema = table(1:0, `field`val`date, [STRING, STRING, DATE]) 35 | db = database("dfs://stock") 36 | db.createTable(schema, `session_state) 37 | session_state = loadTable("dfs://stock", "session_state") 38 | data = table("tag" as field, "中期大涨" as val, 9999.12.31 as date) 39 | session_state.append!(data) 40 | data = table("concept" as field, "整车" as val, 9999.12.31 as date) 41 | session_state.append!(data) 42 | data = table("date" as field, "2022.04.26" as val, 2022.04.26 as date) 43 | session_state.append!(data) 44 | } 45 | """ 46 | conn.run("session_state = loadTable('dfs://stock', 'session_state')") 47 | tag = conn.run("select val from session_state where field = 'tag'").val[0] 48 | concept = conn.run("select val from session_state where field = 'concept'").val[0] 49 | date = conn.run("select val from session_state where field = 'date'").val[0] 50 | st.session_state["tag"] = tag 51 | st.session_state["concept"] = concept 52 | st.session_state["date"] = date 53 | 54 | if "prefs" not in st.session_state: 55 | st.session_state["prefs"] = "高位" 56 | 57 | st.session_state['ts_code'] = None 58 | 59 | 60 | 61 | # 时间控制, 时间的加减. 62 | 63 | def date_控制(): 64 | """最终还是把日期, 以字符串的形式表达出来了, 不然太难管控, 唯一的优点是,我不需要自己维护股票日历了""" 65 | cur_date = st.session_state["date"] 66 | date = st.text_input("输入日期","") 67 | next_date = st.button("--->") 68 | prev_date = st.button("<---") 69 | 70 | if next_date: 71 | new_date = conn.run(f"temporalAdd({cur_date}, 1, 'SSE')").astype("str").replace("-",".") 72 | st.session_state["date"] = new_date 73 | elif prev_date: 74 | new_date = conn.run(f"temporalAdd({cur_date}, -1, 'SSE')").astype("str").replace("-",".") 75 | st.session_state["date"] = new_date 76 | 77 | cur_date = st.session_state["date"] 78 | date2 = st.date_input("挑选日期", datetime.datetime.strptime(cur_date, "%Y.%m.%d")).strftime("%Y.%m.%d") 79 | if not (next_date or prev_date): 80 | if date != "": 81 | new_date = date 82 | elif date == "" : 83 | new_date = date2 84 | 85 | st.session_state["date"] = new_date 86 | conn.run(f"update session_state set val={new_date} where field='date'") 87 | st.write(new_date) 88 | 89 | if not conn.run(f"transFreq({new_date}, 'SSE') == {new_date}"): 90 | # 最近的一天交易日不是当天 91 | st.warning(f"{new_date}非交易日, 请重新选择") 92 | 93 | 94 | if not "date" in st.session_state: 95 | init_default_config() 96 | 97 | 98 | # date_控制() 99 | 100 | # 图表的展示, 参考app.py中的def 大盘情况_menu(date): 101 | def 大盘情况_menu(date): 102 | 103 | st.write(f"当前日期: {date}") 104 | if date: 105 | st.write("大盘指数") 106 | df = read_kline_df(date, "", type="综合指数") 107 | plot_kline_fig(df, date, height=500) 108 | 109 | col1, col2 = st.columns([1, 1]) 110 | with col1: 111 | st.write("中期高度最高的个股") 112 | conn.run("stock_ind = loadTable('dfs://stock', 'stock_ind'); basic = loadTable('dfs://stock', 'basic')") 113 | 114 | df = conn.run(f"""select b.name, round(a.dl,2) as dl from stock_ind a 115 | left join basic b on a.ts_code = b.ts_code 116 | where a.data_dt={date} and a.rn_dl_desc < 5 and not isNull(a.rn_dl_desc) order by a.dl desc""") 117 | st.dataframe(df) 118 | with col2: 119 | st.write("当日涨幅最大的概念") 120 | conn.run("concept_ind = loadTable('dfs://stock', 'concept_ind')") 121 | df = conn.run(f"""select concept, round(chg,2) as chg from concept_ind where data_dt={date} and rn_chg_desc < 5 order by chg desc""") 122 | st.dataframe(df) 123 | 124 | 125 | st.write("中期大涨的概念") 126 | df, y_label = read_concept_df(date, "中期大涨的概念") 127 | plot_concept_fig(df, y_label, date, height=500) 128 | 129 | st.write("中期大跌的概念") 130 | df, y_label = read_concept_df(date, "中期大跌的概念") 131 | plot_concept_fig(df, y_label, date, height=500) 132 | 133 | with st.expander("当天概念情况", expanded=False): 134 | st.write("当天大涨的概念") 135 | df, y_label = read_concept_df(date, "当天大涨的概念") 136 | plot_concept_fig(df, y_label, date, height=500) 137 | 138 | st.write("当天大跌的概念") 139 | df, y_label = read_concept_df(date, "当天大跌的概念") 140 | plot_concept_fig(df, y_label, date, height=500) 141 | 142 | 143 | 144 | 145 | @st.cache_data 146 | def get_sql(data_dt, tag): 147 | 148 | conn.run("stock_ind = loadTable('dfs://stock', 'stock_ind'); daily_adj = loadTable('dfs://stock', 'daily_adj');") 149 | conn.run("concept_cons = loadTable('dfs://stock', 'concept_cons')") 150 | conn.run("basic = loadTable('dfs://stock', 'basic')") 151 | conn.run(f"t_5 = temporalAdd({data_dt}, 5, 'SSE') // 交易日偏移") 152 | if tag.startswith("概念:"): 153 | concept = st.session_state["concept"] 154 | sql = f"""select b.ts_code as ts_code from concept_cons a 155 | left join basic b on a.代码 = b.symbol 156 | where a.概念名称 like '%{concept}%'""" 157 | return sql 158 | else: 159 | tag_sql_config = { 160 | "最近会涨":f"select ts_code from stock_ind where data_dt=t_5 and rn_zf5_desc < 200", 161 | "已经大涨": f"select ts_code from stock_ind where data_dt={data_dt} and rn_dl_desc <200", 162 | "最近会跌":f"select ts_code from stock_ind where data_dt=t_5 and rn_zf5_asc < 200", 163 | "已经大跌": f"select ts_code from stock_ind where data_dt={data_dt} and rn_dl_asc < 200", 164 | "当日大涨":f"select ts_code from daily_adj where data_dt={data_dt} and chg >=9.5", 165 | "当日大跌":f"select ts_code from daily_adj where data_dt={data_dt} and chg <= -6", 166 | "低位大涨": f"select ts_code from stock_ind where data_dt={data_dt} and zf5_10 < 115 and zf5>120 ", 167 | "均线重叠":f"select ts_code from stock_ind where data_dt={data_dt} and rn_bl_amt_asc < 100", 168 | "有潜力": f"select ts_code from stock_ind where data_dt={data_dt} and rn_ql_desc < 1000", 169 | } 170 | assert tag in tag_sql_config.keys() 171 | return tag_sql_config[tag] 172 | 173 | 174 | @st.cache_data 175 | def merge_set(tags , data_dt, prefs): 176 | """每个条件形成一个SQL, 然后intersect形成交集""" 177 | conn.run("stock_ind = loadTable('dfs://stock', 'stock_ind'); basic = loadTable('dfs://stock', 'basic');") 178 | # conn.upload({'tags':tags}) 179 | # sql_list = [] 180 | conn.run("final = set((select ts_code from basic where 1=0)['ts_code'])") 181 | for tag in tags: 182 | tag_sql = get_sql(data_dt, tag) 183 | conn.run(f"""tmp = set(({tag_sql})['ts_code']);""") 184 | conn.run(f"""if (size(final) == 0) final = tmp else final = final & tmp;""") 185 | 186 | # # sql_list.append(tag_sql) 187 | # conn.run(f"final = {' & '.join(['tag_'+str(i) for i in range(len(tags))])}") 188 | # if len(tags) == 0: 189 | # return conn.run("final = select null as name, null as ts_code") 190 | # elif len(sql_list) == 1: 191 | # final_sql = sql_list[0] 192 | # else: 193 | # final_sql = " INTERSECT ".join(sql_list) 194 | 195 | # if st.session_state["prefs"] == "低位": # 偏好低位 196 | # order_by_col = "a.bl_amt asc" 197 | # elif st.session_state["prefs"] == "中位": # 偏好中位, 也即, 5日线靠近10日线 198 | # order_by_col = "a.bl_mid asc" 199 | # else: # 默认, 偏好高位 200 | # order_by_col = "a.dl desc" 201 | if conn.run("size(final)>0"): 202 | stock_list_df = conn.run(f"""select b.name, b.ts_code from stock_ind a 203 | left join basic b on a.ts_code = b.ts_code 204 | where ts_code in final and a.data_dt={data_dt} order by a.dl desc;""") 205 | else: 206 | stock_list_df = None 207 | 208 | st.write(f"{len(stock_list_df)}个股票") 209 | return stock_list_df 210 | 211 | 212 | #个股情况先靠边, 先做sidebar ,把各种控制标签,做起来. 213 | def 个股情况_menu(date): 214 | 215 | st.write(f"当前日期: {date}") 216 | # if "ts" 217 | # ts_code = st.session_state["ts_code"] 218 | 219 | left_col, right_col = st.columns([ 2, 5]) 220 | with left_col: 221 | if date: 222 | tags = st.session_state["tags"] 223 | stock_list_df = merge_set(tags, date, st.session_state["prefs"]) 224 | if stock_list_df is not None: 225 | plot_grid_fig(stock_list_df, type="股票") 226 | else: 227 | st.session_state['ts_code'] = None 228 | 229 | name = st.session_state["name"] 230 | if name != "" : 231 | conn.run("basic = loadTable('dfs://stock', 'basic')") 232 | ts_code = conn.run(f"select ts_code from basic where name like '%{name}%'").values[0][0] 233 | st.session_state['ts_code'] = ts_code 234 | 235 | with right_col: 236 | date = st.session_state['date'] 237 | ts_code = st.session_state['ts_code'] 238 | if date is not None and ts_code is not None: 239 | conn.run("concept_cons = loadTable('dfs://stock', 'concept_cons');bssic = loadTable('dfs://stock', 'basic') ") 240 | sql = f"select 概念名称 as concept from concept_cons where 代码='{ts_code[:6]}'" 241 | concepts = "||".join(list(conn.run(sql)['concept'])) 242 | st.write(f"概念: {concepts}") 243 | st.write("----------") 244 | stock_name = conn.run(f"select name from basic where ts_code='{ts_code}'").values[0][0] 245 | st.write(stock_name) 246 | df = read_kline_df(date, ts_code, type='股票') 247 | plot_kline_fig(df, date) 248 | 249 | 250 | def sidebar(): 251 | date_控制() 252 | 253 | tag_list = [ 254 | "最近会涨", 255 | "已经大涨", 256 | "最近会跌", 257 | "已经大跌", 258 | "当日大涨", 259 | "当日大跌", 260 | "低位大涨", 261 | "均线重叠", 262 | "有潜力" 263 | ] 264 | name = st.text_input("选股:名称", "") 265 | st.session_state['name'] = name 266 | concept = st.text_input("选股:概念", "") 267 | st.session_state["concept"] = concept 268 | if concept != "" and concept != st.session_state['concept']: 269 | conn.run(f"update session_state set val='{concept}' where filed='concept'") 270 | 271 | st.write("选股:标签") 272 | checks = [st.checkbox(tag, value=True if tag==st.session_state['tag'] else False) for tag in tag_list] 273 | tags = [] 274 | concept = st.session_state["concept"] 275 | if concept != "": 276 | tags.append(f"概念:{concept}") 277 | for check, tag in zip(checks, tag_list): 278 | if check: 279 | tags.append(tag) 280 | st.session_state["tags"] = tags 281 | 282 | st.write("选股:偏好") 283 | st.session_state["prefs"] = st.select_slider("偏好", options=["低位", "中位", "高位"], value="高位", key="偏好") 284 | 285 | 286 | if len(tags) == 1 and tags[0] != st.session_state['tag']: 287 | conn.run(f"update session_state set val='{tags[0]}' where field='tag'") 288 | 289 | if st.button("更新数据库", key="更新数据库"): 290 | update_all() 291 | 292 | def 题材列表_menu_df(date, selected_menu): 293 | conn.run("concept_ind = loadTable('dfs://stock', 'concept_ind')") 294 | if selected_menu == "中期大涨": 295 | # 中期大涨的概念 296 | st.write("中期大涨的概念") 297 | df = conn.run(f"select concept, round(dl,2) as dl from concept_ind where data_dt={date} and rn_dl_desc < 20 order by dl desc") 298 | 299 | elif selected_menu == "中期大跌": 300 | # 中期大跌的概念 301 | st.write("中期大跌的概念") 302 | df = conn.run(f"select concept, round(dl,2) as dl from concept_ind where data_dt={date} and rn_dl_asc < 20 order by dl asc") 303 | 304 | elif selected_menu == "当日大涨": 305 | # 当天大涨的概念 306 | st.write("当天大涨的概念") 307 | df = conn.run(f"select concept, round(chg,2) as chg from concept_ind where data_dt={date} and rn_chg_desc < 20 order by chg desc") 308 | 309 | elif selected_menu == "当日大跌": 310 | # 当天大跌的概念 311 | st.write("当天大跌的概念") 312 | df = conn.run(f"select concept, round(chg,2) as chg from concept_ind where data_dt={date} and rn_chg_asc < 20 order by chg asc") 313 | 314 | return df 315 | 316 | 317 | def 题材列表_menu(date): 318 | st.write(f"当前日期: {date}") 319 | 320 | left_col, right_col = st.columns([ 2, 5]) 321 | with left_col: 322 | selected_menu = option_menu(None, ["中期大涨", "中期大跌", "当日大涨", "当日大跌"], 323 | icons=['list-task', "list-task", "list-task", "list-task"], 324 | menu_icon="cast", default_index=0, orientation="horizontal") 325 | 326 | df = 题材列表_menu_df(date, selected_menu) 327 | if df is not None: 328 | plot_grid_fig(df, type="题材", height=400) 329 | 330 | with right_col: 331 | concept = st.session_state["concept"] 332 | if concept is not None and concept != "": 333 | df, y_label = read_concept_df(date, type="概念K线", concept=concept) 334 | plot_concept_fig(df, y_label, date) 335 | 336 | 337 | with st.sidebar: 338 | sidebar() 339 | 340 | 341 | def main(): 342 | selected_menu = option_menu(None, ["大盘情况", "个股情况", "题材列表"], 343 | icons=['house', "list-task", "list-task"], 344 | menu_icon="cast", default_index=0, orientation="horizontal") 345 | date = st.session_state["date"] 346 | if selected_menu == "大盘情况": 347 | 大盘情况_menu(date) 348 | elif selected_menu == "个股情况": 349 | 个股情况_menu(date) 350 | elif selected_menu == "题材列表": 351 | 题材列表_menu(date) 352 | 353 | main() -------------------------------------------------------------------------------- /mirror_v4.dos: -------------------------------------------------------------------------------- 1 | 2 | 3 | def write_table(tab_name, t) { 4 | db = database("dfs://stock") 5 | if (existsTable("dfs://stock", tab_name)) db.dropTable(tab_name) 6 | pt = db.createPartitionedTable(t, tab_name, `data_dt) 7 | pt.append!(select * from t) 8 | } 9 | 10 | def init_market_index(start_date, end_date) { 11 | db = database("dfs://stock") 12 | pt = db.loadTable(`daily_adj) 13 | t2 = select data_dt, ts_code, open\close as open_pct, 14 | high\close as high_pct, low\close as low_pct, ratios(close) as close_pct, 15 | amt\100000 as amt 16 | from pt where data_dt between start_date:end_date context by ts_code 17 | 18 | t3 = select data_dt, avg(open_pct) as open_pct, avg(high_pct) as high_pct, 19 | avg(low_pct) as low_pct, avg(close_pct) as close_pct, sum(amt) as amt from t2 group by data_dt 20 | 21 | t4 = select data_dt, cumprod(close_pct) as cum_close, amt , 100*(close_pct-1) as chg , 22 | open_pct, high_pct, low_pct, close_pct from t3 23 | update t4 set 24 | cum_open = cum_close*open_pct 25 | from t4 26 | update t4 set cum_high = max(cum_close*(1+ (high_pct-1)\3), max(cum_open, cum_close)), 27 | cum_low = min(cum_close*(1+ (low_pct-1)\3), min(cum_close, cum_open)) 28 | from t4 29 | t5 = select data_dt, cum_open,cum_low,cum_high,cum_close, amt, chg from t4 30 | 31 | tab_name = `market_index 32 | if (existsTable("dfs://stock", tab_name)) db.dropTable(tab_name) 33 | pt2 = db.createPartitionedTable(t5, tab_name, `data_dt) 34 | pt2.append!(select * from t5) 35 | } 36 | // market_index 在新增的时候,需要做什么呢?? 37 | 38 | def update_market_index() { 39 | db = database("dfs://stock") 40 | daily_adj = db.loadTable(`daily_adj) 41 | market_index = db.loadTable(`market_index) 42 | pre_date = select max(data_dt) from market_index 43 | start_date = temporalAdd(select max(data_dt) from market_index, 1, 'SSE') 44 | write_start_date = select max(data_dt) from market_index 45 | end_date = select max(data_dt) from daily_adj 46 | 47 | t2 = select data_dt, ts_code, open\close as open_pct, 48 | high\close as high_pct, low\close as low_pct, ratios(close) as close_pct, 49 | amt\100000 as amt 50 | from daily_adj where data_dt between start_date:end_date context by ts_code 51 | 52 | t3 = select data_dt, avg(open_pct) as open_pct, avg(high_pct) as high_pct, 53 | avg(low_pct) as low_pct, avg(close_pct) as close_pct, sum(amt) as amt from t2 group by data_dt 54 | 55 | t4 = select data_dt, cumprod(close_pct) as cum_close, amt , 100*(close_pct-1) as chg , 56 | open_pct, high_pct, low_pct, close_pct from t3 57 | update t4 set 58 | cum_open = cum_close*open_pct 59 | from t4 60 | update t4 set cum_high = max(cum_close*(1+ (high_pct-1)\3), max(cum_open, cum_close)), 61 | cum_low = min(cum_close*(1+ (low_pct-1)\3), min(cum_close, cum_open)) 62 | from t4 63 | 64 | prev_cum_open = select cum_open from market_index where data_dt = pre_date 65 | prev_cum_close = select cum_close from market_index where data_dt = pre_date 66 | prev_cum_high = select cum_high from market_index where data_dt = pre_date 67 | prev_cum_low = select cum_low from market_index where data_dt = pre_date 68 | t5 = select data_dt, cum_open*prev_cum_open as cum_open, 69 | cum_low*prev_cum_low as cum_low, 70 | cum_high*prev_cum_high as cum_high, 71 | cum_close*prev_cum_close as cum_close, 72 | amt, chg from t4 73 | 74 | market_index.append!(select * from t5 where data_dt > write_start_date) 75 | } 76 | 77 | // 如果要更新,就要 计算当天的昨天的差距, 然后, 把整个流程重走一遍. 78 | 79 | 80 | def init_stock_ma(start_date, end_date) { 81 | db = database("dfs://stock") 82 | daily_adj = db.loadTable(`daily_adj) 83 | tab_name = `stock_ma 84 | t1 = select data_dt, ts_code, chg, close, high, low, open, amt/100000 as amt, mavg(close, 3) as ma3, mavg(close, 5) as ma5, 85 | mavg(close, 10) as ma10, mavg(close, 20) as ma20, mavg(close, 60) as ma60 86 | from daily_adj where data_dt between start_date:end_date context by ts_code 87 | if (existsTable("dfs://stock", tab_name)) db.dropTable(tab_name) 88 | stock_ma = db.createPartitionedTable(t1, `stock_ma, `data_dt) 89 | stock_ma.append!(select * from t1) 90 | } 91 | 92 | def update_stock_ma() { 93 | db = database("dfs://stock") 94 | daily_adj = db.loadTable(`daily_adj) 95 | stock_ma = db.loadTable(`stock_ma) 96 | start_date = temporalAdd(select max(data_dt) from stock_ma, -60, 'SSE') 97 | write_start_date = select max(data_dt) from stock_ma 98 | end_date = select max(data_dt) from daily_adj 99 | t1 = select data_dt, ts_code, chg, close, high, low, open, amt/100000 as amt, mavg(close, 3) as ma3, mavg(close, 5) as ma5, 100 | mavg(close, 10) as ma10, mavg(close, 20) as ma20, mavg(close, 60) as ma60 101 | from daily_adj where data_dt between start_date:end_date context by ts_code 102 | stock_ma.append!(select * from t1 where data_dt > write_start_date) 103 | } 104 | 105 | // 在计算 stock_ma的时候, 如果是追加, 不能只追加当天, 必须把限制日期, 放松到60天以前,不然是没数据的. 106 | // 计算 stock_ind 的时候, 日期要放到 20天前, 然后只取当天的追加进去. 107 | // market_index 108 | 109 | 110 | def init_stock_ind(start_date, end_date) { 111 | tab_name = "stock_ind" 112 | db = database("dfs://stock") 113 | pt = db.loadTable(`stock_ma) 114 | t1 = select data_dt, ts_code, amt, chg, 115 | (close-ma20)/ma20*sqrt(amt+5) as dl, 116 | 100*close/move(close, 5) as zf5, 117 | 100*move(close,5)/move(close, 10) as zf5_10, 118 | abs(ma5-ma20)/ma20 + abs(ma5-ma10)/ma10 as bl, 119 | abs(ma5-ma10)/ma10 as bl_mid 120 | from pt where data_dt between start_date:end_date 121 | context by ts_code 122 | 123 | update t1 set bl_amt = bl * sqrt(amt) from t1 124 | update t1 set ql = mmax(chg, 7) * abs(1\bl_amt) from t1 125 | 126 | update t1 set rn_dl_desc = rank(dl, ascending=false) from t1 context by data_dt 127 | update t1 set rn_dl_asc = rank(dl, ascending=true) from t1 context by data_dt 128 | update t1 set rn_zf5_desc = rank(zf5, ascending=false) from t1 context by data_dt 129 | update t1 set rn_zf5_asc = rank(zf5, ascending=true) from t1 context by data_dt 130 | update t1 set rn_bl_amt_asc = rank(bl_amt, ascending=true) from t1 context by data_dt 131 | update t1 set rn_bl_mid_asc = rank(bl_mid, ascending=true) from t1 context by data_dt 132 | update t1 set rn_ql_desc = rank(ql, ascending=false) from t1 context by data_dt 133 | write_table(tab_name, t1) 134 | } 135 | 136 | def update_stock_ind() { 137 | tab_name = "stock_ind" 138 | db = database("dfs://stock") 139 | stock_ma = db.loadTable(`stock_ma) 140 | stock_ind = db.loadTable(`stock_ind) 141 | start_date = temporalAdd(select max(data_dt) from stock_ind, -20, 'SSE') 142 | write_start_date = select max(data_dt) from stock_ind 143 | end_date = select max(data_dt) from stock_ma 144 | t1 = select data_dt, ts_code, amt, chg, 145 | (close-ma20)/ma20*sqrt(amt+5) as dl, 146 | 100*close/move(close, 5) as zf5, 147 | 100*move(close,5)/move(close, 10) as zf5_10, 148 | abs(ma5-ma20)/ma20 + abs(ma5-ma10)/ma10 as bl, 149 | abs(ma5-ma10)/ma10 as bl_mid 150 | from stock_ma where data_dt between start_date:end_date 151 | context by ts_code 152 | 153 | update t1 set bl_amt = bl * sqrt(amt) from t1 154 | update t1 set ql = mmax(chg, 7) * abs(1\bl_amt) from t1 155 | 156 | update t1 set rn_dl_desc = rank(dl, ascending=false) from t1 context by data_dt 157 | update t1 set rn_dl_asc = rank(dl, ascending=true) from t1 context by data_dt 158 | update t1 set rn_zf5_desc = rank(zf5, ascending=false) from t1 context by data_dt 159 | update t1 set rn_zf5_asc = rank(zf5, ascending=true) from t1 context by data_dt 160 | update t1 set rn_bl_amt_asc = rank(bl_amt, ascending=true) from t1 context by data_dt 161 | update t1 set rn_bl_mid_asc = rank(bl_mid, ascending=true) from t1 context by data_dt 162 | update t1 set rn_ql_desc = rank(ql, ascending=false) from t1 context by data_dt 163 | 164 | stock_ma.append!(select * from t1 where data_dt > write_start_date) 165 | } 166 | 167 | def init_concept_ind(start_date, end_date) { 168 | tab_name = "concept_ind" 169 | db = database("dfs://stock") 170 | stock_ind = db.loadTable(`stock_ind) 171 | concept_cons = db.loadTable("concept_cons") // TODO 172 | t1 = select a.data_dt, b.concept, 173 | sqrt(count(*)) as cnt_sqrt, 174 | avg(signum(dl)*pow(dl ,2)) as dl, 175 | avg(signum(chg)*pow(chg,2)*(amt+5)) as chg, 176 | avg(signum(ql)*pow(ql,2)) as ql 177 | from stock_ind a 178 | left join concept_cons b on a.ts_code = b.ts_code 179 | where a.data_dt between start_date:end_date 180 | group by a.data_dt, b.concept 181 | 182 | update t1 set dl = dl * cnt_sqrt from t1 183 | update t1 set chg = chg * cnt_sqrt from t1 184 | update t1 set ql = ql * cnt_sqrt from t1 185 | 186 | update t1 set rn_dl_desc = rank(dl, ascending=false) from t1 context by data_dt 187 | update t1 set rn_dl_asc = rank(dl, ascending=true) from t1 context by data_dt 188 | update t1 set rn_chg_desc = rank(chg, ascending=false) from t1 context by data_dt 189 | update t1 set rn_chg_asc = rank(chg, ascending=true) from t1 context by data_dt 190 | update t1 set rn_ql_desc = rank(ql, ascending=false) from t1 context by data_dt 191 | update t1 set rn_ql_asc = rank(ql, ascending=true) from t1 context by data_dt 192 | write_table(tab_name, t1) 193 | } 194 | 195 | def update_concept_ind() { 196 | tab_name = "concept_ind" 197 | db = database("dfs://stock") 198 | stock_ind = db.loadTable(`stock_ind) 199 | concept_cons = db.loadTable("concept_cons") // TODO 200 | concept_ind = db.loadTable(`concept_ind) 201 | start_date = temporalAdd(select max(data_dt) from concept_ind, 1, 'SSE') 202 | write_start_date = select max(data_dt) from concept_ind 203 | end_date = select max(data_dt) from stock_ind 204 | 205 | t1 = select a.data_dt, b.concept, 206 | sqrt(count(*)) as cnt_sqrt, 207 | avg(signum(dl)*pow(dl ,2)) as dl, 208 | avg(signum(chg)*pow(chg,2)*(amt+5)) as chg, 209 | avg(signum(ql)*pow(ql,2)) as ql 210 | from stock_ind a 211 | left join concept_cons b on a.ts_code = b.ts_code 212 | where a.data_dt between start_date:end_date 213 | group by a.data_dt, b.concept 214 | 215 | update t1 set dl = dl * cnt_sqrt from t1 216 | update t1 set chg = chg * cnt_sqrt from t1 217 | update t1 set ql = ql * cnt_sqrt from t1 218 | 219 | update t1 set rn_dl_desc = rank(dl, ascending=false) from t1 context by data_dt 220 | update t1 set rn_dl_asc = rank(dl, ascending=true) from t1 context by data_dt 221 | update t1 set rn_chg_desc = rank(chg, ascending=false) from t1 context by data_dt 222 | update t1 set rn_chg_asc = rank(chg, ascending=true) from t1 context by data_dt 223 | update t1 set rn_ql_desc = rank(ql, ascending=false) from t1 context by data_dt 224 | update t1 set rn_ql_asc = rank(ql, ascending=true) from t1 context by data_dt 225 | 226 | concept_ind.append!(select * from t1 where data_dt > write_start_date) 227 | } 228 | 229 | 230 | 231 | // init_market_index() 232 | // init_stock_ma() 233 | // init_stock_ind() 234 | // init_concept_ind() 235 | 236 | 237 | def init_db() { 238 | if existsDatabase("dfs://stock"){ 239 | dropDatabase("dfs://stock") 240 | } 241 | db= database("dfs://stock", RANGE, 2019.01.01 2020.01.01 2021.01.01 2024.01.02) 242 | } 243 | 244 | def load_daily_adj() { 245 | db = database("dfs://stock") 246 | t = ploadText("/data/ddb/server/data/daily_adj.csv") 247 | tab_name = `daily_adj 248 | if (existsTable("dfs://stock", tab_name)) db.dropTable(tab_name) 249 | pt=db.createPartitionedTable(t,tab_name, `data_dt) 250 | pt.tableInsert(select * from t) 251 | } 252 | def load_cal2() { 253 | db = database("dfs://stock") 254 | t = ploadText("/data/ddb/server/data/cal2.csv") 255 | pt = db.createPartitionedTable(t,`cal2, `t0) 256 | pt.tableInsert(select * from t) 257 | } 258 | def load_concept_cons_xgb() { 259 | 260 | db = database("dfs://stock") 261 | t = ploadText("/data/ddb/server/data/concept_cons_xgb.csv") 262 | t2 = ploadText("/data/ddb/server/data/basic.csv") 263 | // 把symbol转成ts_code. 264 | tab_name = `concept_cons_xgb 265 | t3 = select t.desc, ts_code, concept from t 266 | left join t2 on t.symbol = t2.symbol 267 | 268 | if (existsTable("dfs://stock", tab_name)) db.dropTable(tab_name) 269 | pt=db.createTable(t3, tab_name) 270 | pt.tableInsert(select * from t3) 271 | } 272 | def load_daily() { 273 | db = database("dfs://stock") 274 | t = ploadText("/data/ddb/server/data/daily.csv") 275 | // 把symbol转成ts_code. 276 | tab_name = `daily 277 | 278 | if (existsTable("dfs://stock", tab_name)) db.dropTable(tab_name) 279 | pt=db.createTable(t, tab_name) 280 | pt.tableInsert(select * from t) 281 | } 282 | 283 | def load_adj_factor() { 284 | tab_name = 'adj_factor' 285 | db = database("dfs://stock") 286 | t = ploadText("/data/ddb/server/data/adj_factor.csv") 287 | 288 | if (existsTable("dfs://stock", tab_name)) db.dropTable(tab_name) 289 | pt=db.createTable(t, tab_name) 290 | pt.tableInsert(select * from t) 291 | } 292 | 293 | 294 | def load_basic() { 295 | db = database("dfs://stock") 296 | t = ploadText("/data/ddb/server/data/basic.csv") 297 | tab_name = `basic 298 | if (existsTable("dfs://stock", tab_name)) db.dropTable(tab_name) 299 | pt=db.createTable(t, tab_name) 300 | pt.tableInsert(select * from t) 301 | } 302 | 303 | def load_data() { 304 | // 会删除之前的所有数据 305 | login(`admin, `123456) 306 | init_db() 307 | load_daily_adj() 308 | load_cal2() 309 | load_concept_cons_xgb() 310 | } 311 | 312 | 313 | def make_daily_adj () { 314 | start_date = 2010.01.01 315 | end_date = 2023.02.27 316 | db = database("dfs://stock") 317 | daily = loadTable('dfs://stock', 'daily') 318 | adj_factor = loadTable('dfs://stock', 'adj_factor') 319 | //select * from daily limit 10 // ts_code, trade_date, open, high, low, close, pre_close, change, pct_chg, vol, amount 320 | 321 | //select * from adj_factor limit 10 // ts_code, trade_date, adj_factor 322 | 323 | //select * from daily_adj limit 10 // data_dt, ts_code, chg, open, high, low, close, amt 324 | 325 | t1 = select distinct a.trade_date as data_dt, a.ts_code, a.pct_chg as chg, 326 | a.open*b.adj_factor as open, 327 | a.high*b.adj_factor as high, 328 | a.low*b.adj_factor as low, 329 | a.close*b.adj_factor as close, 330 | a.amount as amt 331 | from daily a 332 | left join adj_factor b on a.ts_code = b.ts_code and a.trade_date = b.trade_date 333 | where a.trade_date between start_date:end_date 334 | 335 | delete from daily_adj where data_dt between start_date:end_date 336 | daily_adj.append!(select * from t1) 337 | 338 | select distinct data_dt from daily_adj order by data_dt asc 339 | } 340 | 341 | 342 | market_index = loadTable('dfs://stock','market_index'); 343 | stock_ma = loadTable('dfs://stock', 'stock_ma'); 344 | stock_ind = loadTable('dfs://stock', 'stock_ind'); 345 | concept_ind = loadTable('dfs://stock', 'concept_ind'); 346 | daily_adj = loadTable('dfs://stock', 'daily_adj') 347 | daily = loadTable('dfs://stock', 'daily') 348 | adj_factor = loadTable('dfs://stock', 'adj_factor') 349 | 350 | def merge_set(sql_list) { 351 | if (size(sql_list) == 0) return null 352 | final = set([]) 353 | for (sql in sql_list) { 354 | tmp = sql.execute().toDataFrame() 355 | } 356 | } -------------------------------------------------------------------------------- /v4.ipynb: -------------------------------------------------------------------------------- 1 | { 2 | "cells": [ 3 | { 4 | "cell_type": "code", 5 | "execution_count": 9, 6 | "metadata": {}, 7 | "outputs": [], 8 | "source": [ 9 | "import numpy as np\n", 10 | "import pandas as pd\n", 11 | "import akshare as ak\n", 12 | "import dolphindb as ddb\n", 13 | "import tushare as ts\n", 14 | "import time \n", 15 | "import requests\n", 16 | "import datetime\n", 17 | "conn = ddb.session()\n", 18 | "conn.connect(\"localhost\", 8848, \"admin\", \"123456\") # 如果是在 dolphindb-1 上运行, 则需要将 localhost 改为 dolphindb-1\n", 19 | "conn.run(\"\"\"\n", 20 | "db=database(\"dfs://stock\");\n", 21 | "market_index = loadTable('dfs://stock','market_index');\n", 22 | "stock_ma = loadTable('dfs://stock', 'stock_ma');\n", 23 | "stock_ind = loadTable('dfs://stock', 'stock_ind');\n", 24 | "concept_ind = loadTable('dfs://stock', 'concept_ind');\n", 25 | "daily_adj = loadTable('dfs://stock', 'daily_adj')\n", 26 | "\"\"\")\n", 27 | "conn.runFile(\"update_ddb_tables.dos\")" 28 | ] 29 | }, 30 | { 31 | "cell_type": "code", 32 | "execution_count": 5, 33 | "metadata": {}, 34 | "outputs": [], 35 | "source": [ 36 | "token = \"f1ce53736e3b6777425d3df97c05e7460a55534db8ece60114c9e2a3\"\n", 37 | "ts.set_token(token)\n", 38 | "pro = ts.pro_api()" 39 | ] 40 | }, 41 | { 42 | "cell_type": "code", 43 | "execution_count": 6, 44 | "metadata": {}, 45 | "outputs": [ 46 | { 47 | "name": "stdout", 48 | "output_type": "stream", 49 | "text": [ 50 | "更新完成2023.02.28的daily_adj数据\n" 51 | ] 52 | }, 53 | { 54 | "name": "stderr", 55 | "output_type": "stream", 56 | "text": [ 57 | "/var/folders/n1/p3lkv10n0ds8849ms4mn11cw0000gq/T/ipykernel_78003/1098383343.py:15: SettingWithCopyWarning: \n", 58 | "A value is trying to be set on a copy of a slice from a DataFrame\n", 59 | "\n", 60 | "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n", 61 | " df.pct_chg[df.pct_chg > 21] = 21\n", 62 | "/Users/cf/anaconda3/lib/python3.9/site-packages/pandas/core/generic.py:8870: SettingWithCopyWarning: \n", 63 | "A value is trying to be set on a copy of a slice from a DataFrame\n", 64 | "\n", 65 | "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n", 66 | " return self._update_inplace(result)\n", 67 | "/var/folders/n1/p3lkv10n0ds8849ms4mn11cw0000gq/T/ipykernel_78003/1098383343.py:16: SettingWithCopyWarning: \n", 68 | "A value is trying to be set on a copy of a slice from a DataFrame\n", 69 | "\n", 70 | "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n", 71 | " df.pct_chg[df.pct_chg < -21] = -21\n" 72 | ] 73 | } 74 | ], 75 | "source": [ 76 | "def update_daily_adj(date):\n", 77 | " # 先下载 daily 数据\n", 78 | " # 判断是否已经有date的数据\n", 79 | " conn.run(\"daily_adj = loadTable('dfs://stock', `daily_adj)\")\n", 80 | " if conn.run(f\"select 1 from daily_adj where data_dt={date} limit 1\").shape[0] == 0:\n", 81 | " print(f\"已经存在{date}的daily_adj数据\")\n", 82 | " else:\n", 83 | " # 下载数据\n", 84 | " df = pro.daily(start_date=date.replace(\".\",\"\"), end_date=date.replace(\".\",\"\"), adj='hfq')\n", 85 | " if df.shape[0] == 0:\n", 86 | " print(f\"{date}没有数据\")\n", 87 | " return\n", 88 | " df = df.query(\"not ts_code.str.contains('.BJ')\", engine='python')\n", 89 | " # 把chg>20的刷成21, 把chg<-20的刷成-21\n", 90 | " df.pct_chg[df.pct_chg > 21] = 21\n", 91 | " df.pct_chg[df.pct_chg < -21] = -21 \n", 92 | " # to daily_adj\n", 93 | " new_order = ['trade_date', 'ts_code', 'pct_chg', 'open','high', 'low', 'close', 'amount']\n", 94 | " df = df.reindex(columns=new_order)\n", 95 | " df.rename(columns={'trade_date':'data_dt', 'amount':'amt', \"pct_chg\":\"chg\"}, inplace=True)\n", 96 | "\n", 97 | " # 日期从string转成datetime,然后上传.\n", 98 | " df['data_dt'] = pd.to_datetime(df['data_dt'], format='%Y%m%d').values.astype('datetime64[D]')\n", 99 | " df['ts_code'] = df['ts_code'].astype(\"object\")\n", 100 | " conn.upload({'daily_adj_tmp':df})\n", 101 | " conn.run(\"tableInsert(daily_adj,(select * from daily_adj_tmp))\")\n", 102 | " \n", 103 | " print(f\"更新完成{date}的daily_adj数据\")\n", 104 | "\n", 105 | "update_daily_adj(\"2023.02.28\")" 106 | ] 107 | }, 108 | { 109 | "cell_type": "code", 110 | "execution_count": 10, 111 | "metadata": {}, 112 | "outputs": [ 113 | { 114 | "name": "stdout", 115 | "output_type": "stream", 116 | "text": [ 117 | "2023.02.28 --- 2023.06.29\n", 118 | "2023.02.28\n" 119 | ] 120 | }, 121 | { 122 | "name": "stderr", 123 | "output_type": "stream", 124 | "text": [ 125 | "/var/folders/n1/p3lkv10n0ds8849ms4mn11cw0000gq/T/ipykernel_78003/1098383343.py:15: SettingWithCopyWarning: \n", 126 | "A value is trying to be set on a copy of a slice from a DataFrame\n", 127 | "\n", 128 | "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n", 129 | " df.pct_chg[df.pct_chg > 21] = 21\n", 130 | "/Users/cf/anaconda3/lib/python3.9/site-packages/pandas/core/generic.py:8870: SettingWithCopyWarning: \n", 131 | "A value is trying to be set on a copy of a slice from a DataFrame\n", 132 | "\n", 133 | "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n", 134 | " return self._update_inplace(result)\n", 135 | "/var/folders/n1/p3lkv10n0ds8849ms4mn11cw0000gq/T/ipykernel_78003/1098383343.py:16: SettingWithCopyWarning: \n", 136 | "A value is trying to be set on a copy of a slice from a DataFrame\n", 137 | "\n", 138 | "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n", 139 | " df.pct_chg[df.pct_chg < -21] = -21\n" 140 | ] 141 | }, 142 | { 143 | "name": "stdout", 144 | "output_type": "stream", 145 | "text": [ 146 | "更新完成2023.02.28的daily_adj数据\n", 147 | "2023.03.01\n", 148 | "更新完成2023.03.01的daily_adj数据\n", 149 | "2023.03.02\n", 150 | "更新完成2023.03.02的daily_adj数据\n", 151 | "2023.03.03\n", 152 | "出错了, 重试\n", 153 | "更新完成2023.03.03的daily_adj数据\n", 154 | "2023.03.06\n", 155 | "更新完成2023.03.06的daily_adj数据\n", 156 | "2023.03.07\n", 157 | "更新完成2023.03.07的daily_adj数据\n", 158 | "2023.03.08\n", 159 | "更新完成2023.03.08的daily_adj数据\n", 160 | "2023.03.09\n", 161 | "出错了, 重试\n", 162 | "更新完成2023.03.09的daily_adj数据\n", 163 | "2023.03.10\n", 164 | "更新完成2023.03.10的daily_adj数据\n", 165 | "2023.03.13\n", 166 | "更新完成2023.03.13的daily_adj数据\n", 167 | "2023.03.14\n", 168 | "更新完成2023.03.14的daily_adj数据\n", 169 | "2023.03.15\n", 170 | "更新完成2023.03.15的daily_adj数据\n", 171 | "2023.03.16\n", 172 | "更新完成2023.03.16的daily_adj数据\n", 173 | "2023.03.17\n", 174 | "更新完成2023.03.17的daily_adj数据\n", 175 | "2023.03.20\n", 176 | "更新完成2023.03.20的daily_adj数据\n", 177 | "2023.03.21\n", 178 | "更新完成2023.03.21的daily_adj数据\n", 179 | "2023.03.22\n", 180 | "更新完成2023.03.22的daily_adj数据\n", 181 | "2023.03.23\n", 182 | "更新完成2023.03.23的daily_adj数据\n", 183 | "2023.03.24\n", 184 | "出错了, 重试\n", 185 | "更新完成2023.03.24的daily_adj数据\n", 186 | "2023.03.27\n", 187 | "更新完成2023.03.27的daily_adj数据\n", 188 | "2023.03.28\n", 189 | "更新完成2023.03.28的daily_adj数据\n", 190 | "2023.03.29\n", 191 | "更新完成2023.03.29的daily_adj数据\n", 192 | "2023.03.30\n", 193 | "更新完成2023.03.30的daily_adj数据\n", 194 | "2023.03.31\n", 195 | "更新完成2023.03.31的daily_adj数据\n", 196 | "2023.04.03\n", 197 | "更新完成2023.04.03的daily_adj数据\n", 198 | "2023.04.04\n", 199 | "出错了, 重试\n", 200 | "更新完成2023.04.04的daily_adj数据\n", 201 | "2023.04.06\n", 202 | "更新完成2023.04.06的daily_adj数据\n", 203 | "2023.04.07\n", 204 | "更新完成2023.04.07的daily_adj数据\n", 205 | "2023.04.10\n", 206 | "更新完成2023.04.10的daily_adj数据\n", 207 | "2023.04.11\n", 208 | "更新完成2023.04.11的daily_adj数据\n", 209 | "2023.04.12\n", 210 | "更新完成2023.04.12的daily_adj数据\n", 211 | "2023.04.13\n", 212 | "出错了, 重试\n", 213 | "更新完成2023.04.13的daily_adj数据\n", 214 | "2023.04.14\n", 215 | "更新完成2023.04.14的daily_adj数据\n", 216 | "2023.04.17\n", 217 | "更新完成2023.04.17的daily_adj数据\n", 218 | "2023.04.18\n", 219 | "更新完成2023.04.18的daily_adj数据\n", 220 | "2023.04.19\n", 221 | "更新完成2023.04.19的daily_adj数据\n", 222 | "2023.04.20\n", 223 | "更新完成2023.04.20的daily_adj数据\n", 224 | "2023.04.21\n", 225 | "更新完成2023.04.21的daily_adj数据\n", 226 | "2023.04.24\n", 227 | "更新完成2023.04.24的daily_adj数据\n", 228 | "2023.04.25\n", 229 | "更新完成2023.04.25的daily_adj数据\n", 230 | "2023.04.26\n", 231 | "更新完成2023.04.26的daily_adj数据\n", 232 | "2023.04.27\n", 233 | "更新完成2023.04.27的daily_adj数据\n", 234 | "2023.04.28\n", 235 | "更新完成2023.04.28的daily_adj数据\n", 236 | "2023.05.04\n", 237 | "更新完成2023.05.04的daily_adj数据\n", 238 | "2023.05.05\n", 239 | "出错了, 重试\n", 240 | "更新完成2023.05.05的daily_adj数据\n", 241 | "2023.05.08\n", 242 | "更新完成2023.05.08的daily_adj数据\n", 243 | "2023.05.09\n", 244 | "更新完成2023.05.09的daily_adj数据\n", 245 | "2023.05.10\n", 246 | "更新完成2023.05.10的daily_adj数据\n", 247 | "2023.05.11\n", 248 | "更新完成2023.05.11的daily_adj数据\n", 249 | "2023.05.12\n", 250 | "更新完成2023.05.12的daily_adj数据\n", 251 | "2023.05.15\n", 252 | "更新完成2023.05.15的daily_adj数据\n", 253 | "2023.05.16\n", 254 | "更新完成2023.05.16的daily_adj数据\n", 255 | "2023.05.17\n", 256 | "更新完成2023.05.17的daily_adj数据\n", 257 | "2023.05.18\n", 258 | "出错了, 重试\n", 259 | "更新完成2023.05.18的daily_adj数据\n", 260 | "2023.05.19\n", 261 | "更新完成2023.05.19的daily_adj数据\n", 262 | "2023.05.22\n", 263 | "更新完成2023.05.22的daily_adj数据\n", 264 | "2023.05.23\n", 265 | "更新完成2023.05.23的daily_adj数据\n", 266 | "2023.05.24\n", 267 | "更新完成2023.05.24的daily_adj数据\n", 268 | "2023.05.25\n", 269 | "更新完成2023.05.25的daily_adj数据\n", 270 | "2023.05.26\n", 271 | "更新完成2023.05.26的daily_adj数据\n", 272 | "2023.05.29\n", 273 | "出错了, 重试\n", 274 | "更新完成2023.05.29的daily_adj数据\n", 275 | "2023.05.30\n", 276 | "更新完成2023.05.30的daily_adj数据\n", 277 | "2023.05.31\n", 278 | "更新完成2023.05.31的daily_adj数据\n", 279 | "2023.06.01\n", 280 | "更新完成2023.06.01的daily_adj数据\n", 281 | "2023.06.02\n", 282 | "更新完成2023.06.02的daily_adj数据\n", 283 | "2023.06.05\n", 284 | "更新完成2023.06.05的daily_adj数据\n", 285 | "2023.06.06\n", 286 | "出错了, 重试\n", 287 | "更新完成2023.06.06的daily_adj数据\n", 288 | "2023.06.07\n", 289 | "更新完成2023.06.07的daily_adj数据\n", 290 | "2023.06.08\n", 291 | "更新完成2023.06.08的daily_adj数据\n", 292 | "2023.06.09\n", 293 | "更新完成2023.06.09的daily_adj数据\n", 294 | "2023.06.12\n", 295 | "更新完成2023.06.12的daily_adj数据\n", 296 | "2023.06.13\n", 297 | "更新完成2023.06.13的daily_adj数据\n", 298 | "2023.06.14\n", 299 | "更新完成2023.06.14的daily_adj数据\n", 300 | "2023.06.15\n", 301 | "出错了, 重试\n", 302 | "更新完成2023.06.15的daily_adj数据\n", 303 | "2023.06.16\n", 304 | "更新完成2023.06.16的daily_adj数据\n", 305 | "2023.06.19\n", 306 | "更新完成2023.06.19的daily_adj数据\n", 307 | "2023.06.20\n", 308 | "更新完成2023.06.20的daily_adj数据\n", 309 | "2023.06.21\n", 310 | "出错了, 重试\n", 311 | "更新完成2023.06.21的daily_adj数据\n", 312 | "2023.06.26\n", 313 | "更新完成2023.06.26的daily_adj数据\n", 314 | "2023.06.27\n", 315 | "更新完成2023.06.27的daily_adj数据\n", 316 | "2023.06.28\n", 317 | "更新完成2023.06.28的daily_adj数据\n", 318 | "2023.06.29\n", 319 | "出错了, 重试\n", 320 | "更新完成2023.06.29的daily_adj数据\n", 321 | "更新完成批量daily_adj\n" 322 | ] 323 | } 324 | ], 325 | "source": [ 326 | "\n", 327 | "# 批量更新daily_adj\n", 328 | "def batch_update_daily_adj():\n", 329 | " conn.run(\"daily_adj = loadTable('dfs://stock', `daily_adj)\")\n", 330 | " start_date = np.datetime_as_string(conn.run(\"select max(data_dt) from daily_adj\").values[0][0], unit='D').replace(\"-\",\".\")\n", 331 | " today = datetime.datetime.today().strftime(\"%Y.%m.%d\")\n", 332 | " end_date = conn.run(f\"transFreq({today}, 'SSE')\").astype(\"str\").replace(\"-\",\".\") \n", 333 | " print(start_date,\"---\", end_date)\n", 334 | " download_dates = conn.run(f\"getMarketCalendar('SSE', {start_date}, {end_date})\").tolist()\n", 335 | " \n", 336 | " for date in download_dates:\n", 337 | " date = date.strftime(\"%Y.%m.%d\")\n", 338 | " print(date)\n", 339 | " while True:\n", 340 | " try:\n", 341 | " update_daily_adj(date)\n", 342 | " break\n", 343 | " except requests.exceptions.ConnectTimeout:\n", 344 | " print(\"出错了, 重试\")\n", 345 | " time.sleep(5)\n", 346 | " continue\n", 347 | " print(\"更新完成批量daily_adj\")\n", 348 | "\n", 349 | "batch_update_daily_adj()\n", 350 | " " 351 | ] 352 | }, 353 | { 354 | "cell_type": "code", 355 | "execution_count": 14, 356 | "metadata": {}, 357 | "outputs": [], 358 | "source": [ 359 | "conn.run(\"\"\"\n", 360 | " //update_market_index()\n", 361 | " //update_stock_ma()\n", 362 | " //update_stock_ind()\n", 363 | " update_concept_ind()\n", 364 | "\"\"\")" 365 | ] 366 | }, 367 | { 368 | "cell_type": "code", 369 | "execution_count": 18, 370 | "metadata": {}, 371 | "outputs": [], 372 | "source": [ 373 | "conn.run(\"\"\"\n", 374 | " start_date = 2022.01.01\n", 375 | " end_date = 2022.12.31\n", 376 | " //init_market_index(start_date, end_date)\n", 377 | " init_stock_ma(start_date, end_date)\n", 378 | " init_stock_ind(start_date, end_date)\n", 379 | " init_concept_ind(start_date, end_date) // concept_ind 需要单独启动一遍, 因为换concept_cons了\n", 380 | " \"\"\")\n", 381 | "\n", 382 | " " 383 | ] 384 | }, 385 | { 386 | "cell_type": "code", 387 | "execution_count": 17, 388 | "metadata": {}, 389 | "outputs": [], 390 | "source": [ 391 | "# # 热更新\n", 392 | "conn.run(\"\"\"\n", 393 | "//update_market_index()\n", 394 | "update_stock_ma()\n", 395 | "update_stock_ind()\n", 396 | "update_concept_ind()\n", 397 | "\"\"\")" 398 | ] 399 | }, 400 | { 401 | "cell_type": "code", 402 | "execution_count": 12, 403 | "metadata": {}, 404 | "outputs": [ 405 | { 406 | "data": { 407 | "text/html": [ 408 | "
\n", 409 | "\n", 422 | "\n", 423 | " \n", 424 | " \n", 425 | " \n", 426 | " \n", 427 | " \n", 428 | " \n", 429 | " \n", 430 | " \n", 431 | " \n", 432 | " \n", 433 | " \n", 434 | " \n", 435 | " \n", 436 | " \n", 437 | " \n", 438 | " \n", 439 | " \n", 440 | " \n", 441 | " \n", 442 | " \n", 443 | " \n", 444 | " \n", 445 | " \n", 446 | " \n", 447 | " \n", 448 | " \n", 449 | " \n", 450 | " \n", 451 | " \n", 452 | " \n", 453 | " \n", 454 | " \n", 455 | " \n", 456 | " \n", 457 | " \n", 458 | " \n", 459 | " \n", 460 | " \n", 461 | " \n", 462 | " \n", 463 | " \n", 464 | " \n", 465 | " \n", 466 | " \n", 467 | " \n", 468 | " \n", 469 | " \n", 470 | " \n", 471 | " \n", 472 | " \n", 473 | " \n", 474 | " \n", 475 | " \n", 476 | " \n", 477 | " \n", 478 | " \n", 479 | " \n", 480 | " \n", 481 | " \n", 482 | " \n", 483 | " \n", 484 | " \n", 485 | " \n", 486 | " \n", 487 | " \n", 488 | " \n", 489 | " \n", 490 | " \n", 491 | " \n", 492 | " \n", 493 | " \n", 494 | " \n", 495 | " \n", 496 | " \n", 497 | " \n", 498 | " \n", 499 | " \n", 500 | " \n", 501 | " \n", 502 | " \n", 503 | " \n", 504 | " \n", 505 | " \n", 506 | " \n", 507 | " \n", 508 | " \n", 509 | " \n", 510 | " \n", 511 | " \n", 512 | " \n", 513 | " \n", 514 | " \n", 515 | " \n", 516 | " \n", 517 | " \n", 518 | " \n", 519 | " \n", 520 | " \n", 521 | " \n", 522 | " \n", 523 | " \n", 524 | " \n", 525 | " \n", 526 | " \n", 527 | " \n", 528 | " \n", 529 | " \n", 530 | " \n", 531 | " \n", 532 | " \n", 533 | " \n", 534 | " \n", 535 | " \n", 536 | " \n", 537 | " \n", 538 | " \n", 539 | " \n", 540 | " \n", 541 | " \n", 542 | " \n", 543 | " \n", 544 | " \n", 545 | " \n", 546 | " \n", 547 | "
ts_codesymbolnameareaindustrymarketlist_date
0000001.SZ000001平安银行深圳银行主板19910403
1000002.SZ000002万科A深圳全国地产主板19910129
2000004.SZ000004ST国华深圳软件服务主板19910114
3000005.SZ000005ST星源深圳环境保护主板19901210
4000006.SZ000006深振业A深圳区域地产主板19920427
........................
5016688799.SH688799华纳药厂湖南化学制药科创板20210713
5017688800.SH688800瑞可达江苏元器件科创板20210722
5018688819.SH688819天能股份浙江电气设备科创板20210118
5019688981.SH688981中芯国际上海半导体科创板20200716
5020689009.SH689009九号公司-WD北京摩托车科创板20201029
\n", 548 | "

5021 rows × 7 columns

\n", 549 | "
" 550 | ], 551 | "text/plain": [ 552 | " ts_code symbol name area industry market list_date\n", 553 | "0 000001.SZ 000001 平安银行 深圳 银行 主板 19910403\n", 554 | "1 000002.SZ 000002 万科A 深圳 全国地产 主板 19910129\n", 555 | "2 000004.SZ 000004 ST国华 深圳 软件服务 主板 19910114\n", 556 | "3 000005.SZ 000005 ST星源 深圳 环境保护 主板 19901210\n", 557 | "4 000006.SZ 000006 深振业A 深圳 区域地产 主板 19920427\n", 558 | "... ... ... ... ... ... ... ...\n", 559 | "5016 688799.SH 688799 华纳药厂 湖南 化学制药 科创板 20210713\n", 560 | "5017 688800.SH 688800 瑞可达 江苏 元器件 科创板 20210722\n", 561 | "5018 688819.SH 688819 天能股份 浙江 电气设备 科创板 20210118\n", 562 | "5019 688981.SH 688981 中芯国际 上海 半导体 科创板 20200716\n", 563 | "5020 689009.SH 689009 九号公司-WD 北京 摩托车 科创板 20201029\n", 564 | "\n", 565 | "[5021 rows x 7 columns]" 566 | ] 567 | }, 568 | "execution_count": 12, 569 | "metadata": {}, 570 | "output_type": "execute_result" 571 | } 572 | ], 573 | "source": [ 574 | "conn.run(\"basic = loadTable('dfs://stock', 'basic')\")\n", 575 | "conn.run(\"select * from basic\")" 576 | ] 577 | }, 578 | { 579 | "cell_type": "code", 580 | "execution_count": 30, 581 | "metadata": {}, 582 | "outputs": [ 583 | { 584 | "name": "stdout", 585 | "output_type": "stream", 586 | "text": [ 587 | "已经存在 算力租赁\n", 588 | "已经存在 空间计算\n", 589 | "已经存在 英伟达概念\n", 590 | "已经存在 脑机接口\n", 591 | "已经存在 MR（混合现实）\n", 592 | "已经存在 同花顺中特估100\n", 593 | "已经存在 液冷服务器\n", 594 | "已经存在 太赫兹\n", 595 | "已经存在 猴痘概念\n", 596 | "已经存在 粮食概念\n", 597 | "已经存在 存储芯片\n", 598 | "已经存在 数据要素\n", 599 | "已经存在 烟草\n", 600 | "已经存在 首发新股\n", 601 | "已经存在 一季报预增\n", 602 | "已经存在 MLOps概念\n", 603 | "已经存在 电力物联网\n", 604 | "已经存在 ERP概念\n", 605 | "已经存在 DRG/DIP\n", 606 | "已经存在 超导概念\n", 607 | "已经存在 6G概念\n", 608 | "已经存在 时空大数据\n", 609 | "已经存在 毫米波雷达\n", 610 | "已经存在 数字水印\n", 611 | "已经存在 共封装光学（CPO）\n", 612 | "已经存在 固态电池\n", 613 | "已经存在 ChatGPT概念\n", 614 | "已经存在 成飞概念\n", 615 | "已经存在 国产航母\n", 616 | "已经存在 蒙脱石散\n", 617 | "已经存在 血氧仪\n", 618 | "已经存在 新冠特效药\n", 619 | "已经存在 POE胶膜\n", 620 | "已经存在 熊去氧胆酸\n", 621 | "已经存在 抗病毒面料\n", 622 | "已经存在 数据确权\n", 623 | "已经存在 抗原检测\n", 624 | "已经存在 证金持股\n", 625 | "已经存在 国企改革\n", 626 | "已经存在 深圳国企改革\n", 627 | "已经存在 上海国企改革\n", 628 | "已经存在 央企国企改革\n", 629 | "已经存在 PET铜箔\n", 630 | "已经存在 AIGC概念\n", 631 | "已经存在 高压氧舱\n", 632 | "已经存在 Web3.0\n", 633 | "已经存在 创新药\n", 634 | "已经存在 信创\n", 635 | "已经存在 有机硅概念\n", 636 | "已经存在 空气能热泵\n", 637 | "已经存在 先进封装（Chiplet）\n", 638 | "已经存在 减速器\n", 639 | "已经存在 TOPCON电池\n", 640 | "已经存在 钙钛矿电池\n", 641 | "已经存在 生物质能发电\n", 642 | "已经存在 一体化压铸\n", 643 | "已经存在 F5G概念\n", 644 | "已经存在 比亚迪概念\n", 645 | "已经存在 超超临界发电\n", 646 | "已经存在 方舱医院\n", 647 | "已经存在 噪声防治\n", 648 | "已经存在 露营经济\n", 649 | "已经存在 肝炎概念\n", 650 | "已经存在 统一大市场\n", 651 | "已经存在 MicroLED概念\n", 652 | "已经存在 毛发医疗\n", 653 | "已经存在 恒大概念\n", 654 | "已经存在 疫情监测\n", 655 | "已经存在 华为欧拉\n", 656 | "已经存在 家庭医生\n", 657 | "已经存在 华为鲲鹏\n", 658 | "已经存在 低辐射玻璃（Low-E）\n", 659 | "已经存在 国资云\n", 660 | "已经存在 电子身份证\n", 661 | "已经存在 数字经济\n", 662 | "已经存在 金属铅\n", 663 | "已经存在 托育服务\n", 664 | "已经存在 跨境支付（CIPS）\n", 665 | "已经存在 中俄贸易概念\n", 666 | "已经存在 俄乌冲突概念\n", 667 | "已经存在 东数西算（算力）\n", 668 | "已经存在 智慧灯杆\n", 669 | "已经存在 土壤修复\n", 670 | "已经存在 民爆概念\n", 671 | "已经存在 股权转让\n", 672 | "已经存在 PCB概念\n", 673 | "已经存在 抖音概念\n", 674 | "已经存在 硅能源\n", 675 | "已经存在 重组蛋白\n", 676 | "已经存在 新冠治疗\n", 677 | "已经存在 电子纸\n", 678 | "已经存在 幽门螺杆菌概念\n", 679 | "已经存在 预制菜\n", 680 | "已经存在 沪股通\n", 681 | "已经存在 柔性直流输电\n", 682 | "已经存在 高压快充\n", 683 | "已经存在 汽车热管理\n", 684 | "已经存在 EDR概念\n", 685 | "已经存在 新股与次新股\n", 686 | "已经存在 新型烟草\n", 687 | "已经存在 数据安全\n", 688 | "已经存在 虚拟数字人\n", 689 | "已经存在 智能制造\n", 690 | "已经存在 WiFi 6\n", 691 | "已经存在 换电概念\n", 692 | "已经存在 培育钻石\n", 693 | "已经存在 虚拟电厂\n", 694 | "已经存在 绿色电力\n", 695 | "已经存在 元宇宙\n", 696 | "已经存在 抽水蓄能\n", 697 | "已经存在 NFT概念\n", 698 | "已经存在 PVDF概念\n", 699 | "已经存在 北交所概念\n", 700 | "已经存在 工业母机\n", 701 | "已经存在 专精特新\n", 702 | "已经存在 钠离子电池\n", 703 | "已经存在 CRO概念\n", 704 | "已经存在 牙科医疗\n", 705 | "已经存在 MCU芯片\n", 706 | "已经存在 共同富裕示范区\n", 707 | "已经存在 鸿蒙概念\n", 708 | "已经存在 天然气\n", 709 | "已经存在 食品安全\n", 710 | "已经存在 新材料概念\n", 711 | "已经存在 农业种植\n", 712 | "已经存在 生态农业\n", 713 | "已经存在 特钢概念\n", 714 | "已经存在 期货概念\n", 715 | "已经存在 猪肉\n", 716 | "已经存在 储能\n", 717 | "已经存在 华为汽车\n", 718 | "已经存在 中芯国际概念\n", 719 | "已经存在 光伏建筑一体化\n", 720 | "已经存在 碳中和\n", 721 | "已经存在 数据中心\n", 722 | "已经存在 REITs概念\n", 723 | "已经存在 碳交易\n", 724 | "已经存在 传感器\n", 725 | "已经存在 光刻胶\n", 726 | "已经存在 农机\n", 727 | "已经存在 仿制药一致性评价\n", 728 | "已经存在 无线充电\n", 729 | "已经存在 安防\n", 730 | "已经存在 网络安全\n", 731 | "已经存在 消费电子概念\n", 732 | "已经存在 氢能源\n", 733 | "已经存在 汽车芯片\n", 734 | "已经存在 知识产权保护\n", 735 | "已经存在 机器人概念\n", 736 | "已经存在 煤化工\n", 737 | "已经存在 自由贸易港\n", 738 | "已经存在 智能家居\n", 739 | "已经存在 供应链金融\n", 740 | "已经存在 长三角一体化\n", 741 | "已经存在 富士康概念\n", 742 | "已经存在 数字乡村\n", 743 | "已经存在 禽流感\n", 744 | "已经存在 在线旅游\n", 745 | "已经存在 网络直播\n", 746 | "已经存在 3D打印\n", 747 | "已经存在 租售同权\n", 748 | "已经存在 云游戏\n", 749 | "已经存在 工业4.0\n", 750 | "已经存在 两轮车\n", 751 | "已经存在 ETC\n", 752 | "已经存在 丙烯酸\n", 753 | "已经存在 供销社\n", 754 | "已经存在 军民融合\n", 755 | "已经存在 海工装备\n", 756 | "已经存在 HJT电池\n", 757 | "已经存在 民营医院\n", 758 | "已经存在 智慧政务\n", 759 | "已经存在 新型城镇化\n", 760 | "已经存在 基因测序\n", 761 | "已经存在 在线教育\n", 762 | "已经存在 健康中国\n", 763 | "已经存在 转基因\n", 764 | "已经存在 车联网\n", 765 | "已经存在 云计算\n", 766 | "已经存在 赛马概念\n", 767 | "已经存在 工业互联网\n", 768 | "已经存在 节能环保\n", 769 | "已经存在 举牌\n", 770 | "已经存在 职业教育\n", 771 | "已经存在 垃圾分类\n", 772 | "已经存在 OLED\n", 773 | "已经存在 拼多多概念\n", 774 | "已经存在 互联网金融\n", 775 | "已经存在 新冠检测\n", 776 | "已经存在 同花顺漂亮100\n", 777 | "已经存在 物业管理\n", 778 | "已经存在 特斯拉\n", 779 | "已经存在 钒电池\n", 780 | "已经存在 煤炭概念\n", 781 | "已经存在 医美概念\n", 782 | "已经存在 社区团购\n", 783 | "已经存在 辅助生殖\n", 784 | "已经存在 第三代半导体\n", 785 | "已经存在 科创次新股\n", 786 | "已经存在 核准制次新股\n", 787 | "已经存在 注册制次新股\n", 788 | "已经存在 代糖概念\n", 789 | "已经存在 可降解塑料\n", 790 | "已经存在 汽车拆解概念\n", 791 | "已经存在 NMN概念\n", 792 | "已经存在 海南自贸区\n", 793 | "已经存在 快手概念\n", 794 | "已经存在 室外经济\n", 795 | "已经存在 免税店\n", 796 | "已经存在 国家大基金持股\n", 797 | "已经存在 参股保险\n", 798 | "已经存在 装配式建筑\n", 799 | "已经存在 大飞机\n", 800 | "已经存在 大豆\n", 801 | "已经存在 C2M概念\n", 802 | "已经存在 医疗废物处理\n", 803 | "已经存在 消毒剂\n", 804 | "已经存在 云办公\n", 805 | "已经存在 网红经济\n", 806 | "已经存在 MiniLED\n", 807 | "已经存在 胎压监测\n", 808 | "已经存在 无线耳机\n", 809 | "已经存在 标普道琼斯A股\n", 810 | "已经存在 分拆上市意愿\n", 811 | "已经存在 宠物经济\n", 812 | "已经存在 新零售\n", 813 | "已经存在 创业板重组松绑\n", 814 | "已经存在 黑龙江自贸区\n", 815 | "已经存在 动物疫苗\n", 816 | "已经存在 国产操作系统\n", 817 | "已经存在 华为海思概念股\n", 818 | "已经存在 人造肉\n", 819 | "已经存在 超级真菌\n", 820 | "已经存在 数字孪生\n", 821 | "已经存在 超清视频\n", 822 | "已经存在 柔性屏\n", 823 | "已经存在 养鸡\n", 824 | "已经存在 芬太尼\n", 825 | "已经存在 送转填权\n", 826 | "已经存在 环氧丙烷\n", 827 | "已经存在 独角兽概念\n", 828 | "已经存在 盐湖提锂\n", 829 | "已经存在 动力电池回收\n", 830 | "已经存在 水泥概念\n", 831 | "已经存在 冰雪产业\n", 832 | "已经存在 石墨电极\n", 833 | "已经存在 边缘计算\n", 834 | "已经存在 宁德时代概念\n", 835 | "已经存在 超级品牌\n", 836 | "已经存在 燃料乙醇\n", 837 | "已经存在 无人零售\n", 838 | "已经存在 MSCI概念\n", 839 | "已经存在 特色小镇\n", 840 | "已经存在 雄安新区\n", 841 | "已经存在 钴\n", 842 | "已经存在 共享单车\n", 843 | "已经存在 网约车\n", 844 | "已经存在 人民币贬值受益\n", 845 | "已经存在 可燃冰\n", 846 | "已经存在 燃料电池\n", 847 | "已经存在 区块链\n", 848 | "已经存在 蚂蚁金服概念\n", 849 | "已经存在 债转股(AMC概念)\n", 850 | "已经存在 智能音箱\n", 851 | "已经存在 人工智能\n", 852 | "已经存在 电子竞技\n", 853 | "已经存在 腾讯概念\n", 854 | "已经存在 数字货币\n", 855 | "已经存在 智能物流\n", 856 | "已经存在 青蒿素\n", 857 | "已经存在 杭州亚运会\n", 858 | "已经存在 中船系\n", 859 | "已经存在 虚拟现实\n", 860 | "已经存在 智能交通\n", 861 | "已经存在 互联网医疗\n", 862 | "已经存在 深股通\n", 863 | "已经存在 农村电商\n", 864 | "已经存在 广东自贸区\n", 865 | "已经存在 互联网保险\n", 866 | "已经存在 地下管网\n", 867 | "已经存在 PPP概念\n", 868 | "已经存在 壳资源\n", 869 | "已经存在 体育产业\n", 870 | "已经存在 玉米\n", 871 | "已经存在 金属铜\n", 872 | "已经存在 金属锌\n", 873 | "已经存在 医药电商\n", 874 | "已经存在 跨境电商\n", 875 | "已经存在 阿里巴巴概念\n", 876 | "已经存在 细胞免疫治疗\n", 877 | "已经存在 中韩自贸区\n", 878 | "已经存在 国产软件\n", 879 | "已经存在 集成电路概念\n", 880 | "已经存在 高铁\n", 881 | "已经存在 金属镍\n", 882 | "已经存在 医疗器械概念\n", 883 | "已经存在 白酒概念\n", 884 | "已经存在 啤酒概念\n", 885 | "已经存在 养老概念\n", 886 | "已经存在 粤港澳大湾区\n", 887 | "已经存在 工业大麻\n", 888 | "已经存在 汽车电子\n", 889 | "已经存在 航空发动机\n", 890 | "已经存在 华为概念\n", 891 | "已经存在 摘帽\n", 892 | "已经存在 口罩\n", 893 | "已经存在 信托概念\n", 894 | "已经存在 一带一路\n", 895 | "已经存在 染料\n", 896 | "已经存在 无人驾驶\n", 897 | "已经存在 小米概念\n", 898 | "已经存在 参股银行\n", 899 | "已经存在 机器视觉\n", 900 | "已经存在 百度概念\n", 901 | "已经存在 互联网彩票\n", 902 | "已经存在 眼科医疗\n", 903 | "已经存在 福建自贸区\n", 904 | "已经存在 天津自贸区\n", 905 | "已经存在 冷链物流\n", 906 | "已经存在 智能穿戴\n", 907 | "已经存在 人脸识别\n", 908 | "已经存在 上海自贸区\n", 909 | "已经存在 化肥\n", 910 | "已经存在 金属回收\n", 911 | "已经存在 足球概念\n", 912 | "已经存在 芯片概念\n", 913 | "已经存在 光伏概念\n", 914 | "已经存在 手机游戏\n", 915 | "已经存在 净水概念\n", 916 | "已经存在 乳业\n", 917 | "已经存在 航运概念\n", 918 | "已经存在 超级电容\n", 919 | "已经存在 语音技术\n", 920 | "已经存在 融资融券\n", 921 | "已经存在 三胎概念\n", 922 | "已经存在 高端装备\n", 923 | "已经存在 无人机\n", 924 | "已经存在 ST板块\n", 925 | "已经存在 土地流转\n", 926 | "已经存在 5G\n", 927 | "已经存在 乡村振兴\n", 928 | "已经存在 大数据\n", 929 | "已经存在 量子科技\n", 930 | "已经存在 小金属概念\n", 931 | "已经存在 家用电器\n", 932 | "已经存在 文化传媒\n", 933 | "已经存在 充电桩\n", 934 | "已经存在 稀缺资源\n", 935 | "已经存在 生物医药\n", 936 | "已经存在 锂电池\n", 937 | "已经存在 卫星导航\n", 938 | "已经存在 中字头股票\n", 939 | "已经存在 世界杯\n", 940 | "已经存在 智能医疗\n", 941 | "已经存在 智慧城市\n", 942 | "已经存在 电子商务\n", 943 | "已经存在 移动支付\n", 944 | "已经存在 风电\n", 945 | "已经存在 固废处理\n", 946 | "已经存在 光热发电\n", 947 | "已经存在 钛白粉概念\n", 948 | "已经存在 碳纤维\n", 949 | "已经存在 特高压\n", 950 | "已经存在 通用航空\n", 951 | "已经存在 污水处理\n", 952 | "已经存在 物联网\n", 953 | "已经存在 稀土永磁\n", 954 | "已经存在 新疆振兴\n", 955 | "已经存在 参股新三板\n", 956 | "已经存在 页岩气\n", 957 | "已经存在 新能源汽车\n", 958 | "已经存在 网络游戏\n", 959 | "已经存在 智能电网\n", 960 | "已经存在 流感\n", 961 | "已经存在 节能照明\n", 962 | "已经存在 京津冀一体化\n", 963 | "已经存在 海峡两岸\n", 964 | "已经存在 军工\n", 965 | "已经存在 氟化工概念\n", 966 | "已经存在 横琴新区\n", 967 | "已经存在 磷化工\n", 968 | "已经存在 参股券商\n", 969 | "已经存在 生物疫苗\n", 970 | "已经存在 草甘膦\n", 971 | "已经存在 创投\n", 972 | "已经存在 国家大基金持股 \n", 973 | "已经存在 核电\n", 974 | "已经存在 黄金概念\n", 975 | "已经存在 建筑节能\n", 976 | "已经存在 金改\n", 977 | "已经存在 苹果概念\n", 978 | "已经存在 PM2.5\n", 979 | "已经存在 石墨烯\n", 980 | "已经存在 水利\n", 981 | "更新完成\n" 982 | ] 983 | } 984 | ], 985 | "source": [ 986 | "\n", 987 | "import requests\n", 988 | "import time\n", 989 | "\n", 990 | "\n", 991 | "def update_concept_cons(concept):\n", 992 | " conn.run(\"db = database('dfs://stock');\")\n", 993 | " conn.run(\"basic = loadTable('dfs://stock', 'basic');\")\n", 994 | " if conn.run(f\"existsTable('dfs://stock', 'concept_cons')\"):\n", 995 | " conn.run(\"concept_cons = loadTable('dfs://stock', 'concept_cons');\")\n", 996 | " if conn.run(f\"select * from concept_cons where 概念名称 = '{concept}'\").shape[0] > 0:\n", 997 | " print(\"已经存在\", concept)\n", 998 | " return\n", 999 | " while True:\n", 1000 | " try:\n", 1001 | " print(\"开始下载\", concept)\n", 1002 | " df = ak.stock_board_concept_cons_ths(symbol=concept)\n", 1003 | " df['概念名称'] = concept\n", 1004 | " df = df[['代码', '名称', '概念名称']]\n", 1005 | " conn.upload({\"concept_cons_df\": df})\n", 1006 | " if not conn.run(f\"existsTable('dfs://stock', 'concept_cons')\"):\n", 1007 | " conn.run(\"\"\"\n", 1008 | " t2 = select b.ts_code, c.* from concept_cons_df c\n", 1009 | " left join basic b on b.symbol = c.代码 \n", 1010 | " concept_cons = db.createTable(t2, 'concept_cons');\n", 1011 | " concept_cons.append!(select * from t2);\n", 1012 | " \"\"\")\n", 1013 | " else:\n", 1014 | " conn.run(\"concept_cons = loadTable('dfs://stock', 'concept_cons');\")\n", 1015 | " conn.run(f\"\"\"\n", 1016 | " delete from concept_cons where 概念名称 = '{concept}';\n", 1017 | " t2 = select b.ts_code, c.* from concept_cons_df c\n", 1018 | " left join basic b on b.symbol = c.代码 \n", 1019 | " concept_cons.append!(select * from t2);\n", 1020 | " \"\"\")\n", 1021 | " print(\"下载完成\", concept)\n", 1022 | " break\n", 1023 | " except (requests.exceptions.ConnectTimeout,ValueError):\n", 1024 | " print(\"出错了, 重试\")\n", 1025 | " time.sleep(5)\n", 1026 | " continue\n", 1027 | " \n", 1028 | "\n", 1029 | "def batch_update_concept_cons():\n", 1030 | " \n", 1031 | " \n", 1032 | " conn.run(\"concept_list_new = loadTable('dfs://stock', 'concept_list_new');\")\n", 1033 | " concept_list = conn.run(\"\"\"select 概念名称 as concept from concept_list_new\n", 1034 | " \"\"\")['concept'].values\n", 1035 | " \n", 1036 | " for concept in concept_list:\n", 1037 | " update_concept_cons(concept)\n", 1038 | " print(\"更新完成\")\n", 1039 | "\n", 1040 | "\n", 1041 | "batch_update_concept_cons()\n" 1042 | ] 1043 | }, 1044 | { 1045 | "cell_type": "code", 1046 | "execution_count": 26, 1047 | "metadata": {}, 1048 | "outputs": [], 1049 | "source": [ 1050 | "conn.run(\"db = database('dfs://stock'); dropTable(db, 'concept_cons')\")" 1051 | ] 1052 | }, 1053 | { 1054 | "cell_type": "code", 1055 | "execution_count": 3, 1056 | "metadata": {}, 1057 | "outputs": [], 1058 | "source": [ 1059 | "import tushare as ts\n", 1060 | "token = \"f1ce53736e3b6777425d3df97c05e7460a55534db8ece60114c9e2a3\"\n", 1061 | "ts.set_token(token)\n", 1062 | "pro = ts.pro_api()" 1063 | ] 1064 | } 1065 | ], 1066 | "metadata": { 1067 | "kernelspec": { 1068 | "display_name": "base", 1069 | "language": "python", 1070 | "name": "python3" 1071 | }, 1072 | "language_info": { 1073 | "codemirror_mode": { 1074 | "name": "ipython", 1075 | "version": 3 1076 | }, 1077 | "file_extension": ".py", 1078 | "mimetype": "text/x-python", 1079 | "name": "python", 1080 | "nbconvert_exporter": "python", 1081 | "pygments_lexer": "ipython3", 1082 | "version": "3.9.12" 1083 | }, 1084 | "orig_nbformat": 4, 1085 | "vscode": { 1086 | "interpreter": { 1087 | "hash": "745e22ba4b636ebf6f9c05a7ea7929a766a37732a5f5fd2c5299521845509a85" 1088 | } 1089 | } 1090 | }, 1091 | "nbformat": 4, 1092 | "nbformat_minor": 2 1093 | } 1094 | --------------------------------------------------------------------------------