├── .gitattributes
├── .gitignore
├── README.md
├── __init__.py
├── ngrok服务器搭建.docx
├── rqdata
    ├── __init__.py
    ├── mongo
    │   ├── __init__.py
    │   ├── client_template.py
    │   ├── libs.py
    │   └── mongo_server.py
    └── rpc
    │   ├── __init__.py
    │   ├── client.py
    │   └── server.py
├── scripts
    ├── __init__.py
    ├── future1m.py
    ├── passwd.py
    ├── rice_auto.py
    ├── rq.py
    └── stock1m.py
├── setup.py
└── tests
    ├── __init__.py
    └── test_server.py


/.gitattributes:
--------------------------------------------------------------------------------
1 | # Auto detect text files and perform LF normalization
2 | * text=auto


--------------------------------------------------------------------------------
/.gitignore:
--------------------------------------------------------------------------------
  1 | # Byte-compiled / optimized / DLL files
  2 | __pycache__/
  3 | *.py[cod]
  4 | *$py.class
  5 | 
  6 | # C extensions
  7 | *.so
  8 | 
  9 | .idea
 10 | 
 11 | passwd.py
 12 | 
 13 | # Distribution / packaging
 14 | .Python
 15 | build/
 16 | develop-eggs/
 17 | dist/
 18 | downloads/
 19 | eggs/
 20 | .eggs/
 21 | lib/
 22 | lib64/
 23 | parts/
 24 | sdist/
 25 | var/
 26 | wheels/
 27 | *.egg-info/
 28 | .installed.cfg
 29 | *.egg
 30 | 
 31 | # PyInstaller
 32 | #  Usually these files are written by a python script from a template
 33 | #  before PyInstaller builds the exe, so as to inject date/other infos into it.
 34 | *.manifest
 35 | *.spec
 36 | 
 37 | # Installer logs
 38 | pip-log.txt
 39 | pip-delete-this-directory.txt
 40 | 
 41 | # Unit test / coverage reports
 42 | htmlcov/
 43 | .tox/
 44 | .coverage
 45 | .coverage.*
 46 | .cache
 47 | nosetests.xml
 48 | coverage.xml
 49 | *.cover
 50 | .hypothesis/
 51 | 
 52 | # Translations
 53 | *.mo
 54 | *.pot
 55 | 
 56 | # Django stuff:
 57 | *.log
 58 | local_settings.py
 59 | 
 60 | # Flask stuff:
 61 | instance/
 62 | .webassets-cache
 63 | 
 64 | # Scrapy stuff:
 65 | .scrapy
 66 | 
 67 | # Sphinx documentation
 68 | docs/_build/
 69 | 
 70 | # PyBuilder
 71 | target/
 72 | 
 73 | # Jupyter Notebook
 74 | .ipynb_checkpoints
 75 | 
 76 | # pyenv
 77 | .python-version
 78 | 
 79 | # celery beat schedule file
 80 | celerybeat-schedule
 81 | 
 82 | # SageMath parsed files
 83 | *.sage.py
 84 | 
 85 | # Environments
 86 | .env
 87 | .venv
 88 | env/
 89 | venv/
 90 | ENV/
 91 | 
 92 | # Spyder project settings
 93 | .spyderproject
 94 | .spyproject
 95 | 
 96 | # Rope project settings
 97 | .ropeproject
 98 | 
 99 | # mkdocs documentation
100 | /site
101 | 
102 | # mypy
103 | .mypy_cache/
104 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
 1 | rq-data
 2 | ==============
 3 | 
 4 | 获取ricequant中数据, 其中期货数据默认为vnpy格式。
 5 | 
 6 | <br>
 7 | <br>
 8 | 
 9 | 
10 | 准备工作：
11 | 1. 公网IP的网络（若没有可以使用ngrok工具，详细使用方法自行google或参照本目录下的word教程）
12 | 2. 安装配置mongodb数据库（或根据自己需求，修改rpc的回调函数，存其它数据库，或者csv）
13 | 
14 | 使用步骤：
15 | 1. 在拥有公网ip的电脑上运行rpc服务器端 `rqdata/mongo/mongo_server.py`,并配置其中的mongodb以及rpc服务器端口
16 | 2. 将`rqdata/rpc/client.py` 上传到 ricequant研究中的notebook中，放到文件夹rpc下，重命名为rpc.py
17 | 3. 在研究中新建notebook名为：test.ipynb, 并将`rqdata/scripts/`中`[future1m.py, stock1m.py, rq.py]`中的某个代码copy进去，修改其中的rpc客户端地址和端口，或者参照自行编写其它数据获取代码。（vnpy格式数据代码在future1m.py中）
18 | 
19 | 
20 | ##### 若手动下载直接运行第3步中代码，每日自动更新继续下述配置
21 | 
22 | 准备工作：
23 | 1. 配置selenium环境（下载chromewebdriver，并将其路径添加到环境变量PATH中，自行翻墙下载或见群文件）
24 | 
25 | 
26 | 使用步骤：
27 | 1. 在`rqdata/scripts/passwd.py中配置ricequant的用户名密码
28 | 2. 将`driver.get("https://www.ricequant.com/research/user/user_xxxxx/notebooks/test.ipynb")`中的
29 | url路径改为步骤3中test.ipynb的路径。
30 | 3. 运行`rqdata/scripts/rice_auto.py`
31 | 
32 | 


--------------------------------------------------------------------------------
/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/JaysonAlbert/rq-data/782b979903e7d4c01e5a42f7eac8669c84c3b776/__init__.py


--------------------------------------------------------------------------------
/ngrok服务器搭建.docx:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/JaysonAlbert/rq-data/782b979903e7d4c01e5a42f7eac8669c84c3b776/ngrok服务器搭建.docx


--------------------------------------------------------------------------------
/rqdata/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/JaysonAlbert/rq-data/782b979903e7d4c01e5a42f7eac8669c84c3b776/rqdata/__init__.py


--------------------------------------------------------------------------------
/rqdata/mongo/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/JaysonAlbert/rq-data/782b979903e7d4c01e5a42f7eac8669c84c3b776/rqdata/mongo/__init__.py


--------------------------------------------------------------------------------
/rqdata/mongo/client_template.py:
--------------------------------------------------------------------------------
 1 | # -*- coding: utf-8 -*-
 2 | from rqdata.rpc.client import RpcClient
 3 | 
 4 | if __name__ == '__main__':
 5 |     host = 'localhost'
 6 |     port = 1111
 7 | 
 8 |     client = RpcClient(host, port)
 9 |     date = client.latest_date("CF0000", type='future')
10 |     date = client.latest_date("IC0000", type='future')
11 |     print(date)
12 | 


--------------------------------------------------------------------------------
/rqdata/mongo/libs.py:
--------------------------------------------------------------------------------
 1 | from pymongo import MongoClient
 2 | import pymongo
 3 | import re
 4 | 
 5 | '''
 6 | - `type`: CS for common stock, future for future
 7 | 
 8 | - `code`: 
 9 | '''
10 | 
11 | 
12 | class MongoHelper(object):
13 |     host = "localhost"
14 |     port = 27017
15 | 
16 |     stock_min_db = "A_stock_min"
17 |     stock_min_collection = "A_stock_min"
18 | 
19 |     def __init__(self, host="localhost", port=27017):
20 |         self.host = host
21 |         self.port = port
22 | 
23 |         self.client = MongoClient(self.host, self.port)
24 |         self.stock_db = self.client[self.stock_min_db]
25 | 
26 |         self.future_db = self.client['VnTrader_1Min_Db']
27 | 
28 |     def latest_date(self, code, type="CS"):
29 |         if type == "CS":
30 |             cursor = self.stock_db[code].find().sort(
31 |                 [("dt", pymongo.DESCENDING)]).limit(1)
32 |             try:
33 |                 date = cursor.next()['dt']
34 |             except Exception as e:
35 |                 date = None
36 |             return date
37 |         elif type == "future":
38 |             collection = self.future_db[code]
39 |             cursor = collection.find({}, {"date": 1, "_id": 0}).sort([("datetime", pymongo.DESCENDING)]).limit(1)
40 |             try:
41 |                 date = cursor.next()['date']
42 |             except Exception as e:
43 |                 date = None
44 |             return date
45 | 
46 |     def get_order_book_id(self, code):
47 |         if code[-2:] == "SH":
48 |             return code[:-2] + "XSHG"
49 |         else:
50 |             return code[:-2] + "XSHE"
51 | 
52 |     def get_stock_code(self, code):
53 |         if code[-4:] == "XSHG":
54 |             return code[:-4] + "SH"
55 |         else:
56 |             return code[:-4] + "SZ"
57 | 
58 |     def insert(self, code, data, type="CS"):
59 |         if type == "CS":
60 |             try:
61 |                 self.future_db[code].create_index([('datetime',pymongo.ASCENDING)],unique=True)
62 |             except Exception as e:
63 |                 pass
64 |             self.stock_db[code].insert(data)
65 |             # for d in data:
66 |             #     self.stock_db[code].update({'datetime':d['datetime']},d,upsert=True)
67 |         elif type == "future":
68 |             try:
69 |                 self.future_db[code].create_index([('datetime',pymongo.ASCENDING)],unique=True)
70 |             except Exception as e:
71 |                 pass
72 |             self.future_db[code].insert(data)
73 |             # for d in data:
74 |             #     self.future_db[code].update({'datetime':d['datetime']},d,upsert=True)
75 | 


--------------------------------------------------------------------------------
/rqdata/mongo/mongo_server.py:
--------------------------------------------------------------------------------
 1 | from six import PY2
 2 | 
 3 | if PY2:
 4 |     import SocketServer
 5 | else:
 6 |     import socketserver as SocketServer
 7 | from rqdata.mongo.libs import *
 8 | from rqdata.rpc.server import RpcHandler
 9 | 
10 | 
11 | def run_server():
12 |     mongo_helper = MongoHelper("127.0.0.1", 27017)
13 |     host = '0.0.0.0'
14 |     port = 1111
15 |     RpcHandler.register(mongo_helper.latest_date)
16 |     RpcHandler.register(mongo_helper.insert)
17 |     server = SocketServer.TCPServer((host, port), RpcHandler)
18 |     server.serve_forever()
19 | 


--------------------------------------------------------------------------------
/rqdata/rpc/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/JaysonAlbert/rq-data/782b979903e7d4c01e5a42f7eac8669c84c3b776/rqdata/rpc/__init__.py


--------------------------------------------------------------------------------
/rqdata/rpc/client.py:
--------------------------------------------------------------------------------
 1 | # -*- coding: utf-8 -*-
 2 | 
 3 | import zlib
 4 | import socket
 5 | import struct
 6 | import threading
 7 | import pickle
 8 | 
 9 | 
10 | def send_msg(sock, msg):
11 |     # Prefix each message with a 4-byte length (network byte order)
12 |     msg = zlib.compress(pickle.dumps(msg, protocol=2), 3)
13 |     length = len(msg)
14 |     msg = struct.pack('>I', length) + msg
15 |     sock.sendall(msg)
16 | 
17 | 
18 | def recv_msg(sock):
19 |     # Read message length and unpack it into an integer
20 |     raw_msglen = sock.recv(4)
21 |     if not raw_msglen:
22 |         return None
23 |     msglen = struct.unpack('>I', raw_msglen)[0]
24 |     # Read the message data
25 |     return recvall(sock, msglen)
26 | 
27 | 
28 | def recvall(sock, n):
29 |     # Helper function to recv n bytes or return None if EOF is hit
30 |     total_data = []
31 |     data = b''
32 |     byte_recved = 0
33 |     while byte_recved < n:
34 |         packet = sock.recv(n - byte_recved)
35 |         if not packet:
36 |             return None
37 |         total_data.append(packet)
38 |         byte_recved = byte_recved + len(packet)
39 |     data = b''.join(total_data)
40 |     return pickle.loads(zlib.decompress(data))
41 | 
42 | 
43 | class RpcClient(object):
44 |     def __init__(self, address, port):
45 |         self.address = address
46 |         self.port = port
47 |         self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
48 |         errno = self.sock.connect_ex((address, port))
49 |         if errno != 0:
50 |             print("connectting to server failed")
51 | 
52 |     def __getattr__(self, name):
53 |         """实现远程调用功能"""
54 | 
55 |         # 执行远程调用任务
56 |         def dorpc(*args, **kwargs):
57 |             # 生成请求
58 |             req = [name, args, kwargs]
59 | 
60 |             send_msg(self.sock, req)
61 |             repb = recv_msg(self.sock)
62 |             return repb
63 | 
64 |         return dorpc
65 | 


--------------------------------------------------------------------------------
/rqdata/rpc/server.py:
--------------------------------------------------------------------------------
 1 | # -*- coding: utf-8 -*-
 2 | from rqdata.rpc.client import *
 3 | 
 4 | from six import PY2
 5 | 
 6 | if PY2:
 7 |     import SocketServer
 8 | else:
 9 |     import socketserver as SocketServer
10 | 
11 | 
12 | class RpcHandler(SocketServer.StreamRequestHandler):
13 |     __functions = {}
14 | 
15 |     def handle(self):
16 |         while self.request:
17 |             data = recv_msg(self.request)
18 |             if not data:
19 |                 break
20 |             name, args, kwargs = data
21 |             func = RpcHandler.__functions[name]
22 |             rt = func(*args, **kwargs)
23 |             send_msg(self.request, rt)
24 | 
25 |     @staticmethod
26 |     def register(func):
27 |         RpcHandler.__functions[func.__name__] = func
28 | 
29 | 
30 | def hello(a):
31 |     return 2 * a
32 | 
33 | 
34 | if __name__ == '__main__':
35 |     host = 'localhost'
36 |     port = 1111
37 |     RpcHandler.register(hello)
38 |     server = SocketServer.TCPServer((host, port), RpcHandler)
39 |     server.serve_forever()
40 | 


--------------------------------------------------------------------------------
/scripts/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/JaysonAlbert/rq-data/782b979903e7d4c01e5a42f7eac8669c84c3b776/scripts/__init__.py


--------------------------------------------------------------------------------
/scripts/future1m.py:
--------------------------------------------------------------------------------
 1 | import re
 2 | import socket
 3 | import zlib
 4 | import struct
 5 | from pymongo import MongoClient
 6 | import pymongo
 7 | import datetime
 8 | from rpc.rpc import RpcClient
 9 | 
10 | freq_int = {
11 |     '1m': 1,
12 |     '5m': 2,
13 |     '10m': 3,
14 |     '15m': 4,
15 |     '30m': 5,
16 |     '60m': 6,
17 |     '1d': 7
18 | }
19 | 
20 | 
21 | ###公司mysql数据库格式
22 | def get_future_price1(code, start_date, end_date, frequency):
23 |     df = get_price(code.upper(), start_date=start_date, end_date=end_date, frequency=frequency)
24 |     df['dateInt'] = [int(i.strftime("%Y%m%d")) for i in df.index]
25 |     df['timeInt'] = [int(i.strftime("%H%M%S")) for i in df.index]
26 |     df['realDate'] = range(df.shape[0])
27 |     df['vol'] = df['volume']
28 |     df['openInt'] = df['open_interest']
29 |     df['contract'] = re.match('[a-zA-Z]+', code).group()
30 |     df['cycle'] = freq_int[frequency]
31 |     del df['total_turnover']
32 |     del df['volume']
33 |     del df['trading_date']
34 |     del df['limit_up']
35 |     del df['limit_down']
36 |     del df['basis_spread']
37 |     del df['open_interest']
38 |     return df
39 | 
40 | 
41 | ###vnpy回测mongodb格式
42 | def get_future_price2(code, start_date, end_date, frequency):
43 |     contract = code + '0000'
44 |     df = get_price(code.upper() + '88', start_date=start_date, end_date=end_date, frequency=frequency,
45 |                    adjust_type='post')
46 |     df['vtSymbol'] = contract
47 |     df['symbol'] = contract
48 |     df['date'] = [int(i.strftime("%Y%m%d")) for i in df.index]
49 |     df['time'] = [int(i.strftime("%H%M%S")) for i in df.index]
50 |     df['datetime'] = df.index
51 |     df['openInterest'] = df.open_interest
52 | 
53 |     del df['total_turnover']
54 |     del df['trading_date']
55 |     del df['limit_up']
56 |     del df['limit_down']
57 |     del df['basis_spread']
58 |     del df['open_interest']
59 |     return df
60 | 
61 | 
62 | ###插入到vnpy回测数据库中
63 | def insert_mongo2(client, code, frequency):
64 |     start_date = client.latest_date(code + '0000', type='future')
65 |     if not start_date:
66 |         start_date = '20100104'
67 |         print(code, "no begin_date, set default: 20100104")
68 | 
69 |     else:
70 |         print("\r{}, start date: {}".format(code, start_date), end='')
71 |     start_date = datetime.datetime.strptime(str(start_date), "%Y%m%d") + datetime.timedelta(1)
72 | 
73 |     end_date = datetime.datetime.now().date()
74 |     if start_date.date() > end_date:
75 |         return
76 |     df = get_future_price2(code, start_date, end_date, frequency)
77 |     if df.empty:
78 |         return
79 |     client.insert(code + '0000', df.to_dict('record'), type="future")
80 | 
81 | 
82 | codes = ["IC", "IF", "IH", "T", "TF", "CF", "FG", "MA", "OI", "RM", "SR",
83 |          "TA", "ZC", "a", "c", "i", "j", "jd", "jm", "l", "m", "p", "pp",
84 |          "v", "y", "ag", "al", "au", "bu", "cu", "hc", "ni", "pb", "rb",
85 |          "ru", "sn", "zn"]
86 | 
87 | code = 'rb'
88 | frequency = '1m'
89 | host = "carniejq.cn"
90 | port = 31245
91 | client = RpcClient(host, port)
92 | # insert_mongo2(client,code,frequency)
93 | for code in codes:
94 |     insert_mongo2(client, code, frequency)
95 | print("\rfinished, stoping", end='')
96 | client.stop_all()


--------------------------------------------------------------------------------
/scripts/passwd.py:
--------------------------------------------------------------------------------
1 | user_name = ''
2 | 
3 | passwd = ''
4 | 


--------------------------------------------------------------------------------
/scripts/rice_auto.py:
--------------------------------------------------------------------------------
 1 | # encoding: UTF-8
 2 | from selenium import webdriver
 3 | from selenium.webdriver.common.by import By
 4 | from selenium.webdriver.support.ui import WebDriverWait
 5 | from selenium.webdriver.support import expected_conditions as EC
 6 | import time
 7 | from passwd import *
 8 | from rqdata.mongo.mongo_server import run_server
 9 | from rqdata.rpc.server import RpcHandler
10 | import threading
11 | import subprocess
12 | import sys
13 | 
14 | 
15 | def run_auto():
16 |     driver = webdriver.Chrome()
17 |     driver.get("https://www.ricequant.com/login")
18 |     time.sleep(2)
19 |     try:
20 |         element = WebDriverWait(driver, 10).until(
21 |             EC.presence_of_element_located((By.XPATH, '//*[@id="page-register"]/main/div[2]/form[4]/div[2]/input'))
22 |         )
23 |     finally:
24 |         if user_name == '':
25 |             raise Exception('place configure user name in scripts/passwd.py')
26 |         element.send_keys(user_name)
27 |     element = driver.find_element_by_xpath('//*[@id="page-register"]/main/div[2]/form[4]/div[4]/input')
28 |     if passwd == '':
29 |         raise Exception('please configure passwd in scritps/passwd.py')
30 |     element.send_keys(passwd)
31 |     element = driver.find_element_by_xpath('//div/input[@value="登录"]')
32 |     element.click()
33 |     try:
34 |         element = WebDriverWait(driver, 10).until(
35 |             lambda _: u"策略列表" in driver.page_source
36 |         )
37 |     finally:
38 |         pass
39 |     driver.get("https://www.ricequant.com/research/user/user_310960/notebooks/future.ipynb")
40 |     driver.get("https://www.ricequant.com/research/user/user_310960/notebooks/future.ipynb")
41 |     time.sleep(2)
42 |     element = driver.find_element_by_xpath('//button[@title="run cell, select below"]')
43 |     element.click()
44 |     while not stop:
45 |         pass
46 |     driver.close()
47 | 
48 | 
49 | stop = False
50 | 
51 | 
52 | def stop_all():
53 |     global stop
54 |     stop = True
55 | 
56 | 
57 | if __name__ == '__main__':
58 |     RpcHandler.register(stop_all)
59 |     t = threading.Thread(target=run_server, daemon=True)
60 |     t.start()
61 |     t1 = threading.Thread(target=run_auto, daemon=True)
62 |     t1.start()
63 | 
64 |     p=subprocess.Popen(r'C:\Users\jayso\Desktop\ngrok4.exe -config C:\Users\jayso\Desktop\.ngrok start-all',cwd=r'C:\Users\jayso\Desktop', shell=True, stdout=subprocess.PIPE)
65 | 
66 |     while not stop:
67 |         pass
68 | 
69 |     p.kill()
70 | 


--------------------------------------------------------------------------------
/scripts/rq.py:
--------------------------------------------------------------------------------
 1 | from rpc.rpc import RpcClient
 2 | import datetime
 3 | import logging
 4 | import pandas as pd
 5 | import click
 6 | import time
 7 | 
 8 | '''
 9 | rice quant code to download minute bars data
10 | 
11 | '''
12 | 
13 | 
14 | def get_stock_list():
15 |     df_instruments = all_instruments(type="CS")
16 |     stocks = list(df_instruments["order_book_id"])
17 |     stocks = list(map(lambda x: str(x), stocks))
18 |     return stocks
19 | 
20 | 
21 | def insert_one(client, code, default_start='20050104'):
22 |     start_date = client.latest_date(code)
23 |     start_date = start_date + datetime.timedelta(1) if start_date else default_start
24 |     end_date = pd.to_datetime('today').strftime("%Y%m%d")
25 |     df = get_price(code, start_date=start_date, end_date=end_date, adjust_type='none', frequency='1m')
26 |     if not df.empty:
27 |         df['dt'] = df.index
28 |         client.insert(code, df.to_dict('record'))
29 | 
30 | 
31 | def test():
32 |     host = 'www.carniejq.cn'
33 |     port = 31245
34 |     code = "000007.XSHE"
35 |     client = RpcClient(host, port)
36 |     with click.progressbar(get_stock_list(),
37 |                            label="Fetching minute bars:",
38 |                            item_show_func=lambda e: e if e is None else str(e),
39 |                            ) as bar:
40 |         bar.is_hidden = False
41 |         for code in bar:
42 |             insert_one(client, code, default_start='20171111')
43 |     client.stop_all()
44 | 
45 | 
46 | test()
47 | 


--------------------------------------------------------------------------------
/scripts/stock1m.py:
--------------------------------------------------------------------------------
 1 | from rpc.rpc import RpcClient
 2 | import datetime
 3 | import logging
 4 | import pandas as pd
 5 | 
 6 | 
 7 | def get_wind_code(order_book_id):
 8 |     stock_code = order_book_id[0:6]
 9 |     if order_book_id[9:] == "HG":
10 |         return stock_code + ".SH"
11 |     return stock_code + ".SZ"
12 | 
13 | 
14 | def get_stock_list():
15 |     df_instruments = all_instruments(type="CS")
16 |     stocks = list(df_instruments["order_book_id"])
17 |     stocks = list(map(lambda x: str(x), stocks))
18 |     return stocks
19 | 
20 | 
21 | def insert_stock(code, client):
22 |     begin_date = client.latest_date(code, type='CS')
23 |     if not begin_date:
24 |         logging.info(code + " failed, begin_date returned None")
25 |         return
26 |     begin_date = datetime.datetime.strptime(str(begin_date), "%Y%m%d") + datetime.timedelta(1)
27 |     end_date = datetime.datetime.now().date()
28 | 
29 |     df = get_price(code, start_date=begin_date, end_date=end_date, adjust_type="post",
30 |                    frequency="1d", skip_suspended=True)
31 |     if df.empty:
32 |         logging.info(code + "empty df")
33 |         return
34 |     df['time'] = [int(i.strftime("%H%M%S")) for i in df.index]
35 |     df['date'] = [i.strftime("%Y%m%d") for i in df.index]
36 |     df['amt'] = df['total_turnover']
37 |     del df['total_turnover']
38 |     group_data = []
39 |     for name, sub_group in df.groupby('date'):
40 |         dict_data = sub_group.to_dict('list')
41 |         dict_data.update({'date': sub_group['date'].iloc[0]})
42 |         dict_data.update({'order_book_id': code})
43 |         dict_data.update({'wind_code': get_wind_code(code)})
44 |         dict_data.update({'stock_code': code[:6]})
45 |         group_data.append(dict_data)
46 |     client.insert(group_data)
47 | 
48 | 
49 | def main():
50 |     # insert_stock(code,client)
51 |     logging.basicConfig(filename='stock_1m.log', level=logging.INFO)
52 |     host = 'www.carniejq.cn'
53 |     port = 1111
54 |     code = "000007.XSHE"
55 |     import time
56 | 
57 |     client = RpcClient(host, port)
58 |     count = 0
59 |     stock_list = get_stock_list()
60 |     for stock in stock_list:
61 |         insert_stock(stock, client)
62 |         if count % 50 == 0:
63 |             print("\r %f percent complete" % (float(count) / len(stock_list) * 100), end="")
64 |         count = count + 1
65 |     print("\r finished")
66 |     client.stop_all()


--------------------------------------------------------------------------------
/setup.py:
--------------------------------------------------------------------------------
 1 | #!/usr/bin/env python
 2 | 
 3 | from setuptools import setup, find_packages
 4 | 
 5 | try:
 6 |     import pypandoc
 7 | 
 8 |     long_description = pypandoc.convert('README.md', 'rst')
 9 | except (IOError, ImportError):
10 |     long_description = ''
11 | 
12 | setup(
13 |     name='rq-data',
14 |     version='0.01',
15 |     description='rq data tools',
16 |     long_description=long_description,
17 |     author='Jie Wang',
18 |     author_email='790930856@qq.com',
19 |     url='https://github.com/JaysonAlbert/rq-data',
20 |     packages=find_packages(include=['rq-data', 'rq-data.*']),
21 | 
22 |     install_requires=[
23 |         'selenium',
24 |         'pymongo'
25 |     ]
26 | )
27 | 


--------------------------------------------------------------------------------
/tests/__init__.py:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/JaysonAlbert/rq-data/782b979903e7d4c01e5a42f7eac8669c84c3b776/tests/__init__.py


--------------------------------------------------------------------------------
/tests/test_server.py:
--------------------------------------------------------------------------------
 1 | from six import PY2
 2 | 
 3 | if PY2:
 4 |     import SocketServer
 5 | else:
 6 |     import socketserver as SocketServer
 7 | from rqdata.rpc.server import RpcHandler
 8 | import pickle
 9 | 
10 | host = 'localhost'
11 | port = 1111
12 | RpcHandler.register(print)
13 | server = SocketServer.TCPServer((host, port), RpcHandler)
14 | server.serve_forever()


--------------------------------------------------------------------------------