![\"Open](\"https://colab.research.google.com/assets/colab-badge.svg\"){kind=icon}
"
11 | ]
12 | },
13 | {
14 | "cell_type": "code",
15 | "execution_count": 6,
16 | "metadata": {
17 | "id": "dlNzx2ic10Iu"
18 | },
19 | "outputs": [],
20 | "source": [
21 | "import concurrent.futures\n",
22 | "import os"
23 | ]
24 | },
25 | {
26 | "cell_type": "code",
27 | "source": [
28 | "def env(arr):\n",
29 | " print(arr)\n",
30 | "\n",
31 | "\n",
32 | "def worker(x, y):\n",
33 | " os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"\"\n",
34 | " r = x + y\n",
35 | " return r\n",
36 | "\n",
37 | "with concurrent.futures.ProcessPoolExecutor(max_workers=4) as executor:\n",
38 | " f1 = executor.submit(worker, 2, 5)\n",
39 | " f2 = executor.submit(worker, 2, 5)\n",
40 | " f3 = executor.submit(worker, 2, 5)\n",
41 | " f4 = executor.submit(worker, 2, 5)\n",
42 | " arr = [f1.result(),f2.result(),f3.result(),f4.result()]\n",
43 | " env(arr)"
44 | ],
45 | "metadata": {
46 | "colab": {
47 | "base_uri": "https://localhost:8080/"
48 | },
49 | "id": "8Lf9QnmZaQbu",
50 | "outputId": "3d87e9f6-bca5-4ab3-ac3d-f7a6e55bde81"
51 | },
52 | "execution_count": 8,
53 | "outputs": [
54 | {
55 | "output_type": "stream",
56 | "name": "stdout",
57 | "text": [
58 | "[7, 7, 7, 7]\n"
59 | ]
60 | }
61 | ]
62 | }
63 | ],
64 | "metadata": {
65 | "accelerator": "GPU",
66 | "colab": {
67 | "machine_shape": "hm",
68 | "name": "MultiProsessByChangeCpu.ipynb",
69 | "provenance": [],
70 | "collapsed_sections": [],
71 | "authorship_tag": "ABX9TyMyIv0QpTXSbw9a+mln7yKu",
72 | "include_colab_link": true
73 | },
74 | "kernelspec": {
75 | "display_name": "Python 3",
76 | "name": "python3"
77 | },
78 | "language_info": {
79 | "name": "python"
80 | }
81 | },
82 | "nbformat": 4,
83 | "nbformat_minor": 0
84 | }
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 | # 未来を拓く投資ツール：深層強化学習自動売買ツール
2 |
3 | [](https://www.python.org/)
4 | [](https://www.tensorflow.org/)
5 | [](https://colab.research.google.com/)
6 | [](https://pandas.pydata.org/)
7 | [](https://scikit-learn.org/stable/)
8 | [](https://matplotlib.org/)
9 | [](https://numpy.org/)
10 |
11 | ## はじめに
12 |
13 | 深層強化学習を用いた自動売買ツールは、人工知能の力で投資の未来を拓く革新的なツールです。市場の動きを精緻に分析し、高精度な売買シグナルを生成することで、投資の成功確率を大幅に向上させます。
14 |
15 | ## 特長
16 |
17 | + 高精度な予測: 深層強化学習モデルにより、市場の動きを精緻に分析し、高精度な売買シグナルを生成します。
18 | + 自動化による効率化: 煩雑な売買操作を自動化し、時間と労力を大幅に削減します。
19 | + データに基づいた意思決定: 過去のデータに基づいて学習し、客観的な判断で売買を行います。
20 | + 柔軟なカスタマイズ: パラメータを調整することで、様々な投資戦略に対応できます。
21 |
22 |
23 | ## 利用シーン
24 |
25 | + 株式投資
26 | + FX
27 | + 仮想通貨取引
28 | + その他、様々な金融商品
29 |
30 | ## 収録アルゴリズム
31 | + Q-Learning
32 | + Double Q-Learning
33 | + Prioritized Experience Replay DQN
34 | + Categorical DQN
35 | + DDQN
36 | + Rainbow
37 | + SARSA
38 | + A2C
39 | + A3C
40 | + PPO
41 | + IMPALA
42 | + Ape-X
43 | + GORILA
44 | + PDD-DQN
45 | + MuZero
46 | + R2D2
47 | + DRQN (Burn-In)
48 | + DRQN (Sequential)
49 | + DRQN
50 |
51 | ## リンク一覧(Qiita記事)
52 | ### 各アルゴリズムの詳細:
53 |
54 | + Q-Learning: (https://qiita.com/sugiyama404/items/5ac7043c7d3c2edd043a)
55 | + SARSA: [(https://qiita.com/sugiyama404/items/5ac7043c7d3c2edd043a)]
56 | + Double Q-Learning: [https://qiita.com/sugiyama404/items/6a1958079ea14b4ceae7]
57 | + Prioritized Experience Replay DQN: (https://qiita.com/sugiyama404/items/430457524005e828a53f)
58 | + Categorical DQN: (https://qiita.com/sugiyama404/items/430457524005e828a53f)
59 | + DDQN: (https://qiita.com/sugiyama404/items/430457524005e828a53f)
60 | + Rainbow: (https://qiita.com/sugiyama404/items/430457524005e828a53f)
61 | + A2C: (https://qiita.com/sugiyama404/items/059da76f6d0468339104)
62 | + A3C: (https://qiita.com/sugiyama404/items/059da76f6d0468339104)
63 | + PPO: (https://qiita.com/sugiyama404/items/c27eb01da59aeca0edb6)
64 | + IMPALA: (https://qiita.com/sugiyama404/items/ea99d138abb682032056)
65 | + Ape-X: (https://qiita.com/sugiyama404/items/430457524005e828a53f)
66 | + GORILA: (https://qiita.com/sugiyama404/items/a376247c68bb265ba07a)
67 | + PDD-DQN: (https://qiita.com/sugiyama404/items/430457524005e828a53f)
68 | + MuZero: (https://qiita.com/sugiyama404/items/ebdee31ac2bd38361b06)
69 | + R2D2: (https://qiita.com/sugiyama404/items/737a761554a7d5fdcfdc)
70 | + DRQN (Burn-In): (https://qiita.com/sugiyama404/items/737a761554a7d5fdcfdc)
71 | + DRQN (Sequential): (https://qiita.com/sugiyama404/items/737a761554a7d5fdcfdc)
72 | + DRQN: (https://qiita.com/sugiyama404/items/737a761554a7d5fdcfdc)
73 |
74 |
75 |
76 |
77 |
78 |
79 |
80 |
81 |
82 |
83 |
84 |
--------------------------------------------------------------------------------
/gym_trading.ipynb:
--------------------------------------------------------------------------------
1 | {
2 | "nbformat": 4,
3 | "nbformat_minor": 0,
4 | "metadata": {
5 | "kernelspec": {
6 | "display_name": "Python 3",
7 | "language": "python",
8 | "name": "python3"
9 | },
10 | "language_info": {
11 | "codemirror_mode": {
12 | "name": "ipython",
13 | "version": 3
14 | },
15 | "file_extension": ".py",
16 | "mimetype": "text/x-python",
17 | "name": "python",
18 | "nbconvert_exporter": "python",
19 | "pygments_lexer": "ipython3",
20 | "version": "3.7.3"
21 | },
22 | "colab": {
23 | "name": "gym_trading.ipynb",
24 | "private_outputs": true,
25 | "provenance": [],
26 | "collapsed_sections": [],
27 | "include_colab_link": true
28 | },
29 | "accelerator": "GPU"
30 | },
31 | "cells": [
32 | {
33 | "cell_type": "markdown",
34 | "metadata": {
35 | "id": "view-in-github",
36 | "colab_type": "text"
37 | },
38 | "source": [
39 | ""
40 | ]
41 | },
42 | {
43 | "cell_type": "code",
44 | "metadata": {
45 | "id": "t1H9WnXDrVaW"
46 | },
47 | "source": [
48 | "!pip install gym-anytrading"
49 | ],
50 | "execution_count": null,
51 | "outputs": []
52 | },
53 | {
54 | "cell_type": "code",
55 | "metadata": {
56 | "id": "lsXiJ1Cjz49_"
57 | },
58 | "source": [
59 | "import gym\n",
60 | "import gym_anytrading\n",
61 | "\n",
62 | "import numpy as np\n",
63 | "import pandas as pd\n",
64 | "from matplotlib import pyplot as plt\n",
65 | "from google.colab import drive\n",
66 | "\n",
67 | "import random\n",
68 | "\n",
69 | "drive.mount('/content/drive/')\n",
70 | "nov_dir = 'Colab Notebooks/dataset/reinforcement_learning/'\n",
71 | "nov_path = '/content/drive/My Drive/' + nov_dir + 'sp500_candle.csv'\n",
72 | "\n",
73 | "df = pd.read_csv(nov_path)\n",
74 | "df['Date'] = pd.to_datetime(df['Date'], format = '%Y-%m-%d')\n",
75 | "df.set_index('Date', inplace=True)"
76 | ],
77 | "execution_count": null,
78 | "outputs": []
79 | },
80 | {
81 | "cell_type": "code",
82 | "metadata": {
83 | "id": "loKT5IYzz4-B"
84 | },
85 | "source": [
86 | "env = gym.make('stocks-v0', df=df, frame_bound=(5,100), window_size=5)"
87 | ],
88 | "execution_count": null,
89 | "outputs": []
90 | },
91 | {
92 | "cell_type": "code",
93 | "metadata": {
94 | "id": "Xp9sD3Wrz4-h"
95 | },
96 | "source": [
97 | "state = env.reset()\n",
98 | "while True: \n",
99 | " action = random.randint(0,2)\n",
100 | " n_state, reward, done, info = env.step(action)\n",
101 | " if done: \n",
102 | " print(\"info\", info)\n",
103 | " break\n",
104 | " \n",
105 | "plt.figure(figsize=(15,6))\n",
106 | "plt.cla()\n",
107 | "env.render_all()\n",
108 | "plt.show()"
109 | ],
110 | "execution_count": null,
111 | "outputs": []
112 | }
113 | ]
114 | }
--------------------------------------------------------------------------------
/Control/random_test.ipynb:
--------------------------------------------------------------------------------
1 | {
2 | "nbformat": 4,
3 | "nbformat_minor": 0,
4 | "metadata": {
5 | "colab": {
6 | "name": "random_test.ipynb",
7 | "provenance": [],
8 | "collapsed_sections": [],
9 | "machine_shape": "hm",
10 | "authorship_tag": "ABX9TyM5m0p/UkhYwip300xkGuC0",
11 | "include_colab_link": true
12 | },
13 | "kernelspec": {
14 | "name": "python3",
15 | "display_name": "Python 3"
16 | },
17 | "language_info": {
18 | "name": "python"
19 | },
20 | "accelerator": "GPU"
21 | },
22 | "cells": [
23 | {
24 | "cell_type": "markdown",
25 | "metadata": {
26 | "id": "view-in-github",
27 | "colab_type": "text"
28 | },
29 | "source": [
30 | ""
31 | ]
32 | },
33 | {
34 | "cell_type": "code",
35 | "metadata": {
36 | "colab": {
37 | "base_uri": "https://localhost:8080/"
38 | },
39 | "id": "3NIXg6mTzk0K",
40 | "outputId": "50a441bb-290c-44f8-b37e-7429f998bd4c"
41 | },
42 | "source": [
43 | "import pandas as pd\n",
44 | "import numpy as np\n",
45 | "import random\n",
46 | "from google.colab import drive\n",
47 | "import copy\n",
48 | "\n",
49 | "from datetime import datetime\n",
50 | "from matplotlib import pyplot as plt\n",
51 | "from tensorflow.keras.utils import Progbar\n",
52 | "\n",
53 | "mode = 'test'\n",
54 | "name = 'random'\n",
55 | "level = 1\n",
56 | "if level == 2:\n",
57 | " name += name + 'lv2'\n",
58 | "\n",
59 | "drive.mount('/content/drive/')\n",
60 | "nov_dir = 'Colab Notebooks/dataset/reinforcement_learning/'\n",
61 | "nov_path = '/content/drive/My Drive/' + nov_dir + f'sp500_{mode}.csv'\n",
62 | "\n",
63 | "exp_dir = 'Colab Notebooks/workspace/export/'\n",
64 | "mdl_dir = '/content/drive/My Drive/' + exp_dir + 'models'\n",
65 | "csv_path = '/content/drive/My Drive/' + exp_dir + f'csv_data/{name}_{mode}.csv'\n",
66 | "\n",
67 | "df = pd.read_csv(nov_path)\n",
68 | "df['Date'] = pd.to_datetime(df['Date'], format = '%Y-%m-%d')"
69 | ],
70 | "execution_count": 1,
71 | "outputs": [
72 | {
73 | "output_type": "stream",
74 | "name": "stdout",
75 | "text": [
76 | "Mounted at /content/drive/\n"
77 | ]
78 | }
79 | ]
80 | },
81 | {
82 | "cell_type": "code",
83 | "metadata": {
84 | "id": "MN1DKfV6zauY"
85 | },
86 | "source": [
87 | "class Environment:\n",
88 | " def __init__(self, df, initial_money=100000, mode = 'test', commission = 0):\n",
89 | "\n",
90 | " self.df = df.dropna().reset_index()\n",
91 | "\n",
92 | " self.df_total_steps = len(self.df)-1\n",
93 | " self.initial_money = initial_money\n",
94 | " self.mode = mode\n",
95 | " self.commission = commission\n",
96 | " self.trade_time = None\n",
97 | " self.trade_win = None\n",
98 | " self.brfore_buy_cash = None\n",
99 | " self.action_space = np.array([0, 1, 2]) # buy,hold,sell\n",
100 | " self.hold_a_position = None\n",
101 | " self.now_price = None\n",
102 | " self.cash_in_hand = None\n",
103 | " self.sell_price = None\n",
104 | " self.buy_price = None\n",
105 | "\n",
106 | " self.reset()\n",
107 | " \n",
108 | " def reset(self):\n",
109 | "\n",
110 | " self.trade_time = 0\n",
111 | " self.trade_win = 0\n",
112 | " self.brfore_buy_cash = 0\n",
113 | " self.end_step = self.df_total_steps\n",
114 | " self.now_step = 0\n",
115 | " self.hold_a_position = 0.0\n",
116 | " self.now_price = self.df.loc[self.now_step, 'SP500']\n",
117 | " self.cash_in_hand = self.initial_money\n",
118 | "\n",
119 | " return self._get_now_state()\n",
120 | "\n",
121 | " def step(self, action):\n",
122 | "\n",
123 | " prev_revenue = self._get_revenue()\n",
124 | " self.now_step += 1\n",
125 | " self.now_price = self.df.loc[self.now_step, 'SP500']\n",
126 | " \n",
127 | " done = (self.end_step == self.now_step)\n",
128 | "\n",
129 | " self._trade(action,done)\n",
130 | " cur_revenue = self._get_revenue()\n",
131 | " \n",
132 | " reward = cur_revenue - prev_revenue\n",
133 | "\n",
134 | " info = { 'cur_revenue' : cur_revenue , 'trade_time' : self.trade_time, 'trade_win' : self.trade_win }\n",
135 | "\n",
136 | " return self._get_now_state(), reward, done, info\n",
137 | "\n",
138 | " def _get_now_state(self):\n",
139 | " state = np.empty(3)\n",
140 | " state[0] = self.hold_a_position\n",
141 | " state[1] = self.now_price\n",
142 | " state[2] = self.cash_in_hand\n",
143 | " return state\n",
144 | "\n",
145 | " def _get_revenue(self): \n",
146 | " return self.hold_a_position * self.now_price + self.cash_in_hand\n",
147 | "\n",
148 | " def _trade(self, action,lastorder = False):\n",
149 | " if lastorder:\n",
150 | " self.cash_in_hand += self.now_price * self.hold_a_position\n",
151 | " self.hold_a_position = 0\n",
152 | " self.trade_time += 1\n",
153 | " if self.cash_in_hand > self.brfore_buy_cash:\n",
154 | " self.trade_win += 1\n",
155 | " else:\n",
156 | " if self.action_space[0] == action: # buy\n",
157 | " if self.hold_a_position == 0:\n",
158 | " buy_flag = True\n",
159 | " self.brfore_buy_cash = copy.copy(self.cash_in_hand)\n",
160 | " while buy_flag:\n",
161 | " if self.cash_in_hand > self.now_price:\n",
162 | " self.hold_a_position += 1\n",
163 | " self.cash_in_hand -= self.now_price + self.commission * self.now_price\n",
164 | " else:\n",
165 | " buy_flag = False\n",
166 | " if self.action_space[2] == action: # sell\n",
167 | " if self.hold_a_position != 0:\n",
168 | " self.cash_in_hand += self.now_price * self.hold_a_position - self.commission * self.now_price * self.hold_a_position\n",
169 | " self.hold_a_position = 0\n",
170 | " self.trade_time += 1\n",
171 | " if self.cash_in_hand > self.brfore_buy_cash:\n",
172 | " self.trade_win += 1"
173 | ],
174 | "execution_count": 2,
175 | "outputs": []
176 | },
177 | {
178 | "cell_type": "code",
179 | "metadata": {
180 | "id": "On5S8YtLz3U4"
181 | },
182 | "source": [
183 | "class Main:\n",
184 | " def __init__(self, env, mdl_dir, name, episodes_times = 1000, mode = 'test'):\n",
185 | " self.env = env\n",
186 | " self.mdl_dir = mdl_dir\n",
187 | " self.episodes_times = episodes_times\n",
188 | " self.mode = mode\n",
189 | " self.name = name\n",
190 | "\n",
191 | " with open(csv_path, 'w') as f:\n",
192 | " row = 'FixedProfit,TradeTimes,TradeWin'\n",
193 | " print(row, file=f)\n",
194 | "\n",
195 | "\n",
196 | " def play_game(self):\n",
197 | "\n",
198 | " for episode in range(self.episodes_times):\n",
199 | "\n",
200 | " if (episode % 10 == 0):\n",
201 | " metrics_names = ['FixedProfit','TradeTimes','TradeWin']\n",
202 | " pb_i = Progbar(10, stateful_metrics=metrics_names)\n",
203 | " p_mean,trade_time,win_time = np.array([]),np.array([]),np.array([])\n",
204 | "\n",
205 | " state = self.env.reset()\n",
206 | " done = False\n",
207 | " \n",
208 | " while not done:\n",
209 | " action = random.randrange(3)\n",
210 | " next_state, reward, done, info = self.env.step(action)\n",
211 | " state = next_state\n",
212 | "\n",
213 | " p_mean,trade_time,win_time = np.append(p_mean,info['cur_revenue']),np.append(trade_time,info['trade_time']),np.append(win_time,info['trade_win'])\n",
214 | " values=[('FixedProfit',int(np.mean(p_mean))), ('TradeTimes',int(np.mean(trade_time))), ('TradeWin',int(np.mean(win_time)))]\n",
215 | " pb_i.add(1, values=values)\n",
216 | " with open(csv_path, 'a') as f:\n",
217 | " row = str(info['cur_revenue']) + ',' + str(info['trade_time']) + ',' + str(info['trade_win'])\n",
218 | " print(row, file=f)"
219 | ],
220 | "execution_count": 3,
221 | "outputs": []
222 | },
223 | {
224 | "cell_type": "code",
225 | "metadata": {
226 | "colab": {
227 | "base_uri": "https://localhost:8080/"
228 | },
229 | "id": "pYFNVDDQz9X9",
230 | "outputId": "6f24ad11-1ef3-40db-c418-002e8a92c126"
231 | },
232 | "source": [
233 | "initial_money=1000000\n",
234 | "episodes_times = 100\n",
235 | "commission = 0 if level == 1 else 0.02\n",
236 | "\n",
237 | "env = Environment(df, initial_money = initial_money, mode = mode, commission = commission)\n",
238 | "main = Main(env, mdl_dir, name, episodes_times, mode)\n",
239 | "main.play_game()"
240 | ],
241 | "execution_count": 4,
242 | "outputs": [
243 | {
244 | "output_type": "stream",
245 | "name": "stdout",
246 | "text": [
247 | "10/10 [==============================] - 1s 62ms/step - FixedProfit: 1333910.0000 - TradeTimes: 143.0000 - TradeWin: 88.0000\n",
248 | "10/10 [==============================] - 1s 63ms/step - FixedProfit: 1264110.0000 - TradeTimes: 147.0000 - TradeWin: 90.0000\n",
249 | "10/10 [==============================] - 1s 62ms/step - FixedProfit: 1357613.0000 - TradeTimes: 150.0000 - TradeWin: 92.0000\n",
250 | "10/10 [==============================] - 1s 61ms/step - FixedProfit: 1302067.0000 - TradeTimes: 146.0000 - TradeWin: 90.0000\n",
251 | "10/10 [==============================] - 1s 62ms/step - FixedProfit: 1314876.0000 - TradeTimes: 145.0000 - TradeWin: 88.0000\n",
252 | "10/10 [==============================] - 1s 59ms/step - FixedProfit: 1249883.0000 - TradeTimes: 145.0000 - TradeWin: 89.0000\n",
253 | "10/10 [==============================] - 1s 62ms/step - FixedProfit: 1284587.0000 - TradeTimes: 147.0000 - TradeWin: 88.0000\n",
254 | "10/10 [==============================] - 1s 65ms/step - FixedProfit: 1336275.0000 - TradeTimes: 148.0000 - TradeWin: 90.0000\n",
255 | "10/10 [==============================] - 1s 60ms/step - FixedProfit: 1248156.0000 - TradeTimes: 145.0000 - TradeWin: 91.0000\n",
256 | "10/10 [==============================] - 1s 61ms/step - FixedProfit: 1212798.0000 - TradeTimes: 146.0000 - TradeWin: 89.0000\n"
257 | ]
258 | }
259 | ]
260 | }
261 | ]
262 | }
--------------------------------------------------------------------------------
/Control/random2_test.ipynb:
--------------------------------------------------------------------------------
1 | {
2 | "nbformat": 4,
3 | "nbformat_minor": 0,
4 | "metadata": {
5 | "colab": {
6 | "name": "random2_test.ipynb",
7 | "provenance": [],
8 | "collapsed_sections": [],
9 | "machine_shape": "hm",
10 | "authorship_tag": "ABX9TyOTnvcPA+0Ndv5JJB/CHh8e",
11 | "include_colab_link": true
12 | },
13 | "kernelspec": {
14 | "name": "python3",
15 | "display_name": "Python 3"
16 | },
17 | "language_info": {
18 | "name": "python"
19 | },
20 | "accelerator": "GPU"
21 | },
22 | "cells": [
23 | {
24 | "cell_type": "markdown",
25 | "metadata": {
26 | "id": "view-in-github",
27 | "colab_type": "text"
28 | },
29 | "source": [
30 | ""
31 | ]
32 | },
33 | {
34 | "cell_type": "code",
35 | "metadata": {
36 | "colab": {
37 | "base_uri": "https://localhost:8080/"
38 | },
39 | "id": "3NIXg6mTzk0K",
40 | "outputId": "01176e8a-913c-4779-812c-df8f628e9e4e"
41 | },
42 | "source": [
43 | "import pandas as pd\n",
44 | "import numpy as np\n",
45 | "import random\n",
46 | "from google.colab import drive\n",
47 | "import copy\n",
48 | "\n",
49 | "from datetime import datetime\n",
50 | "from matplotlib import pyplot as plt\n",
51 | "from tensorflow.keras.utils import Progbar\n",
52 | "\n",
53 | "\n",
54 | "mode = 'test'\n",
55 | "name = 'random'\n",
56 | "level = 2\n",
57 | "if level == 2:\n",
58 | " name += name + 'lv2'\n",
59 | "\n",
60 | "drive.mount('/content/drive/')\n",
61 | "nov_dir = 'Colab Notebooks/dataset/reinforcement_learning/'\n",
62 | "nov_path = '/content/drive/My Drive/' + nov_dir + f'sp500_{mode}.csv'\n",
63 | "\n",
64 | "exp_dir = 'Colab Notebooks/workspace/export/'\n",
65 | "mdl_dir = '/content/drive/My Drive/' + exp_dir + 'models'\n",
66 | "csv_path = '/content/drive/My Drive/' + exp_dir + f'csv_data/{name}_{mode}.csv'\n",
67 | "\n",
68 | "df = pd.read_csv(nov_path)\n",
69 | "df['Date'] = pd.to_datetime(df['Date'], format = '%Y-%m-%d')"
70 | ],
71 | "execution_count": null,
72 | "outputs": [
73 | {
74 | "output_type": "stream",
75 | "name": "stdout",
76 | "text": [
77 | "Mounted at /content/drive/\n"
78 | ]
79 | }
80 | ]
81 | },
82 | {
83 | "cell_type": "code",
84 | "metadata": {
85 | "id": "MN1DKfV6zauY"
86 | },
87 | "source": [
88 | "class Environment:\n",
89 | " def __init__(self, df, initial_money=100000, mode = 'test', commission = 0):\n",
90 | "\n",
91 | " self.df = df.dropna().reset_index()\n",
92 | "\n",
93 | " self.df_total_steps = len(self.df)-1\n",
94 | " self.initial_money = initial_money\n",
95 | " self.mode = mode\n",
96 | " self.commission = commission\n",
97 | " self.trade_time = None\n",
98 | " self.trade_win = None\n",
99 | " self.brfore_buy_cash = None\n",
100 | " self.action_space = np.array([0, 1, 2]) # buy,hold,sell\n",
101 | " self.hold_a_position = None\n",
102 | " self.now_price = None\n",
103 | " self.cash_in_hand = None\n",
104 | " self.sell_price = None\n",
105 | " self.buy_price = None\n",
106 | "\n",
107 | " self.reset()\n",
108 | " \n",
109 | " def reset(self):\n",
110 | "\n",
111 | " self.trade_time = 0\n",
112 | " self.trade_win = 0\n",
113 | " self.brfore_buy_cash = 0\n",
114 | " self.end_step = self.df_total_steps\n",
115 | " self.now_step = 0\n",
116 | " self.hold_a_position = 0.0\n",
117 | " self.now_price = self.df.loc[self.now_step, 'SP500']\n",
118 | " self.cash_in_hand = self.initial_money\n",
119 | " self.sell_price = 0\n",
120 | " self.buy_price = 0\n",
121 | "\n",
122 | " return self._get_now_state()\n",
123 | "\n",
124 | " def step(self, action):\n",
125 | "\n",
126 | " self.now_step += 1\n",
127 | " self.now_price = self.df.loc[self.now_step, 'SP500']\n",
128 | " \n",
129 | " done = (self.end_step == self.now_step)\n",
130 | "\n",
131 | " self.sell_price = 0\n",
132 | " self._trade(action,done)\n",
133 | " reward = 0\n",
134 | " if (self.sell_price > 0) and (self.buy_price > 0) and ((self.sell_price - self.buy_price) != 0):\n",
135 | " reward = (self.sell_price - self.buy_price) / self.buy_price\n",
136 | " self.buy_price = 0\n",
137 | " cur_revenue = self._get_revenue()\n",
138 | " \n",
139 | " info = { 'cur_revenue' : cur_revenue , 'trade_time' : self.trade_time, 'trade_win' : self.trade_win }\n",
140 | "\n",
141 | " return self._get_now_state(), reward, done, info\n",
142 | "\n",
143 | " def _get_now_state(self):\n",
144 | " state = np.empty(3)\n",
145 | " state[0] = self.hold_a_position\n",
146 | " state[1] = self.now_price\n",
147 | " state[2] = self.cash_in_hand\n",
148 | " return state\n",
149 | "\n",
150 | " def _get_revenue(self): \n",
151 | " return self.hold_a_position * self.now_price + self.cash_in_hand\n",
152 | "\n",
153 | " def _trade(self, action,lastorder = False):\n",
154 | " if lastorder:\n",
155 | " if self.hold_a_position != 0:\n",
156 | " self.cash_in_hand += self.now_price * self.hold_a_position\n",
157 | " self.hold_a_position = 0\n",
158 | " self.trade_time += 1\n",
159 | " if self.cash_in_hand > self.brfore_buy_cash:\n",
160 | " self.trade_win += 1\n",
161 | " else:\n",
162 | " if self.action_space[0] == action: # buy\n",
163 | " if self.hold_a_position == 0:\n",
164 | " buy_flag = True\n",
165 | " self.brfore_buy_cash = copy.copy(self.cash_in_hand)\n",
166 | " while buy_flag:\n",
167 | " if self.cash_in_hand > self.now_price:\n",
168 | " self.hold_a_position += 1\n",
169 | " self.buy_price += self.now_price\n",
170 | " self.cash_in_hand -= self.now_price + self.commission * self.now_price\n",
171 | " else:\n",
172 | " buy_flag = False\n",
173 | " if self.action_space[2] == action: # sell\n",
174 | " if self.hold_a_position != 0:\n",
175 | " self.sell_price += self.now_price * self.hold_a_position\n",
176 | " self.cash_in_hand += self.now_price * self.hold_a_position - self.commission * self.now_price * self.hold_a_position\n",
177 | " self.hold_a_position = 0\n",
178 | " self.trade_time += 1\n",
179 | " if self.cash_in_hand > self.brfore_buy_cash:\n",
180 | " self.trade_win += 1"
181 | ],
182 | "execution_count": null,
183 | "outputs": []
184 | },
185 | {
186 | "cell_type": "code",
187 | "metadata": {
188 | "id": "On5S8YtLz3U4"
189 | },
190 | "source": [
191 | "class Main:\n",
192 | " def __init__(self, env, mdl_dir, name, episodes_times = 1000, mode = 'test'):\n",
193 | " self.env = env\n",
194 | " self.mdl_dir = mdl_dir\n",
195 | " self.episodes_times = episodes_times\n",
196 | " self.mode = mode\n",
197 | " self.name = name\n",
198 | "\n",
199 | " with open(csv_path, 'w') as f:\n",
200 | " row = 'FixedProfit,TradeTimes,TradeWin'\n",
201 | " print(row, file=f)\n",
202 | "\n",
203 | "\n",
204 | " def play_game(self):\n",
205 | "\n",
206 | " for episode in range(self.episodes_times):\n",
207 | "\n",
208 | " if (episode % 10 == 0):\n",
209 | " metrics_names = ['FixedProfit','TradeTimes','TradeWin']\n",
210 | " if (int(str(self.episodes_times)[:-1])*10 == episode):\n",
211 | " pb_i = Progbar(int(str(self.episodes_times)[-1]), stateful_metrics=metrics_names)\n",
212 | " else:\n",
213 | " pb_i = Progbar(10, stateful_metrics=metrics_names)\n",
214 | " p_mean,trade_time,win_time = np.array([]),np.array([]),np.array([])\n",
215 | "\n",
216 | " state = self.env.reset()\n",
217 | " done = False\n",
218 | " \n",
219 | " while not done:\n",
220 | " action = random.randrange(3)\n",
221 | " next_state, reward, done, info = self.env.step(action)\n",
222 | " state = next_state\n",
223 | "\n",
224 | " p_mean,trade_time,win_time = np.append(p_mean,info['cur_revenue']),np.append(trade_time,info['trade_time']),np.append(win_time,info['trade_win'])\n",
225 | " values=[('FixedProfit',int(np.mean(p_mean))), ('TradeTimes',int(np.mean(trade_time))), ('TradeWin',int(np.mean(win_time)))]\n",
226 | " pb_i.add(1, values=values)\n",
227 | " with open(csv_path, 'a') as f:\n",
228 | " row = str(info['cur_revenue']) + ',' + str(info['trade_time']) + ',' + str(info['trade_win'])\n",
229 | " print(row, file=f)"
230 | ],
231 | "execution_count": null,
232 | "outputs": []
233 | },
234 | {
235 | "cell_type": "code",
236 | "metadata": {
237 | "colab": {
238 | "base_uri": "https://localhost:8080/"
239 | },
240 | "id": "pYFNVDDQz9X9",
241 | "outputId": "83ff2a86-1380-4f94-ab01-e63629b464f9"
242 | },
243 | "source": [
244 | "initial_money=1000000\n",
245 | "episodes_times = 100\n",
246 | "commission = 0 if level == 1 else 0.002\n",
247 | "\n",
248 | "env = Environment(df, initial_money = initial_money, mode = mode, commission = commission)\n",
249 | "main = Main(env, mdl_dir, name, episodes_times, mode)\n",
250 | "main.play_game()"
251 | ],
252 | "execution_count": null,
253 | "outputs": [
254 | {
255 | "output_type": "stream",
256 | "name": "stdout",
257 | "text": [
258 | "10/10 [==============================] - 0s 41ms/step - FixedProfit: 671232.0000 - TradeTimes: 146.0000 - TradeWin: 64.0000\n",
259 | "10/10 [==============================] - 0s 44ms/step - FixedProfit: 621698.0000 - TradeTimes: 144.0000 - TradeWin: 63.0000\n",
260 | "10/10 [==============================] - 0s 43ms/step - FixedProfit: 720766.0000 - TradeTimes: 147.0000 - TradeWin: 65.0000\n",
261 | "10/10 [==============================] - 0s 44ms/step - FixedProfit: 734634.0000 - TradeTimes: 149.0000 - TradeWin: 67.0000\n",
262 | "10/10 [==============================] - 0s 46ms/step - FixedProfit: 709434.0000 - TradeTimes: 146.0000 - TradeWin: 63.0000\n",
263 | "10/10 [==============================] - 0s 43ms/step - FixedProfit: 716473.0000 - TradeTimes: 149.0000 - TradeWin: 67.0000\n",
264 | "10/10 [==============================] - 0s 43ms/step - FixedProfit: 745040.0000 - TradeTimes: 147.0000 - TradeWin: 67.0000\n",
265 | "10/10 [==============================] - 0s 44ms/step - FixedProfit: 723518.0000 - TradeTimes: 146.0000 - TradeWin: 64.0000\n",
266 | "10/10 [==============================] - 0s 42ms/step - FixedProfit: 652959.0000 - TradeTimes: 146.0000 - TradeWin: 64.0000\n",
267 | "10/10 [==============================] - 0s 43ms/step - FixedProfit: 702500.0000 - TradeTimes: 149.0000 - TradeWin: 68.0000\n"
268 | ]
269 | }
270 | ]
271 | }
272 | ]
273 | }
--------------------------------------------------------------------------------
/get_csv.ipynb:
--------------------------------------------------------------------------------
1 | {
2 | "nbformat": 4,
3 | "nbformat_minor": 0,
4 | "metadata": {
5 | "colab": {
6 | "name": "get_csv.ipynb",
7 | "provenance": [],
8 | "collapsed_sections": [],
9 | "authorship_tag": "ABX9TyM44ijTGo+rFgmSGZnsFct4",
10 | "include_colab_link": true
11 | },
12 | "kernelspec": {
13 | "name": "python3",
14 | "display_name": "Python 3"
15 | },
16 | "language_info": {
17 | "name": "python"
18 | }
19 | },
20 | "cells": [
21 | {
22 | "cell_type": "markdown",
23 | "metadata": {
24 | "id": "view-in-github",
25 | "colab_type": "text"
26 | },
27 | "source": [
28 | ""
29 | ]
30 | },
31 | {
32 | "cell_type": "code",
33 | "metadata": {
34 | "colab": {
35 | "base_uri": "https://localhost:8080/"
36 | },
37 | "id": "g0nf8SnkYxVo",
38 | "outputId": "13e9313c-02a7-4177-ab56-21eb6f365932"
39 | },
40 | "source": [
41 | "!pip install yfinance"
42 | ],
43 | "execution_count": null,
44 | "outputs": [
45 | {
46 | "output_type": "stream",
47 | "text": [
48 | "Requirement already satisfied: yfinance in /usr/local/lib/python3.7/dist-packages (0.1.62)\n",
49 | "Requirement already satisfied: numpy>=1.15 in /usr/local/lib/python3.7/dist-packages (from yfinance) (1.19.5)\n",
50 | "Requirement already satisfied: pandas>=0.24 in /usr/local/lib/python3.7/dist-packages (from yfinance) (1.1.5)\n",
51 | "Requirement already satisfied: lxml>=4.5.1 in /usr/local/lib/python3.7/dist-packages (from yfinance) (4.6.3)\n",
52 | "Requirement already satisfied: requests>=2.20 in /usr/local/lib/python3.7/dist-packages (from yfinance) (2.23.0)\n",
53 | "Requirement already satisfied: multitasking>=0.0.7 in /usr/local/lib/python3.7/dist-packages (from yfinance) (0.0.9)\n",
54 | "Requirement already satisfied: python-dateutil>=2.7.3 in /usr/local/lib/python3.7/dist-packages (from pandas>=0.24->yfinance) (2.8.1)\n",
55 | "Requirement already satisfied: pytz>=2017.2 in /usr/local/lib/python3.7/dist-packages (from pandas>=0.24->yfinance) (2018.9)\n",
56 | "Requirement already satisfied: chardet<4,>=3.0.2 in /usr/local/lib/python3.7/dist-packages (from requests>=2.20->yfinance) (3.0.4)\n",
57 | "Requirement already satisfied: idna<3,>=2.5 in /usr/local/lib/python3.7/dist-packages (from requests>=2.20->yfinance) (2.10)\n",
58 | "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /usr/local/lib/python3.7/dist-packages (from requests>=2.20->yfinance) (1.24.3)\n",
59 | "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.7/dist-packages (from requests>=2.20->yfinance) (2021.5.30)\n",
60 | "Requirement already satisfied: six>=1.5 in /usr/local/lib/python3.7/dist-packages (from python-dateutil>=2.7.3->pandas>=0.24->yfinance) (1.15.0)\n"
61 | ],
62 | "name": "stdout"
63 | }
64 | ]
65 | },
66 | {
67 | "cell_type": "code",
68 | "metadata": {
69 | "colab": {
70 | "base_uri": "https://localhost:8080/"
71 | },
72 | "id": "VC3KR94BZ9aI",
73 | "outputId": "66782f90-b6fd-4e45-e4f7-80953f9fc789"
74 | },
75 | "source": [
76 | "import yfinance as yf\n",
77 | "import pandas as pd\n",
78 | "from datetime import datetime\n",
79 | "\n",
80 | "from google.colab import drive\n",
81 | "import copy\n",
82 | "\n",
83 | "drive.mount('/content/drive/')\n",
84 | "exp_dir = 'Colab Notebooks/workspace/export/'\n",
85 | "csv_path2 = '/content/drive/My Drive/' + exp_dir + 'sp500_train.csv'\n",
86 | "csv_path3 = '/content/drive/My Drive/' + exp_dir + 'sp500_test.csv'\n",
87 | "\n",
88 | "\n",
89 | "\n",
90 | "brk = yf.Ticker('^GSPC')\n",
91 | "hist = brk.history(period=\"max\", auto_adjust=True)\n",
92 | "\n",
93 | "df = pd.DataFrame()\n",
94 | "\n",
95 | "df['Date'] = hist.index\n",
96 | "df['SP500'] = hist['Close'].values\n",
97 | "df2 = df\n",
98 | "df3 = df"
99 | ],
100 | "execution_count": null,
101 | "outputs": [
102 | {
103 | "output_type": "stream",
104 | "text": [
105 | "Drive already mounted at /content/drive/; to attempt to forcibly remount, call drive.mount(\"/content/drive/\", force_remount=True).\n"
106 | ],
107 | "name": "stdout"
108 | }
109 | ]
110 | },
111 | {
112 | "cell_type": "markdown",
113 | "metadata": {
114 | "id": "D6zLw5TZEObS"
115 | },
116 | "source": [
117 | "# trainデータ構築"
118 | ]
119 | },
120 | {
121 | "cell_type": "code",
122 | "metadata": {
123 | "colab": {
124 | "base_uri": "https://localhost:8080/",
125 | "height": 450
126 | },
127 | "id": "wkkWzl7Agqis",
128 | "outputId": "5c4271dd-301a-46a3-cb13-d945bd9c1bb2"
129 | },
130 | "source": [
131 | "df2 = df2[(df3['Date'] >= datetime(2015,1,1)) & (df2['Date'] < datetime(2017,6,30))]\n",
132 | "df2 = df2.set_index('Date')\n",
133 | "df2.to_csv(csv_path2)\n",
134 | "df2"
135 | ],
136 | "execution_count": null,
137 | "outputs": [
138 | {
139 | "output_type": "execute_result",
140 | "data": {
141 | "text/html": [
142 | "| \n", 160 | " | SP500 | \n", 161 | "
|---|---|
| Date | \n", 164 | "\n", 165 | " |
| 2015-01-02 | \n", 170 | "2058.199951 | \n", 171 | "
| 2015-01-05 | \n", 174 | "2020.579956 | \n", 175 | "
| 2015-01-06 | \n", 178 | "2002.609985 | \n", 179 | "
| 2015-01-07 | \n", 182 | "2025.900024 | \n", 183 | "
| 2015-01-08 | \n", 186 | "2062.139893 | \n", 187 | "
| ... | \n", 190 | "... | \n", 191 | "
| 2017-06-23 | \n", 194 | "2438.300049 | \n", 195 | "
| 2017-06-26 | \n", 198 | "2439.070068 | \n", 199 | "
| 2017-06-27 | \n", 202 | "2419.379883 | \n", 203 | "
| 2017-06-28 | \n", 206 | "2440.689941 | \n", 207 | "
| 2017-06-29 | \n", 210 | "2419.699951 | \n", 211 | "
628 rows × 1 columns
\n", 215 | "| \n", 291 | " | SP500 | \n", 292 | "
|---|---|
| Date | \n", 295 | "\n", 296 | " |
| 2017-07-03 | \n", 301 | "2429.010010 | \n", 302 | "
| 2017-07-05 | \n", 305 | "2432.540039 | \n", 306 | "
| 2017-07-06 | \n", 309 | "2409.750000 | \n", 310 | "
| 2017-07-07 | \n", 313 | "2425.179932 | \n", 314 | "
| 2017-07-10 | \n", 317 | "2427.429932 | \n", 318 | "
| ... | \n", 321 | "... | \n", 322 | "
| 2020-12-24 | \n", 325 | "3703.060059 | \n", 326 | "
| 2020-12-28 | \n", 329 | "3735.360107 | \n", 330 | "
| 2020-12-29 | \n", 333 | "3727.040039 | \n", 334 | "
| 2020-12-30 | \n", 337 | "3732.040039 | \n", 338 | "
| 2020-12-31 | \n", 341 | "3756.070068 | \n", 342 | "
882 rows × 1 columns
\n", 346 | ""
31 | ]
32 | },
33 | {
34 | "cell_type": "code",
35 | "metadata": {
36 | "colab": {
37 | "base_uri": "https://localhost:8080/"
38 | },
39 | "id": "3NIXg6mTzk0K",
40 | "outputId": "2d7391cd-cc66-40a8-b40d-339f5231b696"
41 | },
42 | "source": [
43 | "import pandas as pd\n",
44 | "import numpy as np\n",
45 | "import random\n",
46 | "from google.colab import drive\n",
47 | "import copy\n",
48 | "\n",
49 | "from datetime import datetime\n",
50 | "import pickle\n",
51 | "\n",
52 | "from tensorflow.keras.models import Sequential\n",
53 | "from tensorflow.keras.layers import Dense, ReLU\n",
54 | "from tensorflow.keras.optimizers import RMSprop\n",
55 | "from sklearn.preprocessing import StandardScaler\n",
56 | "\n",
57 | "from tensorflow.keras.utils import Progbar\n",
58 | "\n",
59 | "mode = 'test'\n",
60 | "name = 'qlearning'\n",
61 | "level = 1\n",
62 | "if level == 2:\n",
63 | " name += name + 'lv2'\n",
64 | "\n",
65 | "drive.mount('/content/drive/')\n",
66 | "nov_dir = 'Colab Notebooks/dataset/reinforcement_learning/'\n",
67 | "nov_path = '/content/drive/My Drive/' + nov_dir + f'sp500_{mode}.csv'\n",
68 | "\n",
69 | "exp_dir = 'Colab Notebooks/workspace/export/'\n",
70 | "mdl_dir = '/content/drive/My Drive/' + exp_dir + 'models'\n",
71 | "csv_path = '/content/drive/My Drive/' + exp_dir + f'csv_data/{name}_{mode}.csv'\n",
72 | "\n",
73 | "df = pd.read_csv(nov_path)\n",
74 | "df['Date'] = pd.to_datetime(df['Date'], format = '%Y-%m-%d')"
75 | ],
76 | "execution_count": 1,
77 | "outputs": [
78 | {
79 | "output_type": "stream",
80 | "name": "stdout",
81 | "text": [
82 | "Mounted at /content/drive/\n"
83 | ]
84 | }
85 | ]
86 | },
87 | {
88 | "cell_type": "code",
89 | "metadata": {
90 | "id": "MN1DKfV6zauY"
91 | },
92 | "source": [
93 | "class Environment:\n",
94 | " def __init__(self, df, initial_money=100000, mode = 'test', commission = 0):\n",
95 | "\n",
96 | " self.df = df.dropna().reset_index()\n",
97 | "\n",
98 | " self.df_total_steps = len(self.df)-1\n",
99 | " self.initial_money = initial_money\n",
100 | " self.mode = mode\n",
101 | " self.commission = commission\n",
102 | " self.trade_time = None\n",
103 | " self.trade_win = None\n",
104 | " self.brfore_buy_cash = None\n",
105 | " self.action_space = np.array([0, 1, 2]) # buy,hold,sell\n",
106 | " self.hold_a_position = None\n",
107 | " self.now_price = None\n",
108 | " self.cash_in_hand = None\n",
109 | " self.sell_price = None\n",
110 | " self.buy_price = None\n",
111 | "\n",
112 | " self.reset()\n",
113 | " \n",
114 | " def reset(self):\n",
115 | "\n",
116 | " self.trade_time = 0\n",
117 | " self.trade_win = 0\n",
118 | " self.brfore_buy_cash = 0\n",
119 | " self.end_step = self.df_total_steps\n",
120 | " self.now_step = 0\n",
121 | " self.hold_a_position = 0.0\n",
122 | " self.now_price = self.df.loc[self.now_step, 'SP500']\n",
123 | " self.cash_in_hand = self.initial_money\n",
124 | " self.sell_price = 0\n",
125 | " self.buy_price = 0\n",
126 | "\n",
127 | " return self._get_now_state()\n",
128 | "\n",
129 | " def step(self, action):\n",
130 | "\n",
131 | " self.now_step += 1\n",
132 | " self.now_price = self.df.loc[self.now_step, 'SP500']\n",
133 | " \n",
134 | " done = (self.end_step == self.now_step)\n",
135 | "\n",
136 | " self.sell_price = 0\n",
137 | " self._trade(action,done)\n",
138 | " reward = 0\n",
139 | " if (self.sell_price > 0) and (self.buy_price > 0) and ((self.sell_price - self.buy_price) != 0):\n",
140 | " reward = (self.sell_price - self.buy_price) / self.buy_price\n",
141 | " self.buy_price = 0\n",
142 | " cur_revenue = self._get_revenue()\n",
143 | " \n",
144 | " info = { 'cur_revenue' : cur_revenue , 'trade_time' : self.trade_time, 'trade_win' : self.trade_win }\n",
145 | "\n",
146 | " return self._get_now_state(), reward, done, info\n",
147 | "\n",
148 | " def _get_now_state(self):\n",
149 | " state = np.empty(3)\n",
150 | " state[0] = self.hold_a_position\n",
151 | " state[1] = self.now_price\n",
152 | " state[2] = self.cash_in_hand\n",
153 | " return state\n",
154 | "\n",
155 | " def _get_revenue(self): \n",
156 | " return self.hold_a_position * self.now_price + self.cash_in_hand\n",
157 | "\n",
158 | " def _trade(self, action,lastorder = False):\n",
159 | " if lastorder:\n",
160 | " if self.hold_a_position != 0:\n",
161 | " self.cash_in_hand += self.now_price * self.hold_a_position\n",
162 | " self.hold_a_position = 0\n",
163 | " self.trade_time += 1\n",
164 | " if self.cash_in_hand > self.brfore_buy_cash:\n",
165 | " self.trade_win += 1\n",
166 | " else:\n",
167 | " if self.action_space[0] == action: # buy\n",
168 | " if self.hold_a_position == 0:\n",
169 | " buy_flag = True\n",
170 | " self.brfore_buy_cash = copy.copy(self.cash_in_hand)\n",
171 | " while buy_flag:\n",
172 | " if self.cash_in_hand > self.now_price:\n",
173 | " self.hold_a_position += 1\n",
174 | " self.buy_price += self.now_price\n",
175 | " self.cash_in_hand -= self.now_price + self.commission * self.now_price\n",
176 | " else:\n",
177 | " buy_flag = False\n",
178 | " if self.action_space[2] == action: # sell\n",
179 | " if self.hold_a_position != 0:\n",
180 | " self.sell_price += self.now_price * self.hold_a_position\n",
181 | " self.cash_in_hand += self.now_price * self.hold_a_position - self.commission * self.now_price * self.hold_a_position\n",
182 | " self.hold_a_position = 0\n",
183 | " self.trade_time += 1\n",
184 | " if self.cash_in_hand > self.brfore_buy_cash:\n",
185 | " self.trade_win += 1"
186 | ],
187 | "execution_count": 2,
188 | "outputs": []
189 | },
190 | {
191 | "cell_type": "code",
192 | "metadata": {
193 | "id": "nGeWOM-ZWNYK"
194 | },
195 | "source": [
196 | "class Brain:\n",
197 | " def __init__(self):\n",
198 | "\n",
199 | " self.gamma = 0.9\n",
200 | " optimizer = RMSprop()\n",
201 | "\n",
202 | " model = Sequential()\n",
203 | " model.add(Dense(3, input_shape=(3,)))\n",
204 | " model.add(ReLU()) \n",
205 | " model.add(Dense(3))\n",
206 | " model.add(ReLU())\n",
207 | " model.add(Dense(3))\n",
208 | " model.compile(loss=\"mse\", optimizer=optimizer)\n",
209 | "\n",
210 | " print((model.summary()))\n",
211 | " self.model = model\n",
212 | "\n",
213 | " def train(self, state, action, reward, next_state, done):\n",
214 | " q = self.model.predict(state) \n",
215 | " next_q = self.model.predict(next_state)\n",
216 | " target = np.copy(q)\n",
217 | " if done:\n",
218 | " target[:, action] = reward\n",
219 | " else:\n",
220 | " target[:, action] = reward + self.gamma*np.max(next_q, axis=1)\n",
221 | " self.model.train_on_batch(state, target)\n",
222 | "\n",
223 | " def predict(self, state):\n",
224 | " return self.model.predict(state)\n",
225 | "\n",
226 | " def load(self, name):\n",
227 | " self.model.load_weights(name)\n",
228 | "\n",
229 | " def save(self, name):\n",
230 | " self.model.save_weights(name)"
231 | ],
232 | "execution_count": 3,
233 | "outputs": []
234 | },
235 | {
236 | "cell_type": "code",
237 | "metadata": {
238 | "id": "QxR4grMVRLCR"
239 | },
240 | "source": [
241 | "class Agent(Brain):\n",
242 | " def __init__(self):\n",
243 | "\n",
244 | " super().__init__()\n",
245 | " self.epsilon = 1.0\n",
246 | " self.epsilon_min = 0.01\n",
247 | " self.r = 0.995\n",
248 | "\n",
249 | " def act(self, state):\n",
250 | " if np.random.rand() <= self.epsilon:\n",
251 | " return np.random.choice(3)\n",
252 | " act_values = self.predict(state)\n",
253 | " if self.epsilon > self.epsilon_min:\n",
254 | " self.epsilon *= self.r\n",
255 | " return np.argmax(act_values[0])"
256 | ],
257 | "execution_count": 4,
258 | "outputs": []
259 | },
260 | {
261 | "cell_type": "code",
262 | "metadata": {
263 | "id": "On5S8YtLz3U4"
264 | },
265 | "source": [
266 | "class Main:\n",
267 | " def __init__(self, env, agent, mdl_dir, name, episodes_times = 1000, mode = 'test'):\n",
268 | " self.env = env\n",
269 | " self.agent = agent\n",
270 | " self.mdl_dir = mdl_dir\n",
271 | " self.scaler = self._standard_scaler(self.env)\n",
272 | " self.episodes_times = episodes_times\n",
273 | " self.mode = mode\n",
274 | " self.name = name\n",
275 | "\n",
276 | " self.df_rec = pd.DataFrame(index=[], columns=['FixedProfit','TradeTimes','TradeWin'])\n",
277 | " if self.mode == 'test':\n",
278 | " self.agent.epsilon = 0.01\n",
279 | "\n",
280 | " def play_game(self):\n",
281 | "\n",
282 | " for episode in range(self.episodes_times):\n",
283 | "\n",
284 | " if (episode % 10 == 0):\n",
285 | " metrics_names = ['FixedProfit','TradeTimes','TradeWin']\n",
286 | " if (int(str(self.episodes_times)[:-1])*10 == episode):\n",
287 | " pb_i = Progbar(int(str(self.episodes_times)[-1]), stateful_metrics=metrics_names)\n",
288 | " else:\n",
289 | " pb_i = Progbar(10, stateful_metrics=metrics_names)\n",
290 | " p_mean,trade_time,win_time = np.array([]),np.array([]),np.array([])\n",
291 | "\n",
292 | " state = self.env.reset()\n",
293 | " state = self.scaler.transform([state])\n",
294 | " done = False\n",
295 | " \n",
296 | " while not done:\n",
297 | " action = self.agent.act(state)\n",
298 | " next_state, reward, done, info = self.env.step(action)\n",
299 | " next_state = self.scaler.transform([next_state])\n",
300 | " reward = self._reward_clipping(reward)\n",
301 | "\n",
302 | " if self.mode == 'train':\n",
303 | " self.agent.train(state, action, reward, next_state, done)\n",
304 | "\n",
305 | " state = next_state\n",
306 | " \n",
307 | " record = pd.Series([info['cur_revenue'],info['trade_time'],info['trade_win']], index=self.df_rec.columns)\n",
308 | " self.df_rec = self.df_rec.append(record, ignore_index=True)\n",
309 | " p_mean,trade_time,win_time = np.append(p_mean,info['cur_revenue']),np.append(trade_time,info['trade_time']),np.append(win_time,info['trade_win'])\n",
310 | " values=[('FixedProfit',int(np.mean(p_mean))), ('TradeTimes',int(np.mean(trade_time))), ('TradeWin',int(np.mean(win_time)))]\n",
311 | " pb_i.add(1, values=values)\n",
312 | "\n",
313 | " if self.mode == 'train':\n",
314 | " self._save()\n",
315 | " self._save_csv()\n",
316 | "\n",
317 | " def _standard_scaler(self, env):\n",
318 | " states = []\n",
319 | " for _ in range(env.df_total_steps):\n",
320 | " action = np.random.choice(env.action_space)\n",
321 | " state, reward, done, info = env.step(action)\n",
322 | " states.append(state)\n",
323 | " if done:\n",
324 | " break\n",
325 | " \n",
326 | " scaler = StandardScaler()\n",
327 | " scaler.fit(states)\n",
328 | " return scaler\n",
329 | "\n",
330 | " def _reward_clipping(self, val):\n",
331 | " result = 1 if val > 0 else 0 if val == 0 else -1\n",
332 | " return result\n",
333 | "\n",
334 | " def _load(self):\n",
335 | " with open('{}/{}.pkl'.format(self.mdl_dir, self.name), 'rb') as f:\n",
336 | " self.scaler = pickle.load(f)\n",
337 | " self.agent.load('{}/{}.h5'.format(self.mdl_dir, self.name))\n",
338 | "\n",
339 | " def _save(self):\n",
340 | " self.agent.save('{}/{}.h5'.format(self.mdl_dir, self.name))\n",
341 | " with open('{}/{}.pkl'.format(self.mdl_dir, self.name), 'wb') as f:\n",
342 | " pickle.dump(self.scaler, f)\n",
343 | "\n",
344 | " def _save_csv(self):\n",
345 | " self.df_rec.to_csv(csv_path)"
346 | ],
347 | "execution_count": 5,
348 | "outputs": []
349 | },
350 | {
351 | "cell_type": "code",
352 | "metadata": {
353 | "colab": {
354 | "base_uri": "https://localhost:8080/"
355 | },
356 | "id": "pYFNVDDQz9X9",
357 | "outputId": "393d1a10-ac6a-48c0-b7e0-c3ad7b593034"
358 | },
359 | "source": [
360 | "initial_money=1000000\n",
361 | "episodes_times = 100\n",
362 | "commission = 0 if level == 1 else 0.002\n",
363 | "\n",
364 | "agent = Agent()\n",
365 | "env = Environment(df, initial_money=initial_money, mode = mode, commission = commission)\n",
366 | "main = Main(env, agent, mdl_dir, name, episodes_times, mode)\n",
367 | "main.play_game()"
368 | ],
369 | "execution_count": 6,
370 | "outputs": [
371 | {
372 | "output_type": "stream",
373 | "name": "stdout",
374 | "text": [
375 | "Model: \"sequential\"\n",
376 | "_________________________________________________________________\n",
377 | "Layer (type) Output Shape Param # \n",
378 | "=================================================================\n",
379 | "dense (Dense) (None, 3) 12 \n",
380 | "_________________________________________________________________\n",
381 | "re_lu (ReLU) (None, 3) 0 \n",
382 | "_________________________________________________________________\n",
383 | "dense_1 (Dense) (None, 3) 12 \n",
384 | "_________________________________________________________________\n",
385 | "re_lu_1 (ReLU) (None, 3) 0 \n",
386 | "_________________________________________________________________\n",
387 | "dense_2 (Dense) (None, 3) 12 \n",
388 | "=================================================================\n",
389 | "Total params: 36\n",
390 | "Trainable params: 36\n",
391 | "Non-trainable params: 0\n",
392 | "_________________________________________________________________\n",
393 | "None\n",
394 | "10/10 [==============================] - 272s 27s/step - FixedProfit: 1397161.0000 - TradeTimes: 4.0000 - TradeWin: 3.0000\n",
395 | "10/10 [==============================] - 272s 27s/step - FixedProfit: 1328466.0000 - TradeTimes: 5.0000 - TradeWin: 4.0000\n",
396 | "10/10 [==============================] - 272s 27s/step - FixedProfit: 1360205.0000 - TradeTimes: 4.0000 - TradeWin: 3.0000\n",
397 | "10/10 [==============================] - 270s 27s/step - FixedProfit: 1376867.0000 - TradeTimes: 6.0000 - TradeWin: 4.0000\n",
398 | "10/10 [==============================] - 269s 27s/step - FixedProfit: 1364904.0000 - TradeTimes: 6.0000 - TradeWin: 5.0000\n",
399 | "10/10 [==============================] - 269s 27s/step - FixedProfit: 1380598.0000 - TradeTimes: 5.0000 - TradeWin: 4.0000\n",
400 | "10/10 [==============================] - 268s 27s/step - FixedProfit: 1319612.0000 - TradeTimes: 5.0000 - TradeWin: 3.0000\n",
401 | "10/10 [==============================] - 268s 27s/step - FixedProfit: 1383077.0000 - TradeTimes: 6.0000 - TradeWin: 4.0000\n",
402 | "10/10 [==============================] - 269s 27s/step - FixedProfit: 1396020.0000 - TradeTimes: 4.0000 - TradeWin: 3.0000\n",
403 | "10/10 [==============================] - 268s 27s/step - FixedProfit: 1366896.0000 - TradeTimes: 5.0000 - TradeWin: 4.0000\n"
404 | ]
405 | }
406 | ]
407 | }
408 | ]
409 | }
--------------------------------------------------------------------------------
/Q-Learning/q_learning_train.ipynb:
--------------------------------------------------------------------------------
1 | {
2 | "nbformat": 4,
3 | "nbformat_minor": 0,
4 | "metadata": {
5 | "colab": {
6 | "name": "q_learning_train.ipynb",
7 | "provenance": [],
8 | "collapsed_sections": [],
9 | "machine_shape": "hm",
10 | "authorship_tag": "ABX9TyMQgWNI/SvpF8ZgCl0gdzq/",
11 | "include_colab_link": true
12 | },
13 | "kernelspec": {
14 | "name": "python3",
15 | "display_name": "Python 3"
16 | },
17 | "language_info": {
18 | "name": "python"
19 | },
20 | "accelerator": "GPU"
21 | },
22 | "cells": [
23 | {
24 | "cell_type": "markdown",
25 | "metadata": {
26 | "id": "view-in-github",
27 | "colab_type": "text"
28 | },
29 | "source": [
30 | ""
31 | ]
32 | },
33 | {
34 | "cell_type": "code",
35 | "metadata": {
36 | "colab": {
37 | "base_uri": "https://localhost:8080/"
38 | },
39 | "id": "3NIXg6mTzk0K",
40 | "outputId": "4df454cf-c0bb-4a0a-b99c-9286af4dd5ea"
41 | },
42 | "source": [
43 | "import pandas as pd\n",
44 | "import numpy as np\n",
45 | "import random\n",
46 | "from google.colab import drive\n",
47 | "import copy\n",
48 | "\n",
49 | "from datetime import datetime\n",
50 | "import pickle\n",
51 | "\n",
52 | "from tensorflow.keras.models import Sequential\n",
53 | "from tensorflow.keras.layers import Dense, ReLU\n",
54 | "from tensorflow.keras.optimizers import RMSprop\n",
55 | "from sklearn.preprocessing import StandardScaler\n",
56 | "\n",
57 | "from tensorflow.keras.utils import Progbar\n",
58 | "\n",
59 | "mode = 'train'\n",
60 | "name = 'qlearning'\n",
61 | "level = 1\n",
62 | "if level == 2:\n",
63 | " name += name + 'lv2'\n",
64 | "\n",
65 | "drive.mount('/content/drive/')\n",
66 | "nov_dir = 'Colab Notebooks/dataset/reinforcement_learning/'\n",
67 | "nov_path = '/content/drive/My Drive/' + nov_dir + f'sp500_{mode}.csv'\n",
68 | "\n",
69 | "exp_dir = 'Colab Notebooks/workspace/export/'\n",
70 | "mdl_dir = '/content/drive/My Drive/' + exp_dir + 'models'\n",
71 | "csv_path = '/content/drive/My Drive/' + exp_dir + f'csv_data/{name}_{mode}.csv'\n",
72 | "\n",
73 | "df = pd.read_csv(nov_path)\n",
74 | "df['Date'] = pd.to_datetime(df['Date'], format = '%Y-%m-%d')"
75 | ],
76 | "execution_count": 1,
77 | "outputs": [
78 | {
79 | "output_type": "stream",
80 | "name": "stdout",
81 | "text": [
82 | "Drive already mounted at /content/drive/; to attempt to forcibly remount, call drive.mount(\"/content/drive/\", force_remount=True).\n"
83 | ]
84 | }
85 | ]
86 | },
87 | {
88 | "cell_type": "code",
89 | "metadata": {
90 | "id": "MN1DKfV6zauY"
91 | },
92 | "source": [
93 | "class Environment:\n",
94 | " def __init__(self, df, initial_money=100000, mode = 'test', commission = 0):\n",
95 | "\n",
96 | " self.df = df.dropna().reset_index()\n",
97 | "\n",
98 | " self.df_total_steps = len(self.df)-1\n",
99 | " self.initial_money = initial_money\n",
100 | " self.mode = mode\n",
101 | " self.commission = commission\n",
102 | " self.trade_time = None\n",
103 | " self.trade_win = None\n",
104 | " self.brfore_buy_cash = None\n",
105 | " self.action_space = np.array([0, 1, 2]) # buy,hold,sell\n",
106 | " self.hold_a_position = None\n",
107 | " self.now_price = None\n",
108 | " self.cash_in_hand = None\n",
109 | " self.sell_price = None\n",
110 | " self.buy_price = None\n",
111 | "\n",
112 | " self.reset()\n",
113 | " \n",
114 | " def reset(self):\n",
115 | "\n",
116 | " self.trade_time = 0\n",
117 | " self.trade_win = 0\n",
118 | " self.brfore_buy_cash = 0\n",
119 | " self.end_step = self.df_total_steps\n",
120 | " self.now_step = 0\n",
121 | " self.hold_a_position = 0.0\n",
122 | " self.now_price = self.df.loc[self.now_step, 'SP500']\n",
123 | " self.cash_in_hand = self.initial_money\n",
124 | " self.sell_price = 0\n",
125 | " self.buy_price = 0\n",
126 | "\n",
127 | " return self._get_now_state()\n",
128 | "\n",
129 | " def step(self, action):\n",
130 | "\n",
131 | " self.now_step += 1\n",
132 | " self.now_price = self.df.loc[self.now_step, 'SP500']\n",
133 | " \n",
134 | " done = (self.end_step == self.now_step)\n",
135 | "\n",
136 | " self.sell_price = 0\n",
137 | " self._trade(action,done)\n",
138 | " reward = 0\n",
139 | " if (self.sell_price > 0) and (self.buy_price > 0) and ((self.sell_price - self.buy_price) != 0):\n",
140 | " reward = (self.sell_price - self.buy_price) / self.buy_price\n",
141 | " self.buy_price = 0\n",
142 | " cur_revenue = self._get_revenue()\n",
143 | " \n",
144 | " info = { 'cur_revenue' : cur_revenue , 'trade_time' : self.trade_time, 'trade_win' : self.trade_win }\n",
145 | "\n",
146 | " return self._get_now_state(), reward, done, info\n",
147 | "\n",
148 | " def _get_now_state(self):\n",
149 | " state = np.empty(3)\n",
150 | " state[0] = self.hold_a_position\n",
151 | " state[1] = self.now_price\n",
152 | " state[2] = self.cash_in_hand\n",
153 | " return state\n",
154 | "\n",
155 | " def _get_revenue(self): \n",
156 | " return self.hold_a_position * self.now_price + self.cash_in_hand\n",
157 | "\n",
158 | " def _trade(self, action,lastorder = False):\n",
159 | " if lastorder:\n",
160 | " if self.hold_a_position != 0:\n",
161 | " self.cash_in_hand += self.now_price * self.hold_a_position\n",
162 | " self.hold_a_position = 0\n",
163 | " self.trade_time += 1\n",
164 | " if self.cash_in_hand > self.brfore_buy_cash:\n",
165 | " self.trade_win += 1\n",
166 | " else:\n",
167 | " if self.action_space[0] == action: # buy\n",
168 | " if self.hold_a_position == 0:\n",
169 | " buy_flag = True\n",
170 | " self.brfore_buy_cash = copy.copy(self.cash_in_hand)\n",
171 | " while buy_flag:\n",
172 | " if self.cash_in_hand > self.now_price:\n",
173 | " self.hold_a_position += 1\n",
174 | " self.buy_price += self.now_price\n",
175 | " self.cash_in_hand -= self.now_price + self.commission * self.now_price\n",
176 | " else:\n",
177 | " buy_flag = False\n",
178 | " if self.action_space[2] == action: # sell\n",
179 | " if self.hold_a_position != 0:\n",
180 | " self.sell_price += self.now_price * self.hold_a_position\n",
181 | " self.cash_in_hand += self.now_price * self.hold_a_position - self.commission * self.now_price * self.hold_a_position\n",
182 | " self.hold_a_position = 0\n",
183 | " self.trade_time += 1\n",
184 | " if self.cash_in_hand > self.brfore_buy_cash:\n",
185 | " self.trade_win += 1"
186 | ],
187 | "execution_count": 2,
188 | "outputs": []
189 | },
190 | {
191 | "cell_type": "code",
192 | "metadata": {
193 | "id": "nGeWOM-ZWNYK"
194 | },
195 | "source": [
196 | "class Brain:\n",
197 | " def __init__(self):\n",
198 | "\n",
199 | " self.gamma = 0.9\n",
200 | " optimizer = RMSprop()\n",
201 | "\n",
202 | " model = Sequential()\n",
203 | " model.add(Dense(3, input_shape=(3,)))\n",
204 | " model.add(ReLU()) \n",
205 | " model.add(Dense(3))\n",
206 | " model.add(ReLU())\n",
207 | " model.add(Dense(3))\n",
208 | " model.compile(loss=\"mse\", optimizer=optimizer)\n",
209 | "\n",
210 | " print((model.summary()))\n",
211 | " self.model = model\n",
212 | "\n",
213 | " def train(self, state, action, reward, next_state, done):\n",
214 | " q = self.model.predict(state) \n",
215 | " next_q = self.model.predict(next_state)\n",
216 | " target = np.copy(q)\n",
217 | " if done:\n",
218 | " target[:, action] = reward\n",
219 | " else:\n",
220 | " target[:, action] = reward + self.gamma*np.max(next_q, axis=1)\n",
221 | " self.model.train_on_batch(state, target)\n",
222 | "\n",
223 | " def predict(self, state):\n",
224 | " return self.model.predict(state)\n",
225 | "\n",
226 | " def load(self, name):\n",
227 | " self.model.load_weights(name)\n",
228 | "\n",
229 | " def save(self, name):\n",
230 | " self.model.save_weights(name)"
231 | ],
232 | "execution_count": 3,
233 | "outputs": []
234 | },
235 | {
236 | "cell_type": "code",
237 | "metadata": {
238 | "id": "QxR4grMVRLCR"
239 | },
240 | "source": [
241 | "class Agent(Brain):\n",
242 | " def __init__(self):\n",
243 | "\n",
244 | " super().__init__()\n",
245 | " self.epsilon = 1.0\n",
246 | " self.epsilon_min = 0.01\n",
247 | " self.r = 0.995\n",
248 | "\n",
249 | " def act(self, state):\n",
250 | " if np.random.rand() <= self.epsilon:\n",
251 | " return np.random.choice(3)\n",
252 | " act_values = self.predict(state)\n",
253 | " if self.epsilon > self.epsilon_min:\n",
254 | " self.epsilon *= self.r\n",
255 | " return np.argmax(act_values[0])"
256 | ],
257 | "execution_count": 4,
258 | "outputs": []
259 | },
260 | {
261 | "cell_type": "code",
262 | "metadata": {
263 | "id": "On5S8YtLz3U4"
264 | },
265 | "source": [
266 | "class Main:\n",
267 | " def __init__(self, env, agent, mdl_dir, name, episodes_times = 1000, mode = 'test'):\n",
268 | " self.env = env\n",
269 | " self.agent = agent\n",
270 | " self.mdl_dir = mdl_dir\n",
271 | " self.scaler = self._standard_scaler(self.env)\n",
272 | " self.episodes_times = episodes_times\n",
273 | " self.mode = mode\n",
274 | " self.name = name\n",
275 | "\n",
276 | " self.df_rec = pd.DataFrame(index=[], columns=['FixedProfit','TradeTimes','TradeWin'])\n",
277 | " if self.mode == 'test':\n",
278 | " self.agent.epsilon = 0.01\n",
279 | "\n",
280 | " def play_game(self):\n",
281 | "\n",
282 | " for episode in range(self.episodes_times):\n",
283 | "\n",
284 | " if (episode % 10 == 0):\n",
285 | " metrics_names = ['FixedProfit','TradeTimes','TradeWin']\n",
286 | " if (int(str(self.episodes_times)[:-1])*10 == episode):\n",
287 | " pb_i = Progbar(int(str(self.episodes_times)[-1]), stateful_metrics=metrics_names)\n",
288 | " else:\n",
289 | " pb_i = Progbar(10, stateful_metrics=metrics_names)\n",
290 | " p_mean,trade_time,win_time = np.array([]),np.array([]),np.array([])\n",
291 | "\n",
292 | " state = self.env.reset()\n",
293 | " state = self.scaler.transform([state])\n",
294 | " done = False\n",
295 | " \n",
296 | " while not done:\n",
297 | " action = self.agent.act(state)\n",
298 | " next_state, reward, done, info = self.env.step(action)\n",
299 | " next_state = self.scaler.transform([next_state])\n",
300 | " reward = self._reward_clipping(reward)\n",
301 | "\n",
302 | " if self.mode == 'train':\n",
303 | " self.agent.train(state, action, reward, next_state, done)\n",
304 | "\n",
305 | " state = next_state\n",
306 | " \n",
307 | " record = pd.Series([info['cur_revenue'],info['trade_time'],info['trade_win']], index=self.df_rec.columns)\n",
308 | " self.df_rec = self.df_rec.append(record, ignore_index=True)\n",
309 | " p_mean,trade_time,win_time = np.append(p_mean,info['cur_revenue']),np.append(trade_time,info['trade_time']),np.append(win_time,info['trade_win'])\n",
310 | " values=[('FixedProfit',int(np.mean(p_mean))), ('TradeTimes',int(np.mean(trade_time))), ('TradeWin',int(np.mean(win_time)))]\n",
311 | " pb_i.add(1, values=values)\n",
312 | "\n",
313 | " if self.mode == 'train':\n",
314 | " self._save()\n",
315 | " self._save_csv()\n",
316 | "\n",
317 | " def _standard_scaler(self, env):\n",
318 | " states = []\n",
319 | " for _ in range(env.df_total_steps):\n",
320 | " action = np.random.choice(env.action_space)\n",
321 | " state, reward, done, info = env.step(action)\n",
322 | " states.append(state)\n",
323 | " if done:\n",
324 | " break\n",
325 | " \n",
326 | " scaler = StandardScaler()\n",
327 | " scaler.fit(states)\n",
328 | " return scaler\n",
329 | "\n",
330 | " def _reward_clipping(self, val):\n",
331 | " result = 1 if val > 0 else 0 if val == 0 else -1\n",
332 | " return result\n",
333 | "\n",
334 | " def _load(self):\n",
335 | " with open('{}/{}.pkl'.format(self.mdl_dir, self.name), 'rb') as f:\n",
336 | " self.scaler = pickle.load(f)\n",
337 | " self.agent.load('{}/{}.h5'.format(self.mdl_dir, self.name))\n",
338 | "\n",
339 | " def _save(self):\n",
340 | " self.agent.save('{}/{}.h5'.format(self.mdl_dir, self.name))\n",
341 | " with open('{}/{}.pkl'.format(self.mdl_dir, self.name), 'wb') as f:\n",
342 | " pickle.dump(self.scaler, f)\n",
343 | "\n",
344 | " def _save_csv(self):\n",
345 | " self.df_rec.to_csv(csv_path)"
346 | ],
347 | "execution_count": 5,
348 | "outputs": []
349 | },
350 | {
351 | "cell_type": "code",
352 | "metadata": {
353 | "colab": {
354 | "base_uri": "https://localhost:8080/"
355 | },
356 | "id": "pYFNVDDQz9X9",
357 | "outputId": "5600d6e0-4e72-4d95-c6fc-3eae609724ca"
358 | },
359 | "source": [
360 | "initial_money=1000000\n",
361 | "episodes_times = 100\n",
362 | "commission = 0 if level == 1 else 0.002\n",
363 | "\n",
364 | "agent = Agent()\n",
365 | "env = Environment(df, initial_money=initial_money, mode = mode, commission = commission)\n",
366 | "main = Main(env, agent, mdl_dir, name, episodes_times, mode)\n",
367 | "main.play_game()"
368 | ],
369 | "execution_count": 6,
370 | "outputs": [
371 | {
372 | "output_type": "stream",
373 | "name": "stdout",
374 | "text": [
375 | "Model: \"sequential\"\n",
376 | "_________________________________________________________________\n",
377 | "Layer (type) Output Shape Param # \n",
378 | "=================================================================\n",
379 | "dense (Dense) (None, 3) 12 \n",
380 | "_________________________________________________________________\n",
381 | "re_lu (ReLU) (None, 3) 0 \n",
382 | "_________________________________________________________________\n",
383 | "dense_1 (Dense) (None, 3) 12 \n",
384 | "_________________________________________________________________\n",
385 | "re_lu_1 (ReLU) (None, 3) 0 \n",
386 | "_________________________________________________________________\n",
387 | "dense_2 (Dense) (None, 3) 12 \n",
388 | "=================================================================\n",
389 | "Total params: 36\n",
390 | "Trainable params: 36\n",
391 | "Non-trainable params: 0\n",
392 | "_________________________________________________________________\n",
393 | "None\n",
394 | "10/10 [==============================] - 559s 56s/step - FixedProfit: 1053527.0000 - TradeTimes: 104.0000 - TradeWin: 57.0000\n",
395 | "10/10 [==============================] - 558s 56s/step - FixedProfit: 1151240.0000 - TradeTimes: 103.0000 - TradeWin: 57.0000\n",
396 | "10/10 [==============================] - 558s 56s/step - FixedProfit: 1172527.0000 - TradeTimes: 101.0000 - TradeWin: 54.0000\n",
397 | "10/10 [==============================] - 559s 56s/step - FixedProfit: 1070221.0000 - TradeTimes: 104.0000 - TradeWin: 57.0000\n",
398 | "10/10 [==============================] - 561s 56s/step - FixedProfit: 1148363.0000 - TradeTimes: 101.0000 - TradeWin: 58.0000\n",
399 | "10/10 [==============================] - 559s 56s/step - FixedProfit: 1097206.0000 - TradeTimes: 104.0000 - TradeWin: 57.0000\n",
400 | "10/10 [==============================] - 558s 56s/step - FixedProfit: 1111337.0000 - TradeTimes: 103.0000 - TradeWin: 56.0000\n",
401 | "10/10 [==============================] - 562s 56s/step - FixedProfit: 1162683.0000 - TradeTimes: 103.0000 - TradeWin: 57.0000\n",
402 | "10/10 [==============================] - 556s 56s/step - FixedProfit: 1157609.0000 - TradeTimes: 102.0000 - TradeWin: 57.0000\n",
403 | "10/10 [==============================] - 549s 55s/step - FixedProfit: 1057679.0000 - TradeTimes: 104.0000 - TradeWin: 55.0000\n"
404 | ]
405 | }
406 | ]
407 | }
408 | ]
409 | }
--------------------------------------------------------------------------------
/SARSA/sarsa_test.ipynb:
--------------------------------------------------------------------------------
1 | {
2 | "nbformat": 4,
3 | "nbformat_minor": 0,
4 | "metadata": {
5 | "colab": {
6 | "name": "sarsa_test.ipynb",
7 | "provenance": [],
8 | "collapsed_sections": [],
9 | "machine_shape": "hm",
10 | "authorship_tag": "ABX9TyNHq0IRyB/PxzuB3rbes9Uk",
11 | "include_colab_link": true
12 | },
13 | "kernelspec": {
14 | "name": "python3",
15 | "display_name": "Python 3"
16 | },
17 | "language_info": {
18 | "name": "python"
19 | },
20 | "accelerator": "GPU"
21 | },
22 | "cells": [
23 | {
24 | "cell_type": "markdown",
25 | "metadata": {
26 | "id": "view-in-github",
27 | "colab_type": "text"
28 | },
29 | "source": [
30 | ""
31 | ]
32 | },
33 | {
34 | "cell_type": "code",
35 | "metadata": {
36 | "colab": {
37 | "base_uri": "https://localhost:8080/"
38 | },
39 | "id": "3NIXg6mTzk0K",
40 | "outputId": "f28b3a3b-406a-433b-bb61-70e856c542c9"
41 | },
42 | "source": [
43 | "import pandas as pd\n",
44 | "import numpy as np\n",
45 | "import random\n",
46 | "from google.colab import drive\n",
47 | "import copy\n",
48 | "\n",
49 | "from datetime import datetime\n",
50 | "import pickle\n",
51 | "\n",
52 | "from tensorflow.keras.models import Sequential\n",
53 | "from tensorflow.keras.layers import Dense, ReLU\n",
54 | "from tensorflow.keras.optimizers import RMSprop\n",
55 | "from sklearn.preprocessing import StandardScaler\n",
56 | "\n",
57 | "from tensorflow.keras.utils import Progbar\n",
58 | "\n",
59 | "mode = 'test'\n",
60 | "name = 'sarsa'\n",
61 | "level = 1\n",
62 | "if level == 2:\n",
63 | " name += name + 'lv2'\n",
64 | "\n",
65 | "drive.mount('/content/drive/')\n",
66 | "nov_dir = 'Colab Notebooks/dataset/reinforcement_learning/'\n",
67 | "nov_path = '/content/drive/My Drive/' + nov_dir + f'sp500_{mode}.csv'\n",
68 | "\n",
69 | "exp_dir = 'Colab Notebooks/workspace/export/'\n",
70 | "mdl_dir = '/content/drive/My Drive/' + exp_dir + 'models'\n",
71 | "csv_path = '/content/drive/My Drive/' + exp_dir + f'csv_data/{name}_{mode}.csv'\n",
72 | "\n",
73 | "df = pd.read_csv(nov_path)\n",
74 | "df['Date'] = pd.to_datetime(df['Date'], format = '%Y-%m-%d')"
75 | ],
76 | "execution_count": 1,
77 | "outputs": [
78 | {
79 | "output_type": "stream",
80 | "name": "stdout",
81 | "text": [
82 | "Mounted at /content/drive/\n"
83 | ]
84 | }
85 | ]
86 | },
87 | {
88 | "cell_type": "code",
89 | "metadata": {
90 | "id": "MN1DKfV6zauY"
91 | },
92 | "source": [
93 | "class Environment:\n",
94 | " def __init__(self, df, initial_money=100000, mode = 'test', commission = 0):\n",
95 | "\n",
96 | " self.df = df.dropna().reset_index()\n",
97 | "\n",
98 | " self.df_total_steps = len(self.df)-1\n",
99 | " self.initial_money = initial_money\n",
100 | " self.mode = mode\n",
101 | " self.commission = commission\n",
102 | " self.trade_time = None\n",
103 | " self.trade_win = None\n",
104 | " self.brfore_buy_cash = None\n",
105 | " self.action_space = np.array([0, 1, 2]) # buy,hold,sell\n",
106 | " self.hold_a_position = None\n",
107 | " self.now_price = None\n",
108 | " self.cash_in_hand = None\n",
109 | " self.sell_price = None\n",
110 | " self.buy_price = None\n",
111 | "\n",
112 | " self.reset()\n",
113 | " \n",
114 | " def reset(self):\n",
115 | "\n",
116 | " self.trade_time = 0\n",
117 | " self.trade_win = 0\n",
118 | " self.brfore_buy_cash = 0\n",
119 | " self.end_step = self.df_total_steps\n",
120 | " self.now_step = 0\n",
121 | " self.hold_a_position = 0.0\n",
122 | " self.now_price = self.df.loc[self.now_step, 'SP500']\n",
123 | " self.cash_in_hand = self.initial_money\n",
124 | " self.sell_price = 0\n",
125 | " self.buy_price = 0\n",
126 | "\n",
127 | " return self._get_now_state()\n",
128 | "\n",
129 | " def step(self, action):\n",
130 | "\n",
131 | " self.now_step += 1\n",
132 | " self.now_price = self.df.loc[self.now_step, 'SP500']\n",
133 | " \n",
134 | " done = (self.end_step == self.now_step)\n",
135 | "\n",
136 | " self.sell_price = 0\n",
137 | " self._trade(action,done)\n",
138 | " reward = 0\n",
139 | " if (self.sell_price > 0) and (self.buy_price > 0) and ((self.sell_price - self.buy_price) != 0):\n",
140 | " reward = (self.sell_price - self.buy_price) / self.buy_price\n",
141 | " self.buy_price = 0\n",
142 | " cur_revenue = self._get_revenue()\n",
143 | " \n",
144 | " info = { 'cur_revenue' : cur_revenue , 'trade_time' : self.trade_time, 'trade_win' : self.trade_win }\n",
145 | "\n",
146 | " return self._get_now_state(), reward, done, info\n",
147 | "\n",
148 | " def _get_now_state(self):\n",
149 | " state = np.empty(3)\n",
150 | " state[0] = self.hold_a_position\n",
151 | " state[1] = self.now_price\n",
152 | " state[2] = self.cash_in_hand\n",
153 | " return state\n",
154 | "\n",
155 | " def _get_revenue(self): \n",
156 | " return self.hold_a_position * self.now_price + self.cash_in_hand\n",
157 | "\n",
158 | " def _trade(self, action,lastorder = False):\n",
159 | " if lastorder:\n",
160 | " if self.hold_a_position != 0:\n",
161 | " self.cash_in_hand += self.now_price * self.hold_a_position\n",
162 | " self.hold_a_position = 0\n",
163 | " self.trade_time += 1\n",
164 | " if self.cash_in_hand > self.brfore_buy_cash:\n",
165 | " self.trade_win += 1\n",
166 | " else:\n",
167 | " if self.action_space[0] == action: # buy\n",
168 | " if self.hold_a_position == 0:\n",
169 | " buy_flag = True\n",
170 | " self.brfore_buy_cash = copy.copy(self.cash_in_hand)\n",
171 | " while buy_flag:\n",
172 | " if self.cash_in_hand > self.now_price:\n",
173 | " self.hold_a_position += 1\n",
174 | " self.buy_price += self.now_price\n",
175 | " self.cash_in_hand -= self.now_price + self.commission * self.now_price\n",
176 | " else:\n",
177 | " buy_flag = False\n",
178 | " if self.action_space[2] == action: # sell\n",
179 | " if self.hold_a_position != 0:\n",
180 | " self.sell_price += self.now_price * self.hold_a_position\n",
181 | " self.cash_in_hand += self.now_price * self.hold_a_position - self.commission * self.now_price * self.hold_a_position\n",
182 | " self.hold_a_position = 0\n",
183 | " self.trade_time += 1\n",
184 | " if self.cash_in_hand > self.brfore_buy_cash:\n",
185 | " self.trade_win += 1"
186 | ],
187 | "execution_count": 2,
188 | "outputs": []
189 | },
190 | {
191 | "cell_type": "code",
192 | "metadata": {
193 | "id": "nGeWOM-ZWNYK"
194 | },
195 | "source": [
196 | "class Brain:\n",
197 | " def __init__(self):\n",
198 | "\n",
199 | " self.gamma = 0.9\n",
200 | " \n",
201 | " model = Sequential()\n",
202 | " model.add(Dense(3, input_shape=(3,)))\n",
203 | " model.add(ReLU()) \n",
204 | " model.add(Dense(3))\n",
205 | " model.add(ReLU())\n",
206 | " model.add(Dense(3))\n",
207 | " model.compile(loss=\"mse\", optimizer=RMSprop())\n",
208 | "\n",
209 | " print((model.summary()))\n",
210 | " self.model = model\n",
211 | "\n",
212 | " def train(self, state, action, reward, next_state, next_act, done):\n",
213 | " q = self.model.predict(state) \n",
214 | " target = np.copy(q)\n",
215 | " if done:\n",
216 | " target[:, action] = reward\n",
217 | " else:\n",
218 | " target[:, action] = reward + self.gamma*next_act\n",
219 | " self.model.train_on_batch(state, target)\n",
220 | "\n",
221 | " def predict(self, state):\n",
222 | " return self.model.predict(state)\n",
223 | "\n",
224 | " def load(self, name):\n",
225 | " self.model.load_weights(name)\n",
226 | "\n",
227 | " def save(self, name):\n",
228 | " self.model.save_weights(name)"
229 | ],
230 | "execution_count": 3,
231 | "outputs": []
232 | },
233 | {
234 | "cell_type": "code",
235 | "metadata": {
236 | "id": "QxR4grMVRLCR"
237 | },
238 | "source": [
239 | "class Agent(Brain):\n",
240 | " def __init__(self):\n",
241 | "\n",
242 | " super().__init__()\n",
243 | " self.epsilon = 1.0\n",
244 | " self.epsilon_min = 0.01\n",
245 | " self.r = 0.995\n",
246 | "\n",
247 | " def act(self, state, act, reward, info, next_state, done, mode = 'train'):\n",
248 | "\n",
249 | " next_act = self._next_act(state)\n",
250 | " if mode == 'train':\n",
251 | " self.train(state, act, reward, next_state, next_act, done)\n",
252 | "\n",
253 | " return next_act\n",
254 | "\n",
255 | " def _next_act(self, state):\n",
256 | " if np.random.rand() <= self.epsilon:\n",
257 | " return np.random.choice(3)\n",
258 | " act_values = self.predict(state)\n",
259 | " if self.epsilon > self.epsilon_min:\n",
260 | " self.epsilon *= self.r\n",
261 | " return np.argmax(act_values[0])"
262 | ],
263 | "execution_count": 4,
264 | "outputs": []
265 | },
266 | {
267 | "cell_type": "code",
268 | "metadata": {
269 | "id": "On5S8YtLz3U4"
270 | },
271 | "source": [
272 | "class Main:\n",
273 | " def __init__(self, env, agent, mdl_dir, name, episodes_times = 1000, mode = 'test'):\n",
274 | " self.env = env\n",
275 | " self.agent = agent\n",
276 | " self.mdl_dir = mdl_dir\n",
277 | " self.scaler = self._standard_scaler(self.env)\n",
278 | " self.episodes_times = episodes_times\n",
279 | " self.mode = mode\n",
280 | " self.name = name\n",
281 | "\n",
282 | " self.df_rec = pd.DataFrame(index=[], columns=['FixedProfit','TradeTimes','TradeWin'])\n",
283 | "\n",
284 | " if self.mode == 'test':\n",
285 | " with open('{}/{}.pkl'.format(self.mdl_dir, self.name), 'rb') as f:\n",
286 | " self.scaler = pickle.load(f)\n",
287 | " self.agent.epsilon = 0.01\n",
288 | " self.agent.load('{}/{}.h5'.format(self.mdl_dir, self.name))\n",
289 | "\n",
290 | " def play_game(self):\n",
291 | "\n",
292 | " for episode in range(self.episodes_times):\n",
293 | "\n",
294 | " if (episode % 10 == 0):\n",
295 | " metrics_names = ['FixedProfit','TradeTimes','TradeWin']\n",
296 | " if (int(str(self.episodes_times)[:-1])*10 == episode):\n",
297 | " pb_i = Progbar(int(str(self.episodes_times)[-1]), stateful_metrics=metrics_names)\n",
298 | " else:\n",
299 | " pb_i = Progbar(10, stateful_metrics=metrics_names)\n",
300 | " p_mean,trade_time,win_time = np.array([]),np.array([]),np.array([])\n",
301 | "\n",
302 | " state = self.env.reset()\n",
303 | " state = self.scaler.transform([state])\n",
304 | " done = False\n",
305 | " reward = 0.0\n",
306 | " info = None\n",
307 | " next_state = copy.copy(state)\n",
308 | " next_act = 1\n",
309 | " act = 1\n",
310 | " \n",
311 | " while not done:\n",
312 | " action = self.agent.act(state, act, reward, info, next_state, done, mode)\n",
313 | " state = next_state\n",
314 | " next_state, reward, done, info = self.env.step(action)\n",
315 | " next_state = self.scaler.transform([next_state])\n",
316 | " reward = self._reward_clipping(reward)\n",
317 | " act = copy.copy(action)\n",
318 | " state = next_state\n",
319 | "\n",
320 | " record = pd.Series([info['cur_revenue'],info['trade_time'],info['trade_win']], index=self.df_rec.columns)\n",
321 | " p_mean,trade_time,win_time = np.append(p_mean,info['cur_revenue']),np.append(trade_time,info['trade_time']),np.append(win_time,info['trade_win'])\n",
322 | " values=[('FixedProfit',int(np.mean(p_mean))), ('TradeTimes',int(np.mean(trade_time))), ('TradeWin',int(np.mean(win_time)))]\n",
323 | " pb_i.add(1, values=values)\n",
324 | " self.df_rec = self.df_rec.append(record, ignore_index=True)\n",
325 | "\n",
326 | " if self.mode == 'train':\n",
327 | " self._save_scaler()\n",
328 | " self._save_csv()\n",
329 | "\n",
330 | " def _standard_scaler(self, env):\n",
331 | " states = []\n",
332 | " for _ in range(env.df_total_steps):\n",
333 | " action = np.random.choice(env.action_space)\n",
334 | " state, reward, done, info = env.step(action)\n",
335 | " states.append(state)\n",
336 | " if done:\n",
337 | " break\n",
338 | " \n",
339 | " scaler = StandardScaler()\n",
340 | " scaler.fit(states)\n",
341 | " return scaler\n",
342 | "\n",
343 | " def _reward_clipping(self, val):\n",
344 | " result = 1 if val > 0 else 0 if val == 0 else -1\n",
345 | " return result\n",
346 | "\n",
347 | " def _save_scaler(self):\n",
348 | " self.agent.save('{}/{}.h5'.format(self.mdl_dir, self.name))\n",
349 | " with open('{}/{}.pkl'.format(self.mdl_dir, self.name), 'wb') as f:\n",
350 | " pickle.dump(self.scaler, f)\n",
351 | "\n",
352 | " def _save_csv(self):\n",
353 | " self.df_rec.to_csv(csv_path)"
354 | ],
355 | "execution_count": 5,
356 | "outputs": []
357 | },
358 | {
359 | "cell_type": "code",
360 | "metadata": {
361 | "colab": {
362 | "base_uri": "https://localhost:8080/"
363 | },
364 | "id": "pYFNVDDQz9X9",
365 | "outputId": "fc0ac466-0e72-4f29-dc33-6898db7d8e12"
366 | },
367 | "source": [
368 | "initial_money=1000000\n",
369 | "episodes_times = 100\n",
370 | "commission = 0 if level == 1 else 0.002\n",
371 | "\n",
372 | "agent = Agent()\n",
373 | "env = Environment(df, initial_money=initial_money, mode = mode, commission = commission)\n",
374 | "main = Main(env, agent, mdl_dir, name, episodes_times, mode)\n",
375 | "main.play_game()"
376 | ],
377 | "execution_count": 6,
378 | "outputs": [
379 | {
380 | "output_type": "stream",
381 | "name": "stdout",
382 | "text": [
383 | "Model: \"sequential\"\n",
384 | "_________________________________________________________________\n",
385 | "Layer (type) Output Shape Param # \n",
386 | "=================================================================\n",
387 | "dense (Dense) (None, 3) 12 \n",
388 | "_________________________________________________________________\n",
389 | "re_lu (ReLU) (None, 3) 0 \n",
390 | "_________________________________________________________________\n",
391 | "dense_1 (Dense) (None, 3) 12 \n",
392 | "_________________________________________________________________\n",
393 | "re_lu_1 (ReLU) (None, 3) 0 \n",
394 | "_________________________________________________________________\n",
395 | "dense_2 (Dense) (None, 3) 12 \n",
396 | "=================================================================\n",
397 | "Total params: 36\n",
398 | "Trainable params: 36\n",
399 | "Non-trainable params: 0\n",
400 | "_________________________________________________________________\n",
401 | "None\n",
402 | "10/10 [==============================] - 362s 36s/step - FixedProfit: 1110323.0000 - TradeTimes: 2.0000 - TradeWin: 1.0000\n",
403 | "10/10 [==============================] - 360s 36s/step - FixedProfit: 1130943.0000 - TradeTimes: 2.0000 - TradeWin: 1.0000\n",
404 | "10/10 [==============================] - 365s 36s/step - FixedProfit: 1225114.0000 - TradeTimes: 2.0000 - TradeWin: 2.0000\n",
405 | "10/10 [==============================] - 367s 37s/step - FixedProfit: 1168616.0000 - TradeTimes: 2.0000 - TradeWin: 1.0000\n",
406 | "10/10 [==============================] - 365s 37s/step - FixedProfit: 1189799.0000 - TradeTimes: 2.0000 - TradeWin: 2.0000\n",
407 | "10/10 [==============================] - 362s 36s/step - FixedProfit: 1126362.0000 - TradeTimes: 2.0000 - TradeWin: 2.0000\n",
408 | "10/10 [==============================] - 363s 36s/step - FixedProfit: 1144452.0000 - TradeTimes: 1.0000 - TradeWin: 1.0000\n",
409 | "10/10 [==============================] - 361s 36s/step - FixedProfit: 1169263.0000 - TradeTimes: 2.0000 - TradeWin: 1.0000\n",
410 | "10/10 [==============================] - 364s 36s/step - FixedProfit: 1172059.0000 - TradeTimes: 2.0000 - TradeWin: 2.0000\n",
411 | "10/10 [==============================] - 364s 36s/step - FixedProfit: 1127558.0000 - TradeTimes: 2.0000 - TradeWin: 1.0000\n"
412 | ]
413 | }
414 | ]
415 | }
416 | ]
417 | }
--------------------------------------------------------------------------------
/SARSA/sarsa_train.ipynb:
--------------------------------------------------------------------------------
1 | {
2 | "nbformat": 4,
3 | "nbformat_minor": 0,
4 | "metadata": {
5 | "colab": {
6 | "name": "sarsa_train.ipynb",
7 | "provenance": [],
8 | "collapsed_sections": [],
9 | "machine_shape": "hm",
10 | "authorship_tag": "ABX9TyOuUmBqr5xyzGqGXDZQBje4",
11 | "include_colab_link": true
12 | },
13 | "kernelspec": {
14 | "name": "python3",
15 | "display_name": "Python 3"
16 | },
17 | "language_info": {
18 | "name": "python"
19 | },
20 | "accelerator": "GPU"
21 | },
22 | "cells": [
23 | {
24 | "cell_type": "markdown",
25 | "metadata": {
26 | "id": "view-in-github",
27 | "colab_type": "text"
28 | },
29 | "source": [
30 | ""
31 | ]
32 | },
33 | {
34 | "cell_type": "code",
35 | "metadata": {
36 | "colab": {
37 | "base_uri": "https://localhost:8080/"
38 | },
39 | "id": "3NIXg6mTzk0K",
40 | "outputId": "6ed40601-4f26-4af8-c9d5-55da53144774"
41 | },
42 | "source": [
43 | "import pandas as pd\n",
44 | "import numpy as np\n",
45 | "import random\n",
46 | "from google.colab import drive\n",
47 | "import copy\n",
48 | "\n",
49 | "from datetime import datetime\n",
50 | "import pickle\n",
51 | "\n",
52 | "from tensorflow.keras.models import Sequential\n",
53 | "from tensorflow.keras.layers import Dense, ReLU\n",
54 | "from tensorflow.keras.optimizers import RMSprop\n",
55 | "from sklearn.preprocessing import StandardScaler\n",
56 | "\n",
57 | "from tensorflow.keras.utils import Progbar\n",
58 | "\n",
59 | "mode = 'train'\n",
60 | "name = 'sarsa'\n",
61 | "level = 1\n",
62 | "if level == 2:\n",
63 | " name += name + 'lv2'\n",
64 | "\n",
65 | "drive.mount('/content/drive/')\n",
66 | "nov_dir = 'Colab Notebooks/dataset/reinforcement_learning/'\n",
67 | "nov_path = '/content/drive/My Drive/' + nov_dir + f'sp500_{mode}.csv'\n",
68 | "\n",
69 | "exp_dir = 'Colab Notebooks/workspace/export/'\n",
70 | "mdl_dir = '/content/drive/My Drive/' + exp_dir + 'models'\n",
71 | "csv_path = '/content/drive/My Drive/' + exp_dir + f'csv_data/{name}_{mode}.csv'\n",
72 | "\n",
73 | "df = pd.read_csv(nov_path)\n",
74 | "df['Date'] = pd.to_datetime(df['Date'], format = '%Y-%m-%d')"
75 | ],
76 | "execution_count": 1,
77 | "outputs": [
78 | {
79 | "output_type": "stream",
80 | "name": "stdout",
81 | "text": [
82 | "Drive already mounted at /content/drive/; to attempt to forcibly remount, call drive.mount(\"/content/drive/\", force_remount=True).\n"
83 | ]
84 | }
85 | ]
86 | },
87 | {
88 | "cell_type": "code",
89 | "metadata": {
90 | "id": "MN1DKfV6zauY"
91 | },
92 | "source": [
93 | "class Environment:\n",
94 | " def __init__(self, df, initial_money=100000, mode = 'test', commission = 0):\n",
95 | "\n",
96 | " self.df = df.dropna().reset_index()\n",
97 | "\n",
98 | " self.df_total_steps = len(self.df)-1\n",
99 | " self.initial_money = initial_money\n",
100 | " self.mode = mode\n",
101 | " self.commission = commission\n",
102 | " self.trade_time = None\n",
103 | " self.trade_win = None\n",
104 | " self.brfore_buy_cash = None\n",
105 | " self.action_space = np.array([0, 1, 2]) # buy,hold,sell\n",
106 | " self.hold_a_position = None\n",
107 | " self.now_price = None\n",
108 | " self.cash_in_hand = None\n",
109 | " self.sell_price = None\n",
110 | " self.buy_price = None\n",
111 | "\n",
112 | " self.reset()\n",
113 | " \n",
114 | " def reset(self):\n",
115 | "\n",
116 | " self.trade_time = 0\n",
117 | " self.trade_win = 0\n",
118 | " self.brfore_buy_cash = 0\n",
119 | " self.end_step = self.df_total_steps\n",
120 | " self.now_step = 0\n",
121 | " self.hold_a_position = 0.0\n",
122 | " self.now_price = self.df.loc[self.now_step, 'SP500']\n",
123 | " self.cash_in_hand = self.initial_money\n",
124 | " self.sell_price = 0\n",
125 | " self.buy_price = 0\n",
126 | "\n",
127 | " return self._get_now_state()\n",
128 | "\n",
129 | " def step(self, action):\n",
130 | "\n",
131 | " self.now_step += 1\n",
132 | " self.now_price = self.df.loc[self.now_step, 'SP500']\n",
133 | " \n",
134 | " done = (self.end_step == self.now_step)\n",
135 | "\n",
136 | " self.sell_price = 0\n",
137 | " self._trade(action,done)\n",
138 | " reward = 0\n",
139 | " if (self.sell_price > 0) and (self.buy_price > 0) and ((self.sell_price - self.buy_price) != 0):\n",
140 | " reward = (self.sell_price - self.buy_price) / self.buy_price\n",
141 | " self.buy_price = 0\n",
142 | " cur_revenue = self._get_revenue()\n",
143 | " \n",
144 | " info = { 'cur_revenue' : cur_revenue , 'trade_time' : self.trade_time, 'trade_win' : self.trade_win }\n",
145 | "\n",
146 | " return self._get_now_state(), reward, done, info\n",
147 | "\n",
148 | " def _get_now_state(self):\n",
149 | " state = np.empty(3)\n",
150 | " state[0] = self.hold_a_position\n",
151 | " state[1] = self.now_price\n",
152 | " state[2] = self.cash_in_hand\n",
153 | " return state\n",
154 | "\n",
155 | " def _get_revenue(self): \n",
156 | " return self.hold_a_position * self.now_price + self.cash_in_hand\n",
157 | "\n",
158 | " def _trade(self, action,lastorder = False):\n",
159 | " if lastorder:\n",
160 | " if self.hold_a_position != 0:\n",
161 | " self.cash_in_hand += self.now_price * self.hold_a_position\n",
162 | " self.hold_a_position = 0\n",
163 | " self.trade_time += 1\n",
164 | " if self.cash_in_hand > self.brfore_buy_cash:\n",
165 | " self.trade_win += 1\n",
166 | " else:\n",
167 | " if self.action_space[0] == action: # buy\n",
168 | " if self.hold_a_position == 0:\n",
169 | " buy_flag = True\n",
170 | " self.brfore_buy_cash = copy.copy(self.cash_in_hand)\n",
171 | " while buy_flag:\n",
172 | " if self.cash_in_hand > self.now_price:\n",
173 | " self.hold_a_position += 1\n",
174 | " self.buy_price += self.now_price\n",
175 | " self.cash_in_hand -= self.now_price + self.commission * self.now_price\n",
176 | " else:\n",
177 | " buy_flag = False\n",
178 | " if self.action_space[2] == action: # sell\n",
179 | " if self.hold_a_position != 0:\n",
180 | " self.sell_price += self.now_price * self.hold_a_position\n",
181 | " self.cash_in_hand += self.now_price * self.hold_a_position - self.commission * self.now_price * self.hold_a_position\n",
182 | " self.hold_a_position = 0\n",
183 | " self.trade_time += 1\n",
184 | " if self.cash_in_hand > self.brfore_buy_cash:\n",
185 | " self.trade_win += 1"
186 | ],
187 | "execution_count": 2,
188 | "outputs": []
189 | },
190 | {
191 | "cell_type": "code",
192 | "metadata": {
193 | "id": "nGeWOM-ZWNYK"
194 | },
195 | "source": [
196 | "class Brain:\n",
197 | " def __init__(self):\n",
198 | "\n",
199 | " self.gamma = 0.9\n",
200 | " \n",
201 | " model = Sequential()\n",
202 | " model.add(Dense(3, input_shape=(3,)))\n",
203 | " model.add(ReLU()) \n",
204 | " model.add(Dense(3))\n",
205 | " model.add(ReLU())\n",
206 | " model.add(Dense(3))\n",
207 | " model.compile(loss=\"mse\", optimizer=RMSprop())\n",
208 | "\n",
209 | " print((model.summary()))\n",
210 | " self.model = model\n",
211 | "\n",
212 | " def train(self, state, action, reward, next_state, next_act, done):\n",
213 | " q = self.model.predict(state) \n",
214 | " target = np.copy(q)\n",
215 | " if done:\n",
216 | " target[:, action] = reward\n",
217 | " else:\n",
218 | " target[:, action] = reward + self.gamma*next_act\n",
219 | " self.model.train_on_batch(state, target)\n",
220 | "\n",
221 | " def predict(self, state):\n",
222 | " return self.model.predict(state)\n",
223 | "\n",
224 | " def load(self, name):\n",
225 | " self.model.load_weights(name)\n",
226 | "\n",
227 | " def save(self, name):\n",
228 | " self.model.save_weights(name)"
229 | ],
230 | "execution_count": 3,
231 | "outputs": []
232 | },
233 | {
234 | "cell_type": "code",
235 | "metadata": {
236 | "id": "QxR4grMVRLCR"
237 | },
238 | "source": [
239 | "class Agent(Brain):\n",
240 | " def __init__(self):\n",
241 | "\n",
242 | " super().__init__()\n",
243 | " self.epsilon = 1.0\n",
244 | " self.epsilon_min = 0.01\n",
245 | " self.r = 0.995\n",
246 | "\n",
247 | " def act(self, state, act, reward, info, next_state, done, mode = 'train'):\n",
248 | "\n",
249 | " next_act = self._next_act(state)\n",
250 | " if mode == 'train':\n",
251 | " self.train(state, act, reward, next_state, next_act, done)\n",
252 | "\n",
253 | " return next_act\n",
254 | "\n",
255 | " def _next_act(self, state):\n",
256 | " if np.random.rand() <= self.epsilon:\n",
257 | " return np.random.choice(3)\n",
258 | " act_values = self.predict(state)\n",
259 | " if self.epsilon > self.epsilon_min:\n",
260 | " self.epsilon *= self.r\n",
261 | " return np.argmax(act_values[0])"
262 | ],
263 | "execution_count": 4,
264 | "outputs": []
265 | },
266 | {
267 | "cell_type": "code",
268 | "metadata": {
269 | "id": "On5S8YtLz3U4"
270 | },
271 | "source": [
272 | "class Main:\n",
273 | " def __init__(self, env, agent, mdl_dir, name, episodes_times = 1000, mode = 'test'):\n",
274 | " self.env = env\n",
275 | " self.agent = agent\n",
276 | " self.mdl_dir = mdl_dir\n",
277 | " self.scaler = self._standard_scaler(self.env)\n",
278 | " self.episodes_times = episodes_times\n",
279 | " self.mode = mode\n",
280 | " self.name = name\n",
281 | "\n",
282 | " self.df_rec = pd.DataFrame(index=[], columns=['FixedProfit','TradeTimes','TradeWin'])\n",
283 | "\n",
284 | " if self.mode == 'test':\n",
285 | " with open('{}/{}.pkl'.format(self.mdl_dir, self.name), 'rb') as f:\n",
286 | " self.scaler = pickle.load(f)\n",
287 | " self.agent.epsilon = 0.01\n",
288 | " self.agent.load('{}/{}.h5'.format(self.mdl_dir, self.name))\n",
289 | "\n",
290 | " def play_game(self):\n",
291 | "\n",
292 | " for episode in range(self.episodes_times):\n",
293 | "\n",
294 | " if (episode % 10 == 0):\n",
295 | " metrics_names = ['FixedProfit','TradeTimes','TradeWin']\n",
296 | " if (int(str(self.episodes_times)[:-1])*10 == episode):\n",
297 | " pb_i = Progbar(int(str(self.episodes_times)[-1]), stateful_metrics=metrics_names)\n",
298 | " else:\n",
299 | " pb_i = Progbar(10, stateful_metrics=metrics_names)\n",
300 | " p_mean,trade_time,win_time = np.array([]),np.array([]),np.array([])\n",
301 | "\n",
302 | " state = self.env.reset()\n",
303 | " state = self.scaler.transform([state])\n",
304 | " done = False\n",
305 | " reward = 0.0\n",
306 | " info = None\n",
307 | " next_state = copy.copy(state)\n",
308 | " next_act = 1\n",
309 | " act = 1\n",
310 | " \n",
311 | " while not done:\n",
312 | " action = self.agent.act(state, act, reward, info, next_state, done, mode)\n",
313 | " state = next_state\n",
314 | " next_state, reward, done, info = self.env.step(action)\n",
315 | " next_state = self.scaler.transform([next_state])\n",
316 | " reward = self._reward_clipping(reward)\n",
317 | " act = copy.copy(action)\n",
318 | " state = next_state\n",
319 | "\n",
320 | " record = pd.Series([info['cur_revenue'],info['trade_time'],info['trade_win']], index=self.df_rec.columns)\n",
321 | " p_mean,trade_time,win_time = np.append(p_mean,info['cur_revenue']),np.append(trade_time,info['trade_time']),np.append(win_time,info['trade_win'])\n",
322 | " values=[('FixedProfit',int(np.mean(p_mean))), ('TradeTimes',int(np.mean(trade_time))), ('TradeWin',int(np.mean(win_time)))]\n",
323 | " pb_i.add(1, values=values)\n",
324 | " self.df_rec = self.df_rec.append(record, ignore_index=True)\n",
325 | "\n",
326 | " if self.mode == 'train':\n",
327 | " self._save_scaler()\n",
328 | " self._save_csv()\n",
329 | "\n",
330 | " def _standard_scaler(self, env):\n",
331 | " states = []\n",
332 | " for _ in range(env.df_total_steps):\n",
333 | " action = np.random.choice(env.action_space)\n",
334 | " state, reward, done, info = env.step(action)\n",
335 | " states.append(state)\n",
336 | " if done:\n",
337 | " break\n",
338 | " \n",
339 | " scaler = StandardScaler()\n",
340 | " scaler.fit(states)\n",
341 | " return scaler\n",
342 | "\n",
343 | " def _reward_clipping(self, val):\n",
344 | " result = 1 if val > 0 else 0 if val == 0 else -1\n",
345 | " return result\n",
346 | "\n",
347 | " def _save_scaler(self):\n",
348 | " self.agent.save('{}/{}.h5'.format(self.mdl_dir, self.name))\n",
349 | " with open('{}/{}.pkl'.format(self.mdl_dir, self.name), 'wb') as f:\n",
350 | " pickle.dump(self.scaler, f)\n",
351 | "\n",
352 | " def _save_csv(self):\n",
353 | " self.df_rec.to_csv(csv_path)"
354 | ],
355 | "execution_count": 5,
356 | "outputs": []
357 | },
358 | {
359 | "cell_type": "code",
360 | "metadata": {
361 | "colab": {
362 | "base_uri": "https://localhost:8080/"
363 | },
364 | "id": "pYFNVDDQz9X9",
365 | "outputId": "1eb7434b-7add-4358-fd4f-6ac9a4efe7e8"
366 | },
367 | "source": [
368 | "initial_money=1000000\n",
369 | "episodes_times = 100\n",
370 | "commission = 0 if level == 1 else 0.002\n",
371 | "\n",
372 | "agent = Agent()\n",
373 | "env = Environment(df, initial_money=initial_money, mode = mode, commission = commission)\n",
374 | "main = Main(env, agent, mdl_dir, name, episodes_times, mode)\n",
375 | "main.play_game()"
376 | ],
377 | "execution_count": 6,
378 | "outputs": [
379 | {
380 | "output_type": "stream",
381 | "name": "stdout",
382 | "text": [
383 | "Model: \"sequential\"\n",
384 | "_________________________________________________________________\n",
385 | "Layer (type) Output Shape Param # \n",
386 | "=================================================================\n",
387 | "dense (Dense) (None, 3) 12 \n",
388 | "_________________________________________________________________\n",
389 | "re_lu (ReLU) (None, 3) 0 \n",
390 | "_________________________________________________________________\n",
391 | "dense_1 (Dense) (None, 3) 12 \n",
392 | "_________________________________________________________________\n",
393 | "re_lu_1 (ReLU) (None, 3) 0 \n",
394 | "_________________________________________________________________\n",
395 | "dense_2 (Dense) (None, 3) 12 \n",
396 | "=================================================================\n",
397 | "Total params: 36\n",
398 | "Trainable params: 36\n",
399 | "Non-trainable params: 0\n",
400 | "_________________________________________________________________\n",
401 | "None\n",
402 | "10/10 [==============================] - 296s 29s/step - FixedProfit: 1058093.0000 - TradeTimes: 101.0000 - TradeWin: 55.0000\n",
403 | "10/10 [==============================] - 288s 29s/step - FixedProfit: 1060879.0000 - TradeTimes: 103.0000 - TradeWin: 55.0000\n",
404 | "10/10 [==============================] - 291s 29s/step - FixedProfit: 1075286.0000 - TradeTimes: 103.0000 - TradeWin: 58.0000\n",
405 | "10/10 [==============================] - 289s 29s/step - FixedProfit: 1112158.0000 - TradeTimes: 106.0000 - TradeWin: 60.0000\n",
406 | "10/10 [==============================] - 285s 29s/step - FixedProfit: 1122539.0000 - TradeTimes: 106.0000 - TradeWin: 59.0000\n",
407 | "10/10 [==============================] - 292s 29s/step - FixedProfit: 1103105.0000 - TradeTimes: 106.0000 - TradeWin: 60.0000\n",
408 | "10/10 [==============================] - 292s 29s/step - FixedProfit: 1101880.0000 - TradeTimes: 107.0000 - TradeWin: 56.0000\n",
409 | "10/10 [==============================] - 297s 30s/step - FixedProfit: 1130877.0000 - TradeTimes: 105.0000 - TradeWin: 58.0000\n",
410 | "10/10 [==============================] - 298s 30s/step - FixedProfit: 1105993.0000 - TradeTimes: 103.0000 - TradeWin: 58.0000\n",
411 | "10/10 [==============================] - 296s 30s/step - FixedProfit: 1108156.0000 - TradeTimes: 106.0000 - TradeWin: 59.0000\n"
412 | ]
413 | }
414 | ]
415 | }
416 | ]
417 | }
--------------------------------------------------------------------------------
/data/sp500_train.csv:
--------------------------------------------------------------------------------
1 | Date,SP500
2 | 2015-01-02,2058.199951171875
3 | 2015-01-05,2020.5799560546875
4 | 2015-01-06,2002.6099853515625
5 | 2015-01-07,2025.9000244140625
6 | 2015-01-08,2062.139892578125
7 | 2015-01-09,2044.81005859375
8 | 2015-01-12,2028.260009765625
9 | 2015-01-13,2023.030029296875
10 | 2015-01-14,2011.27001953125
11 | 2015-01-15,1992.6700439453125
12 | 2015-01-16,2019.4200439453125
13 | 2015-01-20,2022.550048828125
14 | 2015-01-21,2032.1199951171875
15 | 2015-01-22,2063.14990234375
16 | 2015-01-23,2051.820068359375
17 | 2015-01-26,2057.090087890625
18 | 2015-01-27,2029.550048828125
19 | 2015-01-28,2002.1600341796875
20 | 2015-01-29,2021.25
21 | 2015-01-30,1994.989990234375
22 | 2015-02-02,2020.8499755859375
23 | 2015-02-03,2050.030029296875
24 | 2015-02-04,2041.510009765625
25 | 2015-02-05,2062.52001953125
26 | 2015-02-06,2055.469970703125
27 | 2015-02-09,2046.739990234375
28 | 2015-02-10,2068.590087890625
29 | 2015-02-11,2068.530029296875
30 | 2015-02-12,2088.47998046875
31 | 2015-02-13,2096.989990234375
32 | 2015-02-17,2100.340087890625
33 | 2015-02-18,2099.679931640625
34 | 2015-02-19,2097.449951171875
35 | 2015-02-20,2110.300048828125
36 | 2015-02-23,2109.659912109375
37 | 2015-02-24,2115.47998046875
38 | 2015-02-25,2113.860107421875
39 | 2015-02-26,2110.739990234375
40 | 2015-02-27,2104.5
41 | 2015-03-02,2117.389892578125
42 | 2015-03-03,2107.780029296875
43 | 2015-03-04,2098.530029296875
44 | 2015-03-05,2101.0400390625
45 | 2015-03-06,2071.260009765625
46 | 2015-03-09,2079.429931640625
47 | 2015-03-10,2044.1600341796875
48 | 2015-03-11,2040.239990234375
49 | 2015-03-12,2065.949951171875
50 | 2015-03-13,2053.39990234375
51 | 2015-03-16,2081.18994140625
52 | 2015-03-17,2074.280029296875
53 | 2015-03-18,2099.5
54 | 2015-03-19,2089.27001953125
55 | 2015-03-20,2108.10009765625
56 | 2015-03-23,2104.419921875
57 | 2015-03-24,2091.5
58 | 2015-03-25,2061.050048828125
59 | 2015-03-26,2056.14990234375
60 | 2015-03-27,2061.02001953125
61 | 2015-03-30,2086.239990234375
62 | 2015-03-31,2067.889892578125
63 | 2015-04-01,2059.68994140625
64 | 2015-04-02,2066.9599609375
65 | 2015-04-06,2080.6201171875
66 | 2015-04-07,2076.330078125
67 | 2015-04-08,2081.89990234375
68 | 2015-04-09,2091.179931640625
69 | 2015-04-10,2102.06005859375
70 | 2015-04-13,2092.429931640625
71 | 2015-04-14,2095.840087890625
72 | 2015-04-15,2106.6298828125
73 | 2015-04-16,2104.989990234375
74 | 2015-04-17,2081.179931640625
75 | 2015-04-20,2100.39990234375
76 | 2015-04-21,2097.2900390625
77 | 2015-04-22,2107.9599609375
78 | 2015-04-23,2112.929931640625
79 | 2015-04-24,2117.68994140625
80 | 2015-04-27,2108.919921875
81 | 2015-04-28,2114.760009765625
82 | 2015-04-29,2106.85009765625
83 | 2015-04-30,2085.510009765625
84 | 2015-05-01,2108.2900390625
85 | 2015-05-04,2114.489990234375
86 | 2015-05-05,2089.4599609375
87 | 2015-05-06,2080.14990234375
88 | 2015-05-07,2088.0
89 | 2015-05-08,2116.10009765625
90 | 2015-05-11,2105.330078125
91 | 2015-05-12,2099.1201171875
92 | 2015-05-13,2098.47998046875
93 | 2015-05-14,2121.10009765625
94 | 2015-05-15,2122.72998046875
95 | 2015-05-18,2129.199951171875
96 | 2015-05-19,2127.830078125
97 | 2015-05-20,2125.85009765625
98 | 2015-05-21,2130.820068359375
99 | 2015-05-22,2126.06005859375
100 | 2015-05-26,2104.199951171875
101 | 2015-05-27,2123.47998046875
102 | 2015-05-28,2120.7900390625
103 | 2015-05-29,2107.389892578125
104 | 2015-06-01,2111.72998046875
105 | 2015-06-02,2109.60009765625
106 | 2015-06-03,2114.070068359375
107 | 2015-06-04,2095.840087890625
108 | 2015-06-05,2092.830078125
109 | 2015-06-08,2079.280029296875
110 | 2015-06-09,2080.14990234375
111 | 2015-06-10,2105.199951171875
112 | 2015-06-11,2108.860107421875
113 | 2015-06-12,2094.110107421875
114 | 2015-06-15,2084.429931640625
115 | 2015-06-16,2096.2900390625
116 | 2015-06-17,2100.43994140625
117 | 2015-06-18,2121.239990234375
118 | 2015-06-19,2109.989990234375
119 | 2015-06-22,2122.85009765625
120 | 2015-06-23,2124.199951171875
121 | 2015-06-24,2108.580078125
122 | 2015-06-25,2102.31005859375
123 | 2015-06-26,2101.489990234375
124 | 2015-06-29,2057.639892578125
125 | 2015-06-30,2063.110107421875
126 | 2015-07-01,2077.419921875
127 | 2015-07-02,2076.780029296875
128 | 2015-07-06,2068.760009765625
129 | 2015-07-07,2081.340087890625
130 | 2015-07-08,2046.6800537109375
131 | 2015-07-09,2051.31005859375
132 | 2015-07-10,2076.6201171875
133 | 2015-07-13,2099.60009765625
134 | 2015-07-14,2108.949951171875
135 | 2015-07-15,2107.39990234375
136 | 2015-07-16,2124.2900390625
137 | 2015-07-17,2126.639892578125
138 | 2015-07-20,2128.280029296875
139 | 2015-07-21,2119.2099609375
140 | 2015-07-22,2114.14990234375
141 | 2015-07-23,2102.14990234375
142 | 2015-07-24,2079.64990234375
143 | 2015-07-27,2067.639892578125
144 | 2015-07-28,2093.25
145 | 2015-07-29,2108.570068359375
146 | 2015-07-30,2108.6298828125
147 | 2015-07-31,2103.840087890625
148 | 2015-08-03,2098.0400390625
149 | 2015-08-04,2093.320068359375
150 | 2015-08-05,2099.840087890625
151 | 2015-08-06,2083.56005859375
152 | 2015-08-07,2077.570068359375
153 | 2015-08-10,2104.179931640625
154 | 2015-08-11,2084.070068359375
155 | 2015-08-12,2086.050048828125
156 | 2015-08-13,2083.389892578125
157 | 2015-08-14,2091.5400390625
158 | 2015-08-17,2102.43994140625
159 | 2015-08-18,2096.919921875
160 | 2015-08-19,2079.610107421875
161 | 2015-08-20,2035.72998046875
162 | 2015-08-21,1970.8900146484375
163 | 2015-08-24,1893.2099609375
164 | 2015-08-25,1867.6099853515625
165 | 2015-08-26,1940.510009765625
166 | 2015-08-27,1987.6600341796875
167 | 2015-08-28,1988.8699951171875
168 | 2015-08-31,1972.1800537109375
169 | 2015-09-01,1913.8499755859375
170 | 2015-09-02,1948.8599853515625
171 | 2015-09-03,1951.1300048828125
172 | 2015-09-04,1921.219970703125
173 | 2015-09-08,1969.4100341796875
174 | 2015-09-09,1942.0400390625
175 | 2015-09-10,1952.2900390625
176 | 2015-09-11,1961.050048828125
177 | 2015-09-14,1953.030029296875
178 | 2015-09-15,1978.0899658203125
179 | 2015-09-16,1995.31005859375
180 | 2015-09-17,1990.199951171875
181 | 2015-09-18,1958.030029296875
182 | 2015-09-21,1966.969970703125
183 | 2015-09-22,1942.739990234375
184 | 2015-09-23,1938.760009765625
185 | 2015-09-24,1932.239990234375
186 | 2015-09-25,1931.3399658203125
187 | 2015-09-28,1881.77001953125
188 | 2015-09-29,1884.0899658203125
189 | 2015-09-30,1920.030029296875
190 | 2015-10-01,1923.8199462890625
191 | 2015-10-02,1951.3599853515625
192 | 2015-10-05,1987.050048828125
193 | 2015-10-06,1979.9200439453125
194 | 2015-10-07,1995.8299560546875
195 | 2015-10-08,2013.4300537109375
196 | 2015-10-09,2014.8900146484375
197 | 2015-10-12,2017.4599609375
198 | 2015-10-13,2003.68994140625
199 | 2015-10-14,1994.239990234375
200 | 2015-10-15,2023.8599853515625
201 | 2015-10-16,2033.1099853515625
202 | 2015-10-19,2033.6600341796875
203 | 2015-10-20,2030.77001953125
204 | 2015-10-21,2018.93994140625
205 | 2015-10-22,2052.510009765625
206 | 2015-10-23,2075.14990234375
207 | 2015-10-26,2071.179931640625
208 | 2015-10-27,2065.889892578125
209 | 2015-10-28,2090.35009765625
210 | 2015-10-29,2089.409912109375
211 | 2015-10-30,2079.360107421875
212 | 2015-11-02,2104.050048828125
213 | 2015-11-03,2109.7900390625
214 | 2015-11-04,2102.31005859375
215 | 2015-11-05,2099.929931640625
216 | 2015-11-06,2099.199951171875
217 | 2015-11-09,2078.580078125
218 | 2015-11-10,2081.719970703125
219 | 2015-11-11,2075.0
220 | 2015-11-12,2045.969970703125
221 | 2015-11-13,2023.0400390625
222 | 2015-11-16,2053.18994140625
223 | 2015-11-17,2050.43994140625
224 | 2015-11-18,2083.580078125
225 | 2015-11-19,2081.239990234375
226 | 2015-11-20,2089.169921875
227 | 2015-11-23,2086.590087890625
228 | 2015-11-24,2089.139892578125
229 | 2015-11-25,2088.8701171875
230 | 2015-11-27,2090.110107421875
231 | 2015-11-30,2080.409912109375
232 | 2015-12-01,2102.6298828125
233 | 2015-12-02,2079.510009765625
234 | 2015-12-03,2049.6201171875
235 | 2015-12-04,2091.68994140625
236 | 2015-12-07,2077.070068359375
237 | 2015-12-08,2063.590087890625
238 | 2015-12-09,2047.6199951171875
239 | 2015-12-10,2052.22998046875
240 | 2015-12-11,2012.3699951171875
241 | 2015-12-14,2021.93994140625
242 | 2015-12-15,2043.4100341796875
243 | 2015-12-16,2073.070068359375
244 | 2015-12-17,2041.8900146484375
245 | 2015-12-18,2005.550048828125
246 | 2015-12-21,2021.1500244140625
247 | 2015-12-22,2038.969970703125
248 | 2015-12-23,2064.2900390625
249 | 2015-12-24,2060.989990234375
250 | 2015-12-28,2056.5
251 | 2015-12-29,2078.360107421875
252 | 2015-12-30,2063.360107421875
253 | 2015-12-31,2043.93994140625
254 | 2016-01-04,2012.6600341796875
255 | 2016-01-05,2016.7099609375
256 | 2016-01-06,1990.260009765625
257 | 2016-01-07,1943.0899658203125
258 | 2016-01-08,1922.030029296875
259 | 2016-01-11,1923.6700439453125
260 | 2016-01-12,1938.6800537109375
261 | 2016-01-13,1890.280029296875
262 | 2016-01-14,1921.8399658203125
263 | 2016-01-15,1880.3299560546875
264 | 2016-01-19,1881.3299560546875
265 | 2016-01-20,1859.3299560546875
266 | 2016-01-21,1868.989990234375
267 | 2016-01-22,1906.9000244140625
268 | 2016-01-25,1877.0799560546875
269 | 2016-01-26,1903.6300048828125
270 | 2016-01-27,1882.949951171875
271 | 2016-01-28,1893.3599853515625
272 | 2016-01-29,1940.239990234375
273 | 2016-02-01,1939.3800048828125
274 | 2016-02-02,1903.030029296875
275 | 2016-02-03,1912.530029296875
276 | 2016-02-04,1915.449951171875
277 | 2016-02-05,1880.050048828125
278 | 2016-02-08,1853.43994140625
279 | 2016-02-09,1852.2099609375
280 | 2016-02-10,1851.8599853515625
281 | 2016-02-11,1829.0799560546875
282 | 2016-02-12,1864.780029296875
283 | 2016-02-16,1895.5799560546875
284 | 2016-02-17,1926.8199462890625
285 | 2016-02-18,1917.8299560546875
286 | 2016-02-19,1917.780029296875
287 | 2016-02-22,1945.5
288 | 2016-02-23,1921.27001953125
289 | 2016-02-24,1929.800048828125
290 | 2016-02-25,1951.699951171875
291 | 2016-02-26,1948.050048828125
292 | 2016-02-29,1932.22998046875
293 | 2016-03-01,1978.3499755859375
294 | 2016-03-02,1986.449951171875
295 | 2016-03-03,1993.4000244140625
296 | 2016-03-04,1999.989990234375
297 | 2016-03-07,2001.760009765625
298 | 2016-03-08,1979.260009765625
299 | 2016-03-09,1989.260009765625
300 | 2016-03-10,1989.5699462890625
301 | 2016-03-11,2022.18994140625
302 | 2016-03-14,2019.6400146484375
303 | 2016-03-15,2015.9300537109375
304 | 2016-03-16,2027.219970703125
305 | 2016-03-17,2040.5899658203125
306 | 2016-03-18,2049.580078125
307 | 2016-03-21,2051.60009765625
308 | 2016-03-22,2049.800048828125
309 | 2016-03-23,2036.7099609375
310 | 2016-03-24,2035.93994140625
311 | 2016-03-28,2037.050048828125
312 | 2016-03-29,2055.010009765625
313 | 2016-03-30,2063.949951171875
314 | 2016-03-31,2059.739990234375
315 | 2016-04-01,2072.780029296875
316 | 2016-04-04,2066.1298828125
317 | 2016-04-05,2045.1700439453125
318 | 2016-04-06,2066.659912109375
319 | 2016-04-07,2041.9100341796875
320 | 2016-04-08,2047.5999755859375
321 | 2016-04-11,2041.989990234375
322 | 2016-04-12,2061.719970703125
323 | 2016-04-13,2082.419921875
324 | 2016-04-14,2082.780029296875
325 | 2016-04-15,2080.72998046875
326 | 2016-04-18,2094.340087890625
327 | 2016-04-19,2100.800048828125
328 | 2016-04-20,2102.39990234375
329 | 2016-04-21,2091.47998046875
330 | 2016-04-22,2091.580078125
331 | 2016-04-25,2087.7900390625
332 | 2016-04-26,2091.699951171875
333 | 2016-04-27,2095.14990234375
334 | 2016-04-28,2075.81005859375
335 | 2016-04-29,2065.300048828125
336 | 2016-05-02,2081.429931640625
337 | 2016-05-03,2063.3701171875
338 | 2016-05-04,2051.1201171875
339 | 2016-05-05,2050.6298828125
340 | 2016-05-06,2057.139892578125
341 | 2016-05-09,2058.68994140625
342 | 2016-05-10,2084.389892578125
343 | 2016-05-11,2064.4599609375
344 | 2016-05-12,2064.110107421875
345 | 2016-05-13,2046.6099853515625
346 | 2016-05-16,2066.659912109375
347 | 2016-05-17,2047.2099609375
348 | 2016-05-18,2047.6300048828125
349 | 2016-05-19,2040.0400390625
350 | 2016-05-20,2052.320068359375
351 | 2016-05-23,2048.0400390625
352 | 2016-05-24,2076.06005859375
353 | 2016-05-25,2090.5400390625
354 | 2016-05-26,2090.10009765625
355 | 2016-05-27,2099.06005859375
356 | 2016-05-31,2096.949951171875
357 | 2016-06-01,2099.330078125
358 | 2016-06-02,2105.260009765625
359 | 2016-06-03,2099.1298828125
360 | 2016-06-06,2109.409912109375
361 | 2016-06-07,2112.1298828125
362 | 2016-06-08,2119.1201171875
363 | 2016-06-09,2115.47998046875
364 | 2016-06-10,2096.070068359375
365 | 2016-06-13,2079.06005859375
366 | 2016-06-14,2075.320068359375
367 | 2016-06-15,2071.5
368 | 2016-06-16,2077.989990234375
369 | 2016-06-17,2071.219970703125
370 | 2016-06-20,2083.25
371 | 2016-06-21,2088.89990234375
372 | 2016-06-22,2085.449951171875
373 | 2016-06-23,2113.320068359375
374 | 2016-06-24,2037.4100341796875
375 | 2016-06-27,2000.5400390625
376 | 2016-06-28,2036.0899658203125
377 | 2016-06-29,2070.77001953125
378 | 2016-06-30,2098.860107421875
379 | 2016-07-01,2102.949951171875
380 | 2016-07-05,2088.550048828125
381 | 2016-07-06,2099.72998046875
382 | 2016-07-07,2097.89990234375
383 | 2016-07-08,2129.89990234375
384 | 2016-07-11,2137.159912109375
385 | 2016-07-12,2152.139892578125
386 | 2016-07-13,2152.429931640625
387 | 2016-07-14,2163.75
388 | 2016-07-15,2161.739990234375
389 | 2016-07-18,2166.889892578125
390 | 2016-07-19,2163.780029296875
391 | 2016-07-20,2173.02001953125
392 | 2016-07-21,2165.169921875
393 | 2016-07-22,2175.030029296875
394 | 2016-07-25,2168.47998046875
395 | 2016-07-26,2169.179931640625
396 | 2016-07-27,2166.580078125
397 | 2016-07-28,2170.06005859375
398 | 2016-07-29,2173.60009765625
399 | 2016-08-01,2170.840087890625
400 | 2016-08-02,2157.030029296875
401 | 2016-08-03,2163.7900390625
402 | 2016-08-04,2164.25
403 | 2016-08-05,2182.8701171875
404 | 2016-08-08,2180.889892578125
405 | 2016-08-09,2181.739990234375
406 | 2016-08-10,2175.489990234375
407 | 2016-08-11,2185.7900390625
408 | 2016-08-12,2184.050048828125
409 | 2016-08-15,2190.14990234375
410 | 2016-08-16,2178.14990234375
411 | 2016-08-17,2182.219970703125
412 | 2016-08-18,2187.02001953125
413 | 2016-08-19,2183.8701171875
414 | 2016-08-22,2182.639892578125
415 | 2016-08-23,2186.89990234375
416 | 2016-08-24,2175.43994140625
417 | 2016-08-25,2172.469970703125
418 | 2016-08-26,2169.0400390625
419 | 2016-08-29,2180.3798828125
420 | 2016-08-30,2176.1201171875
421 | 2016-08-31,2170.949951171875
422 | 2016-09-01,2170.860107421875
423 | 2016-09-02,2179.97998046875
424 | 2016-09-06,2186.47998046875
425 | 2016-09-07,2186.159912109375
426 | 2016-09-08,2181.300048828125
427 | 2016-09-09,2127.81005859375
428 | 2016-09-12,2159.0400390625
429 | 2016-09-13,2127.02001953125
430 | 2016-09-14,2125.77001953125
431 | 2016-09-15,2147.260009765625
432 | 2016-09-16,2139.159912109375
433 | 2016-09-19,2139.1201171875
434 | 2016-09-20,2139.760009765625
435 | 2016-09-21,2163.1201171875
436 | 2016-09-22,2177.179931640625
437 | 2016-09-23,2164.68994140625
438 | 2016-09-26,2146.10009765625
439 | 2016-09-27,2159.929931640625
440 | 2016-09-28,2171.3701171875
441 | 2016-09-29,2151.1298828125
442 | 2016-09-30,2168.27001953125
443 | 2016-10-03,2161.199951171875
444 | 2016-10-04,2150.489990234375
445 | 2016-10-05,2159.72998046875
446 | 2016-10-06,2160.77001953125
447 | 2016-10-07,2153.739990234375
448 | 2016-10-10,2163.659912109375
449 | 2016-10-11,2136.72998046875
450 | 2016-10-12,2139.179931640625
451 | 2016-10-13,2132.550048828125
452 | 2016-10-14,2132.97998046875
453 | 2016-10-17,2126.5
454 | 2016-10-18,2139.60009765625
455 | 2016-10-19,2144.2900390625
456 | 2016-10-20,2141.340087890625
457 | 2016-10-21,2141.159912109375
458 | 2016-10-24,2151.330078125
459 | 2016-10-25,2143.159912109375
460 | 2016-10-26,2139.429931640625
461 | 2016-10-27,2133.0400390625
462 | 2016-10-28,2126.409912109375
463 | 2016-10-31,2126.14990234375
464 | 2016-11-01,2111.719970703125
465 | 2016-11-02,2097.93994140625
466 | 2016-11-03,2088.659912109375
467 | 2016-11-04,2085.179931640625
468 | 2016-11-07,2131.52001953125
469 | 2016-11-08,2139.56005859375
470 | 2016-11-09,2163.260009765625
471 | 2016-11-10,2167.47998046875
472 | 2016-11-11,2164.449951171875
473 | 2016-11-14,2164.199951171875
474 | 2016-11-15,2180.389892578125
475 | 2016-11-16,2176.93994140625
476 | 2016-11-17,2187.1201171875
477 | 2016-11-18,2181.89990234375
478 | 2016-11-21,2198.179931640625
479 | 2016-11-22,2202.93994140625
480 | 2016-11-23,2204.719970703125
481 | 2016-11-25,2213.35009765625
482 | 2016-11-28,2201.719970703125
483 | 2016-11-29,2204.659912109375
484 | 2016-11-30,2198.81005859375
485 | 2016-12-01,2191.080078125
486 | 2016-12-02,2191.949951171875
487 | 2016-12-05,2204.7099609375
488 | 2016-12-06,2212.22998046875
489 | 2016-12-07,2241.35009765625
490 | 2016-12-08,2246.18994140625
491 | 2016-12-09,2259.530029296875
492 | 2016-12-12,2256.9599609375
493 | 2016-12-13,2271.719970703125
494 | 2016-12-14,2253.280029296875
495 | 2016-12-15,2262.030029296875
496 | 2016-12-16,2258.070068359375
497 | 2016-12-19,2262.530029296875
498 | 2016-12-20,2270.760009765625
499 | 2016-12-21,2265.179931640625
500 | 2016-12-22,2260.9599609375
501 | 2016-12-23,2263.7900390625
502 | 2016-12-27,2268.8798828125
503 | 2016-12-28,2249.919921875
504 | 2016-12-29,2249.260009765625
505 | 2016-12-30,2238.830078125
506 | 2017-01-03,2257.830078125
507 | 2017-01-04,2270.75
508 | 2017-01-05,2269.0
509 | 2017-01-06,2276.97998046875
510 | 2017-01-09,2268.89990234375
511 | 2017-01-10,2268.89990234375
512 | 2017-01-11,2275.320068359375
513 | 2017-01-12,2270.43994140625
514 | 2017-01-13,2274.639892578125
515 | 2017-01-17,2267.889892578125
516 | 2017-01-18,2271.889892578125
517 | 2017-01-19,2263.68994140625
518 | 2017-01-20,2271.31005859375
519 | 2017-01-23,2265.199951171875
520 | 2017-01-24,2280.070068359375
521 | 2017-01-25,2298.3701171875
522 | 2017-01-26,2296.679931640625
523 | 2017-01-27,2294.68994140625
524 | 2017-01-30,2280.89990234375
525 | 2017-01-31,2278.8701171875
526 | 2017-02-01,2279.550048828125
527 | 2017-02-02,2280.85009765625
528 | 2017-02-03,2297.419921875
529 | 2017-02-06,2292.56005859375
530 | 2017-02-07,2293.080078125
531 | 2017-02-08,2294.669921875
532 | 2017-02-09,2307.8701171875
533 | 2017-02-10,2316.10009765625
534 | 2017-02-13,2328.25
535 | 2017-02-14,2337.580078125
536 | 2017-02-15,2349.25
537 | 2017-02-16,2347.219970703125
538 | 2017-02-17,2351.159912109375
539 | 2017-02-21,2365.3798828125
540 | 2017-02-22,2362.820068359375
541 | 2017-02-23,2363.81005859375
542 | 2017-02-24,2367.340087890625
543 | 2017-02-27,2369.75
544 | 2017-02-28,2363.639892578125
545 | 2017-03-01,2395.9599609375
546 | 2017-03-02,2381.919921875
547 | 2017-03-03,2383.1201171875
548 | 2017-03-06,2375.31005859375
549 | 2017-03-07,2368.389892578125
550 | 2017-03-08,2362.97998046875
551 | 2017-03-09,2364.8701171875
552 | 2017-03-10,2372.60009765625
553 | 2017-03-13,2373.469970703125
554 | 2017-03-14,2365.449951171875
555 | 2017-03-15,2385.260009765625
556 | 2017-03-16,2381.3798828125
557 | 2017-03-17,2378.25
558 | 2017-03-20,2373.469970703125
559 | 2017-03-21,2344.02001953125
560 | 2017-03-22,2348.449951171875
561 | 2017-03-23,2345.9599609375
562 | 2017-03-24,2343.97998046875
563 | 2017-03-27,2341.590087890625
564 | 2017-03-28,2358.570068359375
565 | 2017-03-29,2361.1298828125
566 | 2017-03-30,2368.06005859375
567 | 2017-03-31,2362.719970703125
568 | 2017-04-03,2358.840087890625
569 | 2017-04-04,2360.159912109375
570 | 2017-04-05,2352.949951171875
571 | 2017-04-06,2357.489990234375
572 | 2017-04-07,2355.5400390625
573 | 2017-04-10,2357.159912109375
574 | 2017-04-11,2353.780029296875
575 | 2017-04-12,2344.929931640625
576 | 2017-04-13,2328.949951171875
577 | 2017-04-17,2349.010009765625
578 | 2017-04-18,2342.18994140625
579 | 2017-04-19,2338.169921875
580 | 2017-04-20,2355.840087890625
581 | 2017-04-21,2348.68994140625
582 | 2017-04-24,2374.14990234375
583 | 2017-04-25,2388.610107421875
584 | 2017-04-26,2387.449951171875
585 | 2017-04-27,2388.77001953125
586 | 2017-04-28,2384.199951171875
587 | 2017-05-01,2388.330078125
588 | 2017-05-02,2391.169921875
589 | 2017-05-03,2388.1298828125
590 | 2017-05-04,2389.52001953125
591 | 2017-05-05,2399.2900390625
592 | 2017-05-08,2399.3798828125
593 | 2017-05-09,2396.919921875
594 | 2017-05-10,2399.6298828125
595 | 2017-05-11,2394.43994140625
596 | 2017-05-12,2390.89990234375
597 | 2017-05-15,2402.320068359375
598 | 2017-05-16,2400.669921875
599 | 2017-05-17,2357.030029296875
600 | 2017-05-18,2365.719970703125
601 | 2017-05-19,2381.72998046875
602 | 2017-05-22,2394.02001953125
603 | 2017-05-23,2398.419921875
604 | 2017-05-24,2404.389892578125
605 | 2017-05-25,2415.070068359375
606 | 2017-05-26,2415.820068359375
607 | 2017-05-30,2412.909912109375
608 | 2017-05-31,2411.800048828125
609 | 2017-06-01,2430.06005859375
610 | 2017-06-02,2439.070068359375
611 | 2017-06-05,2436.10009765625
612 | 2017-06-06,2429.330078125
613 | 2017-06-07,2433.139892578125
614 | 2017-06-08,2433.7900390625
615 | 2017-06-09,2431.77001953125
616 | 2017-06-12,2429.389892578125
617 | 2017-06-13,2440.35009765625
618 | 2017-06-14,2437.919921875
619 | 2017-06-15,2432.4599609375
620 | 2017-06-16,2433.14990234375
621 | 2017-06-19,2453.4599609375
622 | 2017-06-20,2437.030029296875
623 | 2017-06-21,2435.610107421875
624 | 2017-06-22,2434.5
625 | 2017-06-23,2438.300048828125
626 | 2017-06-26,2439.070068359375
627 | 2017-06-27,2419.3798828125
628 | 2017-06-28,2440.68994140625
629 | 2017-06-29,2419.699951171875
630 |
--------------------------------------------------------------------------------
/m_process_random.ipynb:
--------------------------------------------------------------------------------
1 | {
2 | "nbformat": 4,
3 | "nbformat_minor": 0,
4 | "metadata": {
5 | "colab": {
6 | "name": "m_process_random.ipynb",
7 | "provenance": [],
8 | "collapsed_sections": [],
9 | "machine_shape": "hm",
10 | "authorship_tag": "ABX9TyN2h8hymPvpE5VAdS9EzmZH",
11 | "include_colab_link": true
12 | },
13 | "kernelspec": {
14 | "name": "python3",
15 | "display_name": "Python 3"
16 | },
17 | "language_info": {
18 | "name": "python"
19 | },
20 | "accelerator": "GPU"
21 | },
22 | "cells": [
23 | {
24 | "cell_type": "markdown",
25 | "metadata": {
26 | "id": "view-in-github",
27 | "colab_type": "text"
28 | },
29 | "source": [
30 | ""
31 | ]
32 | },
33 | {
34 | "cell_type": "code",
35 | "metadata": {
36 | "colab": {
37 | "base_uri": "https://localhost:8080/"
38 | },
39 | "id": "3NIXg6mTzk0K",
40 | "outputId": "ad2b647b-b878-4072-b22f-897a64797989"
41 | },
42 | "source": [
43 | "import pandas as pd\n",
44 | "import numpy as np\n",
45 | "import random\n",
46 | "from google.colab import drive\n",
47 | "import copy\n",
48 | "\n",
49 | "from datetime import datetime\n",
50 | "from multiprocessing import Process\n",
51 | "\n",
52 | "mode = 'test'\n",
53 | "name = 'm_process_random'\n",
54 | "\n",
55 | "drive.mount('/content/drive/')\n",
56 | "nov_dir = 'Colab Notebooks/dataset/reinforcement_learning/'\n",
57 | "nov_path = '/content/drive/My Drive/' + nov_dir + f'sp500_{mode}.csv'\n",
58 | "exp_dir = 'Colab Notebooks/workspace/export/'\n",
59 | "csv_path = '/content/drive/My Drive/' + exp_dir + f'csv_data/{name}_{mode}.csv'\n",
60 | "\n",
61 | "df = pd.read_csv(nov_path)\n",
62 | "df['Date'] = pd.to_datetime(df['Date'], format = '%Y-%m-%d')"
63 | ],
64 | "execution_count": 1,
65 | "outputs": [
66 | {
67 | "output_type": "stream",
68 | "text": [
69 | "Mounted at /content/drive/\n"
70 | ],
71 | "name": "stdout"
72 | }
73 | ]
74 | },
75 | {
76 | "cell_type": "code",
77 | "metadata": {
78 | "id": "MN1DKfV6zauY"
79 | },
80 | "source": [
81 | "class Environment:\n",
82 | " def __init__(self, df, initial_money=100000, mode = 'test'):\n",
83 | "\n",
84 | " self.df = df.dropna().reset_index()\n",
85 | "\n",
86 | " self.df_total_steps = len(self.df)-1\n",
87 | " self.initial_money = initial_money\n",
88 | " self.mode = mode\n",
89 | " self.trade_time = None\n",
90 | " self.trade_win = None\n",
91 | " self.brfore_buy_cash = None\n",
92 | " self.action_space = np.array([0, 1, 2]) # buy,hold,sell\n",
93 | " self.hold_a_position = None\n",
94 | " self.now_price = None\n",
95 | " self.cash_in_hand = None\n",
96 | "\n",
97 | " self.reset()\n",
98 | " \n",
99 | " def reset(self):\n",
100 | "\n",
101 | " self.trade_time = 0\n",
102 | " self.trade_win = 0\n",
103 | " self.brfore_buy_cash = 0\n",
104 | " self.end_step = self.df_total_steps\n",
105 | " self.now_step = 0\n",
106 | " self.hold_a_position = 0.0\n",
107 | " self.now_price = self.df.loc[self.now_step, 'SP500']\n",
108 | " self.cash_in_hand = self.initial_money\n",
109 | "\n",
110 | " return self._get_now_state()\n",
111 | "\n",
112 | " def step(self, action):\n",
113 | "\n",
114 | " prev_revenue = self._get_revenue()\n",
115 | " self.now_step += 1\n",
116 | " self.now_price = self.df.loc[self.now_step, 'SP500']\n",
117 | " \n",
118 | " done = (self.end_step == self.now_step)\n",
119 | "\n",
120 | " self._trade(action,done)\n",
121 | " cur_revenue = self._get_revenue()\n",
122 | " \n",
123 | " reward = cur_revenue - prev_revenue\n",
124 | "\n",
125 | " if self.mode == 'test':\n",
126 | " info = { 'cur_revenue' : cur_revenue , 'trade_time' : self.trade_time, 'trade_win' : self.trade_win }\n",
127 | " else:\n",
128 | " info = { 'cur_revenue' : cur_revenue }\n",
129 | "\n",
130 | " return self._get_now_state(), reward, done, info\n",
131 | "\n",
132 | " def _get_now_state(self):\n",
133 | " state = np.empty(3)\n",
134 | " state[0] = self.hold_a_position\n",
135 | " state[1] = self.now_price\n",
136 | " state[2] = self.cash_in_hand\n",
137 | " return state\n",
138 | "\n",
139 | " def _get_revenue(self): \n",
140 | " return self.hold_a_position * self.now_price + self.cash_in_hand\n",
141 | "\n",
142 | " def _trade(self, action,lastorder = False):\n",
143 | " if lastorder:\n",
144 | " self.cash_in_hand += self.now_price * self.hold_a_position\n",
145 | " self.hold_a_position = 0\n",
146 | " if self.mode == 'test':\n",
147 | " self.trade_time += 1\n",
148 | " if self.cash_in_hand > self.brfore_buy_cash:\n",
149 | " self.trade_win += 1\n",
150 | " else:\n",
151 | " if self.action_space[0] == action: # buy\n",
152 | " if self.hold_a_position == 0:\n",
153 | " buy_flag = True\n",
154 | " if self.mode == 'test':\n",
155 | " self.brfore_buy_cash = copy.copy(self.cash_in_hand)\n",
156 | " while buy_flag:\n",
157 | " if self.cash_in_hand > self.now_price:\n",
158 | " self.hold_a_position += 1\n",
159 | " self.cash_in_hand -= self.now_price\n",
160 | " else:\n",
161 | " buy_flag = False\n",
162 | " if self.action_space[2] == action: # sell\n",
163 | " if self.hold_a_position != 0:\n",
164 | " self.cash_in_hand += self.now_price * self.hold_a_position\n",
165 | " self.hold_a_position = 0\n",
166 | " if self.mode == 'test':\n",
167 | " self.trade_time += 1\n",
168 | " if self.cash_in_hand > self.brfore_buy_cash:\n",
169 | " self.trade_win += 1"
170 | ],
171 | "execution_count": 2,
172 | "outputs": []
173 | },
174 | {
175 | "cell_type": "code",
176 | "metadata": {
177 | "id": "On5S8YtLz3U4"
178 | },
179 | "source": [
180 | "class Main:\n",
181 | " def __init__(self, env, episodes_times = 1000, mode = 'test'):\n",
182 | " self.env = env\n",
183 | " self.episodes_times = episodes_times\n",
184 | " self.mode = mode\n",
185 | "\n",
186 | " if self.mode == 'test':\n",
187 | " with open(csv_path, 'w') as f:\n",
188 | " row = 'FixedProfit,TradeTimes,TradeWin'\n",
189 | " print(row, file=f)\n",
190 | " else:\n",
191 | " with open(csv_path, 'w') as f:\n",
192 | " row = 'FixedProfit'\n",
193 | " print(row, file=f)\n",
194 | "\n",
195 | " def play_game(self):\n",
196 | "\n",
197 | " for episode in range(self.episodes_times):\n",
198 | " state = self.env.reset()\n",
199 | " done = False\n",
200 | " start_time = datetime.now()\n",
201 | " \n",
202 | " while not done:\n",
203 | " action = random.randrange(3)\n",
204 | " state, reward, done, info = self.env.step(action)\n",
205 | " \n",
206 | " play_time = datetime.now() - start_time\n",
207 | " if self.mode == 'test':\n",
208 | " print(f\"Episode: {episode + 1}/{episodes_times} RapTime: {play_time} FixedProfit: {info['cur_revenue']:.0f} TradeTimes: {info['trade_time']} TradeWin: {info['trade_win']}\")\n",
209 | " with open(csv_path, 'a') as f:\n",
210 | " row = str(info['cur_revenue']) + ',' + str(info['trade_time']) + ',' + str(info['trade_win'])\n",
211 | " print(row, file=f)\n",
212 | " else:\n",
213 | " print(f\"Episode: {episode + 1}/{self.episodes_times} RapTime: {play_time} FixedProfit: {info['cur_revenue']:.0f}\")\n",
214 | " with open(csv_path, 'a') as f:\n",
215 | " row = str(info['cur_revenue'])\n",
216 | " print(row, file=f)"
217 | ],
218 | "execution_count": 3,
219 | "outputs": []
220 | },
221 | {
222 | "cell_type": "code",
223 | "metadata": {
224 | "colab": {
225 | "base_uri": "https://localhost:8080/"
226 | },
227 | "id": "pYFNVDDQz9X9",
228 | "outputId": "e56b4950-70fb-40ed-c625-34681ef01e08"
229 | },
230 | "source": [
231 | "initial_money = 1000000\n",
232 | "episodes_times = 25\n",
233 | "\n",
234 | "envs = []\n",
235 | "for i in range(4):\n",
236 | " env = Environment(df, initial_money=initial_money,mode = mode)\n",
237 | " main = Main(env, episodes_times, mode)\n",
238 | " envs.append(main)\n",
239 | "\n",
240 | "worker = []\n",
241 | "for env in envs:\n",
242 | " p = Process(target=env.play_game)\n",
243 | " worker.append(p)\n",
244 | " p.start()\n",
245 | "\n",
246 | "for w in worker:\n",
247 | " w.join()"
248 | ],
249 | "execution_count": 4,
250 | "outputs": [
251 | {
252 | "output_type": "stream",
253 | "text": [
254 | "Episode: 1/25 RapTime: 0:00:00.052921 FixedProfit: 1002706 TradeTimes: 152 TradeWin: 88\n",
255 | "Episode: 1/25 RapTime: 0:00:00.079429 FixedProfit: 1737040 TradeTimes: 148 TradeWin: 100\n",
256 | "Episode: 1/25 RapTime: 0:00:00.073406 FixedProfit: 1480464 TradeTimes: 148 TradeWin: 91\n",
257 | "Episode: 1/25 RapTime: 0:00:00.082164 FixedProfit: 1321282 TradeTimes: 133 TradeWin: 86\n",
258 | "Episode: 2/25 RapTime: 0:00:00.060588 FixedProfit: 1173427 TradeTimes: 143 TradeWin: 84\n",
259 | "Episode: 2/25 RapTime: 0:00:00.067478 FixedProfit: 1163608 TradeTimes: 158 TradeWin: 93\n",
260 | "Episode: 2/25 RapTime: 0:00:00.069834 FixedProfit: 1192880 TradeTimes: 140 TradeWin: 87\n",
261 | "Episode: 3/25 RapTime: 0:00:00.067062 FixedProfit: 1033315 TradeTimes: 143 TradeWin: 82\n",
262 | "Episode: 2/25 RapTime: 0:00:00.070084 FixedProfit: 1500615 TradeTimes: 158 TradeWin: 108\n",
263 | "Episode: 3/25 RapTime: 0:00:00.066757 FixedProfit: 1108755 TradeTimes: 151 TradeWin: 93\n",
264 | "Episode: 3/25 RapTime: 0:00:00.061718 FixedProfit: 1157477 TradeTimes: 133 TradeWin: 73\n",
265 | "Episode: 3/25 RapTime: 0:00:00.070343 FixedProfit: 1496789 TradeTimes: 146 TradeWin: 90\n",
266 | "Episode: 4/25 RapTime: 0:00:00.079128 FixedProfit: 1299665 TradeTimes: 159 TradeWin: 97\n",
267 | "Episode: 4/25 RapTime: 0:00:00.074119 FixedProfit: 1249797 TradeTimes: 150 TradeWin: 97\n",
268 | "Episode: 4/25 RapTime: 0:00:00.068562 FixedProfit: 1123937 TradeTimes: 143 TradeWin: 91\n",
269 | "Episode: 4/25 RapTime: 0:00:00.073187 FixedProfit: 1297917 TradeTimes: 155 TradeWin: 99\n",
270 | "Episode: 5/25 RapTime: 0:00:00.065115 FixedProfit: 1222155 TradeTimes: 137 TradeWin: 81\n",
271 | "Episode: 5/25 RapTime: 0:00:00.048593 FixedProfit: 963624 TradeTimes: 145 TradeWin: 87\n",
272 | "Episode: 5/25 RapTime: 0:00:00.065598 FixedProfit: 1123560 TradeTimes: 150 TradeWin: 104\n",
273 | "Episode: 6/25 RapTime: 0:00:00.070500 FixedProfit: 1342545 TradeTimes: 149 TradeWin: 94\n",
274 | "Episode: 5/25 RapTime: 0:00:00.071435 FixedProfit: 1418906 TradeTimes: 149 TradeWin: 94\n",
275 | "Episode: 6/25 RapTime: 0:00:00.072602 FixedProfit: 1359623 TradeTimes: 138 TradeWin: 82\n",
276 | "Episode: 6/25 RapTime: 0:00:00.064598 FixedProfit: 824793 TradeTimes: 164 TradeWin: 91\n",
277 | "Episode: 6/25 RapTime: 0:00:00.058760 FixedProfit: 1049222 TradeTimes: 136 TradeWin: 79\n",
278 | "Episode: 7/25 RapTime: 0:00:00.062372 FixedProfit: 827802 TradeTimes: 147 TradeWin: 88\n",
279 | "Episode: 7/25 RapTime: 0:00:00.072701 FixedProfit: 1139580 TradeTimes: 147 TradeWin: 84\n",
280 | "Episode: 7/25 RapTime: 0:00:00.061429 FixedProfit: 876319 TradeTimes: 141 TradeWin: 80\n",
281 | "Episode: 7/25 RapTime: 0:00:00.066050 FixedProfit: 1238760 TradeTimes: 152 TradeWin: 89\n",
282 | "Episode: 8/25 RapTime: 0:00:00.072673 FixedProfit: 1653583 TradeTimes: 139 TradeWin: 97\n",
283 | "Episode: 8/25 RapTime: 0:00:00.076260 FixedProfit: 1661513 TradeTimes: 139 TradeWin: 88\n",
284 | "Episode: 8/25 RapTime: 0:00:00.073900 FixedProfit: 1520226 TradeTimes: 151 TradeWin: 87\n",
285 | "Episode: 8/25 RapTime: 0:00:00.077237 FixedProfit: 1347726 TradeTimes: 157 TradeWin: 107\n",
286 | "Episode: 9/25 RapTime: 0:00:00.075502 FixedProfit: 1662996 TradeTimes: 149 TradeWin: 92\n",
287 | "Episode: 9/25 RapTime: 0:00:00.074161 FixedProfit: 1266522 TradeTimes: 163 TradeWin: 104\n",
288 | "Episode: 9/25 RapTime: 0:00:00.074824 FixedProfit: 1411491 TradeTimes: 145 TradeWin: 84\n",
289 | "Episode: 9/25 RapTime: 0:00:00.065268 FixedProfit: 1329916 TradeTimes: 146 TradeWin: 86\n",
290 | "Episode: 10/25 RapTime: 0:00:00.070241 FixedProfit: 1398157 TradeTimes: 159 TradeWin: 98\n",
291 | "Episode: 10/25 RapTime: 0:00:00.081070 FixedProfit: 1179821 TradeTimes: 149 TradeWin: 100\n",
292 | "Episode: 10/25 RapTime: 0:00:00.071051 FixedProfit: 1465133 TradeTimes: 152 TradeWin: 97\n",
293 | "Episode: 10/25 RapTime: 0:00:00.079841 FixedProfit: 1339174 TradeTimes: 150 TradeWin: 90\n",
294 | "Episode: 11/25 RapTime: 0:00:00.065267 FixedProfit: 1004332 TradeTimes: 141 TradeWin: 86\n",
295 | "Episode: 11/25 RapTime: 0:00:00.069977 FixedProfit: 1427704 TradeTimes: 143 TradeWin: 90\n",
296 | "Episode: 11/25 RapTime: 0:00:00.078982 FixedProfit: 1496240 TradeTimes: 154 TradeWin: 103\n",
297 | "Episode: 11/25 RapTime: 0:00:00.072883 FixedProfit: 991634 TradeTimes: 142 TradeWin: 84\n",
298 | "Episode: 12/25 RapTime: 0:00:00.082258 FixedProfit: 1713360 TradeTimes: 158 TradeWin: 103\n",
299 | "Episode: 12/25 RapTime: 0:00:00.075590 FixedProfit: 1369063 TradeTimes: 157 TradeWin: 93\n",
300 | "Episode: 12/25 RapTime: 0:00:00.064561 FixedProfit: 901086 TradeTimes: 141 TradeWin: 78\n",
301 | "Episode: 12/25 RapTime: 0:00:00.065664 FixedProfit: 990961 TradeTimes: 145 TradeWin: 89\n",
302 | "Episode: 13/25 RapTime: 0:00:00.078437 FixedProfit: 1075606 TradeTimes: 145 TradeWin: 89\n",
303 | "Episode: 13/25 RapTime: 0:00:00.074328 FixedProfit: 1505757 TradeTimes: 147 TradeWin: 95\n",
304 | "Episode: 13/25 RapTime: 0:00:00.071317 FixedProfit: 1546115 TradeTimes: 134 TradeWin: 85\n",
305 | "Episode: 13/25 RapTime: 0:00:00.074440 FixedProfit: 1848318 TradeTimes: 135 TradeWin: 96\n",
306 | "Episode: 14/25 RapTime: 0:00:00.064905 FixedProfit: 1177096 TradeTimes: 147 TradeWin: 94\n",
307 | "Episode: 14/25 RapTime: 0:00:00.071063 FixedProfit: 1689874 TradeTimes: 131 TradeWin: 84\n",
308 | "Episode: 14/25 RapTime: 0:00:00.066678 FixedProfit: 1466261 TradeTimes: 148 TradeWin: 94\n",
309 | "Episode: 14/25 RapTime: 0:00:00.065399 FixedProfit: 972004 TradeTimes: 135 TradeWin: 83\n",
310 | "Episode: 15/25 RapTime: 0:00:00.070818 FixedProfit: 1142938 TradeTimes: 157 TradeWin: 99\n",
311 | "Episode: 15/25 RapTime: 0:00:00.070248 FixedProfit: 1214932 TradeTimes: 151 TradeWin: 98\n",
312 | "Episode: 15/25 RapTime: 0:00:00.071788 FixedProfit: 1181455 TradeTimes: 139 TradeWin: 84\n",
313 | "Episode: 15/25 RapTime: 0:00:00.056895 FixedProfit: 800655 TradeTimes: 144 TradeWin: 84\n",
314 | "Episode: 16/25 RapTime: 0:00:00.070433 FixedProfit: 1272085 TradeTimes: 140 TradeWin: 86\n",
315 | "Episode: 16/25 RapTime: 0:00:00.068363 FixedProfit: 1235574 TradeTimes: 148 TradeWin: 91\n",
316 | "Episode: 16/25 RapTime: 0:00:00.066376 FixedProfit: 1267781 TradeTimes: 152 TradeWin: 92\n",
317 | "Episode: 16/25 RapTime: 0:00:00.070560 FixedProfit: 1226830 TradeTimes: 156 TradeWin: 98\n",
318 | "Episode: 17/25 RapTime: 0:00:00.078644 FixedProfit: 1091463 TradeTimes: 145 TradeWin: 80\n",
319 | "Episode: 17/25 RapTime: 0:00:00.067324 FixedProfit: 1181263 TradeTimes: 150 TradeWin: 98\n",
320 | "Episode: 17/25 RapTime: 0:00:00.082621 FixedProfit: 1136486 TradeTimes: 156 TradeWin: 94\n",
321 | "Episode: 17/25 RapTime: 0:00:00.062785 FixedProfit: 982740 TradeTimes: 142 TradeWin: 88\n",
322 | "Episode: 18/25 RapTime: 0:00:00.064311 FixedProfit: 1280137 TradeTimes: 143 TradeWin: 87\n",
323 | "Episode: 18/25 RapTime: 0:00:00.073478 FixedProfit: 1534196 TradeTimes: 156 TradeWin: 95\n",
324 | "Episode: 18/25 RapTime: 0:00:00.070948 FixedProfit: 1415123 TradeTimes: 139 TradeWin: 85\n",
325 | "Episode: 18/25 RapTime: 0:00:00.070957 FixedProfit: 1178606 TradeTimes: 155 TradeWin: 94\n",
326 | "Episode: 19/25 RapTime: 0:00:00.065677 FixedProfit: 1179217 TradeTimes: 147 TradeWin: 92\n",
327 | "Episode: 19/25 RapTime: 0:00:00.062577 FixedProfit: 868661 TradeTimes: 149 TradeWin: 87\n",
328 | "Episode: 19/25 RapTime: 0:00:00.069872 FixedProfit: 1061586 TradeTimes: 135 TradeWin: 80\n",
329 | "Episode: 19/25 RapTime: 0:00:00.070396 FixedProfit: 1154277 TradeTimes: 146 TradeWin: 95\n",
330 | "Episode: 20/25 RapTime: 0:00:00.066884 FixedProfit: 1443539 TradeTimes: 148 TradeWin: 96\n",
331 | "Episode: 20/25 RapTime: 0:00:00.069988 FixedProfit: 1293469 TradeTimes: 131 TradeWin: 89\n",
332 | "Episode: 20/25 RapTime: 0:00:00.066032 FixedProfit: 898514 TradeTimes: 148 TradeWin: 91\n",
333 | "Episode: 20/25 RapTime: 0:00:00.082895 FixedProfit: 1352182 TradeTimes: 143 TradeWin: 95\n",
334 | "Episode: 21/25 RapTime: 0:00:00.058852 FixedProfit: 981841 TradeTimes: 141 TradeWin: 79\n",
335 | "Episode: 21/25 RapTime: 0:00:00.061248 FixedProfit: 1094897 TradeTimes: 155 TradeWin: 93\n",
336 | "Episode: 21/25 RapTime: 0:00:00.080416 FixedProfit: 1240076 TradeTimes: 154 TradeWin: 93\n",
337 | "Episode: 21/25 RapTime: 0:00:00.063774 FixedProfit: 1041677 TradeTimes: 139 TradeWin: 81\n",
338 | "Episode: 22/25 RapTime: 0:00:00.067297 FixedProfit: 1319550 TradeTimes: 144 TradeWin: 84\n",
339 | "Episode: 22/25 RapTime: 0:00:00.063706 FixedProfit: 930046 TradeTimes: 153 TradeWin: 88\n",
340 | "Episode: 22/25 RapTime: 0:00:00.074681 FixedProfit: 1524666 TradeTimes: 141 TradeWin: 89\n",
341 | "Episode: 22/25 RapTime: 0:00:00.071157 FixedProfit: 1234928 TradeTimes: 154 TradeWin: 95\n",
342 | "Episode: 23/25 RapTime: 0:00:00.065100 FixedProfit: 977319 TradeTimes: 144 TradeWin: 79\n",
343 | "Episode: 23/25 RapTime: 0:00:00.072066 FixedProfit: 1124590 TradeTimes: 148 TradeWin: 82\n",
344 | "Episode: 23/25 RapTime: 0:00:00.078799 FixedProfit: 1436248 TradeTimes: 157 TradeWin: 99\n",
345 | "Episode: 23/25 RapTime: 0:00:00.066646 FixedProfit: 1032711 TradeTimes: 144 TradeWin: 86\n",
346 | "Episode: 24/25 RapTime: 0:00:00.070393 FixedProfit: 1234225 TradeTimes: 146 TradeWin: 97\n",
347 | "Episode: 24/25 RapTime: 0:00:00.070934 FixedProfit: 1173280 TradeTimes: 153 TradeWin: 94\n",
348 | "Episode: 24/25 RapTime: 0:00:00.064679 FixedProfit: 903624 TradeTimes: 147 TradeWin: 93\n",
349 | "Episode: 24/25 RapTime: 0:00:00.068371 FixedProfit: 1018899 TradeTimes: 145 TradeWin: 81\n",
350 | "Episode: 25/25 RapTime: 0:00:00.067611 FixedProfit: 988935 TradeTimes: 147 TradeWin: 83\n",
351 | "Episode: 25/25 RapTime: 0:00:00.050670 FixedProfit: 931574 TradeTimes: 140 TradeWin: 85\n",
352 | "Episode: 25/25 RapTime: 0:00:00.054648 FixedProfit: 1003928 TradeTimes: 148 TradeWin: 83\n",
353 | "Episode: 25/25 RapTime: 0:00:00.060029 FixedProfit: 1333170 TradeTimes: 158 TradeWin: 93\n"
354 | ],
355 | "name": "stdout"
356 | }
357 | ]
358 | }
359 | ]
360 | }
--------------------------------------------------------------------------------
/m_thread_random.ipynb:
--------------------------------------------------------------------------------
1 | {
2 | "nbformat": 4,
3 | "nbformat_minor": 0,
4 | "metadata": {
5 | "colab": {
6 | "name": "m_thread_random.ipynb",
7 | "provenance": [],
8 | "collapsed_sections": [],
9 | "machine_shape": "hm",
10 | "authorship_tag": "ABX9TyOdhoPHoa3OVVWuQMci5j93",
11 | "include_colab_link": true
12 | },
13 | "kernelspec": {
14 | "name": "python3",
15 | "display_name": "Python 3"
16 | },
17 | "language_info": {
18 | "name": "python"
19 | },
20 | "accelerator": "GPU"
21 | },
22 | "cells": [
23 | {
24 | "cell_type": "markdown",
25 | "metadata": {
26 | "id": "view-in-github",
27 | "colab_type": "text"
28 | },
29 | "source": [
30 | ""
31 | ]
32 | },
33 | {
34 | "cell_type": "code",
35 | "metadata": {
36 | "colab": {
37 | "base_uri": "https://localhost:8080/"
38 | },
39 | "id": "3NIXg6mTzk0K",
40 | "outputId": "3ad07176-ec78-4003-a0c1-ea15d3fc40ab"
41 | },
42 | "source": [
43 | "import pandas as pd\n",
44 | "import numpy as np\n",
45 | "import random\n",
46 | "from google.colab import drive\n",
47 | "import copy\n",
48 | "\n",
49 | "from datetime import datetime\n",
50 | "from concurrent.futures import ThreadPoolExecutor\n",
51 | "\n",
52 | "mode = 'test'\n",
53 | "\n",
54 | "drive.mount('/content/drive/')\n",
55 | "nov_dir = 'Colab Notebooks/dataset/reinforcement_learning/'\n",
56 | "nov_path = '/content/drive/My Drive/' + nov_dir + f'sp500_{mode}.csv'\n",
57 | "exp_dir = 'Colab Notebooks/workspace/export/'\n",
58 | "csv_path = '/content/drive/My Drive/' + exp_dir + f'csv_data/m_thread_random_{mode}.csv'\n",
59 | "\n",
60 | "df = pd.read_csv(nov_path)\n",
61 | "df['Date'] = pd.to_datetime(df['Date'], format = '%Y-%m-%d')"
62 | ],
63 | "execution_count": 1,
64 | "outputs": [
65 | {
66 | "output_type": "stream",
67 | "text": [
68 | "Mounted at /content/drive/\n"
69 | ],
70 | "name": "stdout"
71 | }
72 | ]
73 | },
74 | {
75 | "cell_type": "code",
76 | "metadata": {
77 | "id": "MN1DKfV6zauY"
78 | },
79 | "source": [
80 | "class Environment:\n",
81 | " def __init__(self, df, initial_money=100000, mode = 'test'):\n",
82 | "\n",
83 | " self.df = df.dropna().reset_index()\n",
84 | "\n",
85 | " self.df_total_steps = len(self.df)-1\n",
86 | " self.initial_money = initial_money\n",
87 | " self.mode = mode\n",
88 | " self.trade_time = None\n",
89 | " self.trade_win = None\n",
90 | " self.brfore_buy_cash = None\n",
91 | " self.action_space = np.array([0, 1, 2]) # buy,hold,sell\n",
92 | " self.hold_a_position = None\n",
93 | " self.now_price = None\n",
94 | " self.cash_in_hand = None\n",
95 | "\n",
96 | " self.reset()\n",
97 | " \n",
98 | " def reset(self):\n",
99 | "\n",
100 | " self.trade_time = 0\n",
101 | " self.trade_win = 0\n",
102 | " self.brfore_buy_cash = 0\n",
103 | " self.end_step = self.df_total_steps\n",
104 | " self.now_step = 0\n",
105 | " self.hold_a_position = 0.0\n",
106 | " self.now_price = self.df.loc[self.now_step, 'SP500']\n",
107 | " self.cash_in_hand = self.initial_money\n",
108 | "\n",
109 | " return self._get_now_state()\n",
110 | "\n",
111 | " def step(self, action):\n",
112 | "\n",
113 | " prev_revenue = self._get_revenue()\n",
114 | " self.now_step += 1\n",
115 | " self.now_price = self.df.loc[self.now_step, 'SP500']\n",
116 | " \n",
117 | " done = (self.end_step == self.now_step)\n",
118 | "\n",
119 | " self._trade(action,done)\n",
120 | " cur_revenue = self._get_revenue()\n",
121 | " \n",
122 | " reward = cur_revenue - prev_revenue\n",
123 | "\n",
124 | " if self.mode == 'test':\n",
125 | " info = { 'cur_revenue' : cur_revenue , 'trade_time' : self.trade_time, 'trade_win' : self.trade_win }\n",
126 | " else:\n",
127 | " info = { 'cur_revenue' : cur_revenue }\n",
128 | "\n",
129 | " return self._get_now_state(), reward, done, info\n",
130 | "\n",
131 | " def _get_now_state(self):\n",
132 | " state = np.empty(3)\n",
133 | " state[0] = self.hold_a_position\n",
134 | " state[1] = self.now_price\n",
135 | " state[2] = self.cash_in_hand\n",
136 | " return state\n",
137 | "\n",
138 | " def _get_revenue(self): \n",
139 | " return self.hold_a_position * self.now_price + self.cash_in_hand\n",
140 | "\n",
141 | " def _trade(self, action,lastorder = False):\n",
142 | " if lastorder:\n",
143 | " self.cash_in_hand += self.now_price * self.hold_a_position\n",
144 | " self.hold_a_position = 0\n",
145 | " if self.mode == 'test':\n",
146 | " self.trade_time += 1\n",
147 | " if self.cash_in_hand > self.brfore_buy_cash:\n",
148 | " self.trade_win += 1\n",
149 | " else:\n",
150 | " if self.action_space[0] == action: # buy\n",
151 | " if self.hold_a_position == 0:\n",
152 | " buy_flag = True\n",
153 | " if self.mode == 'test':\n",
154 | " self.brfore_buy_cash = copy.copy(self.cash_in_hand)\n",
155 | " while buy_flag:\n",
156 | " if self.cash_in_hand > self.now_price:\n",
157 | " self.hold_a_position += 1\n",
158 | " self.cash_in_hand -= self.now_price\n",
159 | " else:\n",
160 | " buy_flag = False\n",
161 | " if self.action_space[2] == action: # sell\n",
162 | " if self.hold_a_position != 0:\n",
163 | " self.cash_in_hand += self.now_price * self.hold_a_position\n",
164 | " self.hold_a_position = 0\n",
165 | " if self.mode == 'test':\n",
166 | " self.trade_time += 1\n",
167 | " if self.cash_in_hand > self.brfore_buy_cash:\n",
168 | " self.trade_win += 1"
169 | ],
170 | "execution_count": 2,
171 | "outputs": []
172 | },
173 | {
174 | "cell_type": "code",
175 | "metadata": {
176 | "id": "On5S8YtLz3U4"
177 | },
178 | "source": [
179 | "class Main:\n",
180 | " def __init__(self, env, episodes_times = 1000, mode = 'test'):\n",
181 | " self.env = env\n",
182 | " self.episodes_times = episodes_times\n",
183 | " self.mode = mode\n",
184 | "\n",
185 | " if self.mode == 'test':\n",
186 | " with open(csv_path, 'w') as f:\n",
187 | " row = 'FixedProfit,TradeTimes,TradeWin'\n",
188 | " print(row, file=f)\n",
189 | " else:\n",
190 | " with open(csv_path, 'w') as f:\n",
191 | " row = 'FixedProfit'\n",
192 | " print(row, file=f)\n",
193 | "\n",
194 | " def play_game(self):\n",
195 | "\n",
196 | " for episode in range(self.episodes_times):\n",
197 | " state = self.env.reset()\n",
198 | " done = False\n",
199 | " start_time = datetime.now()\n",
200 | " \n",
201 | " while not done:\n",
202 | " action = random.randrange(3)\n",
203 | " state, reward, done, info = self.env.step(action)\n",
204 | " \n",
205 | " play_time = datetime.now() - start_time\n",
206 | " if self.mode == 'test':\n",
207 | " print(f\"Episode: {episode + 1}/{episodes_times} RapTime: {play_time} FixedProfit: {info['cur_revenue']:.0f} TradeTimes: {info['trade_time']} TradeWin: {info['trade_win']}\")\n",
208 | " with open(csv_path, 'a') as f:\n",
209 | " row = str(info['cur_revenue']) + ',' + str(info['trade_time']) + ',' + str(info['trade_win'])\n",
210 | " print(row, file=f)\n",
211 | " else:\n",
212 | " print(f\"Episode: {episode + 1}/{self.episodes_times} RapTime: {play_time} FixedProfit: {info['cur_revenue']:.0f}\")\n",
213 | " with open(csv_path, 'a') as f:\n",
214 | " row = str(info['cur_revenue'])\n",
215 | " print(row, file=f)"
216 | ],
217 | "execution_count": 3,
218 | "outputs": []
219 | },
220 | {
221 | "cell_type": "code",
222 | "metadata": {
223 | "colab": {
224 | "base_uri": "https://localhost:8080/"
225 | },
226 | "id": "pYFNVDDQz9X9",
227 | "outputId": "caf883b7-6b83-4496-e7b8-ab9c7b9af7bc"
228 | },
229 | "source": [
230 | "initial_money = 1000000\n",
231 | "episodes_times = 25\n",
232 | "\n",
233 | "thread_num = 4\n",
234 | "datas = []\n",
235 | "for i in range(thread_num):\n",
236 | " env = Environment(df, initial_money=initial_money,mode = mode)\n",
237 | " main = Main(env, episodes_times, mode)\n",
238 | " datas.append(main)\n",
239 | "\n",
240 | "features = []\n",
241 | "with ThreadPoolExecutor(max_workers=thread_num) as executor:\n",
242 | " for env in datas:\n",
243 | " job = lambda: env.play_game()\n",
244 | " features.append(executor.submit(job))"
245 | ],
246 | "execution_count": 4,
247 | "outputs": [
248 | {
249 | "output_type": "stream",
250 | "text": [
251 | "Episode: 1/25 RapTime: 0:00:00.086261 FixedProfit: 1362232 TradeTimes: 154 TradeWin: 101\n",
252 | "Episode: 1/25 RapTime: 0:00:00.099141 FixedProfit: 1130834 TradeTimes: 145 TradeWin: 91\n",
253 | "Episode: 1/25 RapTime: 0:00:00.072756 FixedProfit: 1389796 TradeTimes: 150 TradeWin: 92\n",
254 | "Episode: 1/25 RapTime: 0:00:00.098554 FixedProfit: 1301946 TradeTimes: 154 TradeWin: 98\n",
255 | "Episode: 2/25 RapTime: 0:00:00.066621 FixedProfit: 942857 TradeTimes: 143 TradeWin: 87\n",
256 | "Episode: 2/25 RapTime: 0:00:00.102152 FixedProfit: 1588988 TradeTimes: 148 TradeWin: 95\n",
257 | "Episode: 2/25 RapTime: 0:00:00.124457 FixedProfit: 931027 TradeTimes: 156 TradeWin: 93\n",
258 | "Episode: 2/25 RapTime: 0:00:00.065103 FixedProfit: 1446612 TradeTimes: 147 TradeWin: 93\n",
259 | "Episode: 3/25 RapTime: 0:00:00.050021 FixedProfit: 914607 TradeTimes: 152 TradeWin: 94\n",
260 | "Episode: 3/25 RapTime: 0:00:00.115055 FixedProfit: 1281313 TradeTimes: 139 TradeWin: 87\n",
261 | "Episode: 4/25 RapTime: 0:00:00.099364 FixedProfit: 1039809 TradeTimes: 141 TradeWin: 89\n",
262 | "Episode: 3/25 RapTime: 0:00:00.121659 FixedProfit: 1147343 TradeTimes: 149 TradeWin: 93\n",
263 | "Episode: 4/25 RapTime: 0:00:00.056179 FixedProfit: 1012182 TradeTimes: 139 TradeWin: 83\n",
264 | "Episode: 3/25 RapTime: 0:00:00.123785 FixedProfit: 1479979 TradeTimes: 158 TradeWin: 103\n",
265 | "Episode: 4/25 RapTime: 0:00:00.068004 FixedProfit: 946468 TradeTimes: 145 TradeWin: 88\n",
266 | "Episode: 5/25 RapTime: 0:00:00.108318 FixedProfit: 1244772 TradeTimes: 149 TradeWin: 95\n",
267 | "Episode: 4/25 RapTime: 0:00:00.067349 FixedProfit: 1079226 TradeTimes: 141 TradeWin: 80\n",
268 | "Episode: 5/25 RapTime: 0:00:00.120117 FixedProfit: 1146583 TradeTimes: 147 TradeWin: 89\n",
269 | "Episode: 5/25 RapTime: 0:00:00.079158 FixedProfit: 1108722 TradeTimes: 154 TradeWin: 101\n",
270 | "Episode: 6/25 RapTime: 0:00:00.088812 FixedProfit: 1495946 TradeTimes: 160 TradeWin: 95\n",
271 | "Episode: 6/25 RapTime: 0:00:00.080257 FixedProfit: 1222622 TradeTimes: 131 TradeWin: 87\n",
272 | "Episode: 5/25 RapTime: 0:00:00.131867 FixedProfit: 1193901 TradeTimes: 157 TradeWin: 95\n",
273 | "Episode: 6/25 RapTime: 0:00:00.077531 FixedProfit: 965865 TradeTimes: 145 TradeWin: 91\n",
274 | "Episode: 7/25 RapTime: 0:00:00.066488 FixedProfit: 1168944 TradeTimes: 142 TradeWin: 83\n",
275 | "Episode: 7/25 RapTime: 0:00:00.072269 FixedProfit: 1154487 TradeTimes: 151 TradeWin: 91\n",
276 | "Episode: 7/25 RapTime: 0:00:00.056733 FixedProfit: 1577257 TradeTimes: 137 TradeWin: 85\n",
277 | "Episode: 8/25 RapTime: 0:00:00.082979 FixedProfit: 942581 TradeTimes: 136 TradeWin: 83\n",
278 | "Episode: 6/25 RapTime: 0:00:00.164498 FixedProfit: 1107150 TradeTimes: 149 TradeWin: 94\n",
279 | "Episode: 8/25 RapTime: 0:00:00.140101 FixedProfit: 1122411 TradeTimes: 148 TradeWin: 79\n",
280 | "Episode: 8/25 RapTime: 0:00:00.127384 FixedProfit: 1504781 TradeTimes: 150 TradeWin: 96\n",
281 | "Episode: 9/25 RapTime: 0:00:00.102475 FixedProfit: 1109599 TradeTimes: 146 TradeWin: 88\n",
282 | "Episode: 7/25 RapTime: 0:00:00.087239 FixedProfit: 1542265 TradeTimes: 149 TradeWin: 88\n",
283 | "Episode: 9/25 RapTime: 0:00:00.087768 FixedProfit: 957056 TradeTimes: 150 TradeWin: 90\n",
284 | "Episode: 9/25 RapTime: 0:00:00.103999 FixedProfit: 975484 TradeTimes: 162 TradeWin: 88\n",
285 | "Episode: 10/25 RapTime: 0:00:00.064340 FixedProfit: 976893 TradeTimes: 149 TradeWin: 88\n",
286 | "Episode: 8/25 RapTime: 0:00:00.090174 FixedProfit: 1133943 TradeTimes: 156 TradeWin: 90\n",
287 | "Episode: 10/25 RapTime: 0:00:00.083194 FixedProfit: 1048701 TradeTimes: 140 TradeWin: 82\n",
288 | "Episode: 10/25 RapTime: 0:00:00.113577 FixedProfit: 1175638 TradeTimes: 148 TradeWin: 85\n",
289 | "Episode: 9/25 RapTime: 0:00:00.099695 FixedProfit: 1351292 TradeTimes: 150 TradeWin: 94\n",
290 | "Episode: 11/25 RapTime: 0:00:00.060064 FixedProfit: 928666 TradeTimes: 147 TradeWin: 80\n",
291 | "Episode: 11/25 RapTime: 0:00:00.171308 FixedProfit: 1210540 TradeTimes: 139 TradeWin: 89\n",
292 | "Episode: 11/25 RapTime: 0:00:00.121189 FixedProfit: 1356599 TradeTimes: 147 TradeWin: 94\n",
293 | "Episode: 10/25 RapTime: 0:00:00.075928 FixedProfit: 946272 TradeTimes: 157 TradeWin: 93\n",
294 | "Episode: 12/25 RapTime: 0:00:00.070361 FixedProfit: 1537121 TradeTimes: 138 TradeWin: 86\n",
295 | "Episode: 12/25 RapTime: 0:00:00.103781 FixedProfit: 1161019 TradeTimes: 145 TradeWin: 84\n",
296 | "Episode: 11/25 RapTime: 0:00:00.082293 FixedProfit: 1076491 TradeTimes: 141 TradeWin: 90\n",
297 | "Episode: 12/25 RapTime: 0:00:00.062007 FixedProfit: 1230975 TradeTimes: 153 TradeWin: 103\n",
298 | "Episode: 13/25 RapTime: 0:00:00.074932 FixedProfit: 945821 TradeTimes: 148 TradeWin: 94\n",
299 | "Episode: 12/25 RapTime: 0:00:00.144890 FixedProfit: 1406771 TradeTimes: 149 TradeWin: 94Episode: 13/25 RapTime: 0:00:00.117807 FixedProfit: 1499813 TradeTimes: 145 TradeWin: 96\n",
300 | "\n",
301 | "Episode: 14/25 RapTime: 0:00:00.072116 FixedProfit: 1236654 TradeTimes: 157 TradeWin: 95\n",
302 | "Episode: 13/25 RapTime: 0:00:00.041074 FixedProfit: 1052767 TradeTimes: 138 TradeWin: 79\n",
303 | "Episode: 14/25 RapTime: 0:00:00.080192 FixedProfit: 1276135 TradeTimes: 133 TradeWin: 83\n",
304 | "Episode: 13/25 RapTime: 0:00:00.110192 FixedProfit: 1041105 TradeTimes: 148 TradeWin: 92\n",
305 | "Episode: 15/25 RapTime: 0:00:00.076495 FixedProfit: 1228657 TradeTimes: 159 TradeWin: 93\n",
306 | "Episode: 15/25 RapTime: 0:00:00.050009 FixedProfit: 1468747 TradeTimes: 151 TradeWin: 98\n",
307 | "Episode: 14/25 RapTime: 0:00:00.158753 FixedProfit: 958625 TradeTimes: 138 TradeWin: 86\n",
308 | "Episode: 16/25 RapTime: 0:00:00.059080 FixedProfit: 814063 TradeTimes: 141 TradeWin: 86\n",
309 | "Episode: 14/25 RapTime: 0:00:00.102390 FixedProfit: 1105198 TradeTimes: 149 TradeWin: 91\n",
310 | "Episode: 15/25 RapTime: 0:00:00.089389 FixedProfit: 1280783 TradeTimes: 154 TradeWin: 97\n",
311 | "Episode: 17/25 RapTime: 0:00:00.086485 FixedProfit: 1595872 TradeTimes: 146 TradeWin: 88\n",
312 | "Episode: 16/25 RapTime: 0:00:00.137570 FixedProfit: 1237100 TradeTimes: 148 TradeWin: 89\n",
313 | "Episode: 15/25 RapTime: 0:00:00.113888 FixedProfit: 1149534 TradeTimes: 150 TradeWin: 90\n",
314 | "Episode: 18/25 RapTime: 0:00:00.073996 FixedProfit: 1236333 TradeTimes: 149 TradeWin: 98\n",
315 | "Episode: 16/25 RapTime: 0:00:00.095807 FixedProfit: 1440544 TradeTimes: 153 TradeWin: 101\n",
316 | "Episode: 17/25 RapTime: 0:00:00.114673 FixedProfit: 1015757 TradeTimes: 147 TradeWin: 88\n",
317 | "Episode: 17/25 RapTime: 0:00:00.083689 FixedProfit: 1193151 TradeTimes: 140 TradeWin: 89Episode: 19/25 RapTime: 0:00:00.118764 FixedProfit: 1422511 TradeTimes: 146 TradeWin: 87\n",
318 | "\n",
319 | "Episode: 18/25 RapTime: 0:00:00.121209 FixedProfit: 1293347 TradeTimes: 151 TradeWin: 96Episode: 16/25 RapTime: 0:00:00.154701 FixedProfit: 984565 TradeTimes: 157 TradeWin: 94\n",
320 | "\n",
321 | "Episode: 20/25 RapTime: 0:00:00.076676 FixedProfit: 1296325 TradeTimes: 140 TradeWin: 94\n",
322 | "Episode: 18/25 RapTime: 0:00:00.113225 FixedProfit: 1079306 TradeTimes: 149 TradeWin: 84Episode: 17/25 RapTime: 0:00:00.096159 FixedProfit: 1178413 TradeTimes: 148 TradeWin: 88\n",
323 | "\n",
324 | "Episode: 19/25 RapTime: 0:00:00.066481 FixedProfit: 1424164 TradeTimes: 144 TradeWin: 93\n",
325 | "Episode: 21/25 RapTime: 0:00:00.055252 FixedProfit: 1012889 TradeTimes: 149 TradeWin: 80\n",
326 | "Episode: 19/25 RapTime: 0:00:00.156929 FixedProfit: 1242162 TradeTimes: 148 TradeWin: 88\n",
327 | "Episode: 22/25 RapTime: 0:00:00.078286 FixedProfit: 957736 TradeTimes: 150 TradeWin: 97\n",
328 | "Episode: 20/25 RapTime: 0:00:00.098795 FixedProfit: 1353068 TradeTimes: 149 TradeWin: 94\n",
329 | "Episode: 18/25 RapTime: 0:00:00.145312 FixedProfit: 1152248 TradeTimes: 157 TradeWin: 95\n",
330 | "Episode: 23/25 RapTime: 0:00:00.138011 FixedProfit: 1477000 TradeTimes: 146 TradeWin: 94\n",
331 | "Episode: 19/25 RapTime: 0:00:00.099362 FixedProfit: 952766 TradeTimes: 148 TradeWin: 91\n",
332 | "Episode: 21/25 RapTime: 0:00:00.147248 FixedProfit: 1414550 TradeTimes: 152 TradeWin: 91\n",
333 | "Episode: 20/25 RapTime: 0:00:00.095948 FixedProfit: 1489344 TradeTimes: 145 TradeWin: 87\n",
334 | "Episode: 24/25 RapTime: 0:00:00.075627 FixedProfit: 1347581 TradeTimes: 161 TradeWin: 94\n",
335 | "Episode: 20/25 RapTime: 0:00:00.076765 FixedProfit: 1465480 TradeTimes: 149 TradeWin: 93\n",
336 | "Episode: 22/25 RapTime: 0:00:00.058319 FixedProfit: 1083109 TradeTimes: 145 TradeWin: 88\n",
337 | "Episode: 21/25 RapTime: 0:00:00.084646 FixedProfit: 1052634 TradeTimes: 156 TradeWin: 95\n",
338 | "Episode: 25/25 RapTime: 0:00:00.102263 FixedProfit: 1303245 TradeTimes: 135 TradeWin: 86\n",
339 | "Episode: 23/25 RapTime: 0:00:00.102644 FixedProfit: 1433922 TradeTimes: 152 TradeWin: 94\n",
340 | "Episode: 21/25 RapTime: 0:00:00.102501 FixedProfit: 1381392 TradeTimes: 156 TradeWin: 100\n",
341 | "Episode: 22/25 RapTime: 0:00:00.066291 FixedProfit: 1137023 TradeTimes: 138 TradeWin: 74\n",
342 | "Episode: 24/25 RapTime: 0:00:00.048731 FixedProfit: 1287181 TradeTimes: 136 TradeWin: 81\n",
343 | "Episode: 23/25 RapTime: 0:00:00.094476 FixedProfit: 1441789 TradeTimes: 153 TradeWin: 97Episode: 22/25 RapTime: 0:00:00.106966 FixedProfit: 1252618 TradeTimes: 157 TradeWin: 93\n",
344 | "Episode: 25/25 RapTime: 0:00:00.063090 FixedProfit: 1351692 TradeTimes: 145 TradeWin: 88\n",
345 | "\n",
346 | "Episode: 23/25 RapTime: 0:00:00.068968 FixedProfit: 1458071 TradeTimes: 144 TradeWin: 95\n",
347 | "Episode: 24/25 RapTime: 0:00:00.061335 FixedProfit: 1104920 TradeTimes: 155 TradeWin: 98\n",
348 | "Episode: 24/25 RapTime: 0:00:00.076966 FixedProfit: 1539684 TradeTimes: 146 TradeWin: 96Episode: 25/25 RapTime: 0:00:00.071110 FixedProfit: 1212191 TradeTimes: 157 TradeWin: 98\n",
349 | "\n",
350 | "Episode: 25/25 RapTime: 0:00:00.042593 FixedProfit: 1315315 TradeTimes: 156 TradeWin: 103\n"
351 | ],
352 | "name": "stdout"
353 | }
354 | ]
355 | }
356 | ]
357 | }
--------------------------------------------------------------------------------
/Double_Q-Learning/w_q_learning_test.ipynb:
--------------------------------------------------------------------------------
1 | {
2 | "nbformat": 4,
3 | "nbformat_minor": 0,
4 | "metadata": {
5 | "colab": {
6 | "name": "w_q_learning_test.ipynb",
7 | "provenance": [],
8 | "collapsed_sections": [],
9 | "machine_shape": "hm",
10 | "authorship_tag": "ABX9TyNfxRg4Eveo8bQL4+2NyHK1",
11 | "include_colab_link": true
12 | },
13 | "kernelspec": {
14 | "name": "python3",
15 | "display_name": "Python 3"
16 | },
17 | "language_info": {
18 | "name": "python"
19 | },
20 | "accelerator": "GPU"
21 | },
22 | "cells": [
23 | {
24 | "cell_type": "markdown",
25 | "metadata": {
26 | "id": "view-in-github",
27 | "colab_type": "text"
28 | },
29 | "source": [
30 | ""
31 | ]
32 | },
33 | {
34 | "cell_type": "code",
35 | "metadata": {
36 | "colab": {
37 | "base_uri": "https://localhost:8080/"
38 | },
39 | "id": "3NIXg6mTzk0K",
40 | "outputId": "9f657ca7-fd3c-448b-ea76-6db2aec1a497"
41 | },
42 | "source": [
43 | "import pandas as pd\n",
44 | "import numpy as np\n",
45 | "import random\n",
46 | "from google.colab import drive\n",
47 | "import copy\n",
48 | "\n",
49 | "from datetime import datetime\n",
50 | "import pickle\n",
51 | "\n",
52 | "from tensorflow.keras.models import Sequential\n",
53 | "from tensorflow.keras.layers import Dense, ReLU\n",
54 | "from tensorflow.keras.optimizers import RMSprop\n",
55 | "from sklearn.preprocessing import StandardScaler\n",
56 | "\n",
57 | "from tensorflow.keras.utils import Progbar\n",
58 | "\n",
59 | "mode = 'test'\n",
60 | "name = 'w_qlearning'\n",
61 | "level = 1\n",
62 | "if level == 2:\n",
63 | " name += name + 'lv2'\n",
64 | "\n",
65 | "drive.mount('/content/drive/')\n",
66 | "nov_dir = 'Colab Notebooks/dataset/reinforcement_learning/'\n",
67 | "nov_path = '/content/drive/My Drive/' + nov_dir + f'sp500_{mode}.csv'\n",
68 | "\n",
69 | "exp_dir = 'Colab Notebooks/workspace/export/'\n",
70 | "mdl_dir = '/content/drive/My Drive/' + exp_dir + 'models'\n",
71 | "csv_path = '/content/drive/My Drive/' + exp_dir + f'csv_data/{name}_{mode}.csv'\n",
72 | "\n",
73 | "df = pd.read_csv(nov_path)\n",
74 | "df['Date'] = pd.to_datetime(df['Date'], format = '%Y-%m-%d')"
75 | ],
76 | "execution_count": null,
77 | "outputs": [
78 | {
79 | "output_type": "stream",
80 | "name": "stdout",
81 | "text": [
82 | "Mounted at /content/drive/\n"
83 | ]
84 | }
85 | ]
86 | },
87 | {
88 | "cell_type": "code",
89 | "metadata": {
90 | "id": "MN1DKfV6zauY"
91 | },
92 | "source": [
93 | "class Environment:\n",
94 | " def __init__(self, df, initial_money=100000, mode = 'test', commission = 0):\n",
95 | "\n",
96 | " self.df = df.dropna().reset_index()\n",
97 | "\n",
98 | " self.df_total_steps = len(self.df)-1\n",
99 | " self.initial_money = initial_money\n",
100 | " self.mode = mode\n",
101 | " self.commission = commission\n",
102 | " self.trade_time = None\n",
103 | " self.trade_win = None\n",
104 | " self.brfore_buy_cash = None\n",
105 | " self.action_space = np.array([0, 1, 2]) # buy,hold,sell\n",
106 | " self.hold_a_position = None\n",
107 | " self.now_price = None\n",
108 | " self.cash_in_hand = None\n",
109 | " self.sell_price = None\n",
110 | " self.buy_price = None\n",
111 | "\n",
112 | " self.reset()\n",
113 | " \n",
114 | " def reset(self):\n",
115 | "\n",
116 | " self.trade_time = 0\n",
117 | " self.trade_win = 0\n",
118 | " self.brfore_buy_cash = 0\n",
119 | " self.end_step = self.df_total_steps\n",
120 | " self.now_step = 0\n",
121 | " self.hold_a_position = 0.0\n",
122 | " self.now_price = self.df.loc[self.now_step, 'SP500']\n",
123 | " self.cash_in_hand = self.initial_money\n",
124 | " self.sell_price = 0\n",
125 | " self.buy_price = 0\n",
126 | "\n",
127 | " return self._get_now_state()\n",
128 | "\n",
129 | " def step(self, action):\n",
130 | "\n",
131 | " self.now_step += 1\n",
132 | " self.now_price = self.df.loc[self.now_step, 'SP500']\n",
133 | " \n",
134 | " done = (self.end_step == self.now_step)\n",
135 | "\n",
136 | " self.sell_price = 0\n",
137 | " self._trade(action,done)\n",
138 | " reward = 0\n",
139 | " if (self.sell_price > 0) and (self.buy_price > 0) and ((self.sell_price - self.buy_price) != 0):\n",
140 | " reward = (self.sell_price - self.buy_price) / self.buy_price\n",
141 | " self.buy_price = 0\n",
142 | " cur_revenue = self._get_revenue()\n",
143 | " \n",
144 | " info = { 'cur_revenue' : cur_revenue , 'trade_time' : self.trade_time, 'trade_win' : self.trade_win }\n",
145 | "\n",
146 | " return self._get_now_state(), reward, done, info\n",
147 | "\n",
148 | " def _get_now_state(self):\n",
149 | " state = np.empty(3)\n",
150 | " state[0] = self.hold_a_position\n",
151 | " state[1] = self.now_price\n",
152 | " state[2] = self.cash_in_hand\n",
153 | " return state\n",
154 | "\n",
155 | " def _get_revenue(self): \n",
156 | " return self.hold_a_position * self.now_price + self.cash_in_hand\n",
157 | "\n",
158 | " def _trade(self, action,lastorder = False):\n",
159 | " if lastorder:\n",
160 | " if self.hold_a_position != 0:\n",
161 | " self.cash_in_hand += self.now_price * self.hold_a_position\n",
162 | " self.hold_a_position = 0\n",
163 | " self.trade_time += 1\n",
164 | " if self.cash_in_hand > self.brfore_buy_cash:\n",
165 | " self.trade_win += 1\n",
166 | " else:\n",
167 | " if self.action_space[0] == action: # buy\n",
168 | " if self.hold_a_position == 0:\n",
169 | " buy_flag = True\n",
170 | " self.brfore_buy_cash = copy.copy(self.cash_in_hand)\n",
171 | " while buy_flag:\n",
172 | " if self.cash_in_hand > self.now_price:\n",
173 | " self.hold_a_position += 1\n",
174 | " self.buy_price += self.now_price\n",
175 | " self.cash_in_hand -= self.now_price + self.commission * self.now_price\n",
176 | " else:\n",
177 | " buy_flag = False\n",
178 | " if self.action_space[2] == action: # sell\n",
179 | " if self.hold_a_position != 0:\n",
180 | " self.sell_price += self.now_price * self.hold_a_position\n",
181 | " self.cash_in_hand += self.now_price * self.hold_a_position - self.commission * self.now_price * self.hold_a_position\n",
182 | " self.hold_a_position = 0\n",
183 | " self.trade_time += 1\n",
184 | " if self.cash_in_hand > self.brfore_buy_cash:\n",
185 | " self.trade_win += 1"
186 | ],
187 | "execution_count": null,
188 | "outputs": []
189 | },
190 | {
191 | "cell_type": "code",
192 | "metadata": {
193 | "id": "nGeWOM-ZWNYK"
194 | },
195 | "source": [
196 | "class Brain:\n",
197 | " def __init__(self):\n",
198 | "\n",
199 | " self.gamma = 0.9\n",
200 | "\n",
201 | " model = Sequential()\n",
202 | " model.add(Dense(3, input_shape=(3,)))\n",
203 | " model.add(ReLU()) \n",
204 | " model.add(Dense(3))\n",
205 | " model.add(ReLU())\n",
206 | " model.add(Dense(3))\n",
207 | " model.compile(loss=\"mse\", optimizer=RMSprop())\n",
208 | " model.summary()\n",
209 | " self.model = model\n",
210 | " \n",
211 | " model_2 = Sequential()\n",
212 | " model_2.add(Dense(3, input_shape=(3,)))\n",
213 | " model_2.add(ReLU()) \n",
214 | " model_2.add(Dense(3))\n",
215 | " model_2.add(ReLU()) \n",
216 | " model_2.add(Dense(3))\n",
217 | " model_2.compile(loss=\"mse\", optimizer=RMSprop())\n",
218 | " model_2.summary()\n",
219 | " self.model_2 = model_2\n",
220 | "\n",
221 | " def train(self, state, action, reward, next_state, done, s_flag):\n",
222 | "\n",
223 | " if s_flag == 11:\n",
224 | " q = self.model.predict(state) \n",
225 | " next_q = self.model_2.predict(next_state)\n",
226 | " target = np.copy(q)\n",
227 | "\n",
228 | " target[:, action] = reward + (1 - done) * self.gamma*np.amax(next_q, axis=1)\n",
229 | " self.model.train_on_batch(state, target)\n",
230 | " else:\n",
231 | " q = self.model_2.predict(state) \n",
232 | " next_q = self.model.predict(next_state)\n",
233 | " target = np.copy(q)\n",
234 | "\n",
235 | " target[:, action] = reward + (1 - done) * self.gamma*np.amax(next_q, axis=1)\n",
236 | " self.model_2.train_on_batch(state, target)\n",
237 | "\n",
238 | "\n",
239 | " def _predict(self, state, s_flag = 12):\n",
240 | " values = None\n",
241 | " q1 = self.model.predict(state)\n",
242 | " q2 = self.model_2.predict(state)\n",
243 | " if s_flag == 12:\n",
244 | " values = np.array([q1[0,a] + q2[0,a] for a in range(3)])\n",
245 | " elif s_flag == 11:\n",
246 | " values = np.array([q1[0,a] + q1[0,a] for a in range(3)])\n",
247 | " else:\n",
248 | " values = np.array([q2[0,a] + q2[0,a] for a in range(3)])\n",
249 | " return values\n",
250 | "\n",
251 | " def load(self, name, name2):\n",
252 | " self.model.load_weights(name)\n",
253 | " self.model_2.load_weights(name2)\n",
254 | "\n",
255 | " def save(self, name, name2):\n",
256 | " self.model.save_weights(name)\n",
257 | " self.model_2.save_weights(name2)"
258 | ],
259 | "execution_count": null,
260 | "outputs": []
261 | },
262 | {
263 | "cell_type": "code",
264 | "metadata": {
265 | "id": "QxR4grMVRLCR"
266 | },
267 | "source": [
268 | "class Agent(Brain):\n",
269 | " def __init__(self):\n",
270 | " super().__init__()\n",
271 | " self.epsilon = 1.0\n",
272 | " self.epsilon_min = 0.01\n",
273 | " self.r = 0.995\n",
274 | "\n",
275 | " def act(self, state,s_flag=12):\n",
276 | " if np.random.rand() <= self.epsilon:\n",
277 | " return np.random.choice(3)\n",
278 | " act_values = self._predict(state,s_flag)\n",
279 | " if self.epsilon > self.epsilon_min:\n",
280 | " self.epsilon *= self.r\n",
281 | " return np.argmax(act_values)"
282 | ],
283 | "execution_count": null,
284 | "outputs": []
285 | },
286 | {
287 | "cell_type": "code",
288 | "metadata": {
289 | "id": "On5S8YtLz3U4"
290 | },
291 | "source": [
292 | "class Main:\n",
293 | " def __init__(self, env, agent, mdl_dir, name, episodes_times = 1000, mode = 'test'):\n",
294 | " self.env = env\n",
295 | " self.agent = agent\n",
296 | " self.mdl_dir = mdl_dir\n",
297 | " self.scaler = self._standard_scaler(self.env)\n",
298 | " self.episodes_times = episodes_times\n",
299 | " self.mode = mode\n",
300 | " self.name = name\n",
301 | " self.df_rec = pd.DataFrame(index=[], columns=['FixedProfit','TradeTimes','TradeWin'])\n",
302 | "\n",
303 | " if self.mode == 'test':\n",
304 | " self._load()\n",
305 | " self.agent.epsilon = 0.01\n",
306 | "\n",
307 | " def play_game(self):\n",
308 | "\n",
309 | " for episode in range(self.episodes_times):\n",
310 | "\n",
311 | " if (episode % 10 == 0):\n",
312 | " metrics_names = ['FixedProfit','TradeTimes','TradeWin']\n",
313 | " if (int(str(self.episodes_times)[:-1])*10 == episode):\n",
314 | " pb_i = Progbar(int(str(self.episodes_times)[-1]), stateful_metrics=metrics_names)\n",
315 | " else:\n",
316 | " pb_i = Progbar(10, stateful_metrics=metrics_names)\n",
317 | " p_mean,trade_time,win_time = np.array([]),np.array([]),np.array([])\n",
318 | "\n",
319 | " state = self.env.reset()\n",
320 | " state = self.scaler.transform([state])\n",
321 | " done = False\n",
322 | " \n",
323 | " while not done:\n",
324 | " action = self.agent.act(state)\n",
325 | " next_state, reward, done, info = self.env.step(action)\n",
326 | " next_state = self.scaler.transform([next_state])\n",
327 | " reward = self._reward_clipping(reward)\n",
328 | "\n",
329 | " if self.mode == 'train':\n",
330 | " s_flag = 11 if np.random.random() <= 0.5 else 22\n",
331 | " agent.train(state, action, reward, next_state, done, s_flag)\n",
332 | "\n",
333 | " state = next_state\n",
334 | "\n",
335 | " record = pd.Series([info['cur_revenue'],info['trade_time'],info['trade_win']], index=self.df_rec.columns)\n",
336 | " self.df_rec = self.df_rec.append(record, ignore_index=True)\n",
337 | "\n",
338 | " p_mean,trade_time,win_time = np.append(p_mean,info['cur_revenue']),np.append(trade_time,info['trade_time']),np.append(win_time,info['trade_win'])\n",
339 | " values=[('FixedProfit',int(np.mean(p_mean))), ('TradeTimes',int(np.mean(trade_time))), ('TradeWin',int(np.mean(win_time)))]\n",
340 | " pb_i.add(1, values=values)\n",
341 | "\n",
342 | " if self.mode == 'train':\n",
343 | " self._save()\n",
344 | " self._save_csv()\n",
345 | "\n",
346 | " def _standard_scaler(self, env):\n",
347 | " states = []\n",
348 | " for _ in range(env.df_total_steps):\n",
349 | " action = np.random.choice(env.action_space)\n",
350 | " state, reward, done, info = env.step(action)\n",
351 | " states.append(state)\n",
352 | " if done:\n",
353 | " break\n",
354 | " \n",
355 | " scaler = StandardScaler()\n",
356 | " scaler.fit(states)\n",
357 | " return scaler\n",
358 | "\n",
359 | " def _reward_clipping(self, val):\n",
360 | " if val > 0:\n",
361 | " return 1\n",
362 | " elif val == 0:\n",
363 | " return 0\n",
364 | " else:\n",
365 | " return -1\n",
366 | "\n",
367 | " def _load(self):\n",
368 | " with open('{}/{}.pkl'.format(self.mdl_dir, self.name), 'rb') as f:\n",
369 | " self.scaler = pickle.load(f)\n",
370 | " self.agent.load('{}/{}.h5'.format(self.mdl_dir, self.name), '{}/{}_2.h5'.format(self.mdl_dir, self.name))\n",
371 | "\n",
372 | " def _save(self):\n",
373 | " self.agent.save('{}/{}.h5'.format(self.mdl_dir, self.name), '{}/{}_2.h5'.format(self.mdl_dir, self.name))\n",
374 | " with open('{}/{}.pkl'.format(self.mdl_dir, self.name), 'wb') as f:\n",
375 | " pickle.dump(self.scaler, f)\n",
376 | "\n",
377 | " def _save_csv(self):\n",
378 | " self.df_rec.to_csv(csv_path)"
379 | ],
380 | "execution_count": null,
381 | "outputs": []
382 | },
383 | {
384 | "cell_type": "code",
385 | "metadata": {
386 | "colab": {
387 | "base_uri": "https://localhost:8080/"
388 | },
389 | "id": "pYFNVDDQz9X9",
390 | "outputId": "7b56765e-c57d-4e68-8719-ff36e6b1aca4"
391 | },
392 | "source": [
393 | "initial_money=1000000\n",
394 | "episodes_times = 100\n",
395 | "commission = 0 if level == 1 else 0.002\n",
396 | "\n",
397 | "agent = Agent()\n",
398 | "env = Environment(df, initial_money = initial_money, mode = mode, commission = commission)\n",
399 | "main = Main(env, agent, mdl_dir, name, episodes_times, mode)\n",
400 | "main.play_game()"
401 | ],
402 | "execution_count": null,
403 | "outputs": [
404 | {
405 | "output_type": "stream",
406 | "name": "stdout",
407 | "text": [
408 | "Model: \"sequential\"\n",
409 | "_________________________________________________________________\n",
410 | "Layer (type) Output Shape Param # \n",
411 | "=================================================================\n",
412 | "dense (Dense) (None, 3) 12 \n",
413 | "_________________________________________________________________\n",
414 | "re_lu (ReLU) (None, 3) 0 \n",
415 | "_________________________________________________________________\n",
416 | "dense_1 (Dense) (None, 3) 12 \n",
417 | "_________________________________________________________________\n",
418 | "re_lu_1 (ReLU) (None, 3) 0 \n",
419 | "_________________________________________________________________\n",
420 | "dense_2 (Dense) (None, 3) 12 \n",
421 | "=================================================================\n",
422 | "Total params: 36\n",
423 | "Trainable params: 36\n",
424 | "Non-trainable params: 0\n",
425 | "_________________________________________________________________\n",
426 | "Model: \"sequential_1\"\n",
427 | "_________________________________________________________________\n",
428 | "Layer (type) Output Shape Param # \n",
429 | "=================================================================\n",
430 | "dense_3 (Dense) (None, 3) 12 \n",
431 | "_________________________________________________________________\n",
432 | "re_lu_2 (ReLU) (None, 3) 0 \n",
433 | "_________________________________________________________________\n",
434 | "dense_4 (Dense) (None, 3) 12 \n",
435 | "_________________________________________________________________\n",
436 | "re_lu_3 (ReLU) (None, 3) 0 \n",
437 | "_________________________________________________________________\n",
438 | "dense_5 (Dense) (None, 3) 12 \n",
439 | "=================================================================\n",
440 | "Total params: 36\n",
441 | "Trainable params: 36\n",
442 | "Non-trainable params: 0\n",
443 | "_________________________________________________________________\n",
444 | "10/10 [==============================] - 528s 53s/step - FixedProfit: 1454705.0000 - TradeTimes: 78.0000 - TradeWin: 45.0000\n",
445 | "10/10 [==============================] - 524s 52s/step - FixedProfit: 1512547.0000 - TradeTimes: 79.0000 - TradeWin: 46.0000\n",
446 | "10/10 [==============================] - 519s 52s/step - FixedProfit: 1404403.0000 - TradeTimes: 77.0000 - TradeWin: 44.0000\n",
447 | "10/10 [==============================] - 520s 52s/step - FixedProfit: 1493439.0000 - TradeTimes: 77.0000 - TradeWin: 45.0000\n",
448 | "10/10 [==============================] - 521s 52s/step - FixedProfit: 1371999.0000 - TradeTimes: 77.0000 - TradeWin: 45.0000\n",
449 | "10/10 [==============================] - 521s 52s/step - FixedProfit: 1427722.0000 - TradeTimes: 76.0000 - TradeWin: 44.0000\n",
450 | "10/10 [==============================] - 522s 52s/step - FixedProfit: 1440309.0000 - TradeTimes: 77.0000 - TradeWin: 44.0000\n",
451 | "10/10 [==============================] - 524s 52s/step - FixedProfit: 1421383.0000 - TradeTimes: 78.0000 - TradeWin: 45.0000\n",
452 | "10/10 [==============================] - 520s 52s/step - FixedProfit: 1319935.0000 - TradeTimes: 76.0000 - TradeWin: 43.0000\n",
453 | "10/10 [==============================] - 519s 52s/step - FixedProfit: 1391426.0000 - TradeTimes: 77.0000 - TradeWin: 44.0000\n"
454 | ]
455 | }
456 | ]
457 | }
458 | ]
459 | }
--------------------------------------------------------------------------------