├── PID.py
├── README.md
├── parameter.py
├── plot_result.py
├── requirements.txt
├── results
├── ChooseModel_class_pid_CurveType_trapezoidal_Height_1000_DumpSystem_False_RunType_test
│ ├── ecValues.png
│ ├── ecValues.txt
│ ├── iaeValues.png
│ ├── iaeValues.txt
│ ├── radValues.png
│ └── radValues.txt
├── ChooseModel_search_electric_CurveType_trapezoidal_Height_1000_DumpSystem_False_RunType_train
│ ├── epRewards.png
│ └── epRewards.txt
└── ChooseModel_search_pid_parameter_CurveType_trapezoidal_Height_1000_DumpSystem_False_RunType_train
│ ├── actor_model.ckpt.data-00000-of-00001
│ ├── actor_model.ckpt.index
│ ├── actor_model.ckpt.meta
│ ├── checkpoint
│ ├── critic_model.ckpt.data-00000-of-00001
│ ├── critic_model.ckpt.index
│ ├── critic_model.ckpt.meta
│ └── reward.PNG
├── run.py
├── servo_system_env.py
└── utils.py
/PID.py:
--------------------------------------------------------------------------------
1 | # -*- coding: utf-8 -*-
2 | # @Time : 11/30/21 3:17 PM
3 | # @Author : Zhiqiang He
4 | # @Email : tinyzqh@163.com
5 | # @File : PID.py
6 | # @Software: PyCharm
7 |
8 |
9 | class PID(object):
10 | def __init__(self, kp, ki, kd, args):
11 | super(PID).__init__()
12 | self.kp = kp
13 | self.ki = ki
14 | self.kd = kd
15 | self.dt = args.dt
16 |
17 | self.CumulativeError = 0.0
18 | self.LastError = None
19 |
20 | def update(self, error):
21 | """
22 | compute the out of fixed PID parameter.
23 | :param error:
24 | :return:
25 | """
26 | p = self.kp * error
27 | i = self.ki * self.CumulativeError * self.dt
28 | if self.LastError is None:
29 | d = 0.0
30 | else:
31 | d = self.kd * (error - self.LastError) / self.dt
32 |
33 | self.CumulativeError += error
34 | self.LastError = error
35 | return p + i + d
36 |
37 | def update_with_parameter(self, error, kp, ki, kd):
38 | """
39 | compute the out of the learned PID parameter.
40 | :param error:
41 | :param kp:
42 | :param ki:
43 | :param kd:
44 | :return:
45 | """
46 | p = kp * error
47 | i = ki * self.CumulativeError * self.dt
48 | if self.LastError is None:
49 | d = 0.0
50 | else:
51 | d = kd * (error - self.LastError) / self.dt
52 |
53 | self.CumulativeError += error
54 | self.LastError = error
55 | return p + i + d
56 |
57 | if __name__ == "__main__":
58 | pass
--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
1 |
2 | # *强化学习用于PID的参数整定和电流补偿控制。*
3 |
4 | # Introduce
5 |
6 | This is the implementation of the paper “Control Strategy of Speed Servo Systems Based on Deep Reinforcement Learning”.
7 |
8 | For more paper information, please checkout the the paper [Link](https://www.mdpi.com/1999-4893/11/5/65).
9 |
10 |
11 | # Class PID
12 |
13 | The class PID algorithms has no training process.
14 |
15 | ```bash
16 | python run.py --choose_model class_pid --curve_type trapezoidal --height 1000 --run_type test
17 | ```
18 |
19 | Plot the result:
20 |
21 | ```bash
22 | python plot_result.py --choose_model class_pid --curve_type trapezoidal --height 1000 --run_type test
23 | ```
24 |
25 |
26 |
27 |
28 |
29 |
30 |
31 |
32 |
33 |
34 | # Parameters Adjustment Of PID
35 |
36 | Search the parameter of PID Based DDPG Algorithm.
37 |
38 | ```bash
39 | python run.py --choose_model search_pid_parameter --curve_type trapezoidal --height 1000 --run_type train
40 | ```
41 |
42 | experimental operation process:
43 |
44 |
45 |
46 |
47 |
48 |
49 |
50 |
51 | # Electric Current Compensation Of RL-PID
52 |
53 |
54 | ```bash
55 | python run.py --choose_model search_electric --curve_type trapezoidal --height 1000 --run_type train
56 | ```
57 |
58 | test
59 |
60 |
61 | ```bash
62 | python run.py --choose_model search_electric --curve_type trapezoidal --height 1000 --run_type test
63 | ```
64 |
65 | Plot the result:
66 |
67 | ```bash
68 | python plot_result.py --choose_model search_electric --curve_type trapezoidal --height 1000 --run_type train
69 | ```
70 |
71 | the result show:
72 |
73 |
74 |
75 |
76 |
77 |
78 |
79 |
80 | # Dependencies
81 |
82 | The code was tested under Ubuntu 16 and uses these packages:
83 |
84 | - tensorflow-gpu==1.14.0
85 | - atari-py==0.2.6
86 | - gym==0.17.3
87 | - numpy==1.91.3
88 |
89 | more packages described in requirements.txt
90 |
91 |
92 | Citing
93 | ------
94 | If you find this open source release useful, please reference in your paper:
95 |
96 |
97 | > Chen P, He Z, Chen C, et al. (2018).
98 | > Control strategy of speed servo systems based on deep reinforcement learning[J].
99 | > *Algorithms, 2018, 11(5): 65.*.
100 |
--------------------------------------------------------------------------------
/parameter.py:
--------------------------------------------------------------------------------
1 | # -*- coding: utf-8 -*-
2 | # @Time : 11/29/21 11:15 AM
3 | # @Author : Zhiqiang He
4 | # @Email : tinyzqh@163.com
5 | # @File : parameter.py
6 | # @Software: PyCharm
7 |
8 | import argparse
9 |
10 | parser = argparse.ArgumentParser(description="The parameter of all Algorithms")
11 | parser.add_argument("--height", type=int, default=1000, help="the highest value of the tracking curve")
12 | parser.add_argument("--curve_type", type=str, default="trapezoidal", help="type of curve to be tracked, choose of ['trapezoidal', 'sine']")
13 |
14 | parser.add_argument("--dt", type=float, default=0.0001, help="discrete time interval")
15 | parser.add_argument("--start_time", type=float, default=0.0, help="the start time of simulate")
16 | parser.add_argument("--end_time", type=float, default=0.5, help="the end time of simulate")
17 | parser.add_argument("--seed", type=int, default=0, help="the random seed")
18 |
19 | parser.add_argument("--change_times", type=int, default=9, help="the change times of system")
20 | parser.add_argument("--use_jump_system", type=bool, default=False, help="select whether to choose a jump system")
21 | parser.add_argument("--used_system_id", type=int, default=1, help="choose the system id when not use jump system. default in [1, 2, 3]")
22 |
23 | parser.add_argument("--kp", type=float, default=0.5, help="the kp parameter of PID")
24 | parser.add_argument("--ki", type=float, default=2.985, help="the kp parameter of PID")
25 | parser.add_argument("--kd", type=float, default=0.0, help="the kp parameter of PID")
26 |
27 | parser.add_argument("--kp_rl", type=float, default=0.9318, help="the kp parameter of RL Agent Given")
28 | parser.add_argument("--ki_rl", type=float, default=1.4824, help="the ki parameter of RL Agent Given")
29 | parser.add_argument("--kd_rl", type=float, default=0.0, help="the kd parameter of RL Agent Given")
30 |
31 | # parameter of DDPG
32 | parser.add_argument("--max_episodes", type=int, default=20000000, help="")
33 | parser.add_argument("--max_ep_steps", type=int, default=4800, help="")
34 | parser.add_argument("--learning_actor", type=float, default=0.0001, help="the learning rate for actor")
35 | parser.add_argument("--learning_critic", type=float, default=0.0001, help="the learning rate for critic")
36 | parser.add_argument("--gamma", type=float, default=0.95, help="the discount of reward")
37 | parser.add_argument("--replace_iter_a", type=int, default=11000, help="the iter steps of replace the actor target network")
38 | parser.add_argument("--replace_iter_c", type=int, default=10000, help="the iter steps of replace the critic target network")
39 | parser.add_argument("--memory_capacity", type=int, default=50000, help="the capacity of the memory replay buffer")
40 | parser.add_argument("--batch_size", type=int, default=256, help="the batch size for training the network")
41 | parser.add_argument("--var", type=float, default=2.0, help="the control exploration of init")
42 | parser.add_argument("--var_min", type=float, default=0.0, help="the control exploration of end")
43 | parser.add_argument("--reg", type=float, default=2.0, help="the regularization parameters")
44 | parser.add_argument("--pid_parameter_search_action_bound", type=float, default=10.0, help="the action bound of pid search for [-10.0, 10.0]")
45 | parser.add_argument("--electric_compensation_action_bound", type=float, default=2.0, help="the action bound of electric search for [-2.0, 2.0]")
46 | parser.add_argument("--state_dim", type=int, default=6, help="the state dim")
47 | parser.add_argument("--pid_parameter_search_action_dim", type=int, default=2, help="the action dim of PID Parameters Adjustment")
48 | parser.add_argument("--electric_compensation_action_dim", type=int, default=1, help="the action dim of electric compensation.")
49 |
50 | parser.add_argument("--ep_reward_max", type=int, default=1000000, help="the ")
51 | parser.add_argument("--run_type", type=str, default="train", help="choose from ['train', 'test']")
52 | parser.add_argument("--choose_model", type=str, default='search_electric', help="one of ['search_pid_parameter', 'search_electric', 'class_pid']")
53 |
54 | args = parser.parse_args()
--------------------------------------------------------------------------------
/plot_result.py:
--------------------------------------------------------------------------------
1 | # -*- coding: utf-8 -*-
2 | # @Time : 12/8/21 7:47 PM
3 | # @Author : Zhiqiang He
4 | # @Email : tinyzqh@163.com
5 | # @File : plot_result.py
6 | # @Software: PyCharm
7 |
8 | import os
9 | import pandas as pd
10 | import seaborn as sns
11 | from parameter import args
12 | import matplotlib.pyplot as plt
13 |
14 | class plot_result(object):
15 | def __init__(self, root_path):
16 | """
17 | :param root_path:
18 | """
19 | self.root_path = root_path
20 | if self.root_path.split('_')[1] == "class":
21 | self.label = "Classic PID"
22 | else:
23 | self.label = "RL PID"
24 |
25 | def get_files_path(self):
26 | """
27 | # Get All '.txt' files.
28 | :return:
29 | """
30 | file_path_dir = []
31 | for root, dir, files in os.walk(self.root_path):
32 | for file in files:
33 | file_path = os.path.join(root, file)
34 | file_path_dir.append(file_path)
35 | return file_path_dir
36 |
37 | def get_data(self):
38 | files = self.get_files_path()
39 |
40 | DataDict = {}
41 | for file in files:
42 | if file.split('/')[-1].split('.')[-1] == "txt":
43 | print("filename {} processed.".format(file))
44 | DictName = file.split('/')[-1].split('.')[0]
45 | df = pd.read_csv(file, sep=" ")
46 | ColumnsName = df.columns
47 | Dict = {}
48 | Dict[ColumnsName[0]] = df[ColumnsName[0]]
49 | Dict[ColumnsName[1]] = df[ColumnsName[1]]
50 |
51 | DataDict[DictName] = Dict
52 | else:
53 | print("filename {} not .txt file.".format(file))
54 |
55 | return DataDict
56 |
57 | def plot_data(self):
58 | """
59 | plot the all .txt files.
60 | :return:
61 | """
62 | sns.set(style="darkgrid", font_scale=1.5, font='serif', rc={'figure.figsize': (10, 8)})
63 | DataDict = self.get_data()
64 | for key, data in DataDict.items():
65 |
66 | plt.plot(data['times'], data[key], label=self.label)
67 | plt.xlabel("Time (s)")
68 | plt.ylabel(key)
69 | plt.legend()
70 | plt.savefig(self.root_path + '/{}.png'.format(key))
71 | plt.show()
72 |
73 |
74 | if __name__ == "__main__":
75 | current_dir = os.path.dirname(os.path.abspath(__file__))
76 | result_dir = os.path.join('results', 'ChooseModel_{}_CurveType_{}_Height_{}_DumpSystem_{}_RunType_{}'.format(args.choose_model, args.curve_type, args.height, args.use_jump_system, args.run_type))
77 |
78 | root_path = os.path.join(current_dir, result_dir)
79 | file_path_dir = plot_result(root_path=root_path).plot_data()
--------------------------------------------------------------------------------
/requirements.txt:
--------------------------------------------------------------------------------
1 | absl-py==0.11.0
2 | alabaster==0.7.12
3 | atari-py==0.2.6
4 | attrs==20.2.0
5 | Babel==2.8.0
6 | cached-property==1.5.2
7 | certifi==2021.5.30
8 | cffi==1.14.3
9 | chardet==3.0.4
10 | cloudpickle==1.6.0
11 | colorama==0.4.4
12 | cycler==0.10.0
13 | Cython==0.29.21
14 | dataclasses==0.7
15 | deepdiff==5.5.0
16 | dm-env==1.3
17 | dm-tree==0.1.5
18 | docopt==0.6.2
19 | docutils==0.16
20 | entrypoints==0.3
21 | enum34==1.1.10
22 | fasteners==0.15
23 | future==0.18.2
24 | gitdb==4.0.7
25 | GitPython==3.1.17
26 | glfw==2.0.0
27 | gym==0.17.3
28 | h5py==3.0.0
29 | idna==2.10
30 | ImageHash==4.1.0
31 | imageio==2.9.0
32 | imagesize==1.2.0
33 | importlib-metadata==2.0.0
34 | iniconfig==1.1.1
35 | ipdb==0.13.4
36 | jedi==0.17.2
37 | Jinja2==2.11.2
38 | jsonpickle==1.5.2
39 | labmaze==1.0.3
40 | lxml==4.6.1
41 | MarkupSafe==1.1.1
42 | mkl-fft==1.2.0
43 | mkl-random==1.1.1
44 | mkl-service==2.3.0
45 | mock==4.0.3
46 | monotonic==1.5
47 | mpyq==0.2.5
48 | mujoco-py==2.0.2.13
49 | munch==2.5.0
50 | numpy==1.19.3
51 | numpydoc==1.1.0
52 | olefile==0.46
53 | ordered-set==4.0.2
54 | packaging==20.4
55 | patsy==0.5.1
56 | pluggy==0.13.1
57 | portpicker==1.4.0
58 | progressbar==2.5
59 | protobuf==3.13.0
60 | py==1.9.0
61 | py-cpuinfo==8.0.0
62 | pycparser==2.20
63 | pygame==2.0.1
64 | pyglet==1.5.0
65 | PyOpenGL==3.1.5
66 | pyparsing==2.4.7
67 | PySC2==3.0.0
68 | pytest==6.1.2
69 | pytest-instafail==0.3.0
70 | python-dateutil==2.8.1
71 | pytz==2020.1
72 | PyWavelets==1.1.1
73 | PyYAML==5.4.1
74 | requests==2.24.0
75 | retrying==1.3.3
76 | s2clientprotocol==5.0.7.84643.0
77 | s2protocol==5.0.7.84643.0
78 | sacred==0.8.2
79 | scipy==1.5.3
80 | seaborn==0.9.0
81 | six==1.15.0
82 | sk-video==1.1.10
83 | smmap==4.0.0
84 | snowballstemmer==2.0.0
85 | Sphinx==3.2.1
86 | sphinx-rtd-theme==0.5.0
87 | sphinxcontrib-applehelp==1.0.2
88 | sphinxcontrib-devhelp==1.0.2
89 | sphinxcontrib-htmlhelp==1.0.3
90 | sphinxcontrib-jsmath==1.0.1
91 | sphinxcontrib-qthelp==1.0.3
92 | sphinxcontrib-serializinghtml==1.1.4
93 | tensorboardX==2.0
94 | toml==0.10.1
95 | torch==1.8.0+cu111
96 | torchaudio==0.8.0
97 | torchvision==0.9.0+cu111
98 | tornado==6.0.4
99 | urllib3==1.25.11
100 | websocket-client==1.0.1
101 | whichcraft==0.6.1
102 | wrapt==1.12.1
103 | xlwt==1.3.0
104 | zipp==3.4.0
--------------------------------------------------------------------------------
/results/ChooseModel_class_pid_CurveType_trapezoidal_Height_1000_DumpSystem_False_RunType_test/ecValues.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/tinyzqh/control-of-jump-systems-based-on-reinforcement-learning/de23c13aecfb6d8745d6005e32d2083498c4246a/results/ChooseModel_class_pid_CurveType_trapezoidal_Height_1000_DumpSystem_False_RunType_test/ecValues.png
--------------------------------------------------------------------------------
/results/ChooseModel_class_pid_CurveType_trapezoidal_Height_1000_DumpSystem_False_RunType_test/iaeValues.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/tinyzqh/control-of-jump-systems-based-on-reinforcement-learning/de23c13aecfb6d8745d6005e32d2083498c4246a/results/ChooseModel_class_pid_CurveType_trapezoidal_Height_1000_DumpSystem_False_RunType_test/iaeValues.png
--------------------------------------------------------------------------------
/results/ChooseModel_class_pid_CurveType_trapezoidal_Height_1000_DumpSystem_False_RunType_test/radValues.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/tinyzqh/control-of-jump-systems-based-on-reinforcement-learning/de23c13aecfb6d8745d6005e32d2083498c4246a/results/ChooseModel_class_pid_CurveType_trapezoidal_Height_1000_DumpSystem_False_RunType_test/radValues.png
--------------------------------------------------------------------------------
/results/ChooseModel_search_electric_CurveType_trapezoidal_Height_1000_DumpSystem_False_RunType_train/epRewards.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/tinyzqh/control-of-jump-systems-based-on-reinforcement-learning/de23c13aecfb6d8745d6005e32d2083498c4246a/results/ChooseModel_search_electric_CurveType_trapezoidal_Height_1000_DumpSystem_False_RunType_train/epRewards.png
--------------------------------------------------------------------------------
/results/ChooseModel_search_electric_CurveType_trapezoidal_Height_1000_DumpSystem_False_RunType_train/epRewards.txt:
--------------------------------------------------------------------------------
1 | times epRewards
2 | 0.0 3036.7259341814656
3 | 0.0001 3043.6923303989365
4 | 0.0002 3017.9982326663167
5 | 0.00030000000000000003 3066.080714219948
6 | 0.0004 3044.6397142553615
7 | 0.0005 3030.9625944126683
8 | 0.0006000000000000001 3039.743050215572
9 | 0.0007 3023.2854571749554
10 | 0.0008 3021.9944469362213
11 | 0.0009000000000000001 3023.699565360395
12 | 0.001 3076.4207968839874
13 | 0.0011 3102.887173448377
14 | 0.0012000000000000001 3107.991250220807
15 | 0.0013000000000000002 3117.195720022614
16 | 0.0014 3120.8241369752936
17 | 0.0015 3173.737503662069
18 | 0.0016 3198.9838898886583
19 | 0.0017000000000000001 3183.4812869604657
20 | 0.0018000000000000002 3216.7629975642876
21 | 0.0019 3238.452123295176
22 | 0.002 3243.6810075367894
23 | 0.0021000000000000003 3256.372256253036
24 | 0.0022 3288.278534943949
25 | 0.0023 3275.7429848551155
26 | 0.0024000000000000002 3296.4360207573704
27 | 0.0025 3310.9131839014312
28 | 0.0026000000000000003 3323.3703375703217
29 | 0.0027 3334.697914223168
30 | 0.0028 3332.676420728047
31 | 0.0029000000000000002 3353.237062546084
32 | 0.003 3365.5150787415755
33 | 0.0031000000000000003 3371.572243763116
34 | 0.0032 3390.179212621337
35 | 0.0033 3388.4623226772273
36 | 0.0034000000000000002 3391.470616095011
37 | 0.0035 3414.962045300303
38 | 0.0036000000000000003 3409.7790657045175
39 | 0.0037 3426.805094550273
40 | 0.0038 3419.5921372889643
41 | 0.0039000000000000003 3425.411186758219
42 | 0.004 3445.9015376405205
43 | 0.0041 3443.6479604053425
44 | 0.004200000000000001 3440.5658731217713
45 | 0.0043 3448.5756823000484
46 | 0.0044 3451.855521973523
47 | 0.0045000000000000005 3450.7678200050345
48 | 0.0046 3466.9810084088745
49 | 0.0047 3466.3342867587858
50 | 0.0048000000000000004 3464.656076634408
51 | 0.0049 3475.286593095242
52 | 0.005 3467.695263105075
53 | 0.0051 3473.711167060322
54 | 0.005200000000000001 3147.908303880091
55 | 0.0053 2912.7297121397232
56 | 0.0054 2978.4591262697954
57 | 0.0055000000000000005 2958.8481619291356
58 | 0.0056 3251.52941865746
59 | 0.0057 3073.2239386255833
60 | 0.0058000000000000005 3315.4084656909013
61 | 0.0059 3197.405696360599
62 | 0.006 3126.6639081244725
63 | 0.0061 2794.314079034672
64 | 0.006200000000000001 3272.493309024069
65 | 0.0063 3435.0819586473967
66 | 0.0064 3374.9285190488035
67 | 0.006500000000000001 3309.7697911073883
68 | 0.0066 3453.9894058479135
69 | 0.0067 3463.199772708253
70 | 0.0068000000000000005 3501.514879964667
71 | 0.006900000000000001 3503.46129012014
72 | 0.007 3506.5379153397384
73 | 0.0071 3390.7595933273356
74 | 0.007200000000000001 2898.7963078461767
75 | 0.0073 3284.2461593354237
76 | 0.0074 3198.766635574699
77 | 0.007500000000000001 3410.659583874455
78 | 0.0076 3419.1911779384145
79 | 0.0077 3342.6748066543782
80 | 0.0078000000000000005 3450.6064607118037
81 | 0.0079 3294.1354084943405
82 | 0.008 2790.4469210588336
83 | 0.0081 3238.9775974171225
84 | 0.0082 2808.8973239989086
85 | 0.0083 3137.3685749253614
86 | 0.008400000000000001 3440.9653372098137
87 | 0.0085 3119.854133191471
88 | 0.0086 2999.9901846747985
89 | 0.008700000000000001 2989.8744768256897
90 | 0.0088 3191.833236106521
91 | 0.0089 2882.1808677009
92 | 0.009000000000000001 2987.4138457487043
93 | 0.0091 3134.1181841183784
94 | 0.0092 3591.405367783205
95 | 0.009300000000000001 3179.7929546415803
96 | 0.0094 3303.5814377979827
97 | 0.0095 3438.489734777065
98 | 0.009600000000000001 3517.476211431069
99 | 0.0097 3504.740321141496
100 | 0.0098 3542.1352700938232
101 | 0.0099 3586.2483293542136
102 | 0.01 3888.789484069617
103 | 0.010100000000000001 4221.606454841663
104 | 0.0102 3809.264931791187
105 | 0.0103 3202.643387289286
106 | 0.010400000000000001 3838.2202073659823
107 | 0.0105 4354.407101473443
108 | 0.0106 4334.608536034208
109 | 0.010700000000000001 4284.572616653706
110 | 0.0108 4233.562753610873
111 | 0.0109 3698.2831756022006
112 | 0.011000000000000001 3976.1156399211322
113 | 0.0111 4260.359021388144
114 | 0.0112 4302.172354796017
115 | 0.011300000000000001 4136.450394033346
116 | 0.0114 4219.24872109257
117 | 0.0115 4215.643146355247
118 | 0.011600000000000001 4335.709850062812
119 | 0.0117 4298.425634734699
120 | 0.0118 4454.339173897462
121 | 0.0119 4136.8841476271755
122 | 0.012 4450.983725552523
123 | 0.012100000000000001 4374.800641650777
124 | 0.0122 4143.236924900215
125 | 0.0123 4403.714997054108
126 | 0.012400000000000001 4300.67294703903
127 | 0.0125 4303.347312043605
128 | 0.0126 4192.134513294037
129 | 0.012700000000000001 4267.112876036415
130 | 0.0128 4244.056254288995
131 | 0.0129 4285.634246044754
132 | 0.013000000000000001 4264.590208291385
133 | 0.0131 4116.045754020984
134 | 0.0132 4193.414202886074
135 | 0.013300000000000001 4335.121288539964
136 | 0.0134 4306.167440484755
137 | 0.0135 4322.200653052792
138 | 0.013600000000000001 4357.691192883888
139 | 0.0137 4444.072313718211
140 | 0.013800000000000002 4332.999146298944
141 | 0.013900000000000001 4365.995680669751
142 | 0.014 4299.934384964377
143 | 0.014100000000000001 4388.516998653022
144 | 0.0142 4454.348760575999
145 | 0.0143 4453.104561945968
146 | 0.014400000000000001 4324.013924428038
147 | 0.0145 4317.404168189095
148 | 0.0146 4275.8342059019305
149 | 0.014700000000000001 4330.38355609085
150 | 0.0148 4406.393167417569
151 | 0.0149 4386.3324433464995
152 | 0.015000000000000001 4445.215155378947
153 | 0.0151 4391.414760190859
154 | 0.0152 4396.03798533988
155 | 0.015300000000000001 4337.203063404891
156 | 0.0154 4407.030491054112
157 | 0.0155 4377.762681171134
158 | 0.015600000000000001 4489.088323375656
159 | 0.015700000000000002 4318.544772205742
160 | 0.0158 4374.489682789004
161 | 0.0159 4431.596393913651
162 | 0.016 4344.3663744918895
163 | 0.0161 4511.522171346495
164 | 0.0162 4420.312437201572
165 | 0.016300000000000002 4248.771279638011
166 | 0.0164 4458.488222335947
167 | 0.0165 4439.1906612911525
168 | 0.0166 4455.229603827504
169 | 0.0167 4423.384309210541
170 | 0.016800000000000002 4419.23442020806
171 | 0.016900000000000002 4363.019430911425
172 | 0.017 4459.500301640532
173 | 0.0171 4440.803159012662
174 | 0.0172 4331.872704018888
175 | 0.0173 4423.442289082499
176 | 0.017400000000000002 4199.984756019031
177 | 0.0175 4436.266045233103
178 | 0.0176 4392.656748167428
179 | 0.0177 4402.1193077392
180 | 0.0178 4373.027862467928
181 | 0.0179 4469.650402698401
182 | 0.018000000000000002 4398.702202561162
183 | 0.0181 4424.4974548573
184 | 0.0182 4476.2174514045055
185 | 0.0183 4437.450168154308
186 | 0.0184 4487.0568195167125
187 | 0.018500000000000003 4401.0698060739205
188 | 0.018600000000000002 4500.393297490085
189 | 0.0187 4488.970108387174
190 | 0.0188 4485.763001860717
191 | 0.0189 4350.2458625323425
192 | 0.019 4462.591713303403
193 | 0.019100000000000002 4465.413974392815
194 | 0.019200000000000002 4444.8617752028185
195 | 0.0193 4526.353688257017
196 | 0.0194 4489.33134707166
197 | 0.0195 4517.624393675544
198 | 0.0196 4502.034806643863
199 | 0.019700000000000002 4481.6487404602785
200 | 0.0198 4460.529679211568
201 | 0.0199 4494.675303545952
202 | 0.02 4305.796267984588
203 | 0.0201 4487.007006204375
204 | 0.020200000000000003 4442.837179646518
205 | 0.020300000000000002 4476.571451906984
206 | 0.0204 4389.4553223094235
207 | 0.0205 4183.059037881925
208 | 0.0206 4418.899633567956
209 | 0.0207 4308.435531957523
210 | 0.020800000000000003 4360.656501886434
211 | 0.020900000000000002 4438.4547125659765
212 | 0.021 4401.023099375283
213 | 0.0211 4403.837607531937
214 | 0.0212 4381.831131073825
215 | 0.0213 4426.548323296748
216 | 0.021400000000000002 4410.8720725007315
217 | 0.021500000000000002 4420.17939944314
218 | 0.0216 4208.519589445352
219 | 0.0217 4342.744269775053
220 | 0.0218 4367.795661951114
221 | 0.0219 4434.550868713155
222 | 0.022000000000000002 4384.817864479502
223 | 0.0221 4428.947419022296
224 | 0.0222 4426.206895598665
225 | 0.0223 4408.291241633852
226 | 0.0224 4427.313783055031
227 | 0.022500000000000003 4485.568975546635
228 | 0.022600000000000002 4460.840407717722
229 | 0.0227 4461.227076963243
230 | 0.0228 4496.6466565720575
231 | 0.0229 4508.007106797012
232 | 0.023 4504.754133032239
233 | 0.023100000000000002 4392.36115833794
234 | 0.023200000000000002 4421.853174059126
235 | 0.0233 4471.376428643626
236 | 0.0234 4459.720354439567
237 | 0.0235 4436.909775711515
238 | 0.0236 4441.414110733332
239 | 0.023700000000000002 4427.730960290675
240 | 0.0238 4466.986080713704
241 | 0.0239 4374.017641957251
242 | 0.024 4466.514594991542
243 | 0.0241 4455.2573317762
244 | 0.024200000000000003 4446.3013936512825
245 | 0.024300000000000002 4464.634889940811
246 | 0.0244 4468.60525511012
247 | 0.0245 4416.246307597157
248 | 0.0246 4399.576084532948
249 | 0.0247 4355.518808750678
250 | 0.024800000000000003 4385.023441060532
251 | 0.024900000000000002 4313.468310140865
252 | 0.025 4414.6972962651
253 | 0.0251 4355.154362615261
254 | 0.0252 4350.542760850686
255 | 0.0253 4437.916677209027
256 | 0.025400000000000002 4443.611055571595
257 | 0.025500000000000002 4438.5195449537505
258 | 0.0256 4448.629919222049
259 | 0.0257 4427.269707061564
260 | 0.0258 4431.13079720861
261 | 0.025900000000000003 4429.859188189083
262 | 0.026000000000000002 4422.3653392886035
263 | 0.0261 4416.539656078386
264 | 0.0262 4371.542489743679
265 | 0.0263 4363.65331482477
266 | 0.0264 4283.245244550804
267 | 0.026500000000000003 4259.448979742627
268 | 0.026600000000000002 4296.306939271484
269 | 0.0267 4380.4293917764035
270 | 0.0268 4166.556872073827
271 | 0.0269 4375.229812774562
272 | 0.027 4443.95410332818
273 | 0.027100000000000003 4496.533492902397
274 | 0.027200000000000002 4494.383281317909
275 | 0.0273 4519.0151454471925
276 | 0.0274 4407.402606160929
277 | 0.0275 4365.419994518845
278 | 0.027600000000000003 4364.900844169226
279 | 0.027700000000000002 4360.977372246751
280 | 0.027800000000000002 4379.134837443217
281 | 0.0279 4430.3242830650815
282 | 0.028 4432.142012413305
283 | 0.0281 4380.3519438691055
284 | 0.028200000000000003 4424.319372122843
285 | 0.028300000000000002 4440.36708997703
286 | 0.0284 4434.366079719375
287 | 0.0285 4452.035891964858
288 | 0.0286 4390.535047239087
289 | 0.0287 4372.006146074955
290 | 0.028800000000000003 4380.56339660231
291 | 0.028900000000000002 4433.692627085836
292 | 0.029 4462.060537442368
293 | 0.0291 4439.980170574645
294 | 0.0292 4359.376783990062
295 | 0.029300000000000003 4393.770387631898
296 | 0.029400000000000003 4375.961877624661
297 | 0.029500000000000002 4419.050191630778
298 | 0.0296 4430.987008788952
299 | 0.0297 4382.738098557449
300 | 0.0298 4453.93172375097
301 | 0.029900000000000003 4427.136351390884
302 | 0.030000000000000002 4440.676893612471
303 | 0.030100000000000002 4449.962919819516
304 | 0.0302 4434.517759944107
305 | 0.0303 4469.832500970235
306 | 0.0304 4463.507177126829
307 | 0.030500000000000003 4427.125583732383
308 | 0.030600000000000002 4320.984961514471
309 | 0.0307 4415.972895463168
310 | 0.0308 4446.795022597304
311 | 0.0309 4434.547277297687
312 | 0.031 4377.202629091898
313 | 0.031100000000000003 4430.967917930318
314 | 0.031200000000000002 4434.19479828753
315 | 0.0313 4413.378804797773
316 | 0.031400000000000004 4396.1421459219855
317 | 0.0315 4359.089063235155
318 | 0.0316 4363.6063801959135
319 | 0.0317 4366.47729382662
320 | 0.0318 4475.941656967744
321 | 0.031900000000000005 4468.987701758295
322 | 0.032 4439.249479915543
323 | 0.032100000000000004 4356.5480757687765
324 | 0.0322 4422.361359427192
325 | 0.0323 4412.375689007601
326 | 0.0324 4379.89510507919
327 | 0.0325 4433.1256262657025
328 | 0.032600000000000004 4411.831966914949
329 | 0.0327 4478.881455769247
330 | 0.0328 4414.7133981916695
331 | 0.0329 4454.839059771139
332 | 0.033 4413.4824603086845
333 | 0.033100000000000004 4491.801038949697
334 | 0.0332 4429.380712067966
335 | 0.0333 4449.862306662729
336 | 0.0334 4429.635994096248
337 | 0.0335 4352.375933272498
338 | 0.033600000000000005 4358.208561297983
339 | 0.0337 4306.627981949049
340 | 0.033800000000000004 4300.980259301358
341 | 0.0339 4381.452130916939
342 | 0.034 4369.913756532929
343 | 0.0341 4443.444021623991
344 | 0.0342 4273.7262536735925
345 | 0.034300000000000004 4145.581691027162
346 | 0.0344 4345.6989644187515
347 | 0.0345 4127.556474729422
348 | 0.0346 4376.228776721722
349 | 0.0347 4182.605008879098
350 | 0.034800000000000005 4432.24489369397
351 | 0.0349 4346.889833166535
352 | 0.035 4344.434573904728
353 | 0.0351 4343.958337326058
354 | 0.0352 4339.005565313726
355 | 0.035300000000000005 4350.140804455015
356 | 0.0354 4269.864393528522
357 | 0.035500000000000004 4334.6543960521385
358 | 0.0356 4345.669075691819
359 | 0.0357 4422.954561603528
360 | 0.0358 4404.9225526219425
361 | 0.0359 4405.632404518491
362 | 0.036000000000000004 4359.022893759282
363 | 0.0361 4343.50414801412
364 | 0.0362 4336.347319253851
365 | 0.0363 4422.601997361631
366 | 0.0364 4421.49147173143
367 | 0.036500000000000005 4204.083438145154
368 | 0.0366 4429.329551053382
369 | 0.0367 4403.420745015327
370 | 0.0368 4406.223117094903
371 | 0.0369 4399.391942756944
372 | 0.037000000000000005 4334.962925828808
373 | 0.0371 4428.729023538332
374 | 0.037200000000000004 4375.9330049931405
375 | 0.0373 4329.369444234582
376 | 0.0374 4188.756974292507
377 | 0.0375 4337.078133824004
378 | 0.0376 4251.457372639439
379 | 0.037700000000000004 4221.694933663649
380 | 0.0378 4186.3122569770185
381 | 0.0379 4140.531406583709
382 | 0.038 4331.622712245178
383 | 0.0381 4334.638851986008
384 | 0.038200000000000005 4112.547326319051
385 | 0.0383 4104.435943548242
386 | 0.038400000000000004 4263.064642088948
387 | 0.0385 4228.806735229537
388 | 0.0386 4269.02356745655
389 | 0.038700000000000005 4225.816657259688
390 | 0.0388 4339.01083629678
391 | 0.038900000000000004 4379.673938950693
392 | 0.039 4391.634359412413
393 | 0.0391 4391.199380560602
394 | 0.0392 4372.796113309642
395 | 0.0393 4271.9430418696975
396 | 0.039400000000000004 4268.492930462117
397 | 0.0395 4282.433077015161
398 | 0.0396 4240.789164926495
399 | 0.0397 4229.7417893343045
400 | 0.0398 4235.009995305034
401 | 0.039900000000000005 4306.510784634346
402 | 0.04 4378.818868333671
403 | 0.040100000000000004 4330.4549713031365
404 | 0.0402 4377.877828502294
405 | 0.0403 4352.412771309074
406 | 0.040400000000000005 4392.343163092161
407 | 0.0405 4407.635648507039
408 | 0.040600000000000004 4415.8235967458795
409 | 0.0407 4402.1543600599325
410 | 0.0408 4355.563669277138
411 | 0.0409 4315.765524618012
412 | 0.041 4157.521430649726
413 | 0.041100000000000005 4210.128379373147
414 | 0.0412 4346.654263123864
415 | 0.0413 4370.366145219769
416 | 0.0414 4388.907718271681
417 | 0.0415 4325.5500339517375
418 | 0.041600000000000005 4262.405578822564
419 | 0.0417 4322.295886647103
420 | 0.041800000000000004 4257.271116725194
421 | 0.0419 4355.333493666723
422 | 0.042 4308.707085423674
423 | 0.042100000000000005 4308.555403944079
424 | 0.0422 4173.567801506868
425 | 0.042300000000000004 4059.288103132137
426 | 0.0424 4123.947636320296
427 | 0.0425 4302.153884688651
428 | 0.0426 4178.500186184762
429 | 0.0427 4263.113130467948
430 | 0.042800000000000005 4163.471153068018
431 | 0.0429 4189.05570638449
432 | 0.043000000000000003 4232.602209977596
433 | 0.0431 4348.5018761871
434 | 0.0432 4171.518931410294
435 | 0.043300000000000005 4310.956503370057
436 | 0.0434 4262.54459586974
437 | 0.043500000000000004 4331.901916150952
438 | 0.0436 4246.918169427158
439 | 0.0437 4276.865959270239
440 | 0.0438 4272.913002515606
441 | 0.0439 4196.702529804375
442 | 0.044000000000000004 4291.889891727884
443 | 0.0441 4147.828467940942
444 | 0.0442 4207.341059955226
445 | 0.0443 4216.568539157274
446 | 0.0444 4135.162494273318
447 | 0.044500000000000005 4177.1145352570675
448 | 0.0446 4203.784447244005
449 | 0.044700000000000004 4115.187813081391
450 | 0.0448 4343.103409517485
451 | 0.0449 4222.1627251559985
452 | 0.045000000000000005 4247.451766346038
453 | 0.0451 4294.365145933141
454 | 0.045200000000000004 4310.168093222668
455 | 0.0453 4181.809107240569
456 | 0.0454 4161.3734868992005
457 | 0.0455 4145.733689554778
458 | 0.0456 4299.260997052471
459 | 0.045700000000000005 4285.964470753583
460 | 0.0458 4176.29236867967
461 | 0.0459 4244.677524554218
462 | 0.046 4388.729546170196
463 | 0.0461 4398.0408088717995
464 | 0.046200000000000005 4479.374999406967
465 | 0.0463 4429.656561378026
466 | 0.046400000000000004 4370.810878999215
467 | 0.0465 4353.642368318909
468 | 0.0466 4330.17883077504
469 | 0.046700000000000005 4430.75405928842
470 | 0.0468 4499.254674895691
471 | 0.046900000000000004 4462.30609969203
472 | 0.047 4418.200702047398
473 | 0.0471 4471.137675025145
474 | 0.0472 4418.55155321394
475 | 0.0473 4452.112539666745
476 | 0.047400000000000005 4394.602302314124
477 | 0.0475 4468.415862370347
478 | 0.0476 4427.421432860728
479 | 0.0477 4415.768086164587
480 | 0.0478 4429.692405560722
481 | 0.047900000000000005 4376.4878533591245
482 | 0.048 4442.552583691507
483 | 0.048100000000000004 4499.001552321517
484 | 0.0482 4398.44011188676
485 | 0.0483 4332.40200805313
486 | 0.048400000000000006 4348.325152469659
487 | 0.0485 4342.331724012974
488 | 0.048600000000000004 4394.916173686139
489 | 0.0487 4354.269679866479
490 | 0.0488 4325.343255139016
491 | 0.0489 4374.690508427324
492 | 0.049 4357.468703488204
493 | 0.049100000000000005 4335.621863905782
494 | 0.0492 4362.352847747393
495 | 0.049300000000000004 4348.462459620172
496 | 0.0494 4453.103984108865
497 | 0.0495 4359.791209419824
498 | 0.049600000000000005 4446.372883423052
499 | 0.0497 4445.388570033542
500 | 0.049800000000000004 4241.598295928038
501 | 0.0499 4307.115210078385
502 | 0.05 4364.00948905661
503 | 0.050100000000000006 4123.808813376697
504 | 0.0502 4298.881686631464
505 | 0.050300000000000004 4317.500138664449
506 | 0.0504 4144.631903123633
507 | 0.0505 4047.878338627089
508 | 0.0506 4120.522493360287
509 | 0.0507 4215.67559728031
510 | 0.050800000000000005 4238.549086569198
511 | 0.0509 4202.78772525598
512 | 0.051000000000000004 4289.132316217688
513 | 0.0511 4256.708015042447
514 | 0.0512 4315.757117418529
515 | 0.051300000000000005 4350.874590356901
516 | 0.0514 4352.178834021475
517 | 0.051500000000000004 4446.033014063412
518 | 0.0516 4042.888267070249
519 | 0.0517 4400.180972317542
520 | 0.051800000000000006 4431.61891195877
521 | 0.0519 4278.089476219895
522 | 0.052000000000000005 4424.48357803447
523 | 0.0521 4412.788946253158
524 | 0.0522 4418.411135837286
525 | 0.0523 4137.411987893498
526 | 0.0524 4240.097390181752
527 | 0.052500000000000005 4226.474667028535
528 | 0.0526 4285.628532975624
529 | 0.052700000000000004 4269.607271978908
530 | 0.0528 4304.255484626818
531 | 0.0529 4327.9710764743995
532 | 0.053000000000000005 4321.189660833056
533 | 0.0531 4277.497926135756
534 | 0.053200000000000004 4289.829822264831
535 | 0.0533 4249.503090042379
536 | 0.0534 4324.349284581222
537 | 0.053500000000000006 4330.638456063682
538 | 0.0536 4334.9634998064685
539 | 0.053700000000000005 4360.13564167239
540 | 0.0538 4251.118777547551
541 | 0.0539 4341.182966235815
542 | 0.054 4212.93229341088
543 | 0.0541 4333.497038792342
544 | 0.054200000000000005 4081.9366956227072
545 | 0.0543 4225.012897192138
546 | 0.054400000000000004 4332.713964393512
547 | 0.0545 4312.979565226338
548 | 0.0546 4229.3957956514705
549 | 0.054700000000000006 4280.27903935942
550 | 0.0548 4364.697039899631
551 | 0.054900000000000004 4283.721332894117
552 | 0.055 4357.104729906341
553 | 0.0551 4462.424108111948
554 | 0.055200000000000006 4424.625705274606
555 | 0.0553 4399.429707028785
556 | 0.055400000000000005 4463.428520483089
557 | 0.0555 4412.817315032471
558 | 0.055600000000000004 4411.28844792561
559 | 0.0557 4355.062700693683
560 | 0.0558 4379.391334865822
561 | 0.055900000000000005 4445.977863461089
562 | 0.056 4411.0096563875295
563 | 0.056100000000000004 4138.85998221622
564 | 0.0562 4390.870879520354
565 | 0.0563 4353.932107866255
566 | 0.056400000000000006 4386.8481904047185
567 | 0.0565 4292.326597477917
568 | 0.056600000000000004 4397.514785960892
569 | 0.0567 4396.081082847065
570 | 0.0568 4357.187389937863
571 | 0.056900000000000006 4334.354295722543
572 | 0.057 4246.150908591707
573 | 0.057100000000000005 4129.453349961976
574 | 0.0572 4343.233059379258
575 | 0.057300000000000004 4385.46089947163
576 | 0.0574 4365.507430575668
577 | 0.0575 4409.29390341105
578 | 0.057600000000000005 4357.620807638532
579 | 0.0577 4407.605019539845
580 | 0.057800000000000004 4392.005949881429
581 | 0.0579 4367.719448502599
582 | 0.058 4443.672130442621
583 | 0.058100000000000006 4359.59081926723
584 | 0.0582 4472.861415787176
585 | 0.058300000000000005 4331.6631425532705
586 | 0.0584 4219.596190202949
587 | 0.0585 4268.828753575115
588 | 0.058600000000000006 4141.3323128387565
589 | 0.0587 4254.270492222178
590 | 0.058800000000000005 4239.9709881367935
591 | 0.0589 4308.850461184035
592 | 0.059000000000000004 4216.62287202549
593 | 0.0591 4280.903365739939
594 | 0.0592 4357.487494101463
595 | 0.059300000000000005 4357.25738339534
596 | 0.0594 4379.705088927585
597 | 0.059500000000000004 4362.938539968424
598 | 0.0596 4347.152164552888
599 | 0.0597 4349.83084211833
600 | 0.059800000000000006 4306.785257561856
601 | 0.0599 4055.2612115550846
602 | 0.060000000000000005 4277.730937800845
603 | 0.0601 4214.575798780641
604 | 0.060200000000000004 4330.047300707444
605 | 0.0603 4366.793423191057
606 | 0.0604 4385.300063262835
607 | 0.060500000000000005 4348.611965801418
608 | 0.0606 4223.861340846572
609 | 0.060700000000000004 4254.072422488802
610 | 0.0608 4211.424662616365
611 | 0.0609 4251.380458019873
612 | 0.061000000000000006 4197.964499617418
613 | 0.0611 4304.8637395083115
614 | 0.061200000000000004 4240.648451872296
615 | 0.0613 4224.636237065962
616 | 0.0614 4252.377760542809
617 | 0.061500000000000006 4241.055383674843
618 | 0.0616 4383.625771316363
619 | 0.061700000000000005 4372.66545296102
620 | 0.0618 4266.975398604267
621 | 0.061900000000000004 4231.134120014123
622 | 0.062 4232.629845214445
623 | 0.0621 4255.0768786817935
624 | 0.062200000000000005 4180.6902336089315
625 | 0.0623 4333.2589459006
626 | 0.062400000000000004 4262.735785679922
627 | 0.0625 4346.710962715649
628 | 0.0626 4298.222559180647
629 | 0.0627 4273.751417438495
630 | 0.06280000000000001 4253.382880985444
631 | 0.0629 4301.30713499277
632 | 0.063 4308.900761176018
633 | 0.0631 4363.524175018731
634 | 0.0632 4283.69218629247
635 | 0.06330000000000001 4253.149560863505
636 | 0.0634 4360.838458459897
637 | 0.0635 4290.09634677516
638 | 0.0636 4354.63577021503
639 | 0.0637 4297.020372409118
640 | 0.06380000000000001 4237.115599541452
641 | 0.0639 4288.2524380535015
642 | 0.064 4140.8644726961
643 | 0.0641 4348.4415989353
644 | 0.06420000000000001 4335.12263476469
645 | 0.06430000000000001 4370.65313721719
646 | 0.0644 4413.969667067563
647 | 0.0645 4256.280518843431
648 | 0.0646 4336.745579851359
649 | 0.06470000000000001 4072.8293955996605
650 | 0.0648 4283.4896542438455
651 | 0.0649 4187.560070184912
652 | 0.065 4413.173351863145
653 | 0.0651 4358.445934063135
654 | 0.06520000000000001 4385.831335874721
655 | 0.0653 4396.8801432803775
656 | 0.0654 4386.003456518598
657 | 0.0655 4358.341155840287
658 | 0.0656 4380.125730461552
659 | 0.06570000000000001 4371.612146620372
660 | 0.0658 4377.113851318041
661 | 0.0659 4448.544419348971
662 | 0.066 4365.661732296084
663 | 0.0661 4444.882331147769
664 | 0.06620000000000001 4342.362829587742
665 | 0.0663 4498.489595475742
666 | 0.0664 4442.108746609096
667 | 0.0665 4426.3228389170245
668 | 0.0666 4453.72129237053
669 | 0.06670000000000001 4483.914106221596
670 | 0.0668 4319.718275861127
671 | 0.0669 4418.707412834887
672 | 0.067 4447.741278718322
673 | 0.0671 4448.06441123746
674 | 0.06720000000000001 4452.604793605917
675 | 0.0673 4436.480789411513
676 | 0.0674 4416.486283329358
677 | 0.0675 4439.818715396502
678 | 0.06760000000000001 4424.6002908730225
679 | 0.06770000000000001 4348.610813648979
680 | 0.0678 4383.733090642338
681 | 0.0679 4476.763700565697
682 | 0.068 4504.918717464212
683 | 0.06810000000000001 4494.567616475839
684 | 0.0682 4279.6391107750605
685 | 0.0683 4361.4872904650165
686 | 0.0684 4448.563936577072
687 | 0.0685 4349.919395772509
688 | 0.06860000000000001 4381.33621322401
689 | 0.0687 4404.658119140754
690 | 0.0688 4458.600916839798
691 | 0.0689 4472.029308509907
692 | 0.069 4337.739275012263
693 | 0.06910000000000001 4316.2900802178265
694 | 0.0692 4319.247267944124
695 | 0.0693 4299.176280700283
696 | 0.0694 4313.242187670604
697 | 0.0695 3914.7023721359874
698 | 0.06960000000000001 4041.2780106036535
699 | 0.0697 3983.4216551588074
700 | 0.0698 4377.577565764369
701 | 0.0699 3983.46185372566
702 | 0.07 4301.8272920046065
703 | 0.07010000000000001 4247.995884133651
704 | 0.0702 4275.458144609508
705 | 0.0703 4168.10495833878
706 | 0.0704 4372.7960137317405
707 | 0.07050000000000001 4281.88107490447
708 | 0.07060000000000001 4299.015969105602
709 | 0.0707 4281.761519141105
710 | 0.0708 4303.610630207074
711 | 0.0709 4090.7654548693563
712 | 0.07100000000000001 3938.244928367584
713 | 0.0711 4338.751822574905
714 | 0.0712 4320.8978062661245
715 | 0.0713 4091.0789208391298
716 | 0.0714 4326.6688342080615
717 | 0.07150000000000001 4307.398470479484
718 | 0.0716 4354.038262095482
719 | 0.0717 4329.901649431056
720 | 0.0718 4315.9466775201145
721 | 0.0719 4279.651045786429
722 | 0.07200000000000001 4330.906643994076
723 | 0.0721 4299.2217088324305
724 | 0.0722 4129.063770665588
725 | 0.0723 4292.872014467582
726 | 0.0724 4304.7617676699465
727 | 0.07250000000000001 4298.759899123688
728 | 0.0726 4334.738045131656
729 | 0.0727 4324.111771401802
730 | 0.0728 4311.105138857592
731 | 0.0729 4082.9828166347042
732 | 0.07300000000000001 3713.221951685061
733 | 0.0731 4093.358038461848
734 | 0.0732 4104.17906602313
735 | 0.0733 4183.81846847912
736 | 0.0734 4149.079049279116
737 | 0.07350000000000001 4186.837022930128
738 | 0.0736 4026.3009928989704
739 | 0.0737 3887.6651180823455
740 | 0.0738 4078.924793995537
741 | 0.07390000000000001 4300.118272479565
742 | 0.07400000000000001 4066.9059800515456
743 | 0.0741 4162.530004690432
744 | 0.0742 3955.5152121452356
745 | 0.0743 4071.8133609470888
746 | 0.07440000000000001 4063.531969833715
747 | 0.0745 3698.6334921947823
748 | 0.0746 3329.4927210650603
749 | 0.0747 3242.8036399892167
750 | 0.0748 4219.629946585674
751 | 0.07490000000000001 4345.138420006686
752 | 0.075 4423.961331650912
753 | 0.0751 4389.272547663948
754 | 0.0752 4381.432434780911
755 | 0.0753 4336.354856755708
756 | 0.07540000000000001 4332.3310965453275
757 | 0.0755 4370.10142291845
758 | 0.0756 4347.54766551829
759 | 0.0757 4373.108018606616
760 | 0.0758 4334.697828218257
761 | 0.07590000000000001 4391.537243662286
762 | 0.076 4362.6375831497535
763 | 0.0761 4166.253492450144
764 | 0.0762 3963.3331546030013
765 | 0.0763 4306.400844780236
766 | 0.07640000000000001 4254.14503181281
767 | 0.0765 4249.496932942652
768 | 0.0766 4369.927212832938
769 | 0.0767 4360.812720784527
770 | 0.07680000000000001 4381.767839573434
771 | 0.07690000000000001 4389.32335844186
772 | 0.077 4363.388240283617
773 | 0.0771 4196.225841940175
774 | 0.0772 4143.949510772305
775 | 0.07730000000000001 4408.967494835959
776 | 0.07740000000000001 4330.798864006113
777 | 0.0775 4391.610318188781
778 | 0.0776 4455.874026923631
779 | 0.0777 4489.752490167659
780 | 0.07780000000000001 4455.475317839064
781 | 0.0779 4413.218267151873
782 | 0.078 4381.860087824818
783 | 0.0781 4355.828467445527
784 | 0.0782 4424.189529118501
785 | 0.07830000000000001 4480.356140318202
786 | 0.0784 4354.7365131975885
787 | 0.0785 4199.839845325443
788 | 0.0786 4328.082781478067
789 | 0.0787 4475.9726899624875
790 | 0.07880000000000001 4474.456600767045
791 | 0.0789 4503.23120176102
792 | 0.079 4364.981029679964
793 | 0.0791 4447.176789478931
794 | 0.0792 4425.667043477425
795 | 0.07930000000000001 4504.826455218814
796 | 0.0794 4518.339553736891
797 | 0.0795 4436.4924141347565
798 | 0.0796 4485.001759758016
799 | 0.07970000000000001 4457.989430328549
800 | 0.07980000000000001 4315.157309794916
801 | 0.0799 4326.910188854356
802 |
--------------------------------------------------------------------------------
/results/ChooseModel_search_pid_parameter_CurveType_trapezoidal_Height_1000_DumpSystem_False_RunType_train/actor_model.ckpt.data-00000-of-00001:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/tinyzqh/control-of-jump-systems-based-on-reinforcement-learning/de23c13aecfb6d8745d6005e32d2083498c4246a/results/ChooseModel_search_pid_parameter_CurveType_trapezoidal_Height_1000_DumpSystem_False_RunType_train/actor_model.ckpt.data-00000-of-00001
--------------------------------------------------------------------------------
/results/ChooseModel_search_pid_parameter_CurveType_trapezoidal_Height_1000_DumpSystem_False_RunType_train/actor_model.ckpt.index:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/tinyzqh/control-of-jump-systems-based-on-reinforcement-learning/de23c13aecfb6d8745d6005e32d2083498c4246a/results/ChooseModel_search_pid_parameter_CurveType_trapezoidal_Height_1000_DumpSystem_False_RunType_train/actor_model.ckpt.index
--------------------------------------------------------------------------------
/results/ChooseModel_search_pid_parameter_CurveType_trapezoidal_Height_1000_DumpSystem_False_RunType_train/actor_model.ckpt.meta:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/tinyzqh/control-of-jump-systems-based-on-reinforcement-learning/de23c13aecfb6d8745d6005e32d2083498c4246a/results/ChooseModel_search_pid_parameter_CurveType_trapezoidal_Height_1000_DumpSystem_False_RunType_train/actor_model.ckpt.meta
--------------------------------------------------------------------------------
/results/ChooseModel_search_pid_parameter_CurveType_trapezoidal_Height_1000_DumpSystem_False_RunType_train/checkpoint:
--------------------------------------------------------------------------------
1 | model_checkpoint_path: "/home/hzq/PycharmProjects/control-of-jump-systems-based-on-reinforcement-learning/results/ChooseModel_search_pid_parameter_CurveType_trapezoidal_Height_1000_DumpSystem_False_RunType_train/critic_model.ckpt"
2 | all_model_checkpoint_paths: "/home/hzq/PycharmProjects/control-of-jump-systems-based-on-reinforcement-learning/results/ChooseModel_search_pid_parameter_CurveType_trapezoidal_Height_1000_DumpSystem_False_RunType_train/critic_model.ckpt"
3 |
--------------------------------------------------------------------------------
/results/ChooseModel_search_pid_parameter_CurveType_trapezoidal_Height_1000_DumpSystem_False_RunType_train/critic_model.ckpt.data-00000-of-00001:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/tinyzqh/control-of-jump-systems-based-on-reinforcement-learning/de23c13aecfb6d8745d6005e32d2083498c4246a/results/ChooseModel_search_pid_parameter_CurveType_trapezoidal_Height_1000_DumpSystem_False_RunType_train/critic_model.ckpt.data-00000-of-00001
--------------------------------------------------------------------------------
/results/ChooseModel_search_pid_parameter_CurveType_trapezoidal_Height_1000_DumpSystem_False_RunType_train/critic_model.ckpt.index:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/tinyzqh/control-of-jump-systems-based-on-reinforcement-learning/de23c13aecfb6d8745d6005e32d2083498c4246a/results/ChooseModel_search_pid_parameter_CurveType_trapezoidal_Height_1000_DumpSystem_False_RunType_train/critic_model.ckpt.index
--------------------------------------------------------------------------------
/results/ChooseModel_search_pid_parameter_CurveType_trapezoidal_Height_1000_DumpSystem_False_RunType_train/critic_model.ckpt.meta:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/tinyzqh/control-of-jump-systems-based-on-reinforcement-learning/de23c13aecfb6d8745d6005e32d2083498c4246a/results/ChooseModel_search_pid_parameter_CurveType_trapezoidal_Height_1000_DumpSystem_False_RunType_train/critic_model.ckpt.meta
--------------------------------------------------------------------------------
/results/ChooseModel_search_pid_parameter_CurveType_trapezoidal_Height_1000_DumpSystem_False_RunType_train/reward.PNG:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/tinyzqh/control-of-jump-systems-based-on-reinforcement-learning/de23c13aecfb6d8745d6005e32d2083498c4246a/results/ChooseModel_search_pid_parameter_CurveType_trapezoidal_Height_1000_DumpSystem_False_RunType_train/reward.PNG
--------------------------------------------------------------------------------
/run.py:
--------------------------------------------------------------------------------
1 | # -*- coding: utf-8 -*-
2 | # @Time : 12/8/21 3:51 PM
3 | # @Author : Zhiqiang He
4 | # @Email : tinyzqh@163.com
5 | # @File : run.py
6 | # @Software: PyCharm
7 |
8 |
9 | import os
10 | import tensorflow as tf
11 | import numpy as np
12 | from parameter import args
13 | from servo_system_env import ServoSystemEnv
14 | from utils import trapezoidal_function
15 |
16 | with tf.name_scope("S"):
17 | S = tf.placeholder(tf.float32, shape=[None, args.state_dim], name="s")
18 |
19 | with tf.name_scope("R"):
20 | R = tf.placeholder(tf.float32, shape=[None, 1], name='r')
21 |
22 | with tf.name_scope("S_"):
23 | S_ = tf.placeholder(tf.float32, shape=[None, args.state_dim], name="s_")
24 |
25 |
26 | class Actor(object):
27 | def __init__(self, sess, action_dim, action_bound):
28 | self.sess = sess
29 | self.a_dim = action_dim
30 | self.a_bound = action_bound
31 | self.lr = args.learning_actor
32 | self.replace_iter = args.replace_iter_a
33 | self.replace_cnt = 0
34 |
35 | with tf.variable_scope("Actor"):
36 | # input s, output a
37 | self.a = self._build_net(S, scope="eval_net", trainable=True)
38 |
39 | # input s_, output a, get a_ for critic
40 | self.a_ = self._build_net(S, scope="target_net", trainable=True)
41 |
42 | self.e_params = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope="Actor/eval_net")
43 | self.t_params = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope="Actor/target_net")
44 |
45 | self.saver = tf.train.Saver()
46 |
47 | def _build_net(self, input, scope, trainable):
48 |
49 | with tf.variable_scope(scope):
50 | init_w = tf.contrib.layers.xavier_initializer()
51 | init_b = tf.constant_initializer(0.001)
52 |
53 | net1 = tf.layers.dense(inputs=input, units=200, activation=tf.nn.relu6, kernel_initializer=init_w,
54 | kernel_regularizer=tf.contrib.layers.l2_regularizer(scale=args.reg),
55 | bias_initializer=init_b, name="l1", trainable=trainable)
56 |
57 | net2 = tf.layers.dense(inputs=net1, units=200, activation=tf.nn.relu6, kernel_initializer=init_w,
58 | kernel_regularizer=tf.contrib.layers.l2_regularizer(scale=args.reg),
59 | bias_initializer=init_b, name="l2", trainable=trainable)
60 |
61 | net3 = tf.layers.dense(inputs=net2, units=10, activation=tf.nn.relu6, kernel_initializer=init_w,
62 | kernel_regularizer=tf.contrib.layers.l2_regularizer(scale=args.reg),
63 | bias_initializer=init_b, name="l3", trainable=trainable)
64 |
65 | with tf.variable_scope("a"):
66 | actions = tf.layers.dense(inputs=net3, units=self.a_dim, activation=tf.nn.tanh,
67 | kernel_initializer=init_w, kernel_regularizer=tf.contrib.layers.l2_regularizer(scale=args.reg),
68 | name="a", trainable=trainable)
69 |
70 | # Scale output to -action_bound to action_bound.
71 | scaled_a = tf.multiply(actions, self.a_bound, name="scaled_action")
72 |
73 | return scaled_a
74 |
75 | def learn(self, s):
76 | """
77 | Batch Update
78 | :param s:
79 | :return:
80 | """
81 | self.sess.run(self.train_op, feed_dict={S: s})
82 |
83 | if self.replace_cnt % self.replace_iter == 0:
84 | self.sess.run([tf.assign(t, e) for t, e in zip(self.t_params, self.e_params)])
85 |
86 | self.replace_cnt += 1
87 |
88 | def choose_action(self, s):
89 | s = s[np.newaxis, :]
90 | return self.sess.run(self.a, feed_dict={S: s})[0] # single action
91 |
92 | def add_grad_to_graph(self, a_grads):
93 | with tf.variable_scope("policy_grads"):
94 | # ys = policy;
95 | # xs = policy's parameters;
96 | # a_grads = the gradients of the policy to get more Q;
97 | # tf.gradients will calculate dys/dxs with a initial gradients for ys, so this is dq/da * da / dparams
98 | self.policy_grads = tf.gradients(ys=self.a, xs=self.e_params, grad_ys=a_grads)
99 |
100 | with tf.variable_scope("A_train"):
101 | opt = tf.train.RMSPropOptimizer(-self.lr) # (- learning rate) for ascent policy
102 | self.train_op = opt.apply_gradients(zip(self.policy_grads, self.e_params))
103 |
104 |
105 | class Critic(object):
106 | def __init__(self, sess, a, a_, action_dim):
107 | self.sess = sess
108 | self.s_dim = args.state_dim
109 | self.a_dim = action_dim
110 | self.lr = args.learning_critic
111 | self.gamma = args.gamma
112 |
113 | self.replace_iter = args.replace_iter_c
114 | self.replace_cnt = 0
115 |
116 | with tf.variable_scope("Critic"):
117 | # Input (s, a), output q
118 | self.a = a
119 | self.q = self._build_net(S, self.a, "eval_net", trainable=True)
120 |
121 | # Input (s_, a_), output q_ for target
122 | self.q_ = self._build_net(S_, a_, "target_net", trainable=True)
123 |
124 | self.e_params = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope="Critic/eval_net")
125 | self.t_params = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope="Critic/target_net")
126 |
127 | with tf.variable_scope("target_q"):
128 | self.target_q = R + self.gamma * self.q_
129 |
130 | with tf.variable_scope("TD_error"):
131 | self.loss = tf.reduce_mean(tf.squared_difference(self.target_q, self.q))
132 |
133 | with tf.variable_scope("C_train"):
134 | self.train_op = tf.train.RMSPropOptimizer(self.lr).minimize(self.loss)
135 |
136 | with tf.variable_scope("a_grad"):
137 | self.a_grads = tf.gradients(self.q, self.a)[0]
138 |
139 | self.saver = tf.train.Saver()
140 |
141 | def _build_net(self, s, a, scope, trainable):
142 | with tf.variable_scope(scope):
143 | init_w = tf.contrib.layers.xavier_initializer()
144 | init_b = tf.constant_initializer(0.01)
145 |
146 | with tf.variable_scope('l1'):
147 | n_l1 = 200
148 | w1_s = tf.get_variable("w1_s", [self.s_dim, n_l1], initializer=init_w, trainable=trainable)
149 | w1_a = tf.get_variable("w1_a", [self.a_dim, n_l1], initializer=init_w, trainable=trainable)
150 | b1 = tf.get_variable("b1", [1, n_l1], initializer=init_b, trainable=trainable)
151 | net = tf.nn.relu6(tf.matmul(s, w1_s) + tf.matmul(a, w1_a) + b1)
152 |
153 | net1 = tf.layers.dense(net, 200, activation=tf.nn.relu6, kernel_initializer=init_w,
154 | kernel_regularizer=tf.contrib.layers.l2_regularizer(scale=args.reg),
155 | bias_initializer=init_b, name="l2", trainable=trainable)
156 |
157 | net2 = tf.layers.dense(net1, 10, activation=tf.nn.relu6, kernel_initializer=init_w,
158 | kernel_regularizer=tf.contrib.layers.l2_regularizer(scale=args.reg),
159 | bias_initializer=init_b, name="l3", trainable=trainable)
160 |
161 | with tf.variable_scope("q"):
162 | q = tf.layers.dense(net2, 1, kernel_initializer=init_w,
163 | kernel_regularizer=tf.contrib.layers.l2_regularizer(scale=args.reg),
164 | bias_initializer=init_b, trainable=trainable)
165 |
166 | return q
167 |
168 | def learn(self, s, a, r, s_):
169 | self.sess.run(self.train_op, feed_dict={S:s, self.a:a, R:r, S_:s_})
170 |
171 | if self.replace_cnt % self.replace_iter == 0:
172 | self.sess.run([tf.assign(t, e) for t, e in zip(self.t_params, self.e_params)])
173 |
174 | self.replace_cnt += 1
175 |
176 |
177 | class Memory(object):
178 | def __init__(self, capacity, dims):
179 | self.capacity = capacity
180 | self.data = np.zeros(shape=(capacity, dims))
181 | self.cnt = 0
182 |
183 | def store_transition(self, s, a, r, s_):
184 | transition = np.hstack((s, a, [r], s_))
185 | index = self.cnt % self.capacity
186 | self.data[index, :] = transition
187 | self.cnt += 1
188 |
189 | def sample(self, n):
190 | assert self.cnt >= self.capacity, print("Memory has not been fulfilled.")
191 | indices = np.random.choice(self.capacity, size=n)
192 | return self.data[indices, :]
193 |
194 |
195 | class run_base(object):
196 | def __init__(self):
197 | current_dir = os.path.dirname(os.path.abspath(__file__))
198 |
199 | if args.run_type == "train":
200 | result_dir = os.path.join('results', 'ChooseModel_{}_CurveType_{}_Height_{}_DumpSystem_{}_RunType_{}'.format(args.choose_model, args.curve_type, args.height, args.use_jump_system, args.run_type))
201 | self.result_dir = os.path.join(current_dir, result_dir)
202 | else:
203 | # Need to load the train model.
204 | result_dir = os.path.join('results', 'ChooseModel_{}_CurveType_{}_Height_{}_DumpSystem_{}_RunType_{}'.format(args.choose_model, args.curve_type, args.height, args.use_jump_system, "train"))
205 | self.result_dir = os.path.join(current_dir, result_dir)
206 |
207 | os.makedirs(self.result_dir, exist_ok=True)
208 |
209 | if args.choose_model == "search_pid_parameter":
210 | self.max_episodes = 800
211 | self.action_dim = args.pid_parameter_search_action_dim
212 | self.action_bound = args.pid_parameter_search_action_bound
213 | self.var = 10
214 |
215 | tf.set_random_seed(args.seed)
216 | self.sess = tf.Session()
217 |
218 | self.actor = Actor(sess=self.sess, action_dim=self.action_dim, action_bound=self.action_bound)
219 | self.critic = Critic(sess=self.sess, a=self.actor.a, a_=self.actor.a_, action_dim=self.action_dim)
220 |
221 | self.actor.add_grad_to_graph(self.critic.a_grads)
222 |
223 | self.M = Memory(capacity=args.memory_capacity, dims=2 * args.state_dim + self.action_dim + 1)
224 |
225 | self.sess.run(tf.global_variables_initializer())
226 |
227 | elif args.choose_model == "search_electric":
228 |
229 | if args.curve_type == "trapezoidal":
230 | self.max_episodes = 800
231 | elif args.curve_type == "sine":
232 | self.max_episodes = 4000
233 | else:
234 | pass
235 |
236 | self.action_dim = args.electric_compensation_action_dim
237 | self.action_bound = args.electric_compensation_action_bound
238 | self.var = 2
239 |
240 | tf.set_random_seed(args.seed)
241 | self.sess = tf.Session()
242 |
243 | self.actor = Actor(sess=self.sess, action_dim=self.action_dim, action_bound=self.action_bound)
244 | self.critic = Critic(sess=self.sess, a=self.actor.a, a_=self.actor.a_, action_dim=self.action_dim)
245 |
246 | self.actor.add_grad_to_graph(self.critic.a_grads)
247 |
248 | self.M = Memory(capacity=args.memory_capacity, dims=2 * args.state_dim + self.action_dim + 1)
249 |
250 | self.sess.run(tf.global_variables_initializer())
251 |
252 | else:
253 | pass
254 |
255 | self.var_min = 0.01
256 | self.env = ServoSystemEnv()
257 |
258 | def train(self, times):
259 | pass
260 |
261 | def test(self, times):
262 | pass
263 |
264 | def load_model(self):
265 |
266 | self.actor.saver.restore(self.actor.sess, os.path.join(self.result_dir, "actor_model.ckpt"))
267 | self.critic.saver.restore(self.critic.sess, os.path.join(self.result_dir, "critic_model.ckpt"))
268 |
269 | print("load_model_successful.")
270 |
271 | def save_model(self):
272 |
273 | if not os.path.exists(self.result_dir):
274 | os.makedirs(self.result_dir)
275 |
276 | self.actor.saver.save(self.actor.sess, os.path.join(self.result_dir, "actor_model.ckpt"))
277 | self.critic.saver.save(self.critic.sess, os.path.join(self.result_dir, "critic_model.ckpt"))
278 |
279 | print("save_model_successful.")
280 |
281 | def save_result_txt(self, xs, ys, yaxis='radValues'):
282 | """
283 | # Save the data to .txt file.
284 | :param xs:
285 | :param ys:
286 | :param yaxis:
287 | :return:
288 | """
289 |
290 | filename = os.path.join(self.result_dir, yaxis + '.txt')
291 | if isinstance(xs, np.ndarray) and isinstance(ys, list):
292 | if not os.path.exists(filename):
293 | with open(file=filename, mode="a+") as file:
294 | file.write("times {}".format(yaxis))
295 | file.write("\r\n")
296 | else:
297 | print("{} has already existed. added will be doing.".format(filename))
298 | with open(file=filename, mode="a+") as file:
299 | file.write("times, {}".format(yaxis))
300 | file.write("\r\n")
301 | else:
302 | pass
303 |
304 | with open(file=filename, mode="a+") as file:
305 | for index, data in enumerate(zip(xs, ys)):
306 | file.write("{} {}".format(str(data[0]), str(data[1])))
307 | file.write("\r\n")
308 |
309 |
310 | class run_class_pid(run_base):
311 | def __init__(self):
312 | super().__init__()
313 |
314 | def test(self, times):
315 | radValues = []
316 | iaeValues = []
317 | ecValues = []
318 |
319 | Episode_reward = 0
320 |
321 | for sim_time in times:
322 |
323 | error = trapezoidal_function(time=sim_time) - self.env.rad_cur
324 |
325 | ec_cur = self.env.ec_last
326 |
327 | state, reward, done = self.env.step(action=0)
328 |
329 | # append the value
330 | radValues.append(self.env.rad_cur)
331 | iaeValues.append(abs(error) if len(iaeValues) == 0 else abs(error) + iaeValues[-1])
332 | ecValues.append(ec_cur)
333 |
334 | Episode_reward += reward
335 |
336 | self.save_result_txt(xs=times, ys=radValues, yaxis="radValues")
337 | self.save_result_txt(xs=times, ys=iaeValues, yaxis="iaeValues")
338 | self.save_result_txt(xs=times, ys=ecValues, yaxis="ecValues")
339 |
340 |
341 | class run_search_pid_parameter(run_base):
342 | def __init__(self):
343 | super().__init__()
344 | self.action_dim = args.pid_parameter_search_action_dim
345 | self.action_bound = args.pid_parameter_search_action_bound
346 |
347 | def train(self, times):
348 | steps = []
349 | epRewards = []
350 | maxEpReward = 4000.0 # set the base max reward.
351 |
352 | for i in range(self.max_episodes):
353 | s = self.env.reset()
354 | ep_reward = 0
355 |
356 | for j, sim_time in enumerate(times):
357 |
358 | # Add exploration noise
359 | a = self.actor.choose_action(s)
360 | a = np.clip(np.random.normal(a, self.var), -self.action_bound, self.action_bound)
361 |
362 | s_, r, done = self.env.step(a)
363 |
364 | self.M.store_transition(s, a, r, s_)
365 |
366 | if self.M.cnt > args.memory_capacity:
367 | self.var = max([self.var * 0.999995, self.var_min]) # decay the action randomness
368 | b_M = self.M.sample(args.batch_size)
369 |
370 | b_s = b_M[:, :args.state_dim]
371 | b_a = b_M[:, args.state_dim: args.state_dim + self.action_dim]
372 | b_r = b_M[:, - args.state_dim - 1: - args.state_dim]
373 | b_s_ = b_M[:, - args.state_dim:]
374 |
375 | self.critic.learn(b_s, b_a, b_r, b_s_)
376 | self.actor.learn(b_s)
377 |
378 | s = s_
379 | ep_reward += r
380 |
381 | if sim_time == times[-1]:
382 | print("Episode: {}, Reward {}, Explore {} Steps {}".format(i, ep_reward, self.var, j))
383 | epRewards.append(ep_reward)
384 | steps.append(i)
385 |
386 | if ep_reward >= maxEpReward: # Save The Max Reward Model.
387 | print("Get More Episode Reward {}".format(maxEpReward))
388 | maxEpReward = ep_reward
389 | self.save_model()
390 |
391 | self.save_result_txt(xs=steps, ys=epRewards, yaxis="epRewards")
392 |
393 |
394 | class run_search_compensation_electric(run_base):
395 | def __init__(self):
396 | super(run_search_compensation_electric, self).__init__()
397 |
398 | def train(self, times):
399 | steps = []
400 | epRewards = []
401 | maxEpReward = 4000.0 # set the base max reward.
402 |
403 | for i in range(self.max_episodes):
404 | s = self.env.reset()
405 | ep_reward = 0
406 |
407 | for j, sim_time in enumerate(times):
408 |
409 | # Add exploration noise
410 | a = self.actor.choose_action(s)
411 | a = np.clip(np.random.normal(a, self.var), -self.action_bound, self.action_bound)
412 |
413 | s_, r, done = self.env.step(a)
414 |
415 | self.M.store_transition(s, a, r, s_)
416 |
417 | if self.M.cnt > args.memory_capacity:
418 | self.var = max([self.var * 0.999995, self.var_min]) # decay the action randomness
419 | b_M = self.M.sample(args.batch_size)
420 |
421 | b_s = b_M[:, :args.state_dim]
422 | b_a = b_M[:, args.state_dim: args.state_dim + self.action_dim]
423 | b_r = b_M[:, - args.state_dim - 1: - args.state_dim]
424 | b_s_ = b_M[:, - args.state_dim:]
425 |
426 | self.critic.learn(b_s, b_a, b_r, b_s_)
427 | self.actor.learn(b_s)
428 |
429 | s = s_
430 | ep_reward += r
431 |
432 | if sim_time == times[-1]:
433 | print("Episode: {}, Reward {}, Explore {} Steps {}".format(i, ep_reward, self.var, j))
434 | epRewards.append(ep_reward)
435 | steps.append(i)
436 |
437 | if ep_reward >= maxEpReward: # Save The Max Reward Model.
438 | print("Get More Episode Reward {}".format(maxEpReward))
439 | maxEpReward = ep_reward
440 | self.save_model()
441 |
442 | self.save_result_txt(xs=steps, ys=epRewards, yaxis="epRewards")
443 |
444 | def test(self, times):
445 |
446 | self.load_model()
447 |
448 | s = self.env.reset()
449 | ep_reward = 0
450 |
451 | radValues = []
452 | iaeValues = []
453 | ecValues = []
454 |
455 | for j, sim_time in enumerate(times):
456 |
457 | error = trapezoidal_function(time=sim_time) - self.env.rad_cur
458 |
459 | # Add exploration noise
460 | a = self.actor.choose_action(s)
461 | a = np.clip(np.random.normal(a, self.var), -self.action_bound, self.action_bound)
462 |
463 | ec_cur = self.env.ec_last
464 |
465 | s_, r, done = self.env.step(a)
466 |
467 | # append the value
468 | radValues.append(self.env.rad_cur)
469 | iaeValues.append(abs(error) if len(iaeValues) == 0 else abs(error) + iaeValues[-1])
470 | ecValues.append(ec_cur)
471 |
472 | s = s_
473 | ep_reward += r
474 |
475 | self.save_result_txt(xs=times, ys=radValues, yaxis="radValues")
476 | self.save_result_txt(xs=times, ys=iaeValues, yaxis="iaeValues")
477 | self.save_result_txt(xs=times, ys=ecValues, yaxis="ecValues")
478 |
479 | def main():
480 | times = np.arange(args.start_time, args.end_time, args.dt)[:args.max_ep_steps]
481 |
482 | if args.run_type == "train":
483 | if args.choose_model == "search_pid_parameter":
484 | run_search_pid_parameter().train(times=times)
485 | elif args.choose_model == "search_electric":
486 | run_search_compensation_electric().train(times=times)
487 | elif args.choose_model == "class_pid":
488 | run_class_pid().test(times=times)
489 | else:
490 | print("Please Input The Right Choose Model.")
491 | else:
492 | if args.choose_model == "search_pid_parameter":
493 | run_search_pid_parameter().test(times=times)
494 | elif args.choose_model == "search_electric":
495 | run_search_compensation_electric().test(times=times)
496 | elif args.choose_model == "class_pid":
497 | run_class_pid().test(times=times)
498 | else:
499 | print("Please Input The Right Choose Model.")
500 |
501 |
502 | if __name__ == "__main__":
503 | main()
--------------------------------------------------------------------------------
/servo_system_env.py:
--------------------------------------------------------------------------------
1 | # -*- coding: utf-8 -*-
2 | # @Time : 11/30/21 2:34 PM
3 | # @Author : Zhiqiang He
4 | # @Email : tinyzqh@163.com
5 | # @File : servo_system_env.py
6 | # @Software: PyCharm
7 |
8 |
9 | from parameter import args
10 | from utils import trapezoidal_function
11 | from PID import PID
12 | import numpy as np
13 | import gym
14 |
15 |
16 | class ServoSystemEnv(gym.Env):
17 | def __init__(self):
18 | super(ServoSystemEnv, self).__init__()
19 |
20 | self.rad_last = 0.0 # Speed at the previous moment
21 | self.rad_cur = 0.0 # Speed at the moment
22 | self.rad_next = 0.0 # Speed at the next moment
23 | self.rad_next_next = 0.0 # Speed at the next next moment
24 | self.error_last = 0.0 # The error at the previous moment
25 | self.error_cur = 0.0 # The error at the moment
26 |
27 | self.ec_last = 0.0 # electric current at the previous moment
28 | self.cnt = 0 # used to count the step number
29 |
30 | self.rad_threshold = 9000
31 |
32 | self.times = np.arange(args.start_time, args.end_time, args.dt) # generate the time steps.
33 | np.random.seed(args.seed)
34 | self.SystemsList = np.random.randint(0, 3, args.change_times) # Generate the random systems id.
35 | self.SystemsLists = [i for i in self.SystemsList for j in range(int(len(self.times)/args.change_times) + 1)] # extended system id.
36 |
37 | self.state = np.array([self.rad_last, self.rad_cur, self.rad_next, self.rad_next_next, self.error_last, self.error_cur])
38 |
39 |
40 | if args.choose_model == "search_pid_parameter":
41 | self.pid = PID(kp=args.kp, ki=args.ki, kd=args.kd, args=args)
42 | else:
43 | self.pid = PID(kp=args.kp_rl, ki=args.ki_rl, kd=args.kd_rl, args=args)
44 |
45 | def update_rad(self, electric):
46 | """
47 | Update the rad value Based on different Parameters.
48 | :param electric:
49 | :return:
50 | """
51 | if args.use_jump_system:
52 | if self.SystemsLists[self.cnt] == 0:
53 | rad = self._system1(ec_cur = electric)
54 | elif self.SystemsLists[self.cnt] == 1:
55 | rad = self._system2(ec_cur = electric)
56 | else:
57 | assert self.SystemsLists[self.cnt] == 2, print("the system id of SystemsLists is out of range.")
58 | rad = self._system3(ec_cur = electric)
59 | else:
60 | if args.used_system_id == 1:
61 | rad = self._system1(ec_cur = electric)
62 | elif args.used_system_id == 2:
63 | rad = self._system2(ec_cur = electric)
64 | else:
65 | assert args.used_system_id == 3, print("the selected system id is illegal")
66 | rad = self._system2(ec_cur = electric)
67 |
68 | return rad
69 |
70 | def step(self, action):
71 | """
72 | :param action: electric of current input.
73 | :return:
74 | """
75 |
76 | rad_last, rad_cur, rad_next, rad_next_next, error_last, error_cur = self.state
77 |
78 | error = trapezoidal_function(time=self.times[self.cnt]) - rad_cur
79 |
80 | if args.choose_model == "search_pid_parameter":
81 | kp, ki = action
82 | kp, ki = abs(kp), abs(ki)
83 | electric = self.pid.update_with_parameter(error=error, kp=kp, ki=ki, kd=0.0)
84 | else:
85 | electric = self.pid.update(error=error) + float(action)
86 |
87 | rad = self.update_rad(electric=electric)
88 |
89 | # Update the state.
90 | self.rad_last = self.rad_cur
91 | self.rad_cur = rad
92 | self.rad_next = trapezoidal_function(time = self.times[self.cnt + 1])
93 | self.rad_next_next = trapezoidal_function(time = self.times[self.cnt + 2])
94 | self.error_last = self.error_cur
95 | self.error_cur = error
96 |
97 | self.ec_last = electric
98 | self.cnt += 1
99 |
100 | if args.choose_model == "search_pid_parameter":
101 | reward = self._get_search_parameter_immediate_reward(error=error, electric=electric)
102 | else:
103 | reward = self._get_eletric_immediate_reward(error=error, electric=electric)
104 |
105 | done = self._get_termination()
106 |
107 | self.state = np.array([self.rad_last, self.rad_cur, self.rad_next, self.rad_next_next, self.error_last, self.error_cur])
108 |
109 | return self.state, reward, done
110 |
111 | def _get_eletric_immediate_reward(self, error, electric):
112 | """
113 | Design the Reward Function of search immediate reward.
114 | :param error:
115 | :param electric:
116 | :return:
117 | """
118 |
119 | error = min(error, 10000)
120 |
121 | mu = abs(error)
122 |
123 | sigma = 2
124 |
125 | reward = 0.9 * np.exp(-1 / 2 * (mu * mu) / (sigma * sigma)) + 0.1 * np.exp(-abs(electric))
126 |
127 | if np.isnan(reward): reward = 0 # error == inf -> nan -> reward == nan
128 |
129 | return reward
130 |
131 | def _get_search_parameter_immediate_reward(self, error, electric):
132 | """
133 | Design the Reward Function of search PID parameter.
134 | :param error:
135 | :param electric:
136 | :return:
137 | """
138 |
139 | error = min(error, 10000)
140 |
141 | mu = 0.9 * abs(error) + 0.1 * abs(electric)
142 |
143 | sigma = 10
144 |
145 | reward = np.exp(-1 / 2 * (mu * mu) / (sigma * sigma))
146 |
147 | if np.isnan(reward): reward = 0 # error == inf -> nan -> reward == nan
148 |
149 | return reward
150 |
151 | def _get_termination(self):
152 | """
153 | Set the End Conditions.
154 | :return:
155 | """
156 |
157 | done = abs(self.rad_cur) >= self.rad_threshold or abs(self.error_cur) >= args.height * 0.08
158 |
159 | return bool(done)
160 |
161 | def reset(self):
162 | """
163 | #
164 | :return:
165 | """
166 |
167 | self.rad_last = 0.0 # Speed at the previous moment
168 | self.rad_cur = 0.0 # Speed at the moment
169 | self.rad_next = 0.0 # Speed at the next moment
170 | self.rad_next_next = 0.0 # Speed at the next next moment
171 | self.error_last = 0.0 # The error at the previous moment
172 | self.error_cur = 0.0 # The error at the moment
173 |
174 | self.ec_last = 0.0 # electric current at the previous moment
175 |
176 | if args.choose_model == "search_pid_parameter":
177 | self.pid = PID(kp=args.kp, ki=args.ki, kd=args.kd, args=args)
178 | else:
179 | self.pid = PID(kp=args.kp_rl, ki=args.ki_rl, kd=args.kd_rl, args=args)
180 |
181 | self.state = np.array([self.rad_last, self.rad_cur, self.rad_next, self.rad_next_next, self.error_last, self.error_cur])
182 | self.cnt = 0
183 | return self.state
184 |
185 | def _system1(self, ec_cur):
186 | """
187 | # system equation of system 1.
188 | :param ec_cur:
189 | :return:
190 | """
191 | return self.rad_cur - 3.478 * 0.0001 * self.rad_last + 1.388 * ec_cur + 0.1986 * self.ec_last + 0.1 * np.random.normal(0, 1)
192 |
193 | def _system2(self, ec_cur):
194 | """
195 | # system equation of system 2.
196 | :param ec_cur:
197 | :return:
198 | """
199 | return self.rad_cur - 3.366 * 0.0001 * self.rad_last + 0.1263 * ec_cur + 0.01799 * self.ec_last + 0.1 * np.random.normal(0, 1)
200 |
201 | def _system3(self, ec_cur):
202 | """
203 | # system equation of system 3.
204 | :param ec_cur:
205 | :return:
206 | """
207 | return self.rad_cur - 3.478 * 0.0001 * self.rad_last + 1.388 * ec_cur + 0.1986 * self.ec_last + 0.1 * np.random.normal(0, 1) + 0.9148
208 |
209 | if __name__ == "__main__":
210 | env = ServoSystemEnv()
211 | x = np.linspace(-10, 10, 100)
212 | y = [env._get_eletric_immediate_reward(error=i, electric=0) for i in x]
213 | import matplotlib.pyplot as plt
214 | plt.plot(x, y)
215 | plt.show()
216 |
--------------------------------------------------------------------------------
/utils.py:
--------------------------------------------------------------------------------
1 | # -*- coding: utf-8 -*-
2 | # @Time : 11/30/21 3:21 PM
3 | # @Author : Zhiqiang He
4 | # @Email : tinyzqh@163.com
5 | # @File : utils.py
6 | # @Software: PyCharm
7 |
8 | from parameter import args
9 |
10 | def trapezoidal_function(time):
11 | """
12 | Get the value of the trace curve at the time moment.
13 | Total simulation time is 0.5s.
14 | :param time:
15 | :return:
16 | """
17 | height = args.height
18 | if time <= 0.075:
19 | return (height / 0.075) * time
20 | elif 0.075 < time and time <= 0.375:
21 | return height
22 | elif 0.375 < time and time <= 0.45:
23 | return (-height / 0.075) * time + 6 * height
24 | elif 0.45 < time and time <= 0.5:
25 | return 0
26 |
27 | if __name__ == "__main__":
28 | pass
--------------------------------------------------------------------------------