├── PID.py ├── README.md ├── parameter.py ├── plot_result.py ├── requirements.txt ├── results ├── ChooseModel_class_pid_CurveType_trapezoidal_Height_1000_DumpSystem_False_RunType_test │ ├── ecValues.png │ ├── ecValues.txt │ ├── iaeValues.png │ ├── iaeValues.txt │ ├── radValues.png │ └── radValues.txt ├── ChooseModel_search_electric_CurveType_trapezoidal_Height_1000_DumpSystem_False_RunType_train │ ├── epRewards.png │ └── epRewards.txt └── ChooseModel_search_pid_parameter_CurveType_trapezoidal_Height_1000_DumpSystem_False_RunType_train │ ├── actor_model.ckpt.data-00000-of-00001 │ ├── actor_model.ckpt.index │ ├── actor_model.ckpt.meta │ ├── checkpoint │ ├── critic_model.ckpt.data-00000-of-00001 │ ├── critic_model.ckpt.index │ ├── critic_model.ckpt.meta │ └── reward.PNG ├── run.py ├── servo_system_env.py └── utils.py /PID.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # @Time : 11/30/21 3:17 PM 3 | # @Author : Zhiqiang He 4 | # @Email : tinyzqh@163.com 5 | # @File : PID.py 6 | # @Software: PyCharm 7 | 8 | 9 | class PID(object): 10 | def __init__(self, kp, ki, kd, args): 11 | super(PID).__init__() 12 | self.kp = kp 13 | self.ki = ki 14 | self.kd = kd 15 | self.dt = args.dt 16 | 17 | self.CumulativeError = 0.0 18 | self.LastError = None 19 | 20 | def update(self, error): 21 | """ 22 | compute the out of fixed PID parameter. 23 | :param error: 24 | :return: 25 | """ 26 | p = self.kp * error 27 | i = self.ki * self.CumulativeError * self.dt 28 | if self.LastError is None: 29 | d = 0.0 30 | else: 31 | d = self.kd * (error - self.LastError) / self.dt 32 | 33 | self.CumulativeError += error 34 | self.LastError = error 35 | return p + i + d 36 | 37 | def update_with_parameter(self, error, kp, ki, kd): 38 | """ 39 | compute the out of the learned PID parameter. 40 | :param error: 41 | :param kp: 42 | :param ki: 43 | :param kd: 44 | :return: 45 | """ 46 | p = kp * error 47 | i = ki * self.CumulativeError * self.dt 48 | if self.LastError is None: 49 | d = 0.0 50 | else: 51 | d = kd * (error - self.LastError) / self.dt 52 | 53 | self.CumulativeError += error 54 | self.LastError = error 55 | return p + i + d 56 | 57 | if __name__ == "__main__": 58 | pass -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | 2 | # *强化学习用于PID的参数整定和电流补偿控制。* 3 | 4 | # Introduce 5 | 6 | This is the implementation of the paper “Control Strategy of Speed Servo Systems Based on Deep Reinforcement Learning”. 7 | 8 | For more paper information, please checkout the the paper [Link](https://www.mdpi.com/1999-4893/11/5/65). 9 | 10 | 11 | # Class PID 12 | 13 | The class PID algorithms has no training process. 14 | 15 | ```bash 16 | python run.py --choose_model class_pid --curve_type trapezoidal --height 1000 --run_type test 17 | ``` 18 | 19 | Plot the result: 20 | 21 | ```bash 22 | python plot_result.py --choose_model class_pid --curve_type trapezoidal --height 1000 --run_type test 23 | ``` 24 | 25 |
26 | 27 | ecValues 28 | iaeValues 29 | radValues 30 | 31 |
32 | 33 | 34 | # Parameters Adjustment Of PID 35 | 36 | Search the parameter of PID Based DDPG Algorithm. 37 | 38 | ```bash 39 | python run.py --choose_model search_pid_parameter --curve_type trapezoidal --height 1000 --run_type train 40 | ``` 41 | 42 | experimental operation process: 43 | 44 |
45 | 46 | epRewards_fig 47 | 48 |
49 | 50 | 51 | # Electric Current Compensation Of RL-PID 52 | 53 | 54 | ```bash 55 | python run.py --choose_model search_electric --curve_type trapezoidal --height 1000 --run_type train 56 | ``` 57 | 58 | test 59 | 60 | 61 | ```bash 62 | python run.py --choose_model search_electric --curve_type trapezoidal --height 1000 --run_type test 63 | ``` 64 | 65 | Plot the result: 66 | 67 | ```bash 68 | python plot_result.py --choose_model search_electric --curve_type trapezoidal --height 1000 --run_type train 69 | ``` 70 | 71 | the result show: 72 | 73 |
74 | 75 | epRewards 76 | 77 |
78 | 79 | 80 | # Dependencies 81 | 82 | The code was tested under Ubuntu 16 and uses these packages: 83 | 84 | - tensorflow-gpu==1.14.0 85 | - atari-py==0.2.6 86 | - gym==0.17.3 87 | - numpy==1.91.3 88 | 89 | more packages described in requirements.txt 90 | 91 | 92 | Citing 93 | ------ 94 | If you find this open source release useful, please reference in your paper: 95 | 96 | 97 | > Chen P, He Z, Chen C, et al. (2018). 98 | > Control strategy of speed servo systems based on deep reinforcement learning[J]. 99 | > *Algorithms, 2018, 11(5): 65.*. 100 | -------------------------------------------------------------------------------- /parameter.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # @Time : 11/29/21 11:15 AM 3 | # @Author : Zhiqiang He 4 | # @Email : tinyzqh@163.com 5 | # @File : parameter.py 6 | # @Software: PyCharm 7 | 8 | import argparse 9 | 10 | parser = argparse.ArgumentParser(description="The parameter of all Algorithms") 11 | parser.add_argument("--height", type=int, default=1000, help="the highest value of the tracking curve") 12 | parser.add_argument("--curve_type", type=str, default="trapezoidal", help="type of curve to be tracked, choose of ['trapezoidal', 'sine']") 13 | 14 | parser.add_argument("--dt", type=float, default=0.0001, help="discrete time interval") 15 | parser.add_argument("--start_time", type=float, default=0.0, help="the start time of simulate") 16 | parser.add_argument("--end_time", type=float, default=0.5, help="the end time of simulate") 17 | parser.add_argument("--seed", type=int, default=0, help="the random seed") 18 | 19 | parser.add_argument("--change_times", type=int, default=9, help="the change times of system") 20 | parser.add_argument("--use_jump_system", type=bool, default=False, help="select whether to choose a jump system") 21 | parser.add_argument("--used_system_id", type=int, default=1, help="choose the system id when not use jump system. default in [1, 2, 3]") 22 | 23 | parser.add_argument("--kp", type=float, default=0.5, help="the kp parameter of PID") 24 | parser.add_argument("--ki", type=float, default=2.985, help="the kp parameter of PID") 25 | parser.add_argument("--kd", type=float, default=0.0, help="the kp parameter of PID") 26 | 27 | parser.add_argument("--kp_rl", type=float, default=0.9318, help="the kp parameter of RL Agent Given") 28 | parser.add_argument("--ki_rl", type=float, default=1.4824, help="the ki parameter of RL Agent Given") 29 | parser.add_argument("--kd_rl", type=float, default=0.0, help="the kd parameter of RL Agent Given") 30 | 31 | # parameter of DDPG 32 | parser.add_argument("--max_episodes", type=int, default=20000000, help="") 33 | parser.add_argument("--max_ep_steps", type=int, default=4800, help="") 34 | parser.add_argument("--learning_actor", type=float, default=0.0001, help="the learning rate for actor") 35 | parser.add_argument("--learning_critic", type=float, default=0.0001, help="the learning rate for critic") 36 | parser.add_argument("--gamma", type=float, default=0.95, help="the discount of reward") 37 | parser.add_argument("--replace_iter_a", type=int, default=11000, help="the iter steps of replace the actor target network") 38 | parser.add_argument("--replace_iter_c", type=int, default=10000, help="the iter steps of replace the critic target network") 39 | parser.add_argument("--memory_capacity", type=int, default=50000, help="the capacity of the memory replay buffer") 40 | parser.add_argument("--batch_size", type=int, default=256, help="the batch size for training the network") 41 | parser.add_argument("--var", type=float, default=2.0, help="the control exploration of init") 42 | parser.add_argument("--var_min", type=float, default=0.0, help="the control exploration of end") 43 | parser.add_argument("--reg", type=float, default=2.0, help="the regularization parameters") 44 | parser.add_argument("--pid_parameter_search_action_bound", type=float, default=10.0, help="the action bound of pid search for [-10.0, 10.0]") 45 | parser.add_argument("--electric_compensation_action_bound", type=float, default=2.0, help="the action bound of electric search for [-2.0, 2.0]") 46 | parser.add_argument("--state_dim", type=int, default=6, help="the state dim") 47 | parser.add_argument("--pid_parameter_search_action_dim", type=int, default=2, help="the action dim of PID Parameters Adjustment") 48 | parser.add_argument("--electric_compensation_action_dim", type=int, default=1, help="the action dim of electric compensation.") 49 | 50 | parser.add_argument("--ep_reward_max", type=int, default=1000000, help="the ") 51 | parser.add_argument("--run_type", type=str, default="train", help="choose from ['train', 'test']") 52 | parser.add_argument("--choose_model", type=str, default='search_electric', help="one of ['search_pid_parameter', 'search_electric', 'class_pid']") 53 | 54 | args = parser.parse_args() -------------------------------------------------------------------------------- /plot_result.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # @Time : 12/8/21 7:47 PM 3 | # @Author : Zhiqiang He 4 | # @Email : tinyzqh@163.com 5 | # @File : plot_result.py 6 | # @Software: PyCharm 7 | 8 | import os 9 | import pandas as pd 10 | import seaborn as sns 11 | from parameter import args 12 | import matplotlib.pyplot as plt 13 | 14 | class plot_result(object): 15 | def __init__(self, root_path): 16 | """ 17 | :param root_path: 18 | """ 19 | self.root_path = root_path 20 | if self.root_path.split('_')[1] == "class": 21 | self.label = "Classic PID" 22 | else: 23 | self.label = "RL PID" 24 | 25 | def get_files_path(self): 26 | """ 27 | # Get All '.txt' files. 28 | :return: 29 | """ 30 | file_path_dir = [] 31 | for root, dir, files in os.walk(self.root_path): 32 | for file in files: 33 | file_path = os.path.join(root, file) 34 | file_path_dir.append(file_path) 35 | return file_path_dir 36 | 37 | def get_data(self): 38 | files = self.get_files_path() 39 | 40 | DataDict = {} 41 | for file in files: 42 | if file.split('/')[-1].split('.')[-1] == "txt": 43 | print("filename {} processed.".format(file)) 44 | DictName = file.split('/')[-1].split('.')[0] 45 | df = pd.read_csv(file, sep=" ") 46 | ColumnsName = df.columns 47 | Dict = {} 48 | Dict[ColumnsName[0]] = df[ColumnsName[0]] 49 | Dict[ColumnsName[1]] = df[ColumnsName[1]] 50 | 51 | DataDict[DictName] = Dict 52 | else: 53 | print("filename {} not .txt file.".format(file)) 54 | 55 | return DataDict 56 | 57 | def plot_data(self): 58 | """ 59 | plot the all .txt files. 60 | :return: 61 | """ 62 | sns.set(style="darkgrid", font_scale=1.5, font='serif', rc={'figure.figsize': (10, 8)}) 63 | DataDict = self.get_data() 64 | for key, data in DataDict.items(): 65 | 66 | plt.plot(data['times'], data[key], label=self.label) 67 | plt.xlabel("Time (s)") 68 | plt.ylabel(key) 69 | plt.legend() 70 | plt.savefig(self.root_path + '/{}.png'.format(key)) 71 | plt.show() 72 | 73 | 74 | if __name__ == "__main__": 75 | current_dir = os.path.dirname(os.path.abspath(__file__)) 76 | result_dir = os.path.join('results', 'ChooseModel_{}_CurveType_{}_Height_{}_DumpSystem_{}_RunType_{}'.format(args.choose_model, args.curve_type, args.height, args.use_jump_system, args.run_type)) 77 | 78 | root_path = os.path.join(current_dir, result_dir) 79 | file_path_dir = plot_result(root_path=root_path).plot_data() -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | absl-py==0.11.0 2 | alabaster==0.7.12 3 | atari-py==0.2.6 4 | attrs==20.2.0 5 | Babel==2.8.0 6 | cached-property==1.5.2 7 | certifi==2021.5.30 8 | cffi==1.14.3 9 | chardet==3.0.4 10 | cloudpickle==1.6.0 11 | colorama==0.4.4 12 | cycler==0.10.0 13 | Cython==0.29.21 14 | dataclasses==0.7 15 | deepdiff==5.5.0 16 | dm-env==1.3 17 | dm-tree==0.1.5 18 | docopt==0.6.2 19 | docutils==0.16 20 | entrypoints==0.3 21 | enum34==1.1.10 22 | fasteners==0.15 23 | future==0.18.2 24 | gitdb==4.0.7 25 | GitPython==3.1.17 26 | glfw==2.0.0 27 | gym==0.17.3 28 | h5py==3.0.0 29 | idna==2.10 30 | ImageHash==4.1.0 31 | imageio==2.9.0 32 | imagesize==1.2.0 33 | importlib-metadata==2.0.0 34 | iniconfig==1.1.1 35 | ipdb==0.13.4 36 | jedi==0.17.2 37 | Jinja2==2.11.2 38 | jsonpickle==1.5.2 39 | labmaze==1.0.3 40 | lxml==4.6.1 41 | MarkupSafe==1.1.1 42 | mkl-fft==1.2.0 43 | mkl-random==1.1.1 44 | mkl-service==2.3.0 45 | mock==4.0.3 46 | monotonic==1.5 47 | mpyq==0.2.5 48 | mujoco-py==2.0.2.13 49 | munch==2.5.0 50 | numpy==1.19.3 51 | numpydoc==1.1.0 52 | olefile==0.46 53 | ordered-set==4.0.2 54 | packaging==20.4 55 | patsy==0.5.1 56 | pluggy==0.13.1 57 | portpicker==1.4.0 58 | progressbar==2.5 59 | protobuf==3.13.0 60 | py==1.9.0 61 | py-cpuinfo==8.0.0 62 | pycparser==2.20 63 | pygame==2.0.1 64 | pyglet==1.5.0 65 | PyOpenGL==3.1.5 66 | pyparsing==2.4.7 67 | PySC2==3.0.0 68 | pytest==6.1.2 69 | pytest-instafail==0.3.0 70 | python-dateutil==2.8.1 71 | pytz==2020.1 72 | PyWavelets==1.1.1 73 | PyYAML==5.4.1 74 | requests==2.24.0 75 | retrying==1.3.3 76 | s2clientprotocol==5.0.7.84643.0 77 | s2protocol==5.0.7.84643.0 78 | sacred==0.8.2 79 | scipy==1.5.3 80 | seaborn==0.9.0 81 | six==1.15.0 82 | sk-video==1.1.10 83 | smmap==4.0.0 84 | snowballstemmer==2.0.0 85 | Sphinx==3.2.1 86 | sphinx-rtd-theme==0.5.0 87 | sphinxcontrib-applehelp==1.0.2 88 | sphinxcontrib-devhelp==1.0.2 89 | sphinxcontrib-htmlhelp==1.0.3 90 | sphinxcontrib-jsmath==1.0.1 91 | sphinxcontrib-qthelp==1.0.3 92 | sphinxcontrib-serializinghtml==1.1.4 93 | tensorboardX==2.0 94 | toml==0.10.1 95 | torch==1.8.0+cu111 96 | torchaudio==0.8.0 97 | torchvision==0.9.0+cu111 98 | tornado==6.0.4 99 | urllib3==1.25.11 100 | websocket-client==1.0.1 101 | whichcraft==0.6.1 102 | wrapt==1.12.1 103 | xlwt==1.3.0 104 | zipp==3.4.0 -------------------------------------------------------------------------------- /results/ChooseModel_class_pid_CurveType_trapezoidal_Height_1000_DumpSystem_False_RunType_test/ecValues.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tinyzqh/control-of-jump-systems-based-on-reinforcement-learning/de23c13aecfb6d8745d6005e32d2083498c4246a/results/ChooseModel_class_pid_CurveType_trapezoidal_Height_1000_DumpSystem_False_RunType_test/ecValues.png -------------------------------------------------------------------------------- /results/ChooseModel_class_pid_CurveType_trapezoidal_Height_1000_DumpSystem_False_RunType_test/iaeValues.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tinyzqh/control-of-jump-systems-based-on-reinforcement-learning/de23c13aecfb6d8745d6005e32d2083498c4246a/results/ChooseModel_class_pid_CurveType_trapezoidal_Height_1000_DumpSystem_False_RunType_test/iaeValues.png -------------------------------------------------------------------------------- /results/ChooseModel_class_pid_CurveType_trapezoidal_Height_1000_DumpSystem_False_RunType_test/radValues.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tinyzqh/control-of-jump-systems-based-on-reinforcement-learning/de23c13aecfb6d8745d6005e32d2083498c4246a/results/ChooseModel_class_pid_CurveType_trapezoidal_Height_1000_DumpSystem_False_RunType_test/radValues.png -------------------------------------------------------------------------------- /results/ChooseModel_search_electric_CurveType_trapezoidal_Height_1000_DumpSystem_False_RunType_train/epRewards.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tinyzqh/control-of-jump-systems-based-on-reinforcement-learning/de23c13aecfb6d8745d6005e32d2083498c4246a/results/ChooseModel_search_electric_CurveType_trapezoidal_Height_1000_DumpSystem_False_RunType_train/epRewards.png -------------------------------------------------------------------------------- /results/ChooseModel_search_electric_CurveType_trapezoidal_Height_1000_DumpSystem_False_RunType_train/epRewards.txt: -------------------------------------------------------------------------------- 1 | times epRewards 2 | 0.0 3036.7259341814656 3 | 0.0001 3043.6923303989365 4 | 0.0002 3017.9982326663167 5 | 0.00030000000000000003 3066.080714219948 6 | 0.0004 3044.6397142553615 7 | 0.0005 3030.9625944126683 8 | 0.0006000000000000001 3039.743050215572 9 | 0.0007 3023.2854571749554 10 | 0.0008 3021.9944469362213 11 | 0.0009000000000000001 3023.699565360395 12 | 0.001 3076.4207968839874 13 | 0.0011 3102.887173448377 14 | 0.0012000000000000001 3107.991250220807 15 | 0.0013000000000000002 3117.195720022614 16 | 0.0014 3120.8241369752936 17 | 0.0015 3173.737503662069 18 | 0.0016 3198.9838898886583 19 | 0.0017000000000000001 3183.4812869604657 20 | 0.0018000000000000002 3216.7629975642876 21 | 0.0019 3238.452123295176 22 | 0.002 3243.6810075367894 23 | 0.0021000000000000003 3256.372256253036 24 | 0.0022 3288.278534943949 25 | 0.0023 3275.7429848551155 26 | 0.0024000000000000002 3296.4360207573704 27 | 0.0025 3310.9131839014312 28 | 0.0026000000000000003 3323.3703375703217 29 | 0.0027 3334.697914223168 30 | 0.0028 3332.676420728047 31 | 0.0029000000000000002 3353.237062546084 32 | 0.003 3365.5150787415755 33 | 0.0031000000000000003 3371.572243763116 34 | 0.0032 3390.179212621337 35 | 0.0033 3388.4623226772273 36 | 0.0034000000000000002 3391.470616095011 37 | 0.0035 3414.962045300303 38 | 0.0036000000000000003 3409.7790657045175 39 | 0.0037 3426.805094550273 40 | 0.0038 3419.5921372889643 41 | 0.0039000000000000003 3425.411186758219 42 | 0.004 3445.9015376405205 43 | 0.0041 3443.6479604053425 44 | 0.004200000000000001 3440.5658731217713 45 | 0.0043 3448.5756823000484 46 | 0.0044 3451.855521973523 47 | 0.0045000000000000005 3450.7678200050345 48 | 0.0046 3466.9810084088745 49 | 0.0047 3466.3342867587858 50 | 0.0048000000000000004 3464.656076634408 51 | 0.0049 3475.286593095242 52 | 0.005 3467.695263105075 53 | 0.0051 3473.711167060322 54 | 0.005200000000000001 3147.908303880091 55 | 0.0053 2912.7297121397232 56 | 0.0054 2978.4591262697954 57 | 0.0055000000000000005 2958.8481619291356 58 | 0.0056 3251.52941865746 59 | 0.0057 3073.2239386255833 60 | 0.0058000000000000005 3315.4084656909013 61 | 0.0059 3197.405696360599 62 | 0.006 3126.6639081244725 63 | 0.0061 2794.314079034672 64 | 0.006200000000000001 3272.493309024069 65 | 0.0063 3435.0819586473967 66 | 0.0064 3374.9285190488035 67 | 0.006500000000000001 3309.7697911073883 68 | 0.0066 3453.9894058479135 69 | 0.0067 3463.199772708253 70 | 0.0068000000000000005 3501.514879964667 71 | 0.006900000000000001 3503.46129012014 72 | 0.007 3506.5379153397384 73 | 0.0071 3390.7595933273356 74 | 0.007200000000000001 2898.7963078461767 75 | 0.0073 3284.2461593354237 76 | 0.0074 3198.766635574699 77 | 0.007500000000000001 3410.659583874455 78 | 0.0076 3419.1911779384145 79 | 0.0077 3342.6748066543782 80 | 0.0078000000000000005 3450.6064607118037 81 | 0.0079 3294.1354084943405 82 | 0.008 2790.4469210588336 83 | 0.0081 3238.9775974171225 84 | 0.0082 2808.8973239989086 85 | 0.0083 3137.3685749253614 86 | 0.008400000000000001 3440.9653372098137 87 | 0.0085 3119.854133191471 88 | 0.0086 2999.9901846747985 89 | 0.008700000000000001 2989.8744768256897 90 | 0.0088 3191.833236106521 91 | 0.0089 2882.1808677009 92 | 0.009000000000000001 2987.4138457487043 93 | 0.0091 3134.1181841183784 94 | 0.0092 3591.405367783205 95 | 0.009300000000000001 3179.7929546415803 96 | 0.0094 3303.5814377979827 97 | 0.0095 3438.489734777065 98 | 0.009600000000000001 3517.476211431069 99 | 0.0097 3504.740321141496 100 | 0.0098 3542.1352700938232 101 | 0.0099 3586.2483293542136 102 | 0.01 3888.789484069617 103 | 0.010100000000000001 4221.606454841663 104 | 0.0102 3809.264931791187 105 | 0.0103 3202.643387289286 106 | 0.010400000000000001 3838.2202073659823 107 | 0.0105 4354.407101473443 108 | 0.0106 4334.608536034208 109 | 0.010700000000000001 4284.572616653706 110 | 0.0108 4233.562753610873 111 | 0.0109 3698.2831756022006 112 | 0.011000000000000001 3976.1156399211322 113 | 0.0111 4260.359021388144 114 | 0.0112 4302.172354796017 115 | 0.011300000000000001 4136.450394033346 116 | 0.0114 4219.24872109257 117 | 0.0115 4215.643146355247 118 | 0.011600000000000001 4335.709850062812 119 | 0.0117 4298.425634734699 120 | 0.0118 4454.339173897462 121 | 0.0119 4136.8841476271755 122 | 0.012 4450.983725552523 123 | 0.012100000000000001 4374.800641650777 124 | 0.0122 4143.236924900215 125 | 0.0123 4403.714997054108 126 | 0.012400000000000001 4300.67294703903 127 | 0.0125 4303.347312043605 128 | 0.0126 4192.134513294037 129 | 0.012700000000000001 4267.112876036415 130 | 0.0128 4244.056254288995 131 | 0.0129 4285.634246044754 132 | 0.013000000000000001 4264.590208291385 133 | 0.0131 4116.045754020984 134 | 0.0132 4193.414202886074 135 | 0.013300000000000001 4335.121288539964 136 | 0.0134 4306.167440484755 137 | 0.0135 4322.200653052792 138 | 0.013600000000000001 4357.691192883888 139 | 0.0137 4444.072313718211 140 | 0.013800000000000002 4332.999146298944 141 | 0.013900000000000001 4365.995680669751 142 | 0.014 4299.934384964377 143 | 0.014100000000000001 4388.516998653022 144 | 0.0142 4454.348760575999 145 | 0.0143 4453.104561945968 146 | 0.014400000000000001 4324.013924428038 147 | 0.0145 4317.404168189095 148 | 0.0146 4275.8342059019305 149 | 0.014700000000000001 4330.38355609085 150 | 0.0148 4406.393167417569 151 | 0.0149 4386.3324433464995 152 | 0.015000000000000001 4445.215155378947 153 | 0.0151 4391.414760190859 154 | 0.0152 4396.03798533988 155 | 0.015300000000000001 4337.203063404891 156 | 0.0154 4407.030491054112 157 | 0.0155 4377.762681171134 158 | 0.015600000000000001 4489.088323375656 159 | 0.015700000000000002 4318.544772205742 160 | 0.0158 4374.489682789004 161 | 0.0159 4431.596393913651 162 | 0.016 4344.3663744918895 163 | 0.0161 4511.522171346495 164 | 0.0162 4420.312437201572 165 | 0.016300000000000002 4248.771279638011 166 | 0.0164 4458.488222335947 167 | 0.0165 4439.1906612911525 168 | 0.0166 4455.229603827504 169 | 0.0167 4423.384309210541 170 | 0.016800000000000002 4419.23442020806 171 | 0.016900000000000002 4363.019430911425 172 | 0.017 4459.500301640532 173 | 0.0171 4440.803159012662 174 | 0.0172 4331.872704018888 175 | 0.0173 4423.442289082499 176 | 0.017400000000000002 4199.984756019031 177 | 0.0175 4436.266045233103 178 | 0.0176 4392.656748167428 179 | 0.0177 4402.1193077392 180 | 0.0178 4373.027862467928 181 | 0.0179 4469.650402698401 182 | 0.018000000000000002 4398.702202561162 183 | 0.0181 4424.4974548573 184 | 0.0182 4476.2174514045055 185 | 0.0183 4437.450168154308 186 | 0.0184 4487.0568195167125 187 | 0.018500000000000003 4401.0698060739205 188 | 0.018600000000000002 4500.393297490085 189 | 0.0187 4488.970108387174 190 | 0.0188 4485.763001860717 191 | 0.0189 4350.2458625323425 192 | 0.019 4462.591713303403 193 | 0.019100000000000002 4465.413974392815 194 | 0.019200000000000002 4444.8617752028185 195 | 0.0193 4526.353688257017 196 | 0.0194 4489.33134707166 197 | 0.0195 4517.624393675544 198 | 0.0196 4502.034806643863 199 | 0.019700000000000002 4481.6487404602785 200 | 0.0198 4460.529679211568 201 | 0.0199 4494.675303545952 202 | 0.02 4305.796267984588 203 | 0.0201 4487.007006204375 204 | 0.020200000000000003 4442.837179646518 205 | 0.020300000000000002 4476.571451906984 206 | 0.0204 4389.4553223094235 207 | 0.0205 4183.059037881925 208 | 0.0206 4418.899633567956 209 | 0.0207 4308.435531957523 210 | 0.020800000000000003 4360.656501886434 211 | 0.020900000000000002 4438.4547125659765 212 | 0.021 4401.023099375283 213 | 0.0211 4403.837607531937 214 | 0.0212 4381.831131073825 215 | 0.0213 4426.548323296748 216 | 0.021400000000000002 4410.8720725007315 217 | 0.021500000000000002 4420.17939944314 218 | 0.0216 4208.519589445352 219 | 0.0217 4342.744269775053 220 | 0.0218 4367.795661951114 221 | 0.0219 4434.550868713155 222 | 0.022000000000000002 4384.817864479502 223 | 0.0221 4428.947419022296 224 | 0.0222 4426.206895598665 225 | 0.0223 4408.291241633852 226 | 0.0224 4427.313783055031 227 | 0.022500000000000003 4485.568975546635 228 | 0.022600000000000002 4460.840407717722 229 | 0.0227 4461.227076963243 230 | 0.0228 4496.6466565720575 231 | 0.0229 4508.007106797012 232 | 0.023 4504.754133032239 233 | 0.023100000000000002 4392.36115833794 234 | 0.023200000000000002 4421.853174059126 235 | 0.0233 4471.376428643626 236 | 0.0234 4459.720354439567 237 | 0.0235 4436.909775711515 238 | 0.0236 4441.414110733332 239 | 0.023700000000000002 4427.730960290675 240 | 0.0238 4466.986080713704 241 | 0.0239 4374.017641957251 242 | 0.024 4466.514594991542 243 | 0.0241 4455.2573317762 244 | 0.024200000000000003 4446.3013936512825 245 | 0.024300000000000002 4464.634889940811 246 | 0.0244 4468.60525511012 247 | 0.0245 4416.246307597157 248 | 0.0246 4399.576084532948 249 | 0.0247 4355.518808750678 250 | 0.024800000000000003 4385.023441060532 251 | 0.024900000000000002 4313.468310140865 252 | 0.025 4414.6972962651 253 | 0.0251 4355.154362615261 254 | 0.0252 4350.542760850686 255 | 0.0253 4437.916677209027 256 | 0.025400000000000002 4443.611055571595 257 | 0.025500000000000002 4438.5195449537505 258 | 0.0256 4448.629919222049 259 | 0.0257 4427.269707061564 260 | 0.0258 4431.13079720861 261 | 0.025900000000000003 4429.859188189083 262 | 0.026000000000000002 4422.3653392886035 263 | 0.0261 4416.539656078386 264 | 0.0262 4371.542489743679 265 | 0.0263 4363.65331482477 266 | 0.0264 4283.245244550804 267 | 0.026500000000000003 4259.448979742627 268 | 0.026600000000000002 4296.306939271484 269 | 0.0267 4380.4293917764035 270 | 0.0268 4166.556872073827 271 | 0.0269 4375.229812774562 272 | 0.027 4443.95410332818 273 | 0.027100000000000003 4496.533492902397 274 | 0.027200000000000002 4494.383281317909 275 | 0.0273 4519.0151454471925 276 | 0.0274 4407.402606160929 277 | 0.0275 4365.419994518845 278 | 0.027600000000000003 4364.900844169226 279 | 0.027700000000000002 4360.977372246751 280 | 0.027800000000000002 4379.134837443217 281 | 0.0279 4430.3242830650815 282 | 0.028 4432.142012413305 283 | 0.0281 4380.3519438691055 284 | 0.028200000000000003 4424.319372122843 285 | 0.028300000000000002 4440.36708997703 286 | 0.0284 4434.366079719375 287 | 0.0285 4452.035891964858 288 | 0.0286 4390.535047239087 289 | 0.0287 4372.006146074955 290 | 0.028800000000000003 4380.56339660231 291 | 0.028900000000000002 4433.692627085836 292 | 0.029 4462.060537442368 293 | 0.0291 4439.980170574645 294 | 0.0292 4359.376783990062 295 | 0.029300000000000003 4393.770387631898 296 | 0.029400000000000003 4375.961877624661 297 | 0.029500000000000002 4419.050191630778 298 | 0.0296 4430.987008788952 299 | 0.0297 4382.738098557449 300 | 0.0298 4453.93172375097 301 | 0.029900000000000003 4427.136351390884 302 | 0.030000000000000002 4440.676893612471 303 | 0.030100000000000002 4449.962919819516 304 | 0.0302 4434.517759944107 305 | 0.0303 4469.832500970235 306 | 0.0304 4463.507177126829 307 | 0.030500000000000003 4427.125583732383 308 | 0.030600000000000002 4320.984961514471 309 | 0.0307 4415.972895463168 310 | 0.0308 4446.795022597304 311 | 0.0309 4434.547277297687 312 | 0.031 4377.202629091898 313 | 0.031100000000000003 4430.967917930318 314 | 0.031200000000000002 4434.19479828753 315 | 0.0313 4413.378804797773 316 | 0.031400000000000004 4396.1421459219855 317 | 0.0315 4359.089063235155 318 | 0.0316 4363.6063801959135 319 | 0.0317 4366.47729382662 320 | 0.0318 4475.941656967744 321 | 0.031900000000000005 4468.987701758295 322 | 0.032 4439.249479915543 323 | 0.032100000000000004 4356.5480757687765 324 | 0.0322 4422.361359427192 325 | 0.0323 4412.375689007601 326 | 0.0324 4379.89510507919 327 | 0.0325 4433.1256262657025 328 | 0.032600000000000004 4411.831966914949 329 | 0.0327 4478.881455769247 330 | 0.0328 4414.7133981916695 331 | 0.0329 4454.839059771139 332 | 0.033 4413.4824603086845 333 | 0.033100000000000004 4491.801038949697 334 | 0.0332 4429.380712067966 335 | 0.0333 4449.862306662729 336 | 0.0334 4429.635994096248 337 | 0.0335 4352.375933272498 338 | 0.033600000000000005 4358.208561297983 339 | 0.0337 4306.627981949049 340 | 0.033800000000000004 4300.980259301358 341 | 0.0339 4381.452130916939 342 | 0.034 4369.913756532929 343 | 0.0341 4443.444021623991 344 | 0.0342 4273.7262536735925 345 | 0.034300000000000004 4145.581691027162 346 | 0.0344 4345.6989644187515 347 | 0.0345 4127.556474729422 348 | 0.0346 4376.228776721722 349 | 0.0347 4182.605008879098 350 | 0.034800000000000005 4432.24489369397 351 | 0.0349 4346.889833166535 352 | 0.035 4344.434573904728 353 | 0.0351 4343.958337326058 354 | 0.0352 4339.005565313726 355 | 0.035300000000000005 4350.140804455015 356 | 0.0354 4269.864393528522 357 | 0.035500000000000004 4334.6543960521385 358 | 0.0356 4345.669075691819 359 | 0.0357 4422.954561603528 360 | 0.0358 4404.9225526219425 361 | 0.0359 4405.632404518491 362 | 0.036000000000000004 4359.022893759282 363 | 0.0361 4343.50414801412 364 | 0.0362 4336.347319253851 365 | 0.0363 4422.601997361631 366 | 0.0364 4421.49147173143 367 | 0.036500000000000005 4204.083438145154 368 | 0.0366 4429.329551053382 369 | 0.0367 4403.420745015327 370 | 0.0368 4406.223117094903 371 | 0.0369 4399.391942756944 372 | 0.037000000000000005 4334.962925828808 373 | 0.0371 4428.729023538332 374 | 0.037200000000000004 4375.9330049931405 375 | 0.0373 4329.369444234582 376 | 0.0374 4188.756974292507 377 | 0.0375 4337.078133824004 378 | 0.0376 4251.457372639439 379 | 0.037700000000000004 4221.694933663649 380 | 0.0378 4186.3122569770185 381 | 0.0379 4140.531406583709 382 | 0.038 4331.622712245178 383 | 0.0381 4334.638851986008 384 | 0.038200000000000005 4112.547326319051 385 | 0.0383 4104.435943548242 386 | 0.038400000000000004 4263.064642088948 387 | 0.0385 4228.806735229537 388 | 0.0386 4269.02356745655 389 | 0.038700000000000005 4225.816657259688 390 | 0.0388 4339.01083629678 391 | 0.038900000000000004 4379.673938950693 392 | 0.039 4391.634359412413 393 | 0.0391 4391.199380560602 394 | 0.0392 4372.796113309642 395 | 0.0393 4271.9430418696975 396 | 0.039400000000000004 4268.492930462117 397 | 0.0395 4282.433077015161 398 | 0.0396 4240.789164926495 399 | 0.0397 4229.7417893343045 400 | 0.0398 4235.009995305034 401 | 0.039900000000000005 4306.510784634346 402 | 0.04 4378.818868333671 403 | 0.040100000000000004 4330.4549713031365 404 | 0.0402 4377.877828502294 405 | 0.0403 4352.412771309074 406 | 0.040400000000000005 4392.343163092161 407 | 0.0405 4407.635648507039 408 | 0.040600000000000004 4415.8235967458795 409 | 0.0407 4402.1543600599325 410 | 0.0408 4355.563669277138 411 | 0.0409 4315.765524618012 412 | 0.041 4157.521430649726 413 | 0.041100000000000005 4210.128379373147 414 | 0.0412 4346.654263123864 415 | 0.0413 4370.366145219769 416 | 0.0414 4388.907718271681 417 | 0.0415 4325.5500339517375 418 | 0.041600000000000005 4262.405578822564 419 | 0.0417 4322.295886647103 420 | 0.041800000000000004 4257.271116725194 421 | 0.0419 4355.333493666723 422 | 0.042 4308.707085423674 423 | 0.042100000000000005 4308.555403944079 424 | 0.0422 4173.567801506868 425 | 0.042300000000000004 4059.288103132137 426 | 0.0424 4123.947636320296 427 | 0.0425 4302.153884688651 428 | 0.0426 4178.500186184762 429 | 0.0427 4263.113130467948 430 | 0.042800000000000005 4163.471153068018 431 | 0.0429 4189.05570638449 432 | 0.043000000000000003 4232.602209977596 433 | 0.0431 4348.5018761871 434 | 0.0432 4171.518931410294 435 | 0.043300000000000005 4310.956503370057 436 | 0.0434 4262.54459586974 437 | 0.043500000000000004 4331.901916150952 438 | 0.0436 4246.918169427158 439 | 0.0437 4276.865959270239 440 | 0.0438 4272.913002515606 441 | 0.0439 4196.702529804375 442 | 0.044000000000000004 4291.889891727884 443 | 0.0441 4147.828467940942 444 | 0.0442 4207.341059955226 445 | 0.0443 4216.568539157274 446 | 0.0444 4135.162494273318 447 | 0.044500000000000005 4177.1145352570675 448 | 0.0446 4203.784447244005 449 | 0.044700000000000004 4115.187813081391 450 | 0.0448 4343.103409517485 451 | 0.0449 4222.1627251559985 452 | 0.045000000000000005 4247.451766346038 453 | 0.0451 4294.365145933141 454 | 0.045200000000000004 4310.168093222668 455 | 0.0453 4181.809107240569 456 | 0.0454 4161.3734868992005 457 | 0.0455 4145.733689554778 458 | 0.0456 4299.260997052471 459 | 0.045700000000000005 4285.964470753583 460 | 0.0458 4176.29236867967 461 | 0.0459 4244.677524554218 462 | 0.046 4388.729546170196 463 | 0.0461 4398.0408088717995 464 | 0.046200000000000005 4479.374999406967 465 | 0.0463 4429.656561378026 466 | 0.046400000000000004 4370.810878999215 467 | 0.0465 4353.642368318909 468 | 0.0466 4330.17883077504 469 | 0.046700000000000005 4430.75405928842 470 | 0.0468 4499.254674895691 471 | 0.046900000000000004 4462.30609969203 472 | 0.047 4418.200702047398 473 | 0.0471 4471.137675025145 474 | 0.0472 4418.55155321394 475 | 0.0473 4452.112539666745 476 | 0.047400000000000005 4394.602302314124 477 | 0.0475 4468.415862370347 478 | 0.0476 4427.421432860728 479 | 0.0477 4415.768086164587 480 | 0.0478 4429.692405560722 481 | 0.047900000000000005 4376.4878533591245 482 | 0.048 4442.552583691507 483 | 0.048100000000000004 4499.001552321517 484 | 0.0482 4398.44011188676 485 | 0.0483 4332.40200805313 486 | 0.048400000000000006 4348.325152469659 487 | 0.0485 4342.331724012974 488 | 0.048600000000000004 4394.916173686139 489 | 0.0487 4354.269679866479 490 | 0.0488 4325.343255139016 491 | 0.0489 4374.690508427324 492 | 0.049 4357.468703488204 493 | 0.049100000000000005 4335.621863905782 494 | 0.0492 4362.352847747393 495 | 0.049300000000000004 4348.462459620172 496 | 0.0494 4453.103984108865 497 | 0.0495 4359.791209419824 498 | 0.049600000000000005 4446.372883423052 499 | 0.0497 4445.388570033542 500 | 0.049800000000000004 4241.598295928038 501 | 0.0499 4307.115210078385 502 | 0.05 4364.00948905661 503 | 0.050100000000000006 4123.808813376697 504 | 0.0502 4298.881686631464 505 | 0.050300000000000004 4317.500138664449 506 | 0.0504 4144.631903123633 507 | 0.0505 4047.878338627089 508 | 0.0506 4120.522493360287 509 | 0.0507 4215.67559728031 510 | 0.050800000000000005 4238.549086569198 511 | 0.0509 4202.78772525598 512 | 0.051000000000000004 4289.132316217688 513 | 0.0511 4256.708015042447 514 | 0.0512 4315.757117418529 515 | 0.051300000000000005 4350.874590356901 516 | 0.0514 4352.178834021475 517 | 0.051500000000000004 4446.033014063412 518 | 0.0516 4042.888267070249 519 | 0.0517 4400.180972317542 520 | 0.051800000000000006 4431.61891195877 521 | 0.0519 4278.089476219895 522 | 0.052000000000000005 4424.48357803447 523 | 0.0521 4412.788946253158 524 | 0.0522 4418.411135837286 525 | 0.0523 4137.411987893498 526 | 0.0524 4240.097390181752 527 | 0.052500000000000005 4226.474667028535 528 | 0.0526 4285.628532975624 529 | 0.052700000000000004 4269.607271978908 530 | 0.0528 4304.255484626818 531 | 0.0529 4327.9710764743995 532 | 0.053000000000000005 4321.189660833056 533 | 0.0531 4277.497926135756 534 | 0.053200000000000004 4289.829822264831 535 | 0.0533 4249.503090042379 536 | 0.0534 4324.349284581222 537 | 0.053500000000000006 4330.638456063682 538 | 0.0536 4334.9634998064685 539 | 0.053700000000000005 4360.13564167239 540 | 0.0538 4251.118777547551 541 | 0.0539 4341.182966235815 542 | 0.054 4212.93229341088 543 | 0.0541 4333.497038792342 544 | 0.054200000000000005 4081.9366956227072 545 | 0.0543 4225.012897192138 546 | 0.054400000000000004 4332.713964393512 547 | 0.0545 4312.979565226338 548 | 0.0546 4229.3957956514705 549 | 0.054700000000000006 4280.27903935942 550 | 0.0548 4364.697039899631 551 | 0.054900000000000004 4283.721332894117 552 | 0.055 4357.104729906341 553 | 0.0551 4462.424108111948 554 | 0.055200000000000006 4424.625705274606 555 | 0.0553 4399.429707028785 556 | 0.055400000000000005 4463.428520483089 557 | 0.0555 4412.817315032471 558 | 0.055600000000000004 4411.28844792561 559 | 0.0557 4355.062700693683 560 | 0.0558 4379.391334865822 561 | 0.055900000000000005 4445.977863461089 562 | 0.056 4411.0096563875295 563 | 0.056100000000000004 4138.85998221622 564 | 0.0562 4390.870879520354 565 | 0.0563 4353.932107866255 566 | 0.056400000000000006 4386.8481904047185 567 | 0.0565 4292.326597477917 568 | 0.056600000000000004 4397.514785960892 569 | 0.0567 4396.081082847065 570 | 0.0568 4357.187389937863 571 | 0.056900000000000006 4334.354295722543 572 | 0.057 4246.150908591707 573 | 0.057100000000000005 4129.453349961976 574 | 0.0572 4343.233059379258 575 | 0.057300000000000004 4385.46089947163 576 | 0.0574 4365.507430575668 577 | 0.0575 4409.29390341105 578 | 0.057600000000000005 4357.620807638532 579 | 0.0577 4407.605019539845 580 | 0.057800000000000004 4392.005949881429 581 | 0.0579 4367.719448502599 582 | 0.058 4443.672130442621 583 | 0.058100000000000006 4359.59081926723 584 | 0.0582 4472.861415787176 585 | 0.058300000000000005 4331.6631425532705 586 | 0.0584 4219.596190202949 587 | 0.0585 4268.828753575115 588 | 0.058600000000000006 4141.3323128387565 589 | 0.0587 4254.270492222178 590 | 0.058800000000000005 4239.9709881367935 591 | 0.0589 4308.850461184035 592 | 0.059000000000000004 4216.62287202549 593 | 0.0591 4280.903365739939 594 | 0.0592 4357.487494101463 595 | 0.059300000000000005 4357.25738339534 596 | 0.0594 4379.705088927585 597 | 0.059500000000000004 4362.938539968424 598 | 0.0596 4347.152164552888 599 | 0.0597 4349.83084211833 600 | 0.059800000000000006 4306.785257561856 601 | 0.0599 4055.2612115550846 602 | 0.060000000000000005 4277.730937800845 603 | 0.0601 4214.575798780641 604 | 0.060200000000000004 4330.047300707444 605 | 0.0603 4366.793423191057 606 | 0.0604 4385.300063262835 607 | 0.060500000000000005 4348.611965801418 608 | 0.0606 4223.861340846572 609 | 0.060700000000000004 4254.072422488802 610 | 0.0608 4211.424662616365 611 | 0.0609 4251.380458019873 612 | 0.061000000000000006 4197.964499617418 613 | 0.0611 4304.8637395083115 614 | 0.061200000000000004 4240.648451872296 615 | 0.0613 4224.636237065962 616 | 0.0614 4252.377760542809 617 | 0.061500000000000006 4241.055383674843 618 | 0.0616 4383.625771316363 619 | 0.061700000000000005 4372.66545296102 620 | 0.0618 4266.975398604267 621 | 0.061900000000000004 4231.134120014123 622 | 0.062 4232.629845214445 623 | 0.0621 4255.0768786817935 624 | 0.062200000000000005 4180.6902336089315 625 | 0.0623 4333.2589459006 626 | 0.062400000000000004 4262.735785679922 627 | 0.0625 4346.710962715649 628 | 0.0626 4298.222559180647 629 | 0.0627 4273.751417438495 630 | 0.06280000000000001 4253.382880985444 631 | 0.0629 4301.30713499277 632 | 0.063 4308.900761176018 633 | 0.0631 4363.524175018731 634 | 0.0632 4283.69218629247 635 | 0.06330000000000001 4253.149560863505 636 | 0.0634 4360.838458459897 637 | 0.0635 4290.09634677516 638 | 0.0636 4354.63577021503 639 | 0.0637 4297.020372409118 640 | 0.06380000000000001 4237.115599541452 641 | 0.0639 4288.2524380535015 642 | 0.064 4140.8644726961 643 | 0.0641 4348.4415989353 644 | 0.06420000000000001 4335.12263476469 645 | 0.06430000000000001 4370.65313721719 646 | 0.0644 4413.969667067563 647 | 0.0645 4256.280518843431 648 | 0.0646 4336.745579851359 649 | 0.06470000000000001 4072.8293955996605 650 | 0.0648 4283.4896542438455 651 | 0.0649 4187.560070184912 652 | 0.065 4413.173351863145 653 | 0.0651 4358.445934063135 654 | 0.06520000000000001 4385.831335874721 655 | 0.0653 4396.8801432803775 656 | 0.0654 4386.003456518598 657 | 0.0655 4358.341155840287 658 | 0.0656 4380.125730461552 659 | 0.06570000000000001 4371.612146620372 660 | 0.0658 4377.113851318041 661 | 0.0659 4448.544419348971 662 | 0.066 4365.661732296084 663 | 0.0661 4444.882331147769 664 | 0.06620000000000001 4342.362829587742 665 | 0.0663 4498.489595475742 666 | 0.0664 4442.108746609096 667 | 0.0665 4426.3228389170245 668 | 0.0666 4453.72129237053 669 | 0.06670000000000001 4483.914106221596 670 | 0.0668 4319.718275861127 671 | 0.0669 4418.707412834887 672 | 0.067 4447.741278718322 673 | 0.0671 4448.06441123746 674 | 0.06720000000000001 4452.604793605917 675 | 0.0673 4436.480789411513 676 | 0.0674 4416.486283329358 677 | 0.0675 4439.818715396502 678 | 0.06760000000000001 4424.6002908730225 679 | 0.06770000000000001 4348.610813648979 680 | 0.0678 4383.733090642338 681 | 0.0679 4476.763700565697 682 | 0.068 4504.918717464212 683 | 0.06810000000000001 4494.567616475839 684 | 0.0682 4279.6391107750605 685 | 0.0683 4361.4872904650165 686 | 0.0684 4448.563936577072 687 | 0.0685 4349.919395772509 688 | 0.06860000000000001 4381.33621322401 689 | 0.0687 4404.658119140754 690 | 0.0688 4458.600916839798 691 | 0.0689 4472.029308509907 692 | 0.069 4337.739275012263 693 | 0.06910000000000001 4316.2900802178265 694 | 0.0692 4319.247267944124 695 | 0.0693 4299.176280700283 696 | 0.0694 4313.242187670604 697 | 0.0695 3914.7023721359874 698 | 0.06960000000000001 4041.2780106036535 699 | 0.0697 3983.4216551588074 700 | 0.0698 4377.577565764369 701 | 0.0699 3983.46185372566 702 | 0.07 4301.8272920046065 703 | 0.07010000000000001 4247.995884133651 704 | 0.0702 4275.458144609508 705 | 0.0703 4168.10495833878 706 | 0.0704 4372.7960137317405 707 | 0.07050000000000001 4281.88107490447 708 | 0.07060000000000001 4299.015969105602 709 | 0.0707 4281.761519141105 710 | 0.0708 4303.610630207074 711 | 0.0709 4090.7654548693563 712 | 0.07100000000000001 3938.244928367584 713 | 0.0711 4338.751822574905 714 | 0.0712 4320.8978062661245 715 | 0.0713 4091.0789208391298 716 | 0.0714 4326.6688342080615 717 | 0.07150000000000001 4307.398470479484 718 | 0.0716 4354.038262095482 719 | 0.0717 4329.901649431056 720 | 0.0718 4315.9466775201145 721 | 0.0719 4279.651045786429 722 | 0.07200000000000001 4330.906643994076 723 | 0.0721 4299.2217088324305 724 | 0.0722 4129.063770665588 725 | 0.0723 4292.872014467582 726 | 0.0724 4304.7617676699465 727 | 0.07250000000000001 4298.759899123688 728 | 0.0726 4334.738045131656 729 | 0.0727 4324.111771401802 730 | 0.0728 4311.105138857592 731 | 0.0729 4082.9828166347042 732 | 0.07300000000000001 3713.221951685061 733 | 0.0731 4093.358038461848 734 | 0.0732 4104.17906602313 735 | 0.0733 4183.81846847912 736 | 0.0734 4149.079049279116 737 | 0.07350000000000001 4186.837022930128 738 | 0.0736 4026.3009928989704 739 | 0.0737 3887.6651180823455 740 | 0.0738 4078.924793995537 741 | 0.07390000000000001 4300.118272479565 742 | 0.07400000000000001 4066.9059800515456 743 | 0.0741 4162.530004690432 744 | 0.0742 3955.5152121452356 745 | 0.0743 4071.8133609470888 746 | 0.07440000000000001 4063.531969833715 747 | 0.0745 3698.6334921947823 748 | 0.0746 3329.4927210650603 749 | 0.0747 3242.8036399892167 750 | 0.0748 4219.629946585674 751 | 0.07490000000000001 4345.138420006686 752 | 0.075 4423.961331650912 753 | 0.0751 4389.272547663948 754 | 0.0752 4381.432434780911 755 | 0.0753 4336.354856755708 756 | 0.07540000000000001 4332.3310965453275 757 | 0.0755 4370.10142291845 758 | 0.0756 4347.54766551829 759 | 0.0757 4373.108018606616 760 | 0.0758 4334.697828218257 761 | 0.07590000000000001 4391.537243662286 762 | 0.076 4362.6375831497535 763 | 0.0761 4166.253492450144 764 | 0.0762 3963.3331546030013 765 | 0.0763 4306.400844780236 766 | 0.07640000000000001 4254.14503181281 767 | 0.0765 4249.496932942652 768 | 0.0766 4369.927212832938 769 | 0.0767 4360.812720784527 770 | 0.07680000000000001 4381.767839573434 771 | 0.07690000000000001 4389.32335844186 772 | 0.077 4363.388240283617 773 | 0.0771 4196.225841940175 774 | 0.0772 4143.949510772305 775 | 0.07730000000000001 4408.967494835959 776 | 0.07740000000000001 4330.798864006113 777 | 0.0775 4391.610318188781 778 | 0.0776 4455.874026923631 779 | 0.0777 4489.752490167659 780 | 0.07780000000000001 4455.475317839064 781 | 0.0779 4413.218267151873 782 | 0.078 4381.860087824818 783 | 0.0781 4355.828467445527 784 | 0.0782 4424.189529118501 785 | 0.07830000000000001 4480.356140318202 786 | 0.0784 4354.7365131975885 787 | 0.0785 4199.839845325443 788 | 0.0786 4328.082781478067 789 | 0.0787 4475.9726899624875 790 | 0.07880000000000001 4474.456600767045 791 | 0.0789 4503.23120176102 792 | 0.079 4364.981029679964 793 | 0.0791 4447.176789478931 794 | 0.0792 4425.667043477425 795 | 0.07930000000000001 4504.826455218814 796 | 0.0794 4518.339553736891 797 | 0.0795 4436.4924141347565 798 | 0.0796 4485.001759758016 799 | 0.07970000000000001 4457.989430328549 800 | 0.07980000000000001 4315.157309794916 801 | 0.0799 4326.910188854356 802 | -------------------------------------------------------------------------------- /results/ChooseModel_search_pid_parameter_CurveType_trapezoidal_Height_1000_DumpSystem_False_RunType_train/actor_model.ckpt.data-00000-of-00001: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tinyzqh/control-of-jump-systems-based-on-reinforcement-learning/de23c13aecfb6d8745d6005e32d2083498c4246a/results/ChooseModel_search_pid_parameter_CurveType_trapezoidal_Height_1000_DumpSystem_False_RunType_train/actor_model.ckpt.data-00000-of-00001 -------------------------------------------------------------------------------- /results/ChooseModel_search_pid_parameter_CurveType_trapezoidal_Height_1000_DumpSystem_False_RunType_train/actor_model.ckpt.index: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tinyzqh/control-of-jump-systems-based-on-reinforcement-learning/de23c13aecfb6d8745d6005e32d2083498c4246a/results/ChooseModel_search_pid_parameter_CurveType_trapezoidal_Height_1000_DumpSystem_False_RunType_train/actor_model.ckpt.index -------------------------------------------------------------------------------- /results/ChooseModel_search_pid_parameter_CurveType_trapezoidal_Height_1000_DumpSystem_False_RunType_train/actor_model.ckpt.meta: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tinyzqh/control-of-jump-systems-based-on-reinforcement-learning/de23c13aecfb6d8745d6005e32d2083498c4246a/results/ChooseModel_search_pid_parameter_CurveType_trapezoidal_Height_1000_DumpSystem_False_RunType_train/actor_model.ckpt.meta -------------------------------------------------------------------------------- /results/ChooseModel_search_pid_parameter_CurveType_trapezoidal_Height_1000_DumpSystem_False_RunType_train/checkpoint: -------------------------------------------------------------------------------- 1 | model_checkpoint_path: "/home/hzq/PycharmProjects/control-of-jump-systems-based-on-reinforcement-learning/results/ChooseModel_search_pid_parameter_CurveType_trapezoidal_Height_1000_DumpSystem_False_RunType_train/critic_model.ckpt" 2 | all_model_checkpoint_paths: "/home/hzq/PycharmProjects/control-of-jump-systems-based-on-reinforcement-learning/results/ChooseModel_search_pid_parameter_CurveType_trapezoidal_Height_1000_DumpSystem_False_RunType_train/critic_model.ckpt" 3 | -------------------------------------------------------------------------------- /results/ChooseModel_search_pid_parameter_CurveType_trapezoidal_Height_1000_DumpSystem_False_RunType_train/critic_model.ckpt.data-00000-of-00001: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tinyzqh/control-of-jump-systems-based-on-reinforcement-learning/de23c13aecfb6d8745d6005e32d2083498c4246a/results/ChooseModel_search_pid_parameter_CurveType_trapezoidal_Height_1000_DumpSystem_False_RunType_train/critic_model.ckpt.data-00000-of-00001 -------------------------------------------------------------------------------- /results/ChooseModel_search_pid_parameter_CurveType_trapezoidal_Height_1000_DumpSystem_False_RunType_train/critic_model.ckpt.index: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tinyzqh/control-of-jump-systems-based-on-reinforcement-learning/de23c13aecfb6d8745d6005e32d2083498c4246a/results/ChooseModel_search_pid_parameter_CurveType_trapezoidal_Height_1000_DumpSystem_False_RunType_train/critic_model.ckpt.index -------------------------------------------------------------------------------- /results/ChooseModel_search_pid_parameter_CurveType_trapezoidal_Height_1000_DumpSystem_False_RunType_train/critic_model.ckpt.meta: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tinyzqh/control-of-jump-systems-based-on-reinforcement-learning/de23c13aecfb6d8745d6005e32d2083498c4246a/results/ChooseModel_search_pid_parameter_CurveType_trapezoidal_Height_1000_DumpSystem_False_RunType_train/critic_model.ckpt.meta -------------------------------------------------------------------------------- /results/ChooseModel_search_pid_parameter_CurveType_trapezoidal_Height_1000_DumpSystem_False_RunType_train/reward.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/tinyzqh/control-of-jump-systems-based-on-reinforcement-learning/de23c13aecfb6d8745d6005e32d2083498c4246a/results/ChooseModel_search_pid_parameter_CurveType_trapezoidal_Height_1000_DumpSystem_False_RunType_train/reward.PNG -------------------------------------------------------------------------------- /run.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # @Time : 12/8/21 3:51 PM 3 | # @Author : Zhiqiang He 4 | # @Email : tinyzqh@163.com 5 | # @File : run.py 6 | # @Software: PyCharm 7 | 8 | 9 | import os 10 | import tensorflow as tf 11 | import numpy as np 12 | from parameter import args 13 | from servo_system_env import ServoSystemEnv 14 | from utils import trapezoidal_function 15 | 16 | with tf.name_scope("S"): 17 | S = tf.placeholder(tf.float32, shape=[None, args.state_dim], name="s") 18 | 19 | with tf.name_scope("R"): 20 | R = tf.placeholder(tf.float32, shape=[None, 1], name='r') 21 | 22 | with tf.name_scope("S_"): 23 | S_ = tf.placeholder(tf.float32, shape=[None, args.state_dim], name="s_") 24 | 25 | 26 | class Actor(object): 27 | def __init__(self, sess, action_dim, action_bound): 28 | self.sess = sess 29 | self.a_dim = action_dim 30 | self.a_bound = action_bound 31 | self.lr = args.learning_actor 32 | self.replace_iter = args.replace_iter_a 33 | self.replace_cnt = 0 34 | 35 | with tf.variable_scope("Actor"): 36 | # input s, output a 37 | self.a = self._build_net(S, scope="eval_net", trainable=True) 38 | 39 | # input s_, output a, get a_ for critic 40 | self.a_ = self._build_net(S, scope="target_net", trainable=True) 41 | 42 | self.e_params = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope="Actor/eval_net") 43 | self.t_params = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope="Actor/target_net") 44 | 45 | self.saver = tf.train.Saver() 46 | 47 | def _build_net(self, input, scope, trainable): 48 | 49 | with tf.variable_scope(scope): 50 | init_w = tf.contrib.layers.xavier_initializer() 51 | init_b = tf.constant_initializer(0.001) 52 | 53 | net1 = tf.layers.dense(inputs=input, units=200, activation=tf.nn.relu6, kernel_initializer=init_w, 54 | kernel_regularizer=tf.contrib.layers.l2_regularizer(scale=args.reg), 55 | bias_initializer=init_b, name="l1", trainable=trainable) 56 | 57 | net2 = tf.layers.dense(inputs=net1, units=200, activation=tf.nn.relu6, kernel_initializer=init_w, 58 | kernel_regularizer=tf.contrib.layers.l2_regularizer(scale=args.reg), 59 | bias_initializer=init_b, name="l2", trainable=trainable) 60 | 61 | net3 = tf.layers.dense(inputs=net2, units=10, activation=tf.nn.relu6, kernel_initializer=init_w, 62 | kernel_regularizer=tf.contrib.layers.l2_regularizer(scale=args.reg), 63 | bias_initializer=init_b, name="l3", trainable=trainable) 64 | 65 | with tf.variable_scope("a"): 66 | actions = tf.layers.dense(inputs=net3, units=self.a_dim, activation=tf.nn.tanh, 67 | kernel_initializer=init_w, kernel_regularizer=tf.contrib.layers.l2_regularizer(scale=args.reg), 68 | name="a", trainable=trainable) 69 | 70 | # Scale output to -action_bound to action_bound. 71 | scaled_a = tf.multiply(actions, self.a_bound, name="scaled_action") 72 | 73 | return scaled_a 74 | 75 | def learn(self, s): 76 | """ 77 | Batch Update 78 | :param s: 79 | :return: 80 | """ 81 | self.sess.run(self.train_op, feed_dict={S: s}) 82 | 83 | if self.replace_cnt % self.replace_iter == 0: 84 | self.sess.run([tf.assign(t, e) for t, e in zip(self.t_params, self.e_params)]) 85 | 86 | self.replace_cnt += 1 87 | 88 | def choose_action(self, s): 89 | s = s[np.newaxis, :] 90 | return self.sess.run(self.a, feed_dict={S: s})[0] # single action 91 | 92 | def add_grad_to_graph(self, a_grads): 93 | with tf.variable_scope("policy_grads"): 94 | # ys = policy; 95 | # xs = policy's parameters; 96 | # a_grads = the gradients of the policy to get more Q; 97 | # tf.gradients will calculate dys/dxs with a initial gradients for ys, so this is dq/da * da / dparams 98 | self.policy_grads = tf.gradients(ys=self.a, xs=self.e_params, grad_ys=a_grads) 99 | 100 | with tf.variable_scope("A_train"): 101 | opt = tf.train.RMSPropOptimizer(-self.lr) # (- learning rate) for ascent policy 102 | self.train_op = opt.apply_gradients(zip(self.policy_grads, self.e_params)) 103 | 104 | 105 | class Critic(object): 106 | def __init__(self, sess, a, a_, action_dim): 107 | self.sess = sess 108 | self.s_dim = args.state_dim 109 | self.a_dim = action_dim 110 | self.lr = args.learning_critic 111 | self.gamma = args.gamma 112 | 113 | self.replace_iter = args.replace_iter_c 114 | self.replace_cnt = 0 115 | 116 | with tf.variable_scope("Critic"): 117 | # Input (s, a), output q 118 | self.a = a 119 | self.q = self._build_net(S, self.a, "eval_net", trainable=True) 120 | 121 | # Input (s_, a_), output q_ for target 122 | self.q_ = self._build_net(S_, a_, "target_net", trainable=True) 123 | 124 | self.e_params = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope="Critic/eval_net") 125 | self.t_params = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, scope="Critic/target_net") 126 | 127 | with tf.variable_scope("target_q"): 128 | self.target_q = R + self.gamma * self.q_ 129 | 130 | with tf.variable_scope("TD_error"): 131 | self.loss = tf.reduce_mean(tf.squared_difference(self.target_q, self.q)) 132 | 133 | with tf.variable_scope("C_train"): 134 | self.train_op = tf.train.RMSPropOptimizer(self.lr).minimize(self.loss) 135 | 136 | with tf.variable_scope("a_grad"): 137 | self.a_grads = tf.gradients(self.q, self.a)[0] 138 | 139 | self.saver = tf.train.Saver() 140 | 141 | def _build_net(self, s, a, scope, trainable): 142 | with tf.variable_scope(scope): 143 | init_w = tf.contrib.layers.xavier_initializer() 144 | init_b = tf.constant_initializer(0.01) 145 | 146 | with tf.variable_scope('l1'): 147 | n_l1 = 200 148 | w1_s = tf.get_variable("w1_s", [self.s_dim, n_l1], initializer=init_w, trainable=trainable) 149 | w1_a = tf.get_variable("w1_a", [self.a_dim, n_l1], initializer=init_w, trainable=trainable) 150 | b1 = tf.get_variable("b1", [1, n_l1], initializer=init_b, trainable=trainable) 151 | net = tf.nn.relu6(tf.matmul(s, w1_s) + tf.matmul(a, w1_a) + b1) 152 | 153 | net1 = tf.layers.dense(net, 200, activation=tf.nn.relu6, kernel_initializer=init_w, 154 | kernel_regularizer=tf.contrib.layers.l2_regularizer(scale=args.reg), 155 | bias_initializer=init_b, name="l2", trainable=trainable) 156 | 157 | net2 = tf.layers.dense(net1, 10, activation=tf.nn.relu6, kernel_initializer=init_w, 158 | kernel_regularizer=tf.contrib.layers.l2_regularizer(scale=args.reg), 159 | bias_initializer=init_b, name="l3", trainable=trainable) 160 | 161 | with tf.variable_scope("q"): 162 | q = tf.layers.dense(net2, 1, kernel_initializer=init_w, 163 | kernel_regularizer=tf.contrib.layers.l2_regularizer(scale=args.reg), 164 | bias_initializer=init_b, trainable=trainable) 165 | 166 | return q 167 | 168 | def learn(self, s, a, r, s_): 169 | self.sess.run(self.train_op, feed_dict={S:s, self.a:a, R:r, S_:s_}) 170 | 171 | if self.replace_cnt % self.replace_iter == 0: 172 | self.sess.run([tf.assign(t, e) for t, e in zip(self.t_params, self.e_params)]) 173 | 174 | self.replace_cnt += 1 175 | 176 | 177 | class Memory(object): 178 | def __init__(self, capacity, dims): 179 | self.capacity = capacity 180 | self.data = np.zeros(shape=(capacity, dims)) 181 | self.cnt = 0 182 | 183 | def store_transition(self, s, a, r, s_): 184 | transition = np.hstack((s, a, [r], s_)) 185 | index = self.cnt % self.capacity 186 | self.data[index, :] = transition 187 | self.cnt += 1 188 | 189 | def sample(self, n): 190 | assert self.cnt >= self.capacity, print("Memory has not been fulfilled.") 191 | indices = np.random.choice(self.capacity, size=n) 192 | return self.data[indices, :] 193 | 194 | 195 | class run_base(object): 196 | def __init__(self): 197 | current_dir = os.path.dirname(os.path.abspath(__file__)) 198 | 199 | if args.run_type == "train": 200 | result_dir = os.path.join('results', 'ChooseModel_{}_CurveType_{}_Height_{}_DumpSystem_{}_RunType_{}'.format(args.choose_model, args.curve_type, args.height, args.use_jump_system, args.run_type)) 201 | self.result_dir = os.path.join(current_dir, result_dir) 202 | else: 203 | # Need to load the train model. 204 | result_dir = os.path.join('results', 'ChooseModel_{}_CurveType_{}_Height_{}_DumpSystem_{}_RunType_{}'.format(args.choose_model, args.curve_type, args.height, args.use_jump_system, "train")) 205 | self.result_dir = os.path.join(current_dir, result_dir) 206 | 207 | os.makedirs(self.result_dir, exist_ok=True) 208 | 209 | if args.choose_model == "search_pid_parameter": 210 | self.max_episodes = 800 211 | self.action_dim = args.pid_parameter_search_action_dim 212 | self.action_bound = args.pid_parameter_search_action_bound 213 | self.var = 10 214 | 215 | tf.set_random_seed(args.seed) 216 | self.sess = tf.Session() 217 | 218 | self.actor = Actor(sess=self.sess, action_dim=self.action_dim, action_bound=self.action_bound) 219 | self.critic = Critic(sess=self.sess, a=self.actor.a, a_=self.actor.a_, action_dim=self.action_dim) 220 | 221 | self.actor.add_grad_to_graph(self.critic.a_grads) 222 | 223 | self.M = Memory(capacity=args.memory_capacity, dims=2 * args.state_dim + self.action_dim + 1) 224 | 225 | self.sess.run(tf.global_variables_initializer()) 226 | 227 | elif args.choose_model == "search_electric": 228 | 229 | if args.curve_type == "trapezoidal": 230 | self.max_episodes = 800 231 | elif args.curve_type == "sine": 232 | self.max_episodes = 4000 233 | else: 234 | pass 235 | 236 | self.action_dim = args.electric_compensation_action_dim 237 | self.action_bound = args.electric_compensation_action_bound 238 | self.var = 2 239 | 240 | tf.set_random_seed(args.seed) 241 | self.sess = tf.Session() 242 | 243 | self.actor = Actor(sess=self.sess, action_dim=self.action_dim, action_bound=self.action_bound) 244 | self.critic = Critic(sess=self.sess, a=self.actor.a, a_=self.actor.a_, action_dim=self.action_dim) 245 | 246 | self.actor.add_grad_to_graph(self.critic.a_grads) 247 | 248 | self.M = Memory(capacity=args.memory_capacity, dims=2 * args.state_dim + self.action_dim + 1) 249 | 250 | self.sess.run(tf.global_variables_initializer()) 251 | 252 | else: 253 | pass 254 | 255 | self.var_min = 0.01 256 | self.env = ServoSystemEnv() 257 | 258 | def train(self, times): 259 | pass 260 | 261 | def test(self, times): 262 | pass 263 | 264 | def load_model(self): 265 | 266 | self.actor.saver.restore(self.actor.sess, os.path.join(self.result_dir, "actor_model.ckpt")) 267 | self.critic.saver.restore(self.critic.sess, os.path.join(self.result_dir, "critic_model.ckpt")) 268 | 269 | print("load_model_successful.") 270 | 271 | def save_model(self): 272 | 273 | if not os.path.exists(self.result_dir): 274 | os.makedirs(self.result_dir) 275 | 276 | self.actor.saver.save(self.actor.sess, os.path.join(self.result_dir, "actor_model.ckpt")) 277 | self.critic.saver.save(self.critic.sess, os.path.join(self.result_dir, "critic_model.ckpt")) 278 | 279 | print("save_model_successful.") 280 | 281 | def save_result_txt(self, xs, ys, yaxis='radValues'): 282 | """ 283 | # Save the data to .txt file. 284 | :param xs: 285 | :param ys: 286 | :param yaxis: 287 | :return: 288 | """ 289 | 290 | filename = os.path.join(self.result_dir, yaxis + '.txt') 291 | if isinstance(xs, np.ndarray) and isinstance(ys, list): 292 | if not os.path.exists(filename): 293 | with open(file=filename, mode="a+") as file: 294 | file.write("times {}".format(yaxis)) 295 | file.write("\r\n") 296 | else: 297 | print("{} has already existed. added will be doing.".format(filename)) 298 | with open(file=filename, mode="a+") as file: 299 | file.write("times, {}".format(yaxis)) 300 | file.write("\r\n") 301 | else: 302 | pass 303 | 304 | with open(file=filename, mode="a+") as file: 305 | for index, data in enumerate(zip(xs, ys)): 306 | file.write("{} {}".format(str(data[0]), str(data[1]))) 307 | file.write("\r\n") 308 | 309 | 310 | class run_class_pid(run_base): 311 | def __init__(self): 312 | super().__init__() 313 | 314 | def test(self, times): 315 | radValues = [] 316 | iaeValues = [] 317 | ecValues = [] 318 | 319 | Episode_reward = 0 320 | 321 | for sim_time in times: 322 | 323 | error = trapezoidal_function(time=sim_time) - self.env.rad_cur 324 | 325 | ec_cur = self.env.ec_last 326 | 327 | state, reward, done = self.env.step(action=0) 328 | 329 | # append the value 330 | radValues.append(self.env.rad_cur) 331 | iaeValues.append(abs(error) if len(iaeValues) == 0 else abs(error) + iaeValues[-1]) 332 | ecValues.append(ec_cur) 333 | 334 | Episode_reward += reward 335 | 336 | self.save_result_txt(xs=times, ys=radValues, yaxis="radValues") 337 | self.save_result_txt(xs=times, ys=iaeValues, yaxis="iaeValues") 338 | self.save_result_txt(xs=times, ys=ecValues, yaxis="ecValues") 339 | 340 | 341 | class run_search_pid_parameter(run_base): 342 | def __init__(self): 343 | super().__init__() 344 | self.action_dim = args.pid_parameter_search_action_dim 345 | self.action_bound = args.pid_parameter_search_action_bound 346 | 347 | def train(self, times): 348 | steps = [] 349 | epRewards = [] 350 | maxEpReward = 4000.0 # set the base max reward. 351 | 352 | for i in range(self.max_episodes): 353 | s = self.env.reset() 354 | ep_reward = 0 355 | 356 | for j, sim_time in enumerate(times): 357 | 358 | # Add exploration noise 359 | a = self.actor.choose_action(s) 360 | a = np.clip(np.random.normal(a, self.var), -self.action_bound, self.action_bound) 361 | 362 | s_, r, done = self.env.step(a) 363 | 364 | self.M.store_transition(s, a, r, s_) 365 | 366 | if self.M.cnt > args.memory_capacity: 367 | self.var = max([self.var * 0.999995, self.var_min]) # decay the action randomness 368 | b_M = self.M.sample(args.batch_size) 369 | 370 | b_s = b_M[:, :args.state_dim] 371 | b_a = b_M[:, args.state_dim: args.state_dim + self.action_dim] 372 | b_r = b_M[:, - args.state_dim - 1: - args.state_dim] 373 | b_s_ = b_M[:, - args.state_dim:] 374 | 375 | self.critic.learn(b_s, b_a, b_r, b_s_) 376 | self.actor.learn(b_s) 377 | 378 | s = s_ 379 | ep_reward += r 380 | 381 | if sim_time == times[-1]: 382 | print("Episode: {}, Reward {}, Explore {} Steps {}".format(i, ep_reward, self.var, j)) 383 | epRewards.append(ep_reward) 384 | steps.append(i) 385 | 386 | if ep_reward >= maxEpReward: # Save The Max Reward Model. 387 | print("Get More Episode Reward {}".format(maxEpReward)) 388 | maxEpReward = ep_reward 389 | self.save_model() 390 | 391 | self.save_result_txt(xs=steps, ys=epRewards, yaxis="epRewards") 392 | 393 | 394 | class run_search_compensation_electric(run_base): 395 | def __init__(self): 396 | super(run_search_compensation_electric, self).__init__() 397 | 398 | def train(self, times): 399 | steps = [] 400 | epRewards = [] 401 | maxEpReward = 4000.0 # set the base max reward. 402 | 403 | for i in range(self.max_episodes): 404 | s = self.env.reset() 405 | ep_reward = 0 406 | 407 | for j, sim_time in enumerate(times): 408 | 409 | # Add exploration noise 410 | a = self.actor.choose_action(s) 411 | a = np.clip(np.random.normal(a, self.var), -self.action_bound, self.action_bound) 412 | 413 | s_, r, done = self.env.step(a) 414 | 415 | self.M.store_transition(s, a, r, s_) 416 | 417 | if self.M.cnt > args.memory_capacity: 418 | self.var = max([self.var * 0.999995, self.var_min]) # decay the action randomness 419 | b_M = self.M.sample(args.batch_size) 420 | 421 | b_s = b_M[:, :args.state_dim] 422 | b_a = b_M[:, args.state_dim: args.state_dim + self.action_dim] 423 | b_r = b_M[:, - args.state_dim - 1: - args.state_dim] 424 | b_s_ = b_M[:, - args.state_dim:] 425 | 426 | self.critic.learn(b_s, b_a, b_r, b_s_) 427 | self.actor.learn(b_s) 428 | 429 | s = s_ 430 | ep_reward += r 431 | 432 | if sim_time == times[-1]: 433 | print("Episode: {}, Reward {}, Explore {} Steps {}".format(i, ep_reward, self.var, j)) 434 | epRewards.append(ep_reward) 435 | steps.append(i) 436 | 437 | if ep_reward >= maxEpReward: # Save The Max Reward Model. 438 | print("Get More Episode Reward {}".format(maxEpReward)) 439 | maxEpReward = ep_reward 440 | self.save_model() 441 | 442 | self.save_result_txt(xs=steps, ys=epRewards, yaxis="epRewards") 443 | 444 | def test(self, times): 445 | 446 | self.load_model() 447 | 448 | s = self.env.reset() 449 | ep_reward = 0 450 | 451 | radValues = [] 452 | iaeValues = [] 453 | ecValues = [] 454 | 455 | for j, sim_time in enumerate(times): 456 | 457 | error = trapezoidal_function(time=sim_time) - self.env.rad_cur 458 | 459 | # Add exploration noise 460 | a = self.actor.choose_action(s) 461 | a = np.clip(np.random.normal(a, self.var), -self.action_bound, self.action_bound) 462 | 463 | ec_cur = self.env.ec_last 464 | 465 | s_, r, done = self.env.step(a) 466 | 467 | # append the value 468 | radValues.append(self.env.rad_cur) 469 | iaeValues.append(abs(error) if len(iaeValues) == 0 else abs(error) + iaeValues[-1]) 470 | ecValues.append(ec_cur) 471 | 472 | s = s_ 473 | ep_reward += r 474 | 475 | self.save_result_txt(xs=times, ys=radValues, yaxis="radValues") 476 | self.save_result_txt(xs=times, ys=iaeValues, yaxis="iaeValues") 477 | self.save_result_txt(xs=times, ys=ecValues, yaxis="ecValues") 478 | 479 | def main(): 480 | times = np.arange(args.start_time, args.end_time, args.dt)[:args.max_ep_steps] 481 | 482 | if args.run_type == "train": 483 | if args.choose_model == "search_pid_parameter": 484 | run_search_pid_parameter().train(times=times) 485 | elif args.choose_model == "search_electric": 486 | run_search_compensation_electric().train(times=times) 487 | elif args.choose_model == "class_pid": 488 | run_class_pid().test(times=times) 489 | else: 490 | print("Please Input The Right Choose Model.") 491 | else: 492 | if args.choose_model == "search_pid_parameter": 493 | run_search_pid_parameter().test(times=times) 494 | elif args.choose_model == "search_electric": 495 | run_search_compensation_electric().test(times=times) 496 | elif args.choose_model == "class_pid": 497 | run_class_pid().test(times=times) 498 | else: 499 | print("Please Input The Right Choose Model.") 500 | 501 | 502 | if __name__ == "__main__": 503 | main() -------------------------------------------------------------------------------- /servo_system_env.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # @Time : 11/30/21 2:34 PM 3 | # @Author : Zhiqiang He 4 | # @Email : tinyzqh@163.com 5 | # @File : servo_system_env.py 6 | # @Software: PyCharm 7 | 8 | 9 | from parameter import args 10 | from utils import trapezoidal_function 11 | from PID import PID 12 | import numpy as np 13 | import gym 14 | 15 | 16 | class ServoSystemEnv(gym.Env): 17 | def __init__(self): 18 | super(ServoSystemEnv, self).__init__() 19 | 20 | self.rad_last = 0.0 # Speed at the previous moment 21 | self.rad_cur = 0.0 # Speed at the moment 22 | self.rad_next = 0.0 # Speed at the next moment 23 | self.rad_next_next = 0.0 # Speed at the next next moment 24 | self.error_last = 0.0 # The error at the previous moment 25 | self.error_cur = 0.0 # The error at the moment 26 | 27 | self.ec_last = 0.0 # electric current at the previous moment 28 | self.cnt = 0 # used to count the step number 29 | 30 | self.rad_threshold = 9000 31 | 32 | self.times = np.arange(args.start_time, args.end_time, args.dt) # generate the time steps. 33 | np.random.seed(args.seed) 34 | self.SystemsList = np.random.randint(0, 3, args.change_times) # Generate the random systems id. 35 | self.SystemsLists = [i for i in self.SystemsList for j in range(int(len(self.times)/args.change_times) + 1)] # extended system id. 36 | 37 | self.state = np.array([self.rad_last, self.rad_cur, self.rad_next, self.rad_next_next, self.error_last, self.error_cur]) 38 | 39 | 40 | if args.choose_model == "search_pid_parameter": 41 | self.pid = PID(kp=args.kp, ki=args.ki, kd=args.kd, args=args) 42 | else: 43 | self.pid = PID(kp=args.kp_rl, ki=args.ki_rl, kd=args.kd_rl, args=args) 44 | 45 | def update_rad(self, electric): 46 | """ 47 | Update the rad value Based on different Parameters. 48 | :param electric: 49 | :return: 50 | """ 51 | if args.use_jump_system: 52 | if self.SystemsLists[self.cnt] == 0: 53 | rad = self._system1(ec_cur = electric) 54 | elif self.SystemsLists[self.cnt] == 1: 55 | rad = self._system2(ec_cur = electric) 56 | else: 57 | assert self.SystemsLists[self.cnt] == 2, print("the system id of SystemsLists is out of range.") 58 | rad = self._system3(ec_cur = electric) 59 | else: 60 | if args.used_system_id == 1: 61 | rad = self._system1(ec_cur = electric) 62 | elif args.used_system_id == 2: 63 | rad = self._system2(ec_cur = electric) 64 | else: 65 | assert args.used_system_id == 3, print("the selected system id is illegal") 66 | rad = self._system2(ec_cur = electric) 67 | 68 | return rad 69 | 70 | def step(self, action): 71 | """ 72 | :param action: electric of current input. 73 | :return: 74 | """ 75 | 76 | rad_last, rad_cur, rad_next, rad_next_next, error_last, error_cur = self.state 77 | 78 | error = trapezoidal_function(time=self.times[self.cnt]) - rad_cur 79 | 80 | if args.choose_model == "search_pid_parameter": 81 | kp, ki = action 82 | kp, ki = abs(kp), abs(ki) 83 | electric = self.pid.update_with_parameter(error=error, kp=kp, ki=ki, kd=0.0) 84 | else: 85 | electric = self.pid.update(error=error) + float(action) 86 | 87 | rad = self.update_rad(electric=electric) 88 | 89 | # Update the state. 90 | self.rad_last = self.rad_cur 91 | self.rad_cur = rad 92 | self.rad_next = trapezoidal_function(time = self.times[self.cnt + 1]) 93 | self.rad_next_next = trapezoidal_function(time = self.times[self.cnt + 2]) 94 | self.error_last = self.error_cur 95 | self.error_cur = error 96 | 97 | self.ec_last = electric 98 | self.cnt += 1 99 | 100 | if args.choose_model == "search_pid_parameter": 101 | reward = self._get_search_parameter_immediate_reward(error=error, electric=electric) 102 | else: 103 | reward = self._get_eletric_immediate_reward(error=error, electric=electric) 104 | 105 | done = self._get_termination() 106 | 107 | self.state = np.array([self.rad_last, self.rad_cur, self.rad_next, self.rad_next_next, self.error_last, self.error_cur]) 108 | 109 | return self.state, reward, done 110 | 111 | def _get_eletric_immediate_reward(self, error, electric): 112 | """ 113 | Design the Reward Function of search immediate reward. 114 | :param error: 115 | :param electric: 116 | :return: 117 | """ 118 | 119 | error = min(error, 10000) 120 | 121 | mu = abs(error) 122 | 123 | sigma = 2 124 | 125 | reward = 0.9 * np.exp(-1 / 2 * (mu * mu) / (sigma * sigma)) + 0.1 * np.exp(-abs(electric)) 126 | 127 | if np.isnan(reward): reward = 0 # error == inf -> nan -> reward == nan 128 | 129 | return reward 130 | 131 | def _get_search_parameter_immediate_reward(self, error, electric): 132 | """ 133 | Design the Reward Function of search PID parameter. 134 | :param error: 135 | :param electric: 136 | :return: 137 | """ 138 | 139 | error = min(error, 10000) 140 | 141 | mu = 0.9 * abs(error) + 0.1 * abs(electric) 142 | 143 | sigma = 10 144 | 145 | reward = np.exp(-1 / 2 * (mu * mu) / (sigma * sigma)) 146 | 147 | if np.isnan(reward): reward = 0 # error == inf -> nan -> reward == nan 148 | 149 | return reward 150 | 151 | def _get_termination(self): 152 | """ 153 | Set the End Conditions. 154 | :return: 155 | """ 156 | 157 | done = abs(self.rad_cur) >= self.rad_threshold or abs(self.error_cur) >= args.height * 0.08 158 | 159 | return bool(done) 160 | 161 | def reset(self): 162 | """ 163 | # 164 | :return: 165 | """ 166 | 167 | self.rad_last = 0.0 # Speed at the previous moment 168 | self.rad_cur = 0.0 # Speed at the moment 169 | self.rad_next = 0.0 # Speed at the next moment 170 | self.rad_next_next = 0.0 # Speed at the next next moment 171 | self.error_last = 0.0 # The error at the previous moment 172 | self.error_cur = 0.0 # The error at the moment 173 | 174 | self.ec_last = 0.0 # electric current at the previous moment 175 | 176 | if args.choose_model == "search_pid_parameter": 177 | self.pid = PID(kp=args.kp, ki=args.ki, kd=args.kd, args=args) 178 | else: 179 | self.pid = PID(kp=args.kp_rl, ki=args.ki_rl, kd=args.kd_rl, args=args) 180 | 181 | self.state = np.array([self.rad_last, self.rad_cur, self.rad_next, self.rad_next_next, self.error_last, self.error_cur]) 182 | self.cnt = 0 183 | return self.state 184 | 185 | def _system1(self, ec_cur): 186 | """ 187 | # system equation of system 1. 188 | :param ec_cur: 189 | :return: 190 | """ 191 | return self.rad_cur - 3.478 * 0.0001 * self.rad_last + 1.388 * ec_cur + 0.1986 * self.ec_last + 0.1 * np.random.normal(0, 1) 192 | 193 | def _system2(self, ec_cur): 194 | """ 195 | # system equation of system 2. 196 | :param ec_cur: 197 | :return: 198 | """ 199 | return self.rad_cur - 3.366 * 0.0001 * self.rad_last + 0.1263 * ec_cur + 0.01799 * self.ec_last + 0.1 * np.random.normal(0, 1) 200 | 201 | def _system3(self, ec_cur): 202 | """ 203 | # system equation of system 3. 204 | :param ec_cur: 205 | :return: 206 | """ 207 | return self.rad_cur - 3.478 * 0.0001 * self.rad_last + 1.388 * ec_cur + 0.1986 * self.ec_last + 0.1 * np.random.normal(0, 1) + 0.9148 208 | 209 | if __name__ == "__main__": 210 | env = ServoSystemEnv() 211 | x = np.linspace(-10, 10, 100) 212 | y = [env._get_eletric_immediate_reward(error=i, electric=0) for i in x] 213 | import matplotlib.pyplot as plt 214 | plt.plot(x, y) 215 | plt.show() 216 | -------------------------------------------------------------------------------- /utils.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | # @Time : 11/30/21 3:21 PM 3 | # @Author : Zhiqiang He 4 | # @Email : tinyzqh@163.com 5 | # @File : utils.py 6 | # @Software: PyCharm 7 | 8 | from parameter import args 9 | 10 | def trapezoidal_function(time): 11 | """ 12 | Get the value of the trace curve at the time moment. 13 | Total simulation time is 0.5s. 14 | :param time: 15 | :return: 16 | """ 17 | height = args.height 18 | if time <= 0.075: 19 | return (height / 0.075) * time 20 | elif 0.075 < time and time <= 0.375: 21 | return height 22 | elif 0.375 < time and time <= 0.45: 23 | return (-height / 0.075) * time + 6 * height 24 | elif 0.45 < time and time <= 0.5: 25 | return 0 26 | 27 | if __name__ == "__main__": 28 | pass --------------------------------------------------------------------------------