├── Final Weights ├── GTO1_1stnet.zip ├── GTO1_2ndnet.zip ├── GTO2_1stnet.zip ├── GTO2_2ndnet.zip ├── GTO3_1stnet.zip ├── GTO3_2ndnet.zip ├── SuperGTO_1stnet.zip ├── SuperGTO_2ndnet.zip └── inf ├── Mat_env.m ├── Model_training_logs └── a ├── Model_training_weights └── a ├── Paper_output ├── GTO-GEO3dpic.PNG └── superGTO-GEO3dpic.PNG ├── README.md ├── Spacecraft_Env.py ├── __pycache__ ├── Spacecraft_Env.cpython-37.pyc ├── config.cpython-37.pyc ├── enviornment.cpython-37.pyc ├── inf └── scenerios.cpython-37.pyc ├── chkStop.m ├── config.py ├── csv_files ├── Rtimeinput.dat ├── csvlist.dat └── csvlistinitialize.dat ├── earth.png ├── enviornment.py ├── environment.yml ├── environment_info.txt ├── model_1_50.mat ├── paper-outputs ├── GTO-GEO.gif ├── GTO-GEO3dpic.PNG ├── GTO-GEO3dpic.png ├── GTO-GEO_fast (2).mp4 ├── GTO-GEO_fast.mov ├── Orbital_Rotation_2 .png ├── Super GTO-GEO_fast.mp4 ├── SuperGTO-GEO.gif ├── SuperGTO-GEO_fast.mov ├── a ├── a_e.png ├── fig10.PNG ├── fig11.PNG ├── fig12.PNG ├── fig13.PNG ├── fig5.PNG ├── fig6.PNG ├── fig7.PNG ├── fig8.PNG ├── superGTO-GEO3dpic.PNG ├── superGTO-GEO3dpic.png ├── tab3.PNG └── tab6.PNG ├── plots └── a ├── readme.txt ├── satellite.png ├── scenerios.py ├── spacecraftEnivironment.m ├── test.py ├── test.sh └── train.py /Final Weights/GTO1_1stnet.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/talhazaidi13/Cascaded-Deep-Reinforcement-Learning-Based-Multi-Revolution-Low-Thrust-Spacecraft-Orbit-Transfer/289ca90a2726808dc2613b27a37cf0bd0deeed90/Final Weights/GTO1_1stnet.zip -------------------------------------------------------------------------------- /Final Weights/GTO1_2ndnet.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/talhazaidi13/Cascaded-Deep-Reinforcement-Learning-Based-Multi-Revolution-Low-Thrust-Spacecraft-Orbit-Transfer/289ca90a2726808dc2613b27a37cf0bd0deeed90/Final Weights/GTO1_2ndnet.zip -------------------------------------------------------------------------------- /Final Weights/GTO2_1stnet.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/talhazaidi13/Cascaded-Deep-Reinforcement-Learning-Based-Multi-Revolution-Low-Thrust-Spacecraft-Orbit-Transfer/289ca90a2726808dc2613b27a37cf0bd0deeed90/Final Weights/GTO2_1stnet.zip -------------------------------------------------------------------------------- /Final Weights/GTO2_2ndnet.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/talhazaidi13/Cascaded-Deep-Reinforcement-Learning-Based-Multi-Revolution-Low-Thrust-Spacecraft-Orbit-Transfer/289ca90a2726808dc2613b27a37cf0bd0deeed90/Final Weights/GTO2_2ndnet.zip -------------------------------------------------------------------------------- /Final Weights/GTO3_1stnet.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/talhazaidi13/Cascaded-Deep-Reinforcement-Learning-Based-Multi-Revolution-Low-Thrust-Spacecraft-Orbit-Transfer/289ca90a2726808dc2613b27a37cf0bd0deeed90/Final Weights/GTO3_1stnet.zip -------------------------------------------------------------------------------- /Final Weights/GTO3_2ndnet.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/talhazaidi13/Cascaded-Deep-Reinforcement-Learning-Based-Multi-Revolution-Low-Thrust-Spacecraft-Orbit-Transfer/289ca90a2726808dc2613b27a37cf0bd0deeed90/Final Weights/GTO3_2ndnet.zip -------------------------------------------------------------------------------- /Final Weights/SuperGTO_1stnet.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/talhazaidi13/Cascaded-Deep-Reinforcement-Learning-Based-Multi-Revolution-Low-Thrust-Spacecraft-Orbit-Transfer/289ca90a2726808dc2613b27a37cf0bd0deeed90/Final Weights/SuperGTO_1stnet.zip -------------------------------------------------------------------------------- /Final Weights/SuperGTO_2ndnet.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/talhazaidi13/Cascaded-Deep-Reinforcement-Learning-Based-Multi-Revolution-Low-Thrust-Spacecraft-Orbit-Transfer/289ca90a2726808dc2613b27a37cf0bd0deeed90/Final Weights/SuperGTO_2ndnet.zip -------------------------------------------------------------------------------- /Final Weights/inf: -------------------------------------------------------------------------------- 1 | Final weights for all scenerios 2 | -------------------------------------------------------------------------------- /Mat_env.m: -------------------------------------------------------------------------------- 1 | 2 | classdef Mat_env < handle 3 | 4 | properties (SetAccess = private) 5 | 6 | %%%%%%% Computing inputs for the testcase 7 | mu=0; % Gravitational parameter 8 | a=0; % Semi-major axis of initial orbit in km 9 | e0=0; % eccentricity of initial orbit 10 | inc =0; %Inclination of initial orbit in Deg 11 | rp=0; % Perigee radius of initial orbit 12 | ra=0;% Apogee radius of initial orbit 13 | vp=0; % Velocity at the perigee of initial orbit 14 | rc =[]; % Position vector of spacecraft at perigee of the initial orbit in the body fixed frame (3-1-3 rotation) 15 | v=[]; %Velocity vector of spacecraft at perigee of the initial orbit (3-1-3 rotation) 16 | RRR=[]; % Rotation matrix considering RAAN=0 deg and argument of latitude=0 17 | r=[]; % Position vector of spacecraft at perigee of the initial orbit in the Inertial frame 18 | rv = []; 19 | Param =[]; 20 | h0 =0; 21 | hx0 = 0; 22 | hy0 = 0; 23 | ex0 = 0; 24 | ey0 = 0; 25 | state0 = []; % Converting the inertial position and velocity vectors into the suwat's dynamical coordinates 26 | state = []; %[h;hx;hy;ex;ey;phi;time] where m=m0-time*m_dot 27 | %%%%%%%%%%%%%%%%%%%%%%%%%%%% 28 | %The above computution is just required once to convert initial states into 29 | %dynamic coordinates 30 | %%%%%%%%%%%%%%%%%%%%%%%%%%%% 31 | h_limit=0; 32 | F=0; %%0.3115/1000; %in kilo Newtons 33 | I_sp = 0; % in sec 34 | m0=0; % kg 35 | alpha=0; % in radians 36 | beta=0;% in radians 37 | segment=0; % segment length (angle) after which the end states are compouted in radians 38 | 39 | % non-dimensional values for reference 40 | DU = 0; %distance unit, Km 41 | TU = 0; %time unit, s 42 | SU = 0; %speed unit, Km/sec 43 | MU =0; %mass Unit, Kg 44 | HU=0; % angular momentum 45 | FU = 0; %force unit(K-N) 46 | 47 | g0=0; % m/sec^2 48 | m_dot = 0; % in kg/sec 49 | 50 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 51 | % %Terminal desired state values for reference 52 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 53 | % h(end)=129640.2292 %km^2/sec 54 | % hx(end)=0; 55 | % hy(end)=0; 56 | % ex(end)=0; 57 | % ey(end)=0; 58 | end 59 | 60 | methods 61 | function obj = Mat_env () % constructor 62 | 63 | obj.mu=398600.4418; % Gravitational parameter 64 | a=41145.4922; % Semi-major axis of initial orbit in km # maxGEO=42164 initialGTO=24364 /// 40000 65 | e0=0.0071; % eccentricity of initial orbit # maxGEO=0 initialGTO=0.7306 //0.2 66 | inc=4.9085; %Inclination of initial orbit in Deg # # maxGEO=0 initialGTO=28.5 //10 67 | obj.ra=a*(1+e0); 68 | obj.rp=a*(1-e0); 69 | a = (obj.rp+obj.ra)/2; 70 | ex0 = (obj.ra/a-1); 71 | ey0 = 0; 72 | h0 = obj.mu*sqrt(a*(1-ex0^2)); 73 | hx0 = -sin(inc/180*pi)*h0; 74 | hy0 = 0; 75 | m0 = 1; 76 | 77 | state =[h0;hx0;hy0;ex0;ey0;0;0;0]; %[h;hx;hy;ex;ey;phi;time;fuel burnt] 78 | %[h;hx;hy;ex;ey;phi;time] where m=m0-time*m_dot 79 | %%%%%%%%%%%%%%%%%%%%%%%%%%%% 80 | %The above computution is just required once to convert initial states into 81 | %dynamic coordinates 82 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%% 83 | 84 | format long 85 | obj.h_limit= 42164*sqrt(obj.mu/(42164)); 86 | obj.F=1.17/1000;%0.3115/1000; %in kilo Newtons 87 | obj.I_sp = 1800; % in sec 88 | obj.m0=2000; % kg 89 | obj.alpha=0.5; % in radians 90 | obj.beta=0.5;% in radians 91 | obj.segment=10*pi/180; % segment length (angle) after which the end states are compouted in radians 92 | 93 | %obj.g0=9.81; % m/sec^2 94 | %obj.m_dot = -obj.F/obj.I_sp/obj.g0; % in kg/sec 95 | 96 | % non-dimensional values for reference 97 | obj.DU = 42164; %distance unit, Km 98 | obj.TU = sqrt(42164^3/398600); %time unit, s 99 | obj.SU = obj.DU/obj.TU; %speed unit, Km/sec 100 | obj.MU = 5000; %mass Unit, Kg 101 | obj.HU=obj.DU*obj.SU; % angular momentum 102 | obj.FU = obj.MU*obj.DU*1000/obj.TU^2; %force unit(K-N) 103 | 104 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 105 | % %Terminal desired state values for reference 106 | %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% 107 | % h(end)=129640.2292 %km^2/sec 108 | % hx(end)=0; 109 | % hy(end)=0; 110 | % ex(end)=0; 111 | % ey(end)=0; 112 | 113 | %write = [obj.state(1), obj.state(2), obj.state(3), obj.state(4), obj.state(5), obj.state(6), obj.state(7), obj.alpha,obj.beta,obj.F,obj.segment] 114 | %csvwrite("csvlist.dat", write) 115 | end 116 | 117 | 118 | end 119 | 120 | 121 | 122 | methods(Static) 123 | function result = resulting() 124 | %global mu % defined in chkStop 125 | mu = 398600.4418; 126 | M = csvread('E:/RL_project_outputs/Training_testing_code/csv_files/csvlist.dat'); 127 | state = M(1:8)'; 128 | alpha = M(9); 129 | beta = M(10); 130 | F = M(11); 131 | segment = M(12); 132 | m0 = M(13); 133 | I_sp=M(14); 134 | %display(state) 135 | [finalState, finalSpacecraftMass]=spacecraftEnivironment(state,alpha,beta,F,segment,m0,I_sp); % testing the environment function 136 | PropellentBurnt=abs(finalState(1,8)); 137 | finalState1=finalState; 138 | finalSpacecraftMass1=finalSpacecraftMass; 139 | 140 | % non-dimensional values for reference 141 | DU = 42164; %distance unit, Km 142 | TU = sqrt(42164^3/398600); %time unit, s 143 | SU = DU/TU; %speed unit, Km/sec 144 | MU = 5000; %mass Unit, Kg 145 | HU=DU*SU; % angular momentum 146 | FU = MU*DU*1000/TU^2; %force unit(K-N) 147 | e0=0.7306; 148 | 149 | p =finalState(1,1)^2/mu; 150 | e = sqrt(finalState(1,4)^2+finalState(1,5)^2); 151 | a=p/(1-e^2); 152 | i=( (asin(sqrt(finalState(1,2)^2+finalState(1,3)^2)/finalState(1,1)) )/pi)*180; 153 | %i=(asin(sqrt(finalState(1,2)^2+finalState(1,3)^2)/finalState(1,1))) * (pi/180); 154 | 155 | %flag=chkStop( finalState(1,1)/HU,finalState(1,2)/HU,finalState(1,3)/HU,finalState(1,4),finalState(1,5)); 156 | flag=chkStop( finalState(1,1),finalState(1,2),finalState(1,3),finalState(1,4),finalState(1,5)); 157 | 158 | if flag==1 159 | %disp('Matlab 236 :Terminal conditions reached') 160 | % break 161 | end 162 | if i >=89.9 163 | disp('Matlab 240 :Inclination is aprroching 90 deg') 164 | % break 165 | end 166 | if a>= 42164*2 167 | disp('Matlab 244 : Energy above threshold') 168 | % break 169 | end 170 | if e> e0+(e0*0.2) 171 | disp('Matlab 248 :Eccentricity above threshold') 172 | % break 173 | end 174 | 175 | result= [finalState1, finalSpacecraftMass1]; 176 | 177 | end 178 | end 179 | 180 | methods(Static) 181 | function eclipse_flag = chkEclipse() 182 | %global mu % defined in chkStop 183 | mu = 398600.4418; 184 | M = csvread('E:/RL_project_outputs/Training_testing_code/csv_files/csvlist.dat'); 185 | state = M(1:8)'; 186 | alpha = M(9); 187 | beta = M(10); 188 | F = M(11); 189 | segment = M(12); 190 | m0 = M(13); 191 | I_sp=M(14); 192 | %display(state) 193 | rEarth=6378.1363; 194 | 195 | mu=398600.4415; 196 | h = M(1); 197 | hx = M(2); 198 | hy = M(3); 199 | ex = M(4); 200 | ey = M(5); 201 | phi= M(6); 202 | 203 | hz = sqrt(h^2-hx^2-hy^2); 204 | r = h^2/mu/(1+ex*cos(phi)+ey*sin(phi)); 205 | r = [r; zeros(2,1)]; 206 | 207 | % Ro_dash2G( H ) 208 | % =============== Error ===================================================% ===============% =============== 209 | H = [hx;hy;hz]; 210 | [m,n] = size( H ); 211 | if (m~=3)||(n~=1) 212 | error('Input has to be 3x1 vector ([hx;hy;hz])'); 213 | end 214 | % ================ Code ===================================================% ===============% ===============% =============== 215 | hx_1 = H(1); 216 | hy_1 = H(2); 217 | hz_1 = H(3); 218 | 219 | x_dash_hat = [hz_1;0;-hx_1]/sqrt(hx_1^2+hz_1^2); 220 | h_hat = [hx_1;hy_1;hz_1]/sqrt(hx_1^2+hy_1^2+hz_1^2); 221 | xn_dash_hat = [0 -h_hat(3) h_hat(2); 222 | h_hat(3) 0 -h_hat(1); 223 | -h_hat(2) h_hat(1) 0]*x_dash_hat; 224 | Rotation = [x_dash_hat xn_dash_hat h_hat]; 225 | 226 | % ===============% ===============% ===============% ===============% ===============% =============== 227 | % Ro_rnh2dash(theta) 228 | % =============== Error =================================================== 229 | if ~isscalar(phi) 230 | error('theta has to be scalar') 231 | end 232 | % ================ Code =================================================== 233 | Rotation_2 = [cos(phi) -sin(phi) 0; 234 | sin(phi) cos(phi) 0; 235 | 0 0 1]; 236 | % ===============% ===============% ===============% ===============% ===============% =============== 237 | 238 | Rxyz = Rotation* Rotation_2 *r; 239 | if Rxyz(1)>0 && sqrt(Rxyz(2)^2 + Rxyz(3)^2) | 8 | 9 | 10 | ## Files Description 11 | 12 | - `config.py`: Contains the configurations or initial parameters to run the code. 13 | 14 | - `Scenarios.py`: Contains the parameters for six transfer cases, which are as follows: 15 | - GTO-1 to GEO_1st network 16 | - GTO-1 to GEO_2nd network 17 | - GTO-2 to GEO_1st network 18 | - GTO-2 to GEO_2nd network 19 | - Super-GTO to GEO_1st network 20 | - Super-GTO to GEO_2nd network 21 | 22 | - `Spacecraft_env.py`: Contains the gym structured environment, which includes `env.reset()`, `env.step()`, and `env.render()` functions. 23 | 24 | - `environment.py`: Contains custom environment functions. These functions in `environment.py` are called by the gym environment in `Spacecraft_env.py`. 25 | 26 | - `spacecraftEnivironment.m`: A MATLAB code used to calculate `env.step()` values. 27 | 28 | - `environment.yml`: Contains all the required commands to recreate the Conda environment in any local system. This will recreate the exact same environment in which we trained our algorithms. 29 | 30 | - `environment_info.txt`: Contains the versions of all installed packages present in our Conda environment. 31 | 32 | - `test.py`: Python file which can be used to run the scenerios from pre trained weights. 33 | - `test.sh`: Shell file that contains the code to run test.py file. You can select the case number and max number of episode and all other parameters which are defined in config file in here.
34 | e.g If you want to run Case 1 i.e 'GTO-1 to GEO_1st network' then in test.sh file you will write as follow: 35 | ```python 36 | python test.py --case 1 --max_nu_ep 100 37 | ``` 38 | 39 | - `train.py`: Python file which can be used to train the scenarios from scratch. 40 | - `train.sh`: Shell file that contains the code to run train.py file. You can select the case number and max number of episode and all other parameters which are defined in config file in here.
41 | e.g If you want to run Case 1 i.e 'GTO-1 to GEO_1st network' then in train.sh file you will write as follow: 42 | ```python 43 | python train.py --case 1 --sh_flag 0 44 | ``` 45 | Note: Make sure that while training --sh_flag 0 46 | - `Final weights` folder: Contains the final trained weights for all six scenarios. 47 | - `CSV Files` folder: Contains Csv files which is used to communicate between Matlab and python programs data. 48 | - `Plots` : The resulting plots from training or testing the DRL agents will be saved in plots folder. 49 | - `Model_training_weights`: The resulting weights from training the DRL agent will be saved in Model_training_weights folder. 50 | - `Model_training_logs`: The resulting logs from training the DRL agent will be saved in Model_training_logs folder. 51 | 52 | ## Setting up Enviornment: 53 | 54 | 55 | 1. Install Conda: If conda is not installed in your system then install conda. (I used 4.10.1)
56 | 2. Install CUDA & CUDNN: We installed Cuda 11.7. Follow the instructions in https://medium.com/geekculture/install-cuda-and-cudnn-on-windows-linux-52d1501a8805#3e72 57 | 3. create conda environment named as mat_py3.7 and install python 3.7 in it. 58 | ```shell 59 | conda create --name mat_py3.7 python=3.7 60 | ``` 61 | 4. Activate the environment: Use the following command to activate the environment: 62 | ```shell 63 | conda activate mat_py3.7 64 | ``` 65 | 5. Install tensorflow-gpu 2.7.0 as follows: 66 | ```shell 67 | pip uninstall tensorflow-intel 68 | pip install tensorflow-gpu==2.7.0 69 | ``` 70 | if there are any errors appeared while installing then try to remove those errors first e.g as follows:
71 | ```shell 72 | pip install array-record dm-tree etils[enp,epath]>=0.9.0 promise tensorflow-metadata toml 73 | ``` 74 | Verify its installation by checking its version with GPU as follows:
75 | ```shell 76 | python -c "import tensorflow as tf; print(tf.config.experimental.list_physical_devices('GPU'))" 77 | ``` 78 | it should return the GPU details like [PhysicalDevice(name='/physical_device:GPU:0', device_type='GPU')] etc
79 | 80 | 6. Install pytorch with gpu: 81 | we installed torch 2.0.1+cu117.
82 | Please follow the instructions as follows to install torch.
83 | Install PyTorch 2.0.1 with CUDA 11.7:
84 | ```shell 85 | pip install torch==2.0.1+cu117 torchvision==0.15.2+cu117 --extra-index-url https://download.pytorch.org/whl/cu117 86 | ``` 87 | Verify if it installed correctly as follows:
88 | ```shell 89 | python -c "import torch; print(f'Torch Version: {torch.__version__}\nGPU Available: {torch.cuda.is_available()}')" 90 | ``` 91 | It should show the version as 2.0.1+cu117 and GPU Available: True (if there is any GPU)
92 | 93 | 5. Install MATLAB: Install MATLAB on your system. (I am using MATLAB 2021a). If you dont have matlab, you can use the following link to install MATLAB
https://www.mathworks.com/products/new_products/previous_release_overview.html
94 | 6. Navigate to the MATLAB folder: In the activated Conda environment, go to the MATLAB folder by running the following command: 95 | ```shell 96 | cd "" 97 | ``` 98 | Replace with the path to your MATLAB installation folder. By default, the MATLAB folder is located at "C:/Program Files/MATLAB". Make sure to include the double quotes if the path contains spaces. 99 | 7. Go to the MATLAB Engine Python folder: Change the directory to the MATLAB Engine Python folder by running the following command: 100 | ```shell 101 | cd "R2021a\extern\engines\python" 102 | ``` 103 | This will navigate you to the relevant folder containing the MATLAB Engine Python setup file. 104 | 8. Install the MATLAB Engine: To install the MATLAB Engine in your Conda environment, execute the setup.py file by running the following command: 105 | ```shell 106 | python setup.py install 107 | ``` 108 | if this doesnt install directly then open setup.py file and in __main__ replace the version as version="0.1.0" and then save it. 109 | after that just run the command in command line "pip install ." 110 | 111 | This command will install the MATLAB Engine package in your Conda environment. 112 | 10. Verify the installation: To check if the MATLAB Engine is installed correctly, run the following command: 113 | ```shell 114 | python -c "import matlab.engine" 115 | ``` 116 | 117 | 11. Install git: - If git is not installed in system then install git ( I used 2.40.0.windows.1)
118 | - Download git from https://git-scm.com/downloads and install it.
119 | - While installing: On the "Adjusting your PATH environment" page, select the option "Git from the command line and also from 3rd-party software." This ensures that Git is added to your system's PATH 120 | Environment variable, allowing you to use Git from the command prompt or terminal.
121 | - Verify the installation: After the installation is complete, open a new command prompt or terminal window and run the following command to verify the Git version: 122 | ```shell 123 | git --version 124 | ``` 125 | 11. Clone the repository: Run the following command in your command prompt or terminal to clone the GitHub repository to your local system: 126 | 127 | ```shell 128 | git clone https://github.com/talhazaidi13/Cascaded-Deep-Reinforcement-Learning-Based-Multi-Revolution-Low-Thrust-Spacecraft-Orbit-Transfer.git 129 | ``` 130 | Alternatively, you can just download the code files from the above link. 131 | 132 | 12. Navigate to the project directory: Navigate to the project directory on your local system, which contains the cloned repository. In that folder you will find the environment.yml file. you can use the cd command 133 | to navigate to the folder. e.g if environment.yml is at the location of D:\project then 134 | ```shell 135 | cd "D:\project" 136 | ``` 137 | 13. Update conda environment: Update the Conda environment using the environment.yml file. use the following code:
138 | ```shell 139 | conda env update -f environment.yml 140 | ``` 141 | This command will update the Conda environment based on the specifications in the environment.yml file.
142 | 14. Activate the environment: Use the following command to activate the environment: 143 | ```shell 144 | conda activate mat_py3.7 145 | ``` 146 | Please note that the name mat_py3.7 is the name of environment specified in the enviornment.yml file. You can rename it according to you.
147 | 148 | ## Running the code: 149 | 150 | Before running the code, please ensure that you have set the paths for the CSV files. The CSV files serve as the communication link between the MATLAB environment's step function and the Python code. Without correctly setting up the paths, the state values will not be updated. 151 | 152 | - To set the paths for the CSV files, follow these steps: 153 | 154 | 1. Open the Mat_env.m file. 155 | 2. Locate lines #126 and #184 in the Mat_env.m file. 156 | 3. In those lines, modify the path for the csvlist.dat file to match the location of the file on your system.
157 | For example, if the location of the csvlist.dat file on your system is D:/Cascaded-DRL/csv_files/csvlist.dat, update the lines as follows: 158 | ```python 159 | M = csvread('D:/Cascaded-DRL/csv_files/csvlist.dat') 160 | ``` 161 | Replace D:/Cascaded-DRL/csv_files/csvlist.dat with the actual path to the csvlist.dat file on your system. 162 | 163 | - Open Git Bash from the Start menu and Activate your Conda environment by running the appropriate command. For example, if your Conda environment is named "mat_py3.7," you can use the following command: 164 | ```shell 165 | conda activate mat_py3.7 166 | ``` 167 | 168 | - Change the current directory to the folder containing the test.sh or train.sh file using the cd command. For example, if the test.sh file is located D:/Cascaded-DRL, you can use the following command: 169 | ```shell 170 | cd "D:/Cascaded-DRL" 171 | ``` 172 | - Finally, you can run the test.sh or train.sh file for testing with trained weights and training from the scratch, using the bash command: 173 | ```shell 174 | bash test.sh 175 | bash train.sh 176 | ``` 177 | 178 | ## Results 179 | CDRL based GTO to GEO transfer | CDRL based Super-GTO to GEO transfer 180 | :-: | :-: 181 | | 182 | | 183 | | 184 | | 185 | | 186 | 187 | ## Citation 188 | If you find this work beneficial to your research or project, I kindly request that you cite it: 189 | ``` 190 | @article{zaidi2023cascaded, 191 | title={Cascaded Deep Reinforcement Learning-Based Multi-Revolution Low-Thrust Spacecraft Orbit-Transfer}, 192 | author={Zaidi, Syed Muhammad Talha and Chadalavada, Pardhasai and Ullah, Hayat and Munir, Arslan and Dutta, Atri}, 193 | journal={IEEE Access}, 194 | year={2023}, 195 | publisher={IEEE} 196 | } 197 | ``` 198 | -------------------------------------------------------------------------------- /Spacecraft_Env.py: -------------------------------------------------------------------------------- 1 | #################################################################################################################################################################################################### 2 | # for continuos episode 3 | ###################################################################################################################################################################################################### 4 | 5 | import numpy as np 6 | import gym 7 | from gym import spaces 8 | from enviornment import Enviornment 9 | ################################################################## 10 | import math 11 | import numpy as np 12 | import random 13 | from IPython.display import clear_output 14 | from collections import deque 15 | import matlab.engine # IMPORTING MATLAB ENGINE 16 | #import progressbar 17 | import gym 18 | import random 19 | import os 20 | import matplotlib 21 | matplotlib.use('Agg') 22 | import matplotlib.pyplot as plt 23 | from mpl_toolkits import mplot3d 24 | import matplotlib.image as img 25 | from matplotlib.offsetbox import (OffsetImage, AnnotationBbox) 26 | ## enviornment start 27 | import matlab.engine 28 | import csv 29 | #eng = matlab.engine.start_matlab(); 30 | from numpy.core.fromnumeric import shape 31 | from tensorflow.keras import Model 32 | from tensorflow.keras.layers import Dense, Embedding, Reshape 33 | from tensorflow.keras.optimizers import Adam 34 | import os.path 35 | import time 36 | from datetime import date 37 | import matplotlib.cm as cm 38 | import matplotlib.animation as animation 39 | import tensorflow as tf 40 | from scenerios import cases 41 | tf.debugging.set_log_device_placement(True) 42 | 43 | LEN_GOAL = 30 44 | ################################################################## 45 | class Spacecraft_Env(gym.Env): 46 | """ 47 | Custom Environment that follows gym interface. 48 | This is a spacecraft env where agent learns to find the optimal satellite path. 49 | """ 50 | # Because of google colab, we cannot implement the GUI ('human' render mode) 51 | # metadata = {'render.modes': ['console']} 52 | 53 | def __init__(self, args): 54 | super(Spacecraft_Env, self).__init__() 55 | self.max_steps_one_ep = args.max_steps_one_ep 56 | chosen_case = cases[args.case] 57 | eng = matlab.engine.start_matlab(); 58 | #env1 = eng.Mat_env.resulting(); 59 | self.env = Enviornment(eng, args) 60 | self.max_R_time = 500000000000 61 | self.F = chosen_case['force'][0]/1000 62 | self.max_steps_in_ep = chosen_case['max_steps_in_ep'][0] 63 | 64 | number_of_revolutions = 200*50000 65 | self.segment_interval = self.env.segment # in degrees 66 | self.seg1= self.segment_interval # * np.pi/180 # in radians 67 | self.segment = (360/ self.segment_interval) 68 | max_possible_steps_in_episodes = number_of_revolutions * self.segment 69 | self.phi_normalizing_factor = max_possible_steps_in_episodes * self.seg1 70 | self.max_steps = number_of_revolutions * self.segment 71 | 72 | 73 | # define arrays for info 74 | self.h_history = [[]] 75 | self.hx_history = [[]] 76 | self.hy_history = [[]] 77 | self.ex_history = [[]] 78 | self.ey_history = [[]] 79 | self.phi_history = [[]] 80 | self.mass_history = [[]] 81 | self.alpha_history = [[]] 82 | self.beta_history = [[]] 83 | self.thrust_history = [[]] 84 | 85 | 86 | self.ecc_history = [[]] 87 | self.a_history = [[]] 88 | self.inclination_history = [[]] 89 | self.ecc_history_nd = [[]] 90 | self.a_history_nd = [[]] 91 | self.inclination_history_nd = [[]] 92 | 93 | 94 | self.score_data = [] 95 | self.score_detailed_data = [[]] 96 | 97 | 98 | ## 99 | self.seg_count = 0 100 | self.average = 0 101 | self.a_sum = 0 102 | self.time_rev = 0 103 | self.ep_time = 0 104 | self.ep_time_1 = 0 105 | self.T_rev = [[]] 106 | self.T_ep_success = [] 107 | self.T_ep_success.append([]) 108 | ## 109 | self.ep_counter = -1 110 | self.counter = 0 111 | self.MAIN_Episode = 1 112 | self.done_counter = 0 113 | self.ep_counter_SAC_step = -1 114 | self.ep_Cont_len_counter = 0 115 | self.success_counter = 0 116 | self.time_mat = 0 117 | self.acc_reward = 0 118 | 119 | current_dir = os.getcwd() 120 | today = date.today() 121 | self.day = today.strftime('%b-%d-%Y') 122 | self.txt_dir = f"{int(time.time())}" 123 | self.dir_name = self.day + "_" + self.txt_dir 124 | folder_path = os.path.join(current_dir,'plots', self.dir_name) 125 | folders = ['h', 'hx', 'hy', 'ex', 'ey', 'a', 'ecc', 'inc', 'mass', 'phi', 'sum_reward', 'time', 'Successful_episodes'] 126 | folder_paths = [os.path.join(f"{folder_path}/{folder}") for folder in folders] 127 | if not os.path.exists(folder_path): 128 | os.makedirs(folder_path) 129 | for folder_path_1 in folder_paths: 130 | os.makedirs(folder_path_1) 131 | 132 | 133 | self.completeName_successful = os.path.join(folder_path+"/output.dat") 134 | self.write_final_state = os.path.join(folder_path+"/final_state.dat") 135 | self.completeName_all_data= os.path.join(folder_path+"/output_all_data.dat") 136 | self.figure_file_ecc = os.path.join(folder_path + "/_ecc_history") 137 | self.figure_file_a = os.path.join(folder_path + "/_a_history") 138 | self.figure_file_inclination = os.path.join(folder_path + "/_inclination_history") 139 | self.figure_file_reward = os.path.join(folder_path + "/_reward_history") 140 | self.figure_file_time_success = os.path.join(folder_path + "/_success_ep_time") 141 | self.figure_file_time_success_actual = os.path.join(folder_path, "_success_ep_time_actual") 142 | self.folder_path_h = os.path.join(f"{folder_paths[0]}/{'h'}") 143 | self.folder_path_hx = os.path.join(f"{folder_paths[1]}/{'hx'}") 144 | self.folder_path_hy = os.path.join(f"{folder_paths[2]}/{'hy'}") 145 | self.folder_path_ex = os.path.join(f"{folder_paths[3]}/{'ex'}") 146 | self.folder_path_ey = os.path.join(f"{folder_paths[4]}/{'ey'}") 147 | self.folder_path_a = os.path.join(f"{folder_paths[5]}/{'a'}") 148 | self.folder_path_ecc = os.path.join(f"{folder_paths[6]}/{'ecc'}") 149 | self.folder_path_inc = os.path.join(f"{folder_paths[7]}/{'inc'}") 150 | self.folder_path_mass = os.path.join(f"{folder_paths[8]}/{'mass'}") 151 | self.folder_path_phi = os.path.join(f"{folder_paths[9]}/{'phi'}") 152 | self.folder_path_sum_reward = os.path.join(f"{folder_paths[10]}/{'reward'}") 153 | self.folder_path_time = os.path.join(f"{folder_paths[11]}/{'ep_time'}") 154 | self.successful_episodes = os.path.join(f"{folder_paths[12]}") 155 | 156 | 157 | 158 | self.action_space = spaces.Box(low=np.float32([-np.pi, -np.pi/2]), 159 | high=np.float32([np.pi, np.pi/2]), shape=(2,), 160 | dtype=np.float32) 161 | 162 | self.observation_space = spaces.Box(low=-15, high=15, 163 | shape=(7,), dtype=np.float64) 164 | 165 | 166 | 167 | def reset(self): 168 | """ 169 | Important: the observation must be a numpy array 170 | :return: (np.array) 171 | """ 172 | self.state = np.array(self.env.reset_csv()) 173 | self.ND_state = np.array(self.env.DimtoNonDim_states(self.state, self.max_R_time, 2*np.pi)) 174 | self.observation_notime = np.delete(self.ND_state, 6) 175 | 176 | self.ep_Cont_len_counter = self.ep_fixed_len_counter = 0 177 | self.seg_count = 0 178 | self.ep_time = self.ep_time_1 = 0 179 | self.ep_counter += 1 180 | self.ep_counter_SAC_step += 1 181 | self.acc_reward = 0 182 | self.seg_change_indx = [] 183 | 184 | ecc = math.sqrt(self.state[3]**2 + self.state[4]**2) 185 | self.ecc_history.append([ecc]) 186 | self.a_history.append([(self.state[0]**2 / 398600.4418) / (1 - ecc**2)]) 187 | self.inclination_history.append([(math.asin(math.sqrt(self.state[1]**2 + self.state[2]**2) / self.state[0]) / np.pi) * 180]) 188 | 189 | ecc_ND = math.sqrt(self.ND_state[3]**2 + self.ND_state[4]**2) 190 | self.ecc_history_nd.append([ecc_ND]) 191 | self.a_history_nd.append([(self.ND_state[0]**2 / 398600.4418) / (1 - ecc_ND**2)]) 192 | self.inclination_history_nd.append([(((math.asin(math.sqrt((self.ND_state[1]**2)+(self.ND_state[2]**2))/self.ND_state[0])) / np.pi)*180)/10] ) 193 | 194 | self.h_history.append([self.state[0]]) 195 | self.hx_history.append([self.state[1]]) 196 | self.hy_history.append([self.state[2]]) 197 | self.ex_history.append([self.state[3]]) 198 | self.ey_history.append([self.state[4]]) 199 | self.phi_history.append([self.state[5]]) 200 | self.mass_history.append([self.state[-1]]) 201 | self.alpha_history.append([]) 202 | self.beta_history.append([]) 203 | self.thrust_history.append([]) 204 | self.score_data.append([]) 205 | self.score_detailed_data.append([]) 206 | 207 | self.T_rev.append([]) 208 | self.time_mat = 0 209 | 210 | self.observation = self.observation_notime 211 | 212 | return self.observation 213 | 214 | 215 | def step(self, action): 216 | action = np.array([action[0], action[1] ,self.F]) #0.34fixed thrust 0.089 gives error near GEO 217 | ## Convert Non dimensional values to dimensional to p#ass through the matlab step function 218 | self.observation_Notime = [self.observation[0], self.observation[1], self.observation[2], self.observation[3], self.observation[4], self.observation[5], 0, self.observation[6]] 219 | self.D_state = np.array(self.observation_Notime) 220 | self.D_state = self.env.NonDimtoDim_states(self.D_state , self.max_R_time , (2*np.pi)) 221 | self.state1= np.array([self.D_state[0], self.D_state[1], self.D_state[2], self.D_state[3], self.D_state[4], self.D_state[5], self.D_state[6] ,self.D_state[7] ]).reshape((1,8)) 222 | 223 | ## matlab step function 224 | mass = self.D_state[-1] 225 | self.state1[0][-1] = self.state1[0][-1] * -1 # making mass value negative before sending in matlab function env 226 | self.state1[0][6] = self.time_mat 227 | finalmass= self.env.m0 - np.abs(mass) 228 | 229 | self.next_state, self.reward, self.distance, self.dist_aei, terminal_flag, Remaining_time, target_state_parameters, redflag, time_in_days, ecc_nd, a_nd, i_nd, eclipse_flag, segment_flag, time_bef_seg_change, time_after_seg_change,segment = self.env.step(self.state1, action[0], action[1], action[2], self.seg1, finalmass , self.env.I_sp, self.max_R_time, (2*np.pi) ,self.ep_Cont_len_counter) 230 | self.next_state[-1] = self.next_state[-1] * -1 # making mass value positive for our networks 231 | self.time_mat = time_in_days * (60*60*24) 232 | 233 | eclipse_info = 0 234 | if eclipse_flag: 235 | action[2] = 0.0 236 | eclipse_info = 1 237 | 238 | self.next_state = np.array(self.next_state) 239 | self.ND_next_state = np.array(self.env.DimtoNonDim_states (self.next_state , self.max_R_time , (2*np.pi))) 240 | self.observation_next_state_Notime = [self.ND_next_state[0], self.ND_next_state[1], self.ND_next_state[2], self.ND_next_state[3], self.ND_next_state[4], self.ND_next_state[5], self.ND_next_state[7] ] 241 | self.observation = self.observation_next_state_Notime 242 | ## termianl flag 243 | self.done = terminal_flag 244 | # for plotting append data 245 | self.h_history[self.ep_counter-1].append(self.state1[0][0]) 246 | self.hx_history[self.ep_counter-1].append(self.state1[0][1]) 247 | self.hy_history[self.ep_counter-1].append(self.state1[0][2]) 248 | self.ex_history[self.ep_counter-1].append(self.state1[0][3]) 249 | self.ey_history[self.ep_counter-1].append(self.state1[0][4]) 250 | self.phi_history[self.ep_counter-1].append(self.state1[0][5]) 251 | self.ecc_history[self.ep_counter-1].append(target_state_parameters[0]) 252 | self.a_history[self.ep_counter-1].append(target_state_parameters[1]) 253 | self.inclination_history[self.ep_counter-1].append(target_state_parameters[2]) 254 | self.ecc_history_nd[self.ep_counter-1].append(ecc_nd) 255 | self.a_history_nd[self.ep_counter-1].append(a_nd) 256 | self.inclination_history_nd[self.ep_counter-1].append(i_nd) 257 | self.mass_history[self.ep_counter-1].append(finalmass) 258 | self.alpha_history[self.ep_counter-1].append([action[0]]) 259 | self.beta_history[self.ep_counter-1].append([action[1]]) 260 | self.thrust_history[self.ep_counter-1].append([action[2]]) 261 | self.acc_reward = self.acc_reward + self.reward[0] 262 | self.score_data[self.ep_counter-1] = self.acc_reward 263 | self.score_detailed_data[self.ep_counter-1].append(self.acc_reward) 264 | 265 | # Increase counter value 266 | self.ep_Cont_len_counter = self.ep_Cont_len_counter + 1 267 | self.ep_fixed_len_counter = self.ep_fixed_len_counter + 1 268 | self.counter = self.counter + 1 269 | 270 | #### Time from matlab file 271 | if self.done: 272 | self.T_rev[self.ep_counter-1].append(time_in_days) #matlab 273 | ############################## 274 | if self.ep_Cont_len_counter == 10000 and (self.ep_counter-1 ) % 10 == 0 and (self.ep_counter-1 ) != 0: 275 | self.env.plot_variable("Score_value", self.score_data, self.figure_file_reward, self.ep_counter, all_episode_plot_flag=1) 276 | 277 | if ((redflag==1) or (self.ep_Cont_len_counter % 5000 == 0)): 278 | self.env.plot_variable("H", self.h_history, self.folder_path_h, self.ep_counter) 279 | self.env.plot_variable("Hx", self.hx_history, self.folder_path_hx, self.ep_counter) 280 | self.env.plot_variable("Hy", self.hy_history, self.folder_path_hy, self.ep_counter) 281 | self.env.plot_variable("ex", self.ex_history, self.folder_path_ex, self.ep_counter) 282 | self.env.plot_variable("ey", self.ey_history, self.folder_path_ey, self.ep_counter) 283 | self.env.plot_variable("ecc", self.ecc_history, self.folder_path_ecc, self.ep_counter, flag_ter_values=2, tsp=target_state_parameters, tsp_indexes=[3]) 284 | self.env.plot_variable("a", self.a_history, self.folder_path_a, self.ep_counter, flag_ter_values=1, tsp=target_state_parameters, tsp_indexes=[4,5]) 285 | self.env.plot_variable("inc", self.inclination_history, self.folder_path_inc, self.ep_counter, flag_ter_values=2, tsp=target_state_parameters, tsp_indexes=[6]) 286 | self.env.plot_variable("mass", self.mass_history, self.folder_path_mass, self.ep_counter) 287 | self.env.plot_variable("Reward", self.score_detailed_data, self.folder_path_sum_reward, self.ep_counter) 288 | self.env.plot_variable("Phi", self.phi_history, self.folder_path_phi, self.ep_counter) 289 | 290 | if self.done: 291 | self.success_counter = self.success_counter +1 292 | self.T_ep_success.append([]) 293 | self.T_ep_success[-2] = self.T_rev[self.ep_counter-1][-1] 294 | average = 0 295 | done_data = [(self.ep_counter-1), self.ep_Cont_len_counter, self.acc_reward ,self.T_ep_success[-2], self.completeName_successful ] 296 | self.env.writing_Successful_episodes ( int(self.success_counter), int(done_data[0]), int(done_data[1]), 297 | (float(done_data[2])), (float(done_data[3])), 298 | float(42164- (self.a_history[self.ep_counter-1][-1])), float(self.inclination_history[self.ep_counter-1][-1]), 299 | float(self.ecc_history[self.ep_counter-1][-1]), 300 | float(self.h_history[self.ep_counter-1][-1]), float(self.hx_history[self.ep_counter-1][-1]), float(self.hy_history[self.ep_counter-1][-1]), 301 | float(self.ex_history[self.ep_counter-1][-1]), float(self.ey_history[self.ep_counter-1][-1]), 302 | self.completeName_successful ) 303 | self.env.writing_final_states( float(self.h_history[self.ep_counter-1][-1]), float(self.hx_history[self.ep_counter-1][-1]), float(self.hy_history[self.ep_counter-1][-1]), 304 | float(self.ex_history[self.ep_counter-1][-1]), float(self.ey_history[self.ep_counter-1][-1]), self.phi_history[self.ep_counter-1][-1], 305 | float(self.time_mat) , float(self.next_state[-1]*-1) ,self.write_final_state ) 306 | 307 | ############################################################## 308 | self.env.plot_variable("ecc", self.ecc_history, self.figure_file_ecc, self.ep_counter, flag_ter_values=2, tsp=target_state_parameters, tsp_indexes=[3]) 309 | self.env.plot_variable("a", self.a_history, self.figure_file_a, self.ep_counter, flag_ter_values=1, tsp=target_state_parameters, tsp_indexes=[4,5]) 310 | self.env.plot_variable("inc", self.inclination_history, self.figure_file_inclination, self.ep_counter, flag_ter_values=2, tsp=target_state_parameters, tsp_indexes=[6]) 311 | self.env.plot_variable("Score_value", self.score_data, self.figure_file_reward, self.ep_counter, all_episode_plot_flag=1) 312 | 313 | 314 | if not os.path.exists(self.successful_episodes ): 315 | os.makedirs(self.successful_episodes) 316 | self.Successful_Episode_num = os.path.join(self.successful_episodes +"/_ep_"+ str(self.ep_counter-1) ) 317 | if not os.path.exists(self.Successful_Episode_num ): 318 | os.makedirs(self.Successful_Episode_num ) 319 | 320 | self.env.plot_variable("H", self.h_history, self.Successful_Episode_num, self.ep_counter, flag_saving_with_no_ep_nu =1) 321 | self.env.plot_two_variable("hx_hy","hx","hy", self.hx_history,self.hy_history, self.Successful_Episode_num, self.ep_counter, flag_saving_with_no_ep_nu =1) 322 | self.env.plot_two_variable("ex_ey","ex","ey", self.ex_history,self.ey_history, self.Successful_Episode_num, self.ep_counter, flag_saving_with_no_ep_nu =1) 323 | self.env.plot_variable("ecc", self.ecc_history, self.Successful_Episode_num, self.ep_counter, flag_ter_values=2, tsp=target_state_parameters, tsp_indexes=[3], flag_saving_with_no_ep_nu =1) 324 | self.env.plot_variable("a(km)", self.a_history, self.Successful_Episode_num, self.ep_counter, flag_ter_values=1, tsp=target_state_parameters, tsp_indexes=[4,5], flag_saving_with_no_ep_nu =1) 325 | self.env.plot_variable("inc(deg)", self.inclination_history, self.Successful_Episode_num, self.ep_counter, flag_ter_values=2, tsp=target_state_parameters, tsp_indexes=[6], flag_saving_with_no_ep_nu =1) 326 | self.env.plot_variable("mass(kg)", self.mass_history, self.Successful_Episode_num, self.ep_counter, flag_saving_with_no_ep_nu =1) 327 | self.env.plot_variable("Reward", self.score_detailed_data, self.Successful_Episode_num, self.ep_counter, flag_saving_with_no_ep_nu =1) 328 | self.env.plot_variable("Phi(rad)", self.phi_history, self.Successful_Episode_num, self.ep_counter, flag_saving_with_no_ep_nu =1) 329 | self.env.plot_two_variable("actions","alpha(rad)","beta(rad)", self.alpha_history,self.beta_history, self.Successful_Episode_num, self.ep_counter, flag_saving_with_no_ep_nu =1) 330 | self.env.plot_variable("Thrust(KN)", self.thrust_history, self.Successful_Episode_num, self.ep_counter, flag_saving_with_no_ep_nu =1) 331 | self.env.plot_variable("Time(KN)", self.T_rev, self.Successful_Episode_num, self.ep_counter, flag_saving_with_no_ep_nu =1) 332 | 333 | 334 | 335 | # Optionally we can pass additional info, we are not using that for now 336 | info = {} 337 | 338 | 339 | 340 | print("Ep:", self.ep_counter_SAC_step , " Ep_len :", self.ep_fixed_len_counter , " step_reward:" , self.reward[0] , " Acc_reward: ", self.acc_reward, 341 | " ecc:" , float(round(target_state_parameters[0],6)), " a :" , float(round(target_state_parameters[1],4)), " inc:" , float(round(target_state_parameters[2],4)), 342 | " a_diff:" , float(round(42164- (self.a_history[self.ep_counter-1][-1]),4)), " time :", time_in_days, " Seg :", segment*(180/3.14) ," eclipse_info : ", eclipse_info) 343 | 344 | 345 | self.env.writing_all_episodes_data ( int(self.ep_counter_SAC_step) , int(self.ep_fixed_len_counter), 346 | float(round(self.state1[0][0],6)), float(round(self.state1[0][1],6)),float(round(self.state1[0][2],6)),float(round(self.state1[0][3],6)), 347 | float(round(self.state1[0][4],6)),float(round(self.state1[0][5],6)), 348 | float(round(self.observation_Notime[0],6)),float(round(self.observation_Notime[1],6)),float(round(self.observation_Notime[2],6)),float(round(self.observation_Notime[3],6)), 349 | float(round(self.observation_Notime[4],6)),float(round(self.observation_Notime[5],6)), 350 | float(round(target_state_parameters[0],6)), float(round(target_state_parameters[1],6)), float(round(target_state_parameters[2],6)), 351 | float(round(action[0],6)), float(round(action[1],4)),float(round(action[2],4)), 352 | float(round(self.reward[0],4)), float(round(self.acc_reward,4)), float(finalmass), 353 | target_state_parameters[7] , target_state_parameters[8], target_state_parameters[9], segment*(180/3.14), 354 | self.completeName_all_data ) 355 | 356 | if (redflag) or (self.ep_fixed_len_counter > self.max_steps_in_ep ): #50000 with 10 degree , 1000000 with 1 deg 357 | self.done=1 358 | 359 | 360 | return np.array(self.observation), self.reward[0], self.done, info 361 | 362 | 363 | def render(self): 364 | pass 365 | 366 | def close(self): 367 | pass -------------------------------------------------------------------------------- /__pycache__/Spacecraft_Env.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/talhazaidi13/Cascaded-Deep-Reinforcement-Learning-Based-Multi-Revolution-Low-Thrust-Spacecraft-Orbit-Transfer/289ca90a2726808dc2613b27a37cf0bd0deeed90/__pycache__/Spacecraft_Env.cpython-37.pyc -------------------------------------------------------------------------------- /__pycache__/config.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/talhazaidi13/Cascaded-Deep-Reinforcement-Learning-Based-Multi-Revolution-Low-Thrust-Spacecraft-Orbit-Transfer/289ca90a2726808dc2613b27a37cf0bd0deeed90/__pycache__/config.cpython-37.pyc -------------------------------------------------------------------------------- /__pycache__/enviornment.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/talhazaidi13/Cascaded-Deep-Reinforcement-Learning-Based-Multi-Revolution-Low-Thrust-Spacecraft-Orbit-Transfer/289ca90a2726808dc2613b27a37cf0bd0deeed90/__pycache__/enviornment.cpython-37.pyc -------------------------------------------------------------------------------- /__pycache__/inf: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /__pycache__/scenerios.cpython-37.pyc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/talhazaidi13/Cascaded-Deep-Reinforcement-Learning-Based-Multi-Revolution-Low-Thrust-Spacecraft-Orbit-Transfer/289ca90a2726808dc2613b27a37cf0bd0deeed90/__pycache__/scenerios.cpython-37.pyc -------------------------------------------------------------------------------- /chkStop.m: -------------------------------------------------------------------------------- 1 | function flag = ... 2 | chkStop( h,hx,hy,ex,ey) 3 | global mu 4 | mu= 398600.4418; 5 | p = h^2/mu; 6 | 7 | tol_inc = 0.01; % tolerance of inclination +- deg 8 | tol_ecc = 0.0001; % 0.00001 tolerance of eccentricity +0 9 | %tol_a = 0.00001*35786; % tolerance of normalize a +- DU 10 | %tol_a = 35786; % tolerance of normalize a +- DU 11 | target_a = 42164; % tolerance of normalize a +- DU 12 | tol_a = target_a * (0.00001); 13 | %============================================== 14 | ecc = sqrt(ex^2+ey^2); 15 | if ecc < (tol_ecc) 16 | flag_ecc = 1; 17 | else 18 | flag_ecc = 0; 19 | end 20 | %============================================== 21 | a = p/(1-ecc^2); 22 | %if (target_a -tol_a) < a && a < (target_a + (2*tol_a)) 23 | if (target_a) < a && a < (target_a + (2*tol_a)) 24 | flag_a = 1; 25 | else 26 | flag_a = 0; 27 | end 28 | %============================================== 29 | i = ((asin(sqrt(hx^2+hy^2)/h))/pi)*180; 30 | %i = (asin(sqrt(hx^2+hy^2)/h))* (pi/180); 31 | 32 | if i < tol_inc 33 | flag_inc = 1; 34 | else 35 | flag_inc = 0; 36 | end 37 | %============================================== 38 | 39 | 40 | if flag_a && flag_ecc && flag_inc 41 | flag = 1; 42 | else 43 | flag = 0; 44 | end 45 | 46 | end 47 | 48 | -------------------------------------------------------------------------------- /config.py: -------------------------------------------------------------------------------- 1 | import argparse 2 | # Create ArgumentParser object 3 | from scenerios import cases 4 | 5 | parser = argparse.ArgumentParser() 6 | # Define command-line arguments with default values and help messages 7 | 8 | 9 | 10 | # parser.add_argument('--GTO_state', type=float, nargs=7, default=[101055.56404, 8993.12420, -44988.20968, 0.634234170, 0.142292050, 1000.0, 0], help='GTO1 orbit state for first DRL [h,hx,hy,ex,ey,mass,time]') 11 | # parser.add_argument('--Isp', type=int, default=1800, help='Specific Impulse Isp') 12 | # parser.add_argument('--m0', type=int, default=1200, help='Mass of spacecraft Kg') 13 | # parser.add_argument('--force', type=int, default=0.311, help='Constant thrust magnitude in N') 14 | # parser.add_argument('--tol_inc', type=float, default=0.1, help='tolerance for inclination') 15 | # parser.add_argument('--tol_ecc', type=float, default=0.1 , help='tolerance for eccentricity') 16 | # parser.add_argument('--tol_a', type=float, default=0.5 , help='tolerance for semimajoraxis km') 17 | # parser.add_argument('--w1_a', type=float, default=1000 , help='Reward function weights for w_a') 18 | # parser.add_argument('--w1_e', type=float, default=2010, help='Reward function weights for w_ecc') 19 | # parser.add_argument('--w1_i', type=float, default=300 , help='Reward function weights for w_inc') 20 | # parser.add_argument('--w1_a_', type=float, default= 0.010 , help='Reward function weights for w_a_') 21 | # parser.add_argument('--w1_e_', type=float, default=0.000000198, help='Reward function weights for w_ecc_') 22 | # parser.add_argument('--w1_i_', type=float, default=0.0003 , help='Reward function weights for w_inc_') 23 | # parser.add_argument('--c1_a', type=float, default=500 , help='Reward function weights for c_a') 24 | # parser.add_argument('--c1_e', type=float, default=700, help='Reward function weights for c_ecc') 25 | # parser.add_argument('--c1_i', type=float, default=300 , help='Reward function weights for c_inc') 26 | # parser.add_argument('--tau', type=float, default=0.003 , help='Reward function constant tau value') 27 | 28 | 29 | 30 | # parser.add_argument('--GTO_state', type=float, nargs=7, default=[101055.56404, 8993.12420, -44988.20968, 0.634234170, 0.142292050, 1000.0, 0], help='GTO1 orbit state for Second DRL [h,hx,hy,ex,ey,mass,time]') 31 | # parser.add_argument('--Isp', type=int, default=1800, help='Specific Impulse Isp') 32 | # parser.add_argument('--m0', type=int, default=1200, help='Mass of spacecraft Kg') 33 | # parser.add_argument('--force', type=int, default=0.311, help='Constant thrust magnitude in N') 34 | # parser.add_argument('--tol_inc', type=float, default=0.08, help='tolerance for inclination') 35 | # parser.add_argument('--tol_ecc', type=float, default=0.00005 , help='tolerance for eccentricity') 36 | # parser.add_argument('--tol_a', type=float, default=0.2 , help='tolerance for semimajoraxis km') 37 | # parser.add_argument('--w1_a', type=float, default=1000*3 , help='Reward function weights for w_a') 38 | # parser.add_argument('--w1_e', type=float, default=700*3, help='Reward function weights for w_ecc') 39 | # parser.add_argument('--w1_i', type=float, default=200 , help='Reward function weights for w_inc') 40 | # parser.add_argument('--w1_a_', type=float, default= 0.010*3 , help='Reward function weights for w_a_') 41 | # parser.add_argument('--w1_e_', type=float, default=0.00000198*3, help='Reward function weights for w_ecc_') 42 | # parser.add_argument('--w1_i_', type=float, default=0.0003 , help='Reward function weights for w_inc_') 43 | # parser.add_argument('--c1_a', type=float, default=500*3 , help='Reward function weights for c_a') 44 | # parser.add_argument('--c1_e', type=float, default=700*3, help='Reward function weights for c_ecc') 45 | # parser.add_argument('--c1_i', type=float, default=300 , help='Reward function weights for c_inc') 46 | # parser.add_argument('--tau', type=float, default=0.003 , help='Reward function constant tau value') 47 | 48 | # parser.add_argument('--GTO_state', type=float, nargs=7, default=[101055.56404, 8993.12420, -44988.20968, 0.634234170, 0.142292050, 1000.0, 0], help='GTO2 orbit state for first DRL[h,hx,hy,ex,ey,mass,time]') 49 | # parser.add_argument('--Isp', type=int, default=3300, help='Specific Impulse Isp') 50 | # parser.add_argument('--m0', type=int, default=450, help='Mass of spacecraft Kg') 51 | # parser.add_argument('--force', type=int, default=0.2007, help='Constant thrust magnitude in N') 52 | # parser.add_argument('--tol_inc', type=float, default=0.1, help='tolerance for inclination') 53 | # parser.add_argument('--tol_ecc', type=float, default=0.1 , help='tolerance for eccentricity') 54 | # parser.add_argument('--tol_a', type=float, default=50 , help='tolerance for semimajoraxis km') 55 | 56 | # parser.add_argument('--w1_a', type=float, default=1000 , help='Reward function weights for w_a') 57 | # parser.add_argument('--w1_e', type=float, default=2010, help='Reward function weights for w_ecc') 58 | # parser.add_argument('--w1_i', type=float, default=300 , help='Reward function weights for w_inc') 59 | # parser.add_argument('--w1_a_', type=float, default= 0.010 , help='Reward function weights for w_a_') 60 | # parser.add_argument('--w1_e_', type=float, default=0.000000198, help='Reward function weights for w_ecc_') 61 | # parser.add_argument('--w1_i_', type=float, default=0.0003 , help='Reward function weights for w_inc_') 62 | # parser.add_argument('--c1_a', type=float, default=500 , help='Reward function weights for c_a') 63 | # parser.add_argument('--c1_e', type=float, default=700, help='Reward function weights for c_ecc') 64 | # parser.add_argument('--c1_i', type=float, default=300 , help='Reward function weights for c_inc') 65 | # parser.add_argument('--tau', type=float, default=0.003 , help='Reward function constant tau value') 66 | 67 | # parser.add_argument('--GTO_state', type=float, nargs=7, default=[101055.56404, 8993.12420, -44988.20968, 0.634234170, 0.142292050, 1000.0, 0], help='GTO1 orbit state for Second DRL [h,hx,hy,ex,ey,mass,time]') 68 | # parser.add_argument('--Isp', type=int, default=3300, help='Specific Impulse Isp') 69 | # parser.add_argument('--m0', type=int, default=450, help='Mass of spacecraft Kg') 70 | # parser.add_argument('--force', type=int, default=0.2007, help='Constant thrust magnitude in N') 71 | # parser.add_argument('--tol_inc', type=float, default=0.07, help='tolerance for inclination') 72 | # parser.add_argument('--tol_ecc', type=float, default=00.000088 , help='tolerance for eccentricity') 73 | # parser.add_argument('--tol_a', type=float, default=0.09 , help='tolerance for semimajoraxis km') 74 | # parser.add_argument('--tol_inc', type=float, default=0.1, help='tolerance for inclination') 75 | # parser.add_argument('--tol_ecc', type=float, default=0.1 , help='tolerance for eccentricity') 76 | # parser.add_argument('--tol_a', type=float, default=0.5 , help='tolerance for semimajoraxis km') 77 | # parser.add_argument('--w1_a', type=float, default=500*3 , help='Reward function weights for w_a') 78 | # parser.add_argument('--w1_e', type=float, default=700*3, help='Reward function weights for w_ecc') 79 | # parser.add_argument('--w1_i', type=float, default=200 , help='Reward function weights for w_inc') 80 | # parser.add_argument('--w1_a_', type=float, default= 0.010*3 , help='Reward function weights for w_a_') 81 | # parser.add_argument('--w1_e_', type=float, default=0.00000198*3, help='Reward function weights for w_ecc_') 82 | # parser.add_argument('--w1_i_', type=float, default=0.0003 , help='Reward function weights for w_inc_') 83 | # parser.add_argument('--c1_a', type=float, default=50*3 , help='Reward function weights for c_a') 84 | # parser.add_argument('--c1_e', type=float, default=30*3, help='Reward function weights for c_ecc') 85 | # parser.add_argument('--c1_i', type=float, default=5 , help='Reward function weights for c_inc') 86 | # parser.add_argument('--tau', type=float, default=0.003 , help='Reward function constant tau value') 87 | 88 | # parser.add_argument('--GTO_state', type=float, nargs=7, default=[101055.56404, 8993.12420, -44988.20968, 0.634234170, 0.142292050, 1000.0, 0], help='GTO3 orbit state for first DRL [h,hx,hy,ex,ey,mass,time]') 89 | # parser.add_argument('--Isp', type=int, default=3000, help='Specific Impulse Isp') 90 | # parser.add_argument('--m0', type=int, default=2000, help='Mass of spacecraft Kg') 91 | # parser.add_argument('--force', type=int, default=0.35, help='Constant thrust magnitude in N') 92 | # parser.add_argument('--tol_inc', type=float, default=0.1, help='tolerance for inclination') 93 | # parser.add_argument('--tol_ecc', type=float, default=0.01 , help='tolerance for eccentricity') 94 | # parser.add_argument('--tol_a', type=float, default=55 , help='tolerance for semimajoraxis km') 95 | # parser.add_argument('--w1_a', type=float, default=1000 , help='Reward function weights for w_a') 96 | # parser.add_argument('--w1_e', type=float, default=2010, help='Reward function weights for w_ecc') 97 | # parser.add_argument('--w1_i', type=float, default=300 , help='Reward function weights for w_inc') 98 | # parser.add_argument('--w1_a_', type=float, default= 0.010 , help='Reward function weights for w_a_') 99 | # parser.add_argument('--w1_e_', type=float, default=0.000000198, help='Reward function weights for w_ecc_') 100 | # parser.add_argument('--w1_i_', type=float, default=0.0003 , help='Reward function weights for w_inc_') 101 | # parser.add_argument('--c1_a', type=float, default=500 , help='Reward function weights for c_a') 102 | # parser.add_argument('--c1_e', type=float, default=700, help='Reward function weights for c_ecc') 103 | # parser.add_argument('--c1_i', type=float, default=300 , help='Reward function weights for c_inc') 104 | # parser.add_argument('--tau', type=float, default=0.003 , help='Reward function constant tau value') 105 | 106 | # parser.add_argument('--GTO_state', type=float, nargs=7, default=[101055.56404, 8993.12420, -44988.20968, 0.634234170, 0.142292050, 1000.0, 0], help='GTO3 orbit state for Second DRL [h,hx,hy,ex,ey,mass,time]') 107 | # parser.add_argument('--Isp', type=int, default=3000, help='Specific Impulse Isp') 108 | # parser.add_argument('--m0', type=int, default=2000, help='Mass of spacecraft Kg') 109 | # parser.add_argument('--force', type=int, default=0.35, help='Constant thrust magnitude in N') 110 | # parser.add_argument('--tol_inc', type=float, default=0.1, help='tolerance for inclination') 111 | # parser.add_argument('--tol_ecc', type=float, default=0.0009 , help='tolerance for eccentricity') 112 | # parser.add_argument('--tol_a', type=float, default=36 , help='tolerance for semimajoraxis km') 113 | # parser.add_argument('--tol_inc', type=float, default=0.1, help='tolerance for inclination') 114 | # parser.add_argument('--tol_ecc', type=float, default=0.1 , help='tolerance for eccentricity') 115 | # parser.add_argument('--tol_a', type=float, default=0.5 , help='tolerance for semimajoraxis km') 116 | # parser.add_argument('--w1_a', type=float, default=500*3 , help='Reward function weights for w_a') 117 | # parser.add_argument('--w1_e', type=float, default=700*6, help='Reward function weights for w_ecc') 118 | # parser.add_argument('--w1_i', type=float, default=200 , help='Reward function weights for w_inc') 119 | # parser.add_argument('--w1_a_', type=float, default= 0.010*3 , help='Reward function weights for w_a_') 120 | # parser.add_argument('--w1_e_', type=float, default=0.00000198*6, help='Reward function weights for w_ecc_') 121 | # parser.add_argument('--w1_i_', type=float, default=0.0003 , help='Reward function weights for w_inc_') 122 | # parser.add_argument('--c1_a', type=float, default=50*3 , help='Reward function weights for c_a') 123 | # parser.add_argument('--c1_e', type=float, default=30*6, help='Reward function weights for c_ecc') 124 | # parser.add_argument('--c1_i', type=float, default=5 , help='Reward function weights for c_inc') 125 | # parser.add_argument('--tau', type=float, default=0.003 , help='Reward function constant tau value') 126 | 127 | # parser.add_argument('--GTO_state', type=float, nargs=7, default=[101055.56404, 8993.12420, -44988.20968, 0.634234170, 0.142292050, 1000.0, 0], help='Super-GTO orbit state for first DRL [h,hx,hy,ex,ey,mass,time]') 128 | # parser.add_argument('--Isp', type=int, default=3300, help='Specific Impulse Isp') 129 | # parser.add_argument('--m0', type=int, default=1200, help='Mass of spacecraft Kg') 130 | # parser.add_argument('--force', type=int, default=0.4015, help='Constant thrust magnitude in N') 131 | # parser.add_argument('--tol_inc', type=float, default=0.1, help='tolerance for inclination') 132 | # parser.add_argument('--tol_ecc', type=float, default=0.1 , help='tolerance for eccentricity') 133 | # parser.add_argument('--tol_a', type=float, default=50 , help='tolerance for semimajoraxis km') 134 | # parser.add_argument('--tol_inc', type=float, default=0.1, help='tolerance for inclination') 135 | # parser.add_argument('--tol_ecc', type=float, default=0.1 , help='tolerance for eccentricity') 136 | # parser.add_argument('--tol_a', type=float, default=0.5 , help='tolerance for semimajoraxis km') 137 | # parser.add_argument('--w1_a', type=float, default=1000 , help='Reward function weights for w_a') 138 | # parser.add_argument('--w1_e', type=float, default=4010, help='Reward function weights for w_ecc') 139 | # parser.add_argument('--w1_i', type=float, default=300 , help='Reward function weights for w_inc') 140 | # parser.add_argument('--w1_a_', type=float, default= 0.010 , help='Reward function weights for w_a_') 141 | # parser.add_argument('--w1_e_', type=float, default=0.000000000198, help='Reward function weights for w_ecc_') 142 | # parser.add_argument('--w1_i_', type=float, default=0.0003 , help='Reward function weights for w_inc_') 143 | # parser.add_argument('--c1_a', type=float, default=500 , help='Reward function weights for c_a') 144 | # parser.add_argument('--c1_e', type=float, default=2000, help='Reward function weights for c_ecc') 145 | # parser.add_argument('--c1_i', type=float, default=300 , help='Reward function weights for c_inc') 146 | # parser.add_argument('--tau', type=float, default=0.003 , help='Reward function constant tau value') 147 | 148 | # parser.add_argument('--GTO_state', type=float, nargs=7, default=[101055.56404, 8993.12420, -44988.20968, 0.634234170, 0.142292050, 1000.0, 0], help='Super-GTO orbit state for Second DRL [h,hx,hy,ex,ey,mass,time]') 149 | # parser.add_argument('--Isp', type=int, default=3300, help='Specific Impulse Isp') 150 | # parser.add_argument('--m0', type=int, default=1200, help='Mass of spacecraft Kg') 151 | # parser.add_argument('--force', type=int, default=0.4015, help='Constant thrust magnitude in N') 152 | # parser.add_argument('--tol_inc', type=float, default=0.1, help='tolerance for inclination') 153 | # parser.add_argument('--tol_ecc', type=float, default=0.000069 , help='tolerance for eccentricity') 154 | # parser.add_argument('--tol_a', type=float, default=0.5 , help='tolerance for semimajoraxis km') 155 | # parser.add_argument('--tol_inc', type=float, default=0.1, help='tolerance for inclination') 156 | # parser.add_argument('--tol_ecc', type=float, default=0.1 , help='tolerance for eccentricity') 157 | # parser.add_argument('--tol_a', type=float, default=0.5 , help='tolerance for semimajoraxis km') 158 | # parser.add_argument('--w1_a', type=float, default=1000*4 , help='Reward function weights for w_a') 159 | # parser.add_argument('--w1_e', type=float, default=4010*4.5, help='Reward function weights for w_ecc') 160 | # parser.add_argument('--w1_i', type=float, default=300*1.5 , help='Reward function weights for w_inc') 161 | # parser.add_argument('--w1_a_', type=float, default= 0.010*4 , help='Reward function weights for w_a_') 162 | # parser.add_argument('--w1_e_', type=float, default=0.000000000198*4.5, help='Reward function weights for w_ecc_') 163 | # parser.add_argument('--w1_i_', type=float, default=0.0003*1.5 , help='Reward function weights for w_inc_') 164 | # parser.add_argument('--c1_a', type=float, default=500*4 , help='Reward function weights for c_a') 165 | # parser.add_argument('--c1_e', type=float, default=2000*4.5, help='Reward function weights for c_ecc') 166 | # parser.add_argument('--c1_i', type=float, default=300*1.5 , help='Reward function weights for c_inc') 167 | # parser.add_argument('--tau', type=float, default=0.003 , help='Reward function constant tau value') 168 | 169 | parser.add_argument('--case', choices=cases.keys(), default='8', help='Choose one of the predefined cases, 1: GTO1_1st , 2: GTO1_2nd, 3: GTO2_1st , 4: GTO2_2nd, 5: GTO3_1st , 6: GTO3_2nd, 7: superGTO_1st , 8: superGTO_2nd, ') 170 | 171 | for arg_name, arg_vals in cases['1'].items(): 172 | parser.add_argument(f'--{arg_name}_1', type=type(arg_vals[0]), nargs=len(arg_vals), default=arg_vals, help=f'{arg_name} for case1') 173 | 174 | for arg_name, arg_vals in cases['2'].items(): 175 | parser.add_argument(f'--{arg_name}_2', type=type(arg_vals[0]), nargs=len(arg_vals), default=arg_vals, help=f'{arg_name} for case2') 176 | 177 | for arg_name, arg_vals in cases['3'].items(): 178 | parser.add_argument(f'--{arg_name}_3', type=type(arg_vals[0]), nargs=len(arg_vals), default=arg_vals, help=f'{arg_name} for case3') 179 | 180 | for arg_name, arg_vals in cases['4'].items(): 181 | parser.add_argument(f'--{arg_name}_4', type=type(arg_vals[0]), nargs=len(arg_vals), default=arg_vals, help=f'{arg_name} for case4') 182 | 183 | for arg_name, arg_vals in cases['5'].items(): 184 | parser.add_argument(f'--{arg_name}_5', type=type(arg_vals[0]), nargs=len(arg_vals), default=arg_vals, help=f'{arg_name} for case5') 185 | 186 | for arg_name, arg_vals in cases['6'].items(): 187 | parser.add_argument(f'--{arg_name}_6', type=type(arg_vals[0]), nargs=len(arg_vals), default=arg_vals, help=f'{arg_name} for case6') 188 | 189 | for arg_name, arg_vals in cases['7'].items(): 190 | parser.add_argument(f'--{arg_name}_7', type=type(arg_vals[0]), nargs=len(arg_vals), default=arg_vals, help=f'{arg_name} for case7') 191 | 192 | for arg_name, arg_vals in cases['8'].items(): 193 | parser.add_argument(f'--{arg_name}_8', type=type(arg_vals[0]), nargs=len(arg_vals), default=arg_vals, help=f'{arg_name} for case8') 194 | 195 | parser.add_argument('--max_steps_one_ep', type=int, default=20000, help='Max number of steps in one episode ') 196 | parser.add_argument('--max_nu_ep', type=int, default=600, help='Max number of episodes') 197 | parser.add_argument('--weights_save_steps', type=int, default=500, help='Number of steps after which weights will save') 198 | parser.add_argument('--buffer_size', type=int, default=10000, help='size of replay buffer') 199 | parser.add_argument('--gamma', type=float, default=0.99, help='value of discounting parametr gamma') 200 | parser.add_argument('--lr', type=float, default=0.003, help='learning rate') 201 | 202 | 203 | 204 | # Parse the arguments 205 | args = parser.parse_args() -------------------------------------------------------------------------------- /csv_files/Rtimeinput.dat: -------------------------------------------------------------------------------- 1 | 0.9768338724896896,0.007052391916151941,4.9167872222355005,0.5,0.4,0.1,0.8318173392368997 2 | -------------------------------------------------------------------------------- /csv_files/csvlist.dat: -------------------------------------------------------------------------------- 1 | 129648.08125092302,-147.33472530780173,-90.07314133694743,0.0003386210925782429,0.0001757794220578507,4.634988888888848,9095.647992855947,-84.39600705121967,2.3620193004608154,-0.011414170265197754,0.0004015,0.0017444444444444447,1115.6039929487804,3300.0 2 | -------------------------------------------------------------------------------- /csv_files/csvlistinitialize.dat: -------------------------------------------------------------------------------- 1 | 129624.62553,-160.07483,-86.112609,-0.000256813,0.00082013,3.9721,0.0,84.2832,0.5,0.5,0.0004015,0.0017444444444444447,1200.0,3300.0 2 | -------------------------------------------------------------------------------- /earth.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/talhazaidi13/Cascaded-Deep-Reinforcement-Learning-Based-Multi-Revolution-Low-Thrust-Spacecraft-Orbit-Transfer/289ca90a2726808dc2613b27a37cf0bd0deeed90/earth.png -------------------------------------------------------------------------------- /enviornment.py: -------------------------------------------------------------------------------- 1 | import math 2 | from operator import mod 3 | from turtle import distance 4 | import numpy as np 5 | #import progressbar 6 | import gym 7 | import random 8 | import matplotlib.pyplot as plt 9 | ## enviornment start 10 | import matlab.engine 11 | import csv 12 | #eng = matlab.engine.start_matlab(); 13 | 14 | from numpy.core.fromnumeric import shape 15 | 16 | from tensorflow.keras import Model 17 | from tensorflow.keras.layers import Dense, Embedding, Reshape 18 | from tensorflow.keras.optimizers import Adam 19 | 20 | import os.path 21 | import re 22 | import tensorflow as tf 23 | tf.debugging.set_log_device_placement(True) 24 | from config import args 25 | from scenerios import cases 26 | 27 | 28 | #env = eng.Triangle_1.resulting(); 29 | #state = list(env._data) 30 | #print('states [h hx hy ex ey phi m^] : ', state) 31 | 32 | #state_space= shape(state) 33 | #print('State_Space : ',state_space[0]) 34 | 35 | #alpha = 1.5; 36 | #beta = 3; 37 | #F = 2*0.001035; 38 | #segment = (10*3.14)/180 39 | 40 | 41 | class Enviornment: 42 | def __init__(self, eng, args): 43 | self.eng = eng 44 | 45 | chosen_case = cases[args.case] 46 | self.initial_state = np.array((chosen_case['GTO_state'])) # [h;hx;hy;ex;ey;phi;time;fuel_burnt] Initial GTO orbit state 47 | 48 | ################################################################################################ 49 | ######## following initialization for dimenless quantities required for Neural network ######## 50 | self.DU = 42164; # distance unit, Km 51 | self.TU = math.sqrt((42164**3)/398600); # time unit, s 572.92 52 | self.SU = self.DU/self.TU; # speed unit, Km/sec 53 | self.MU = 398600.4418; # mass Unit, Kg 54 | self.HU = self.DU * self.SU; 55 | self.g0 = 9.81; # m/sec^2 56 | 57 | ################################################################################################ 58 | # define path for dat files 59 | current_dir = os.getcwd() 60 | save_path_1 = os.path.join(current_dir, "csv_files") 61 | self.completeName_csvlist = os.path.join(save_path_1, "csvlist.dat") 62 | self.completeName_csvlistinitialize = os.path.join(save_path_1, "csvlistinitialize.dat") 63 | self.completeName_Rtimeinput = os.path.join(save_path_1, "Rtimeinput.dat") 64 | ################################################################################################ 65 | self.mu = 398600.4418 # global mu matlab 66 | 67 | self.segment = 10*(3.14/180) # 10 degree gap convert in radians 68 | self.I_sp = chosen_case['Isp'][0] # in sec 69 | self.m0 = chosen_case['m0'][0] # kg 70 | self.F = chosen_case['force'][0]/1000 71 | self.tol_inc = chosen_case['tol_inc'][0] 72 | self.tol_ecc = chosen_case['tol_ecc'][0] 73 | self.tol_a = chosen_case['tol_a'][0] 74 | self.m_dot = -self.F/self.I_sp/self.g0; # in kg/sec 75 | 76 | self.shadow_flag = chosen_case['sh_flag'][0] 77 | self.transfer_case = args.case 78 | 79 | self.target_inc, self.target_ecc, self.target_a = 0, 0, 42164 80 | self.tol_a_low, self.tol_a_high = self.target_a - self.tol_a, self.target_a + self.tol_a 81 | 82 | w1 = { "a": chosen_case['w1_a'][0], "e": chosen_case['w1_e'][0], "i": chosen_case['w1_i'][0] } 83 | w1_ = { "a_": chosen_case['w1_a_'][0], "e_": chosen_case['w1_e_'][0], "i_": chosen_case['w1_i_'][0] } 84 | c1 = { "a": chosen_case['c1_a'][0], "e": chosen_case['c1_e'][0], "i": chosen_case['c1_i'][0] } 85 | 86 | self.weights = { 87 | "w1": w1, 88 | "w1_": w1_, 89 | "c1": c1, 90 | "tau": chosen_case['tau'][0] 91 | } 92 | 93 | self.segment_flag = 0 94 | self.time_before_seg_change = [] 95 | self.time_after_seg_change = 0 96 | self.time_before_seg_change_1 = 0 97 | 98 | def is_terminal(self,state_5): 99 | h, hx, hy, ex, ey= state_5 100 | mu = self.mu # global mu matlab 101 | 102 | ecc = math.sqrt((ex**2)+(ey**2)) 103 | flag_ecc = 1 if ecc < (self.tol_ecc) else 0 104 | p = (h*h)/mu 105 | a = p/(1-ecc**2) 106 | flag_a = 1 if (self.tol_a_low) < a and a < (self.tol_a_high) else 0 107 | i = ((math.asin(math.sqrt((hx**2)+(hy**2))/h)) / np.pi)*180 108 | flag_inc = 1 if i<(self.tol_inc) else 0 109 | flag = 1 if flag_inc and flag_ecc and flag_a else 0 110 | 111 | p_init = ((self.initial_state[0]**2))/mu; 112 | tol_error_a = self.tol_a_high + (self.tol_a_high* (5/100)) # 5 percent of final target a (SHOULD BE 2 PERCENT) 113 | initial_state_ecc = math.sqrt((self.initial_state[3]**2)+(self.initial_state[4]**2)) 114 | a_init = p_init/(1-initial_state_ecc**2); 115 | 116 | if int(self.transfer_case) > 6: 117 | flag_for_error_a = 1 if (a < 35000) or (a > 52500) else 0 ## only for super GTO case 118 | else: 119 | flag_for_error_a = 1 if (a > tol_error_a) or (a<22000) else 0 ## only for all other GTO cases 120 | red_flag = 1 if flag_for_error_a else 0 121 | monitor_a_flag = 1 if (a < a_init) or (a > self.tol_a_high) else 0 122 | 123 | return flag, ecc, self.tol_ecc, a, self.tol_a_low, self.tol_a_high, i,self.tol_inc , 0, flag_for_error_a, 0,monitor_a_flag, red_flag 124 | 125 | 126 | def get_seg_value(self, state_5): 127 | state_5_1 = state_5[0] 128 | h, hx, hy, ex, ey, _ , _, _= state_5_1 129 | mu = self.mu # global mu matlab 130 | 131 | tol_inc = 0.1; # tolerance of inclination +- deg 0.1 0.01 132 | tol_ecc = 0.01; # 0.00001 tolerance of eccentricity +0 0.01 0.00001 133 | tol_a = 42164 * (5/100) # 5,2 perent 134 | # For First segment change 135 | ecc = math.sqrt((ex**2)+(ey**2)); 136 | flag_ecc_seg = 1 if ecc < (tol_ecc) else 0 137 | p = (h*h)/mu; 138 | a = p/(1-ecc**2); 139 | flag_a_seg = 1 if (self.target_a - tol_a) < a and a < (self.target_a + tol_a) else 0 140 | i = ((math.asin(math.sqrt((hx**2)+(hy**2))/h)) / np.pi)*180; 141 | flag_inc_seg = 1 if i< (tol_inc) else 0 142 | self.segment = 1*(3.14/180) if flag_inc_seg and flag_ecc_seg and flag_a_seg else 10*(3.14/180) 143 | # For second segment change 144 | tol_inc = 0.1; # tolerance of inclination +- deg 0.1 0.01 145 | tol_ecc = 0.01; # 0.00001 tolerance of eccentricity +0 0.01 0.00001 146 | tol_a_2 = 200 # 300, 30 147 | ecc = math.sqrt((ex**2)+(ey**2)); 148 | flag_ecc_seg_2 = 1 if ecc < (tol_ecc) else 0 149 | p = (h*h)/mu; 150 | a = p/(1-ecc**2); 151 | flag_a_seg_2 = 1 if (self.target_a - tol_a) < a and a < (self.target_a + tol_a) else 0 152 | i = ((math.asin(math.sqrt((hx**2)+(hy**2))/h)) / np.pi)*180; 153 | flag_inc_seg_2 = 1 if i< (tol_inc) else 0 154 | if self.transfer_case in [2, 4, 6, 8]: # for 2nd DRL case always it is 0.1 degree 155 | self.segment = 0.1*(3.14/180) 156 | else: 157 | if flag_inc_seg_2 and flag_ecc_seg_2 and flag_a_seg_2: 158 | self.segment = 0.1*(3.14/180) 159 | elif flag_inc_seg and flag_ecc_seg and flag_a_seg: 160 | self.segment = 1*(3.14/180) 161 | else: 162 | self.segment = 10*(3.14/180) 163 | 164 | return self.segment 165 | 166 | 167 | def reset_csv(self): 168 | state = self.initial_state 169 | self.temp = state 170 | self.temp = np.append(self.temp, 0.5) 171 | self.temp = np.append(self.temp, 0.5) 172 | self.temp = np.append(self.temp, self.F) 173 | self.temp = np.append(self.temp, self.segment) 174 | self.temp = np.append(self.temp, self.m0) 175 | self.temp = np.append(self.temp, self.I_sp) 176 | with open(self.completeName_csvlistinitialize , 'w') as csvfile: 177 | csvwriter = csv.writer(csvfile) 178 | csvwriter.writerow(self.temp) 179 | csvfile.close() 180 | #print(a) 181 | return state 182 | 183 | 184 | def step( self, state, alpha, beta, F, segment, m01, I_sp, Max_R_time , phi_normalizing_factor, timestep): 185 | self.temp = state 186 | segment = self.get_seg_value(state) 187 | if args.case in ['2','4','6','8']: 188 | segment = 0.1*(3.14/180) 189 | # for checking eclipse 190 | eclipse_flag = self.eng.Mat_env.chkEclipse() 191 | if eclipse_flag * self.shadow_flag: 192 | F = 0 193 | 194 | self.temp = np.append(self.temp, alpha) 195 | self.temp = np.append(self.temp, beta) 196 | self.temp = np.append(self.temp, F) 197 | self.temp = np.append(self.temp, segment) 198 | self.temp = np.append(self.temp, m01) 199 | self.temp = np.append(self.temp, I_sp) 200 | with open(self.completeName_csvlist, 'w') as csvfile: 201 | csvwriter = csv.writer(csvfile) 202 | csvwriter.writerow(self.temp) 203 | csvfile.close() 204 | 205 | a1 = self.eng.Mat_env.resulting() 206 | #print("136env: state", a1) 207 | a3 = list(a1._data) 208 | t_state=a3[0:8] # getting next state 209 | PropellentBurnt = t_state[7] 210 | time_in_days = t_state[6] / (60*60*24) 211 | if (round(t_state[5], 2) % (round((2*np.pi), 2)) ) == 0: 212 | t_state[5] = 0 213 | 214 | ## Calculating Reward Function (passing ND values) 215 | new_state = [t_state[0], t_state[1], t_state[2], t_state[3], t_state[4]] 216 | ND_state = self.DimtoNonDim_states (t_state , Max_R_time, phi_normalizing_factor) 217 | new_state_ND = [ND_state[0], ND_state[1], ND_state[2], ND_state[3], ND_state[4]] 218 | prev_state = [state[0][0], state[0][1], state[0][2], state[0][3], state[0][4]] 219 | ND_Previous_state = self.DimtoNonDim_states (state[0] , Max_R_time, phi_normalizing_factor) 220 | prev_state_ND = [ND_Previous_state[0], ND_Previous_state[1], ND_Previous_state[2], ND_Previous_state[3], ND_Previous_state[4]] 221 | done,ecc, tol_ecc, a, tol_a_low, tol_a_high, i,tol_inc, flag_ecc, flag_a, flag_inc, monitor_a_flag,redflag =self.is_terminal(t_state[0:5]) 222 | target_state_parameters = [ecc,a, i , tol_ecc, tol_a_low, tol_a_high, tol_inc,flag_ecc, flag_a, flag_inc ] 223 | reward, distance, dist_aei, ecc_new, a_new, i_new, ecc_target, a_target, i_target = Enviornment.Reward(self, prev_state_ND,new_state_ND, prev_state,new_state, done,redflag,monitor_a_flag, timestep) #ex,ey,h,Remaining_time 224 | 225 | return t_state , reward, distance, dist_aei, 1==done , 1, target_state_parameters, redflag, time_in_days , ecc_new, a_new, i_new, eclipse_flag , self.segment_flag, self.time_before_seg_change_1, self.time_after_seg_change, segment 226 | 227 | 228 | 229 | def DimtoNonDim_states (self, state , Max_R_time, phi_normalizing_factor): 230 | h_1, hx_1, hy_1, ex_1, ey_1, phi_1, time_1, fuel_1 = state 231 | h_1 = h_1 / self.HU # h 232 | hx_1 = hx_1 / self.HU # hx 233 | hy_1 = hy_1 / self.HU # hy 234 | phi_1 = phi_1 / phi_normalizing_factor 235 | time_1 = time_1 / Max_R_time # accumulating time 236 | fuel_1 = fuel_1 / self.m0 # fuel burnt 237 | return [h_1, hx_1, hy_1, ex_1, ey_1, phi_1, time_1, fuel_1] 238 | 239 | def NonDimtoDim_states (self, state , Max_R_time, phi_normalizing_factor): 240 | h_1, hx_1, hy_1, ex_1, ey_1, phi_1, time_1, fuel_1 = state 241 | h_1 = h_1 * self.HU # h 242 | hx_1 = hx_1 * self.HU # hx 243 | hy_1 = hy_1 * self.HU # hy 244 | phi_1 = phi_1 * phi_normalizing_factor 245 | time_1 = time_1 * Max_R_time # accumulating time 246 | fuel_1 = fuel_1 * self.m0 # fuel burnt 247 | return [h_1, hx_1, hy_1, ex_1, ey_1, phi_1, time_1, fuel_1] 248 | 249 | 250 | 251 | def Reward (self, prev_state_ND,new_state_ND, prev_state,new_state, done,redflag,monitor_a_flag, timestep): 252 | w1_aei = [self.weights["w1"]["a"] , self.weights["w1"]["e"], self.weights["w1"]["i"]] 253 | w1_aei_ = [self.weights["w1_"]["a_"] , self.weights["w1_"]["e_"], self.weights["w1_"]["i_"] ] 254 | c1_aei = [self.weights["c1"]["a"] , self.weights["c1"]["e"], self.weights["c1"]["i"]] 255 | tauu_aei= self.weights["tau"] 256 | 257 | starget = [ [1*(129640.2292/ self.HU)] ,[0] , [0], [0], [0] ] 258 | 259 | ecc_prev = math.sqrt((prev_state_ND[3])**2 + (prev_state_ND[4])**2) 260 | ecc_new = math.sqrt((new_state_ND[3])**2 + (new_state_ND[4])**2) 261 | ecc_target = math.sqrt((starget[3][0])**2 + (starget[4][0])**2) 262 | 263 | a_prev = (((prev_state_ND[0])**2) /1) / ( 1- (ecc_prev **2)) 264 | a_new = (((new_state_ND[0])**2) /1) / ( 1- (ecc_new **2)) 265 | a_target = (((starget[0][0])**2) /1) / ( 1- (ecc_target **2)) 266 | 267 | i_prev = ((math.asin (math.sqrt((prev_state_ND[1]**2)+(prev_state_ND[2]**2))/prev_state_ND[0])) / np.pi)*180 268 | i_new = ((math.asin (math.sqrt((new_state_ND[1]**2)+(new_state_ND[2]**2))/new_state_ND[0])) / np.pi)*180 269 | i_target = ((math.asin (math.sqrt((starget[1][0]**2)+(starget[2][0]**2))/starget[0][0])) / np.pi)*180 270 | 271 | i_prev = i_prev /10 272 | i_new = i_new /10 273 | i_target = i_target /10 274 | 275 | st_a_e_i_prev = [[a_prev], [ecc_prev], [i_prev]] 276 | st_a_e_i_new = [[a_new], [ecc_new], [i_new]] 277 | st_a_e_i_target = [[a_target], [ecc_target], [i_target]] 278 | 279 | exp_value_t_aei = 0 280 | exp_value_t_plus_1_aei = 0 281 | 282 | for i in range(0,3): 283 | exp_value_t_aei = exp_value_t_aei + ( c1_aei[i] * math.exp(-(w1_aei_[i] * abs(np.subtract(st_a_e_i_prev[i], st_a_e_i_target[i])) ) ) ) 284 | exp_value_t_plus_1_aei = exp_value_t_plus_1_aei + ( c1_aei[i] * math.exp(-(w1_aei_[i] * abs(np.subtract(st_a_e_i_new[i], st_a_e_i_target[i])) ) ) ) 285 | phi_st_aei = - np.dot( np.transpose(np.array(w1_aei)) , abs(np.subtract(st_a_e_i_prev ,st_a_e_i_target))) + exp_value_t_aei -(0.03*monitor_a_flag) 286 | phi_st_plus_1_aei = - np.dot( np.transpose(np.array(w1_aei)) , abs(np.subtract(st_a_e_i_new ,st_a_e_i_target))) + exp_value_t_plus_1_aei -(0.03*monitor_a_flag) 287 | reward_t_aei = phi_st_plus_1_aei - phi_st_aei - tauu_aei + (100 * done) - (5*redflag) -(0.00*monitor_a_flag) 288 | 289 | diff_a = math.sqrt(( st_a_e_i_target[0][0] - st_a_e_i_new[0][0])**2) 290 | diff_b = math.sqrt((st_a_e_i_target[1][0] - st_a_e_i_new[1][0])**2) 291 | diff_c = math.sqrt((st_a_e_i_target[2][0] - st_a_e_i_new[2][0])**2) 292 | dist_aei = diff_a + diff_b + diff_c 293 | return [reward_t_aei[0]], 0, dist_aei, ecc_new, a_new, i_new, ecc_target, a_target, i_target 294 | 295 | 296 | 297 | 298 | def writing_Successful_episodes( self,success_ep_counter, episode, len_episode, score, time,a_last, Inc_last, ecc_last, h_last, hx_last,hy_last,ex_last,ey_last, completeName_successful ): 299 | self.temp = ['Succ_ep_counter : ', success_ep_counter, ' ', ' ', 300 | 'ep : ', episode, ' ', ' ', 'ep_length : ', len_episode, ' ', ' ', 301 | 'score: ', score, ' ', ' ', 'time : ', time, ' ', ' ', 302 | 'targ-a[-1]: ', a_last, ' ', ' ', 303 | 'inc[-1]: ', Inc_last, ' ', ' ', 'ecc[-1]: ', ecc_last, ' ', ' ', 304 | 'h[-1]: ', h_last, ' ', ' ', 'hx[-1]: ', hx_last, ' ', ' ', 305 | 'hy[-1]: ', hy_last, ' ', ' ', 'ex[-1]: ', ex_last, ' ', ' ', 306 | 'ey[-1]: ', ey_last] 307 | with open(completeName_successful , 'a') as csvfile: 308 | csvwriter = csv.writer(csvfile) 309 | csvwriter.writerow(self.temp) 310 | csvfile.close() 311 | #print(a) 312 | 313 | def writing_all_episodes_data( self, episode, ep_fixed_len_counter, h,hx,hy,ex,ey,phi,ND_h,ND_hx,ND_hy,ND_ex,ND_ey,ND_phi,ecc,a,inc, alpha,beta,thrust, reward_step, score,mass, flag_ecc,flag_a,flag_inc,segment,completeName_successful ): 314 | self.temp = ['ep : ', episode, ' ', 'ep_step : ', ep_fixed_len_counter, ' ', 'state : ', h, hx, hy, ex, ey, phi, ' ', 'ecc,a,inc:', ecc, a, inc, ' ', 'Reward,score : ', reward_step, score, ' ', ' ', 315 | 'Normalized_State: ', ND_h, ND_hx, ND_hy, ND_ex, ND_ey, ND_phi, ' ', ' ', 'action_values : ', alpha, beta,thrust, ' ', ' ', 'Mass : ', mass, ' ', ' ', 'flag_ecc : ', flag_ecc, ' ', 316 | 'flag_a : ', flag_a, ' ', 'flag_inc : ', flag_inc, ' ', 'Segment : ', segment] 317 | with open(completeName_successful , 'a') as csvfile: 318 | csvwriter = csv.writer(csvfile) 319 | csvwriter.writerow(self.temp) 320 | csvfile.close() 321 | #print(a) 322 | 323 | def writing_final_states ( self, h,hx,hy,ex,ey,phi,time,mass, completeName_1 ): 324 | self.temp = [] 325 | self.temp = np.append(self.temp, h) 326 | self.temp = np.append(self.temp, hx ) 327 | self.temp = np.append(self.temp, hy) 328 | self.temp = np.append(self.temp, ex) 329 | self.temp = np.append(self.temp, ey) # [h;hx;hy;ex;ey;phi;time;fuel_burnt] 330 | self.temp = np.append(self.temp, phi) 331 | self.temp = np.append(self.temp, time) 332 | self.temp = np.append(self.temp, mass) 333 | 334 | with open(completeName_1 , 'a') as csvfile: 335 | csvwriter = csv.writer(csvfile) 336 | csvwriter.writerow(self.temp) 337 | csvfile.close() 338 | 339 | 340 | def plot_variable(env, name, hist, folder_path, ep_counter, flag_ter_values= None, tsp=None, tsp_indexes=None, all_episode_plot_flag=None, flag_saving_with_no_ep_nu =None): 341 | plt.figure(figsize=(12, 8)) 342 | if all_episode_plot_flag == 1: 343 | plt.plot(hist[0:-2]) 344 | else: 345 | plt.plot(hist[ep_counter-1]) 346 | plt.title(f'state parameters values {name}') 347 | plt.ylabel(f'{name.lower()} values') 348 | if flag_ter_values == 1: 349 | plt.axhline(tsp[tsp_indexes[0]], color='r', linestyle='-') 350 | plt.axhline(tsp[tsp_indexes[1]], color='r', linestyle='-') 351 | elif flag_ter_values == 2: 352 | plt.axhline(tsp[tsp_indexes[0]], color='r', linestyle='-') 353 | 354 | plt.grid(True) 355 | 356 | if flag_saving_with_no_ep_nu == 1: 357 | plt.savefig(folder_path+"/_"+name + ".png") 358 | else: 359 | plt.savefig(folder_path+"_ep_"+str(ep_counter-1) + ".png") 360 | 361 | plt.close() 362 | 363 | def plot_two_variable(env, name, a,b, hist_1, hist_2, folder_path, ep_counter, flag_ter_values= None, tsp=None, tsp_indexes=None , flag_saving_with_no_ep_nu =None ): 364 | plt.figure(figsize=(12, 8)) 365 | plt.plot(hist_1[ep_counter-1], c='b', label=a, linewidth=1.5) 366 | plt.plot(hist_2[ep_counter-1], c='r', label=b, linewidth=1.5) 367 | plt.legend() 368 | plt.grid(True) 369 | plt.title(f'state parameters values {name}') 370 | plt.ylabel(f'{name.lower()} values') 371 | if flag_ter_values == 1: 372 | plt.axhline(tsp[tsp_indexes[0]], color='r', linestyle='-') 373 | plt.axhline(tsp[tsp_indexes[1]], color='r', linestyle='-') 374 | 375 | if flag_saving_with_no_ep_nu == 1: 376 | plt.savefig(folder_path+"/_"+name + ".png") 377 | else: 378 | plt.savefig(folder_path+"_ep_"+str(ep_counter-1) + ".png") 379 | 380 | plt.close() -------------------------------------------------------------------------------- /environment.yml: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/talhazaidi13/Cascaded-Deep-Reinforcement-Learning-Based-Multi-Revolution-Low-Thrust-Spacecraft-Orbit-Transfer/289ca90a2726808dc2613b27a37cf0bd0deeed90/environment.yml -------------------------------------------------------------------------------- /environment_info.txt: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/talhazaidi13/Cascaded-Deep-Reinforcement-Learning-Based-Multi-Revolution-Low-Thrust-Spacecraft-Orbit-Transfer/289ca90a2726808dc2613b27a37cf0bd0deeed90/environment_info.txt -------------------------------------------------------------------------------- /model_1_50.mat: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/talhazaidi13/Cascaded-Deep-Reinforcement-Learning-Based-Multi-Revolution-Low-Thrust-Spacecraft-Orbit-Transfer/289ca90a2726808dc2613b27a37cf0bd0deeed90/model_1_50.mat -------------------------------------------------------------------------------- /paper-outputs/GTO-GEO.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/talhazaidi13/Cascaded-Deep-Reinforcement-Learning-Based-Multi-Revolution-Low-Thrust-Spacecraft-Orbit-Transfer/289ca90a2726808dc2613b27a37cf0bd0deeed90/paper-outputs/GTO-GEO.gif -------------------------------------------------------------------------------- /paper-outputs/GTO-GEO3dpic.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/talhazaidi13/Cascaded-Deep-Reinforcement-Learning-Based-Multi-Revolution-Low-Thrust-Spacecraft-Orbit-Transfer/289ca90a2726808dc2613b27a37cf0bd0deeed90/paper-outputs/GTO-GEO3dpic.PNG -------------------------------------------------------------------------------- /paper-outputs/GTO-GEO3dpic.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/talhazaidi13/Cascaded-Deep-Reinforcement-Learning-Based-Multi-Revolution-Low-Thrust-Spacecraft-Orbit-Transfer/289ca90a2726808dc2613b27a37cf0bd0deeed90/paper-outputs/GTO-GEO3dpic.png -------------------------------------------------------------------------------- /paper-outputs/GTO-GEO_fast (2).mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/talhazaidi13/Cascaded-Deep-Reinforcement-Learning-Based-Multi-Revolution-Low-Thrust-Spacecraft-Orbit-Transfer/289ca90a2726808dc2613b27a37cf0bd0deeed90/paper-outputs/GTO-GEO_fast (2).mp4 -------------------------------------------------------------------------------- /paper-outputs/GTO-GEO_fast.mov: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/talhazaidi13/Cascaded-Deep-Reinforcement-Learning-Based-Multi-Revolution-Low-Thrust-Spacecraft-Orbit-Transfer/289ca90a2726808dc2613b27a37cf0bd0deeed90/paper-outputs/GTO-GEO_fast.mov -------------------------------------------------------------------------------- /paper-outputs/Orbital_Rotation_2 .png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/talhazaidi13/Cascaded-Deep-Reinforcement-Learning-Based-Multi-Revolution-Low-Thrust-Spacecraft-Orbit-Transfer/289ca90a2726808dc2613b27a37cf0bd0deeed90/paper-outputs/Orbital_Rotation_2 .png -------------------------------------------------------------------------------- /paper-outputs/Super GTO-GEO_fast.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/talhazaidi13/Cascaded-Deep-Reinforcement-Learning-Based-Multi-Revolution-Low-Thrust-Spacecraft-Orbit-Transfer/289ca90a2726808dc2613b27a37cf0bd0deeed90/paper-outputs/Super GTO-GEO_fast.mp4 -------------------------------------------------------------------------------- /paper-outputs/SuperGTO-GEO.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/talhazaidi13/Cascaded-Deep-Reinforcement-Learning-Based-Multi-Revolution-Low-Thrust-Spacecraft-Orbit-Transfer/289ca90a2726808dc2613b27a37cf0bd0deeed90/paper-outputs/SuperGTO-GEO.gif -------------------------------------------------------------------------------- /paper-outputs/SuperGTO-GEO_fast.mov: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/talhazaidi13/Cascaded-Deep-Reinforcement-Learning-Based-Multi-Revolution-Low-Thrust-Spacecraft-Orbit-Transfer/289ca90a2726808dc2613b27a37cf0bd0deeed90/paper-outputs/SuperGTO-GEO_fast.mov -------------------------------------------------------------------------------- /paper-outputs/a: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /paper-outputs/a_e.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/talhazaidi13/Cascaded-Deep-Reinforcement-Learning-Based-Multi-Revolution-Low-Thrust-Spacecraft-Orbit-Transfer/289ca90a2726808dc2613b27a37cf0bd0deeed90/paper-outputs/a_e.png -------------------------------------------------------------------------------- /paper-outputs/fig10.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/talhazaidi13/Cascaded-Deep-Reinforcement-Learning-Based-Multi-Revolution-Low-Thrust-Spacecraft-Orbit-Transfer/289ca90a2726808dc2613b27a37cf0bd0deeed90/paper-outputs/fig10.PNG -------------------------------------------------------------------------------- /paper-outputs/fig11.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/talhazaidi13/Cascaded-Deep-Reinforcement-Learning-Based-Multi-Revolution-Low-Thrust-Spacecraft-Orbit-Transfer/289ca90a2726808dc2613b27a37cf0bd0deeed90/paper-outputs/fig11.PNG -------------------------------------------------------------------------------- /paper-outputs/fig12.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/talhazaidi13/Cascaded-Deep-Reinforcement-Learning-Based-Multi-Revolution-Low-Thrust-Spacecraft-Orbit-Transfer/289ca90a2726808dc2613b27a37cf0bd0deeed90/paper-outputs/fig12.PNG -------------------------------------------------------------------------------- /paper-outputs/fig13.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/talhazaidi13/Cascaded-Deep-Reinforcement-Learning-Based-Multi-Revolution-Low-Thrust-Spacecraft-Orbit-Transfer/289ca90a2726808dc2613b27a37cf0bd0deeed90/paper-outputs/fig13.PNG -------------------------------------------------------------------------------- /paper-outputs/fig5.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/talhazaidi13/Cascaded-Deep-Reinforcement-Learning-Based-Multi-Revolution-Low-Thrust-Spacecraft-Orbit-Transfer/289ca90a2726808dc2613b27a37cf0bd0deeed90/paper-outputs/fig5.PNG -------------------------------------------------------------------------------- /paper-outputs/fig6.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/talhazaidi13/Cascaded-Deep-Reinforcement-Learning-Based-Multi-Revolution-Low-Thrust-Spacecraft-Orbit-Transfer/289ca90a2726808dc2613b27a37cf0bd0deeed90/paper-outputs/fig6.PNG -------------------------------------------------------------------------------- /paper-outputs/fig7.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/talhazaidi13/Cascaded-Deep-Reinforcement-Learning-Based-Multi-Revolution-Low-Thrust-Spacecraft-Orbit-Transfer/289ca90a2726808dc2613b27a37cf0bd0deeed90/paper-outputs/fig7.PNG -------------------------------------------------------------------------------- /paper-outputs/fig8.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/talhazaidi13/Cascaded-Deep-Reinforcement-Learning-Based-Multi-Revolution-Low-Thrust-Spacecraft-Orbit-Transfer/289ca90a2726808dc2613b27a37cf0bd0deeed90/paper-outputs/fig8.PNG -------------------------------------------------------------------------------- /paper-outputs/superGTO-GEO3dpic.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/talhazaidi13/Cascaded-Deep-Reinforcement-Learning-Based-Multi-Revolution-Low-Thrust-Spacecraft-Orbit-Transfer/289ca90a2726808dc2613b27a37cf0bd0deeed90/paper-outputs/superGTO-GEO3dpic.PNG -------------------------------------------------------------------------------- /paper-outputs/superGTO-GEO3dpic.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/talhazaidi13/Cascaded-Deep-Reinforcement-Learning-Based-Multi-Revolution-Low-Thrust-Spacecraft-Orbit-Transfer/289ca90a2726808dc2613b27a37cf0bd0deeed90/paper-outputs/superGTO-GEO3dpic.png -------------------------------------------------------------------------------- /paper-outputs/tab3.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/talhazaidi13/Cascaded-Deep-Reinforcement-Learning-Based-Multi-Revolution-Low-Thrust-Spacecraft-Orbit-Transfer/289ca90a2726808dc2613b27a37cf0bd0deeed90/paper-outputs/tab3.PNG -------------------------------------------------------------------------------- /paper-outputs/tab6.PNG: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/talhazaidi13/Cascaded-Deep-Reinforcement-Learning-Based-Multi-Revolution-Low-Thrust-Spacecraft-Orbit-Transfer/289ca90a2726808dc2613b27a37cf0bd0deeed90/paper-outputs/tab6.PNG -------------------------------------------------------------------------------- /plots/a: -------------------------------------------------------------------------------- 1 | 2 | -------------------------------------------------------------------------------- /readme.txt: -------------------------------------------------------------------------------- 1 | 2 | 3 | ############################################__________GTO1- GEO_________########################################################################## 4 | # ## First network weights training start from GEO are as follow. Time: 116.75 days+ 0.25days = 117days 5 | model_path ="E:/RL_project_outputs/plots/finalweights/23197_1671114199_GTO1_1stnet.zip" #Dec-14-2022_1670999182/23197_1671114199.zip for GTO a0.5 e0.1 i0.1 start GTO end following comment line 6 | # ## GTO - GEO first endpoints self.state= np.array([129639.305662,80.60863664,-51.31518440,-0.0001912370795,-0.00133784952,4.88,0,171.924]) #a0.5, e0.1 i0.1 a0.5 e0.01 i0.1 7 | # ## Then with above endpoints, check the whole trajectory and found the point just after the last shadow the new end points from where second netwrok trained is as follow 8 | # ## GTO - GEO first early trajectory endpoints: self.state= np.array([129618.36017,231.805808,-205.689326,-0.000674,0.000658,0.348889,0,173.45981452]) # ,ep_step : 6375, ecc:0.0008 a:5, inc:0.145, time 116.75 days mass 1029.172668937865 9 | ##### Now trained the second network with above start point: in weights multiply a, e with 3. in def_ter i0.1, ecc 0.0002, a 0.6 in def_seg seg=0.1 always and values 0.1,0.1,a5km insecond 0.1, ecc0.01, a0.5 10 | # ## Second network weights training start from above endpoints are as follow 11 | model_path = "E:/OneDrive - Kansas State University/Talha Zaidi Phd/Research/basics code/SAC_stable_baseline/models/Jan-26-2023_1674760417/4670_1674789424.zip" # Time 0.31days inc<0.057 (exact 0.56), ecc<=0.00019, a<0.4km(exact0.34km) 12 | # Another second netwrok, starts from same above points and weights but with more better tolerance values and time, ie tol: ecc<0.00005 a<0.2 inc<0.08, time: 0.25 13 | model_path = "E:/RL_project_outputs/plots/finalweights/7830_1674817940_GTO1_2ndnet.zip" # Hayat comp, weights path D:\NASA_project\models\Jan-26-2023_1674774560\7830_1674817940.zip 14 | ##################################################################################################################################################### 15 | 16 | 17 | # #############################################__________SACpaper GTO- GEO_________########################################################################## 18 | # ### ### Total_time = 145.19+1.24 = 146.43 days with matching or better toelrance values! 19 | # ### ### First model, end conditions, a55_ecc0.01_inc0.1, getseg_0.1,0.1,5%..0.1,0.1,70 20 | # ## First model, end states self.state = np.array([129705.88617642476,-17.661511149943497,82.29267866601921,0.008180683884052145,0.005754129819996143,0.4448333333333312,0,149.19048617296883]) #ecc 0.0099,a47,i0.03716 21 | # model_path = "E:/RL_project_outputs/plots/finalweights/8117_1669700993_SAC_paper_1stnet.zip" # hayat pc "D:/NASA_project/models/Nov-28-2022_1669663038/8117_1669700993.zip" # sac paper _ Time 145.19 days is_ter 0.1 0.01 55 #def_seg 0.1 0.1 5%, 0.1 0.1 70, 22 | # ###### Now trained the second network with above start point: in weights multiply a*3, e*6 .in def_ter i0.1, ecc 0.0009 a36 def_seg seg=0.1 always ep_length_set=5000 23 | model_path = "E:/OneDrive - Kansas State University/Talha Zaidi Phd/Research/basics code/SAC_stable_baseline/models/Feb-01-2023_1675311068_SAC_GTO_2ndnet/11128_1675372524.zip" #Time 1.24, ecc0.00089 i<0.07 a<11 24 | # ###################################################################################################################################################### 25 | 26 | 27 | 28 | # #############################################__________GTO2- GEO_________############################################################################### 29 | # ### First model, end conditions, a50_ecc0.1_inc0.1, getseg_0.1,0.1,5%..0.1,0.1,200 Time: 70.08 + 0.31 days = 70.39 days 30 | # ## First model, end states ,self.state = np.array([129639.13631,-217.707577,-234.640998,-0.000326,0.000801,1.221111,0,36.111]) # final chosen one ep_step : 3716, ecc:0.0008 a:5, inc:0.13, time 70.0 days 31 | # model_path = "E:/RL_project_outputs/plots/finalweights/15440_1674943155_GTO2_1stnet.zip" #witheclipse 69.1 days =>70.2 (70.0) with just after eclipse ecc0.0008 a0.5(5in used-one state) i0.1(0.14) #orignal_path hayatpc: "D:/NASA_project/models/Jan-27-2023_1674866586/15440_1674943155.zip" 32 | # ###### Now trained the second network with above start point: in weights multiply a, e with 3. in def_ter i0.1, ecc 0.00009, a 0.1 in def_seg seg=0.1 always 33 | # ############# Second_Netwrok end conditions, a0.08km_ecc0.000087_inc0.06, getseg_0.1forall 34 | # model_path = "E:/RL_project_outputs/plots/finalweights/11495_1675448339_GTO2_2ndnet.zip " #time0.31days #orignalpath hayatpc: D:/NASA_project/models/Feb-02-2023_1675389717/11495_1675448339.zip 35 | # ###################################################################################################################################################### 36 | 37 | 38 | # ############################################__________superGTO- GEO_________############################################################################### 39 | # ### First model, end conditions, a50_ecc0.1_inc0.1, getseg_0.1,0.1,5%..0.1,0.1,200 Time: 80.6 + 0.27 days = 80.87 days 40 | # ## First model, end states self.state = np.array([129624.62553,-160.07483,-86.112609,-0.000256813,0.00082013,3.97210,0,84.2832]) # final chosen ep:6, ecc:0.002(0.0003-0.0007) a:10, inc:0.1, time 80.6 days 41 | # model_path = "E:/OneDrive - Kansas State University/Talha Zaidi Phd/Research/basics code/SAC_stable_baseline/models/Feb-05-2023_1675579558_SuperGTO_1stnet/49128_1675873753.zip" #a<10 ecc<0.002 (0.0003-0.0007) time 80.6-80.8 42 | # ###### Now trained the second network with above start point: in weights multiply a*4, e*4.5 i*1.5. in def_ter i0.1, ecc 0.0002, a 0.1 in def_seg seg=0.1 always and SHADOW ON WHILE TRAINING 43 | # ############# Second_Netwrok end conditions, a0.5km_ecc0.000069_inc0.1, getseg_0.1forall, and start_point_is self.state = np.array([129624.62553,-160.07483,-86.112609,-0.000256813,0.00082013,3.97210,0,84.2832]) 44 | model_path = "E:/OneDrive - Kansas State University/Talha Zaidi Phd/Research/basics code/SAC_stable_baseline/models/Feb-10-2023_1676078567_SuperGTO_2ndnet/4359_1676106250.zip" #a0.5 ecc0.000069 i0.1 time 0.27 plots:E:\RL_project_outputs\plots\txtdata\Feb-12-2023_1676247819 45 | # ###################################################################################################################################################### 46 | -------------------------------------------------------------------------------- /satellite.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/talhazaidi13/Cascaded-Deep-Reinforcement-Learning-Based-Multi-Revolution-Low-Thrust-Spacecraft-Orbit-Transfer/289ca90a2726808dc2613b27a37cf0bd0deeed90/satellite.png -------------------------------------------------------------------------------- /scenerios.py: -------------------------------------------------------------------------------- 1 | 2 | import argparse 3 | 4 | cases = { 5 | '1': { 6 | 'GTO_state': [67288.41965,0,-32107.258900,0.7306,0,0,0,0], 7 | 'Isp': [1800], 8 | 'm0': [1200], 9 | 'force': [0.311], 10 | 'tol_inc': [0.1], 11 | 'tol_ecc': [0.002], 12 | 'tol_a': [55], #0.5 13 | 'w1_a': [1000], 14 | 'w1_e': [2010], 15 | 'w1_i': [300], 16 | 'w1_a_': [0.010], 17 | 'w1_e_': [0.000000198], 18 | 'w1_i_': [0.0003], 19 | 'c1_a': [500], 20 | 'c1_e': [700], 21 | 'c1_i': [300], 22 | 'tau': [0.003], 23 | 'sh_flag' : [1], 24 | 'weights_path': ["Final Weights/GTO1_1stnet.zip"], 25 | 'max_steps_in_ep': [10000] 26 | }, 27 | 28 | 29 | '2': { 30 | 'GTO_state': [129618.36017,231.805808,-205.689326,-0.000674,0.000658,0.348889,0,173.45981452], 31 | 'Isp': [1800], 32 | 'm0': [1200], 33 | 'force': [0.311], 34 | 'tol_inc': [0.1], 35 | 'tol_ecc': [0.00009], #0.0002 36 | 'tol_a': [0.6], 37 | 'w1_a': [1000*3], 38 | 'w1_e': [700*3], 39 | 'w1_i': [200], 40 | 'w1_a_': [0.010*3], 41 | 'w1_e_': [0.000000198*3], 42 | 'w1_i_': [0.0003], 43 | 'c1_a': [500*3], 44 | 'c1_e': [700*3], 45 | 'c1_i': [300], 46 | 'tau': [0.003], 47 | 'sh_flag' : [1], 48 | 'weights_path': ["Final Weights/GTO1_2ndnet.zip"], 49 | 'max_steps_in_ep': [2000] 50 | }, 51 | 52 | 53 | '3': { 54 | 'GTO_state': [67246.21689,0,-30529.143615,0.7310,0,0,0,0], 55 | 'Isp': [3300], 56 | 'm0': [450], 57 | 'force': [0.2007], 58 | 'tol_inc': [0.1], 59 | 'tol_ecc': [0.1], 60 | 'tol_a': [50], 61 | 'w1_a': [1000], 62 | 'w1_e': [2010], 63 | 'w1_i': [300], 64 | 'w1_a_': [0.010], 65 | 'w1_e_': [0.000000198], 66 | 'w1_i_': [0.0003], 67 | 'c1_a': [500], 68 | 'c1_e': [700], 69 | 'c1_i': [300], 70 | 'tau': [0.003], 71 | 'sh_flag' : [1], 72 | 'weights_path': ["Final Weights/GTO2_1stnet.zip"], 73 | 'max_steps_in_ep': [20000] 74 | }, 75 | 76 | 77 | '4': { 78 | 'GTO_state': [129639.13631,-217.707577,-234.640998,-0.000326,0.000801,1.221111,0,36.111], 79 | 'Isp': [3300], 80 | 'm0': [450], 81 | 'force': [0.2007], 82 | 'tol_inc': [0.08], 83 | 'tol_ecc': [0.000088], 84 | 'tol_a': [0.2], 85 | 'w1_a': [1000*3], 86 | 'w1_e': [700*3], 87 | 'w1_i': [200], 88 | 'w1_a_': [0.010*3], 89 | 'w1_e_': [0.000000198*3], 90 | 'w1_i_': [0.0003], 91 | 'c1_a': [500*3], 92 | 'c1_e': [700*3], 93 | 'c1_i': [300], 94 | 'tau': [0.003], 95 | 'sh_flag' : [1], 96 | 'weights_path': ["Final Weights/GTO2_2ndnet.zip"], 97 | 'max_steps_in_ep': [2000] 98 | 99 | }, 100 | '5': { 101 | 'GTO_state': [68196.3519, 0.0, -8311.0446, 0.725 ,0.0, 0.0, 0.0, 0.0], 102 | 'Isp': [3000], 103 | 'm0': [2000], 104 | 'force': [0.35], 105 | 'tol_inc': [0.1], 106 | 'tol_ecc': [0.01], 107 | 'tol_a': [55], 108 | 'w1_a': [1000], 109 | 'w1_e': [2010], 110 | 'w1_i': [300], 111 | 'w1_a_': [0.010], 112 | 'w1_e_': [0.000000198], 113 | 'w1_i_': [0.0003], 114 | 'c1_a': [500], 115 | 'c1_e': [700], 116 | 'c1_i': [300], 117 | 'tau': [0.003], 118 | 'sh_flag' : [0], 119 | 'weights_path': ["Final Weights/GTO3_1stnet.zip"], 120 | 'max_steps_in_ep': [20000] 121 | }, 122 | 123 | 124 | '6': { 125 | 'GTO_state': [129705.88617642476,-17.661511149943497,82.29267866601921,0.008180683884052145,0.005754129819996143,0.4448333333333312,0,149.19048617296883], 126 | 'Isp': [3000], 127 | 'm0': [2000], 128 | 'force': [0.35], 129 | 'tol_inc': [0.1], 130 | 'tol_ecc': [0.0009], 131 | 'tol_a': [12], 132 | 'w1_a': [1000*3], 133 | 'w1_e': [700*6], 134 | 'w1_i': [200], 135 | 'w1_a_': [0.010*3], 136 | 'w1_e_': [0.000000198*6], 137 | 'w1_i_': [0.0003], 138 | 'c1_a':[ 500*3], 139 | 'c1_e': [700*6], 140 | 'c1_i': [300], 141 | 'tau': [0.003], 142 | 'sh_flag' : [0], 143 | 'weights_path':["Final Weights/GTO3_2ndnet.zip"], 144 | 'max_steps_in_ep': [8000] 145 | }, 146 | 147 | 148 | '7': { 149 | 'GTO_state': [70532.88179,0,-26991.765301,0.8705,0,0,0,0], 150 | 'Isp': [3300], 151 | 'm0': [1200], 152 | 'force':[ 0.4015], 153 | 'tol_inc': [0.1], 154 | 'tol_ecc': [0.002], 155 | 'tol_a': [10], 156 | 'w1_a': [1000], 157 | 'w1_e': [2010*2], 158 | 'w1_i': [300], 159 | 'w1_a_': [0.010], 160 | 'w1_e_': [0.000000198*2], 161 | 'w1_i_':[0.0003], 162 | 'c1_a': [500], 163 | 'c1_e': [1000*2], 164 | 'c1_i': [300], 165 | 'tau': [0.003], 166 | 'sh_flag' : [1], 167 | 'weights_path': ["Final Weights/SuperGTO_1stnet.zip"], 168 | 'max_steps_in_ep': [20000] 169 | }, 170 | 171 | 172 | '8': { 173 | 'GTO_state': [129624.62553,-160.07483,-86.112609,-0.000256813,0.00082013,3.97210,0,84.2832], 174 | 'Isp': [3300], 175 | 'm0': [1200], 176 | 'force': [0.4015], 177 | 'tol_inc': [0.1], 178 | 'tol_ecc': [0.0002], 179 | 'tol_a': [0.5], 180 | 'w1_a': [1000*2*2], 181 | 'w1_e': [2010*2*2], 182 | 'w1_i': [300], 183 | 'w1_a_': [0.010*4], 184 | 'w1_e_': [0.000000198*4], 185 | 'w1_i_': [0.0003], 186 | 'c1_a': [500*4], 187 | 'c1_e': [1000*4], 188 | 'c1_i': [300], 189 | 'tau': [0.003], 190 | 'sh_flag' : [1], 191 | 'weights_path': ["Final Weights/SuperGTO_2ndnet.zip"], 192 | 'max_steps_in_ep': [5000] 193 | } 194 | 195 | } 196 | 197 | 198 | -------------------------------------------------------------------------------- /spacecraftEnivironment.m: -------------------------------------------------------------------------------- 1 | function[finalState, finalSpacecraftMass]=spacecraftEnivironment(state,alpha,beta,F,segment,m0,Isp) 2 | % segment = angle in radians 3 | % F= thrust in kilo newtons 4 | % alpha, beta = thrust angles in radians 5 | %display("line 5: "); 6 | %display(state); 7 | n=10; % integration steps 8 | phi_step=segment/n; 9 | phi_span = state(6):phi_step:n*phi_step+state(6); 10 | ctrl_res=[alpha*ones(length(phi_span)-1,1);beta*ones(length(phi_span)-1,1)]; 11 | [~,tra_d] = ode45(@(phi,x)TwoBody(phi,x,ctrl_res,n,state(6),F,phi_step,m0,Isp)... 12 | ,phi_span,state); 13 | finalState=tra_d(end,:); 14 | finalSpacecraftMass=m0+tra_d(end,8); 15 | end 16 | 17 | function dst_dphi = TwoBody(phi,state,u,n,phi0,Thr,phi_step,m0,Isp) 18 | %global mu % defined in chkStop 19 | mu = 398600.4418; 20 | g0=9.81/1000; % km/sec^2 21 | if (phi-phi0) > 0 22 | if (phi-phi0) >= n*phi_step 23 | alp = u(n); 24 | bet = u(n*2); 25 | else 26 | alp = u(ceil((phi-phi0)/phi_step)); 27 | bet = u(n + ceil((phi-phi0)/phi_step)); 28 | end 29 | 30 | else 31 | alp = u(1); 32 | bet = u(n+1); 33 | 34 | %display("line 33: "); 35 | %display(state); 36 | 37 | end 38 | 39 | [G,f] = Var_mat([state(1:5);phi],mu); 40 | 41 | h = state(1); 42 | hx = state(2); 43 | hy = state(3); 44 | ex = state(4); 45 | ey = state(5); 46 | m = m0 +state(8); 47 | %display("line 45: "); 48 | %display(state); 49 | 50 | %display("line 46: "); 51 | %display(state(1)); 52 | %display(state(2)); 53 | %display(state(3)); 54 | 55 | 56 | 57 | if h < hy 58 | display('h less than hy') 59 | display(h) 60 | display(hx) 61 | display(hy) 62 | error('xxx') 63 | end 64 | 65 | B = 1+ex*cos(phi)+ey*sin(phi); 66 | fr = -sin(alp).*cos(bet); 67 | fn = cos(alp).*cos(bet); 68 | fh = sin(bet); 69 | F = Thr*[fr; fn; fh]; 70 | U = F + [0;0;0]; 71 | 72 | dt_dphi = h^3*m*mu*B*sqrt(h^2-hy^2)/... 73 | (m*mu^3*B^3*sqrt(h^2-hy^2)-h^4*hy*sin(phi)*U(3)); 74 | 75 | 76 | dOrb_dphi = (f(1:5,1) + 1/m*G(1:5,:)*U)*dt_dphi; 77 | dm_dphi=(-Thr/g0/Isp)*dt_dphi; 78 | dst_dphi = [dOrb_dphi;1;dt_dphi;dm_dphi]; 79 | end 80 | 81 | function [G,f] = Var_mat( state,mu ) 82 | %UNTITLED4 Summary of this function goes here 83 | % Detailed explanation goes here 84 | h = state(1); 85 | hx = state(2); 86 | hy = state(3); 87 | ex = state(4); 88 | ey = state(5); 89 | phi = state(6); 90 | 91 | A = ex*sin(phi) - ey*cos(phi); 92 | B = 1 + ex*cos(phi) + ey*sin(phi); 93 | 94 | g12 = h^2/mu/B; 95 | g22 = h*hx/mu/B; 96 | g23 = 1/mu/B/sqrt(h^2-hy^2)*(h^2*sqrt(h^2-hx^2-hy^2)*sin(phi) + ... 97 | h*hx*hy*cos(phi)); %correct 98 | g32 = h*hy/mu/B; %correct 99 | g33 = -h*sqrt(h^2-hy^2)/mu/B*cos(phi); %correct 100 | g41 = h*sin(phi)/mu; %correct 101 | g42 = 2*h*cos(phi)/mu + h*(A/mu/B)*sin(phi); %correct 102 | g43 = h*ey*hy/mu/B/sqrt(h^2-hy^2)*sin(phi); %correct 103 | g51 = -h/mu*cos(phi); %correct 104 | g52 = 2*h/mu*sin(phi) - h*A/mu/B*cos(phi); %correct 105 | g53 = -h*ex*hy/mu/B/sqrt(h^2-hy^2)*sin(phi); %correct 106 | g63 = -h*hy/mu/B/sqrt(h^2-hy^2)*sin(phi); 107 | 108 | G = [0 g12 0; 0 g22 g23; 0 g32 g33; g41 g42 g43; g51 g52 g53; 0 0 g63]; 109 | f = [0;0;0;0;0; mu^2*B^2/h^3]; 110 | end 111 | 112 | 113 | -------------------------------------------------------------------------------- /test.py: -------------------------------------------------------------------------------- 1 | import stable_baselines3 2 | from stable_baselines3 import SAC 3 | from stable_baselines3.common.env_util import make_vec_env 4 | import os 5 | from Spacecraft_Env import Spacecraft_Env 6 | import time 7 | #from enviornment import Enviornment 8 | import numpy as np 9 | import tensorflow as tf 10 | import os.path 11 | import time 12 | from datetime import date 13 | from config import args 14 | from scenerios import cases 15 | 16 | tf.debugging.set_log_device_placement(True) 17 | import argparse 18 | from config import args 19 | 20 | chosen_case = cases[args.case] 21 | model_path = chosen_case['weights_path'][0] 22 | current_dir = os.getcwd() 23 | models_dir = os.path.join(current_dir , model_path) 24 | 25 | 26 | env = Spacecraft_Env(args) 27 | env.reset() 28 | model = SAC.load(model_path, env =env ) 29 | 30 | episodes = args.max_nu_ep 31 | with tf.device('/GPU:0'): 32 | for ep in range (episodes): 33 | obs = env.reset() 34 | done = False 35 | steps = 0 36 | model = SAC.load(model_path, env =env ) 37 | while not done: 38 | steps = steps + 1 39 | # print("Step {}".format(steps)) 40 | action = model.predict(obs) 41 | obs, reward, done, info = env.step(action[0]) 42 | 43 | env.close() 44 | print("All EPISODES DONE ! ") 45 | 46 | -------------------------------------------------------------------------------- /test.sh: -------------------------------------------------------------------------------- 1 | python test.py --case 2 --max_nu_ep 100 2 | -------------------------------------------------------------------------------- /train.py: -------------------------------------------------------------------------------- 1 | import stable_baselines3 2 | from stable_baselines3 import SAC 3 | from stable_baselines3.common.env_util import make_vec_env 4 | import os 5 | from Spacecraft_Env import Spacecraft_Env 6 | import time 7 | #from enviornment import Enviornment 8 | import numpy as np 9 | import tensorflow as tf 10 | import os.path 11 | import time 12 | from datetime import date 13 | 14 | 15 | tf.debugging.set_log_device_placement(True) 16 | import argparse 17 | from config import args 18 | 19 | # Assign the variables to the parsed arguments 20 | max_steps_one_ep = args.max_steps_one_ep 21 | max_nu_ep = args.max_nu_ep 22 | weights_save_steps = args.weights_save_steps 23 | buffer_size = args.buffer_size 24 | gamma = args.gamma 25 | lr = args.lr 26 | 27 | today = date.today() 28 | day = today.strftime('%b-%d-%Y') 29 | txt_dir = f'{int(time.time())}' 30 | dir_name = day + "_" + txt_dir +'/' 31 | current_dir = os.getcwd() 32 | models_dir = os.path.join(current_dir , 'Model_training_weights', dir_name) 33 | logdir = os.path.join(current_dir , 'Model_training_logs', dir_name) 34 | 35 | if not os.path.exists(models_dir): 36 | os.makedirs(models_dir) 37 | if not os.path.exists(logdir): 38 | os.makedirs(logdir) 39 | 40 | env = Spacecraft_Env(args) 41 | env.reset() 42 | 43 | model = SAC('MlpPolicy', env, learning_rate=lr, buffer_size=buffer_size, gamma=gamma, tau=0.005, train_freq=1, verbose=1, tensorboard_log=logdir) 44 | 45 | n_steps = max_steps_one_ep*max_nu_ep 46 | TIMESTEPS = weights_save_steps 47 | with tf.device('/GPU:0'): 48 | for step in range (n_steps): 49 | print("Step {}".format(step + 1)) 50 | model.learn(total_timesteps=TIMESTEPS, reset_num_timesteps=False, log_interval=1, tb_log_name='SAC') 51 | file_name= str(step)+"_"+str(int(time.time())) 52 | model.save(f"{models_dir}/{file_name}") 53 | 54 | print("All EPISODES DONE ! ") 55 | 56 | 57 | 58 | 59 | 60 | 61 | --------------------------------------------------------------------------------