├── log
    ├── re_log_2017-06-12_02:55:09
    └── re_log_2017-06-12_02:59:23
├── pg_re.pyc
├── pg_su.pyc
├── data
    └── .DS_Store
├── environment.pyc
├── other_agents.pyc
├── parameters.pyc
├── pg_network.pyc
├── plot
    └── .DS_Store
├── slow_down_cdf.pyc
├── job_distribution.pyc
├── pg_re_single_core.pyc
├── deeprm state space.png
├── deeprm2 state space.png
├── launcher2.py
├── run_script.py
├── other_agents.py
├── job_distribution.py
├── parameters.py
├── README.md
├── launcher.py
├── pg_su.py
├── slow_down_cdf.py
├── pg_re_single_core_o.py
├── pg_re_single_core.py
├── pg_network.py
├── pg_re_o.py
├── pg_re.py
└── environment.py


/log/re_log_2017-06-12_02:55:09:
--------------------------------------------------------------------------------
1 | 


--------------------------------------------------------------------------------
/log/re_log_2017-06-12_02:59:23:
--------------------------------------------------------------------------------
1 | 


--------------------------------------------------------------------------------
/pg_re.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/BrightFeather/deeprm_conv/HEAD/pg_re.pyc


--------------------------------------------------------------------------------
/pg_su.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/BrightFeather/deeprm_conv/HEAD/pg_su.pyc


--------------------------------------------------------------------------------
/data/.DS_Store:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/BrightFeather/deeprm_conv/HEAD/data/.DS_Store


--------------------------------------------------------------------------------
/environment.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/BrightFeather/deeprm_conv/HEAD/environment.pyc


--------------------------------------------------------------------------------
/other_agents.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/BrightFeather/deeprm_conv/HEAD/other_agents.pyc


--------------------------------------------------------------------------------
/parameters.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/BrightFeather/deeprm_conv/HEAD/parameters.pyc


--------------------------------------------------------------------------------
/pg_network.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/BrightFeather/deeprm_conv/HEAD/pg_network.pyc


--------------------------------------------------------------------------------
/plot/.DS_Store:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/BrightFeather/deeprm_conv/HEAD/plot/.DS_Store


--------------------------------------------------------------------------------
/slow_down_cdf.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/BrightFeather/deeprm_conv/HEAD/slow_down_cdf.pyc


--------------------------------------------------------------------------------
/job_distribution.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/BrightFeather/deeprm_conv/HEAD/job_distribution.pyc


--------------------------------------------------------------------------------
/pg_re_single_core.pyc:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/BrightFeather/deeprm_conv/HEAD/pg_re_single_core.pyc


--------------------------------------------------------------------------------
/deeprm state space.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/BrightFeather/deeprm_conv/HEAD/deeprm state space.png


--------------------------------------------------------------------------------
/deeprm2 state space.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/BrightFeather/deeprm_conv/HEAD/deeprm2 state space.png


--------------------------------------------------------------------------------
/launcher2.py:
--------------------------------------------------------------------------------
 1 | # tidier and simpler launcher without use of command line
 2 | import os
 3 | os.environ["THEANO_FLAGS"] = "device=cpu,floatX=float32"
 4 | import sys
 5 | import getopt
 6 | import matplotlib
 7 | matplotlib.use('Agg')
 8 | 
 9 | import parameters
10 | import pg_re
11 | import pg_re_single_core
12 | import pg_su
13 | import slow_down_cdf
14 | 
15 | pa = parameters.Parameters()
16 | pa.type_exp = "pg_re"
17 | # pa.pg_resume = "data/pg_su_net_file_9990.pkl"
18 | pa.simu_len = 50
19 | pa.num_ex = 10
20 | pa.output_filename= "data/pg_re_conv"
21 | pa.output_freq=2
22 | pg_re_single_core.launch(pa)
23 | 


--------------------------------------------------------------------------------
/run_script.py:
--------------------------------------------------------------------------------
 1 | # /usr/bin/env python
 2 | 
 3 | import os
 4 | 
 5 | simu_len = 200
 6 | 
 7 | for new_job_rate in [0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 1]:
 8 |     for num_seq_per_batch in [20]:
 9 |         for num_ex in [100]:
10 |             for num_nw in [10]:
11 | 
12 |                 file_name = 'data/pg_re_rate_' + str(new_job_rate) + '_simu_len_' + str(simu_len) + '_num_seq_per_batch_' + str(num_seq_per_batch) + '_ex_' + str(num_ex) + '_nw_' + str(num_nw)
13 |                 log = 'log/pg_re_rate_' + str(new_job_rate) + '_simu_len_' + str(simu_len) + '_num_seq_per_batch_' + str(num_seq_per_batch) + '_ex_' + str(num_ex) + '_nw_' + str(num_nw)
14 | 
15 |                 # run experiment
16 |                 os.system('nohup python -u launcher.py --exp_type=pg_re --out_freq=50 --simu_len=' + str(simu_len) + ' --eps_max_len=' + str(simu_len * 4) + ' --num_ex=' + str(num_ex) + ' --new_job_rate=' + str(new_job_rate) + ' --num_seq_per_batch=' + str(num_seq_per_batch) + ' --num_nw=' + str(num_nw) + ' --ofile=' + file_name + ' > ' + log + ' &')
17 | 
18 |                 # plot slowdown
19 |                 # it_num = 100
20 |                 # os.system('nohup python -u launcher.py --exp_type=test --simu_len=' + str(simu_len) + '--num_ex=' + str(num_ex) + ' --new_job_rate=' + str(new_job_rate) + ' --num_seq_per_batch=' + str(num_seq_per_batch) + ' --pg_re=' + file_name + '_' + str(it_num) + '.pkl' + ' &')
21 | 


--------------------------------------------------------------------------------
/other_agents.py:
--------------------------------------------------------------------------------
 1 | import numpy as np
 2 | 
 3 | 
 4 | def get_packer_action(machine, job_slot):
 5 |         align_score = 0
 6 |         act = len(job_slot.slot)  # if no action available, hold
 7 | 
 8 |         for i in xrange(len(job_slot.slot)):
 9 |             new_job = job_slot.slot[i]
10 |             if new_job is not None:  # there is a pending job
11 | 
12 |                 avbl_res = machine.avbl_slot[:new_job.len, :]
13 |                 res_left = avbl_res - new_job.res_vec
14 | 
15 |                 if np.all(res_left[:] >= 0):  # enough resource to allocate
16 | 
17 |                     tmp_align_score = avbl_res[0, :].dot(new_job.res_vec)
18 | 
19 |                     if tmp_align_score > align_score:
20 |                         align_score = tmp_align_score
21 |                         act = i
22 |         return act
23 | 
24 | 
25 | def get_sjf_action(machine, job_slot):
26 |         sjf_score = 0
27 |         act = len(job_slot.slot)  # if no action available, hold
28 | 
29 |         for i in xrange(len(job_slot.slot)):
30 |             new_job = job_slot.slot[i]
31 |             if new_job is not None:  # there is a pending job
32 | 
33 |                 avbl_res = machine.avbl_slot[:new_job.len, :]
34 |                 res_left = avbl_res - new_job.res_vec
35 | 
36 |                 if np.all(res_left[:] >= 0):  # enough resource to allocate
37 | 
38 |                     tmp_sjf_score = 1 / float(new_job.len)
39 | 
40 |                     if tmp_sjf_score > sjf_score:
41 |                         sjf_score = tmp_sjf_score
42 |                         act = i
43 |         return act
44 | 
45 | 
46 | def get_packer_sjf_action(machine, job_slot, knob):  # knob controls which to favor, 1 to packer, 0 to sjf
47 | 
48 |         combined_score = 0
49 |         act = len(job_slot.slot)  # if no action available, hold
50 | 
51 |         for i in xrange(len(job_slot.slot)):
52 |             new_job = job_slot.slot[i]
53 |             if new_job is not None:  # there is a pending job
54 | 
55 |                 avbl_res = machine.avbl_slot[:new_job.len, :]
56 |                 res_left = avbl_res - new_job.res_vec
57 | 
58 |                 if np.all(res_left[:] >= 0):  # enough resource to allocate
59 | 
60 |                     tmp_align_score = avbl_res[0, :].dot(new_job.res_vec)
61 |                     tmp_sjf_score = 1 / float(new_job.len)
62 | 
63 |                     tmp_combined_score = knob * tmp_align_score + (1 - knob) * tmp_sjf_score
64 | 
65 |                     if tmp_combined_score > combined_score:
66 |                         combined_score = tmp_combined_score
67 |                         act = i
68 |         return act
69 | 
70 | 
71 | def get_random_action(job_slot):
72 |     num_act = len(job_slot.slot) + 1  # if no action available,
73 |     act = np.random.randint(num_act)
74 |     return act
75 | 


--------------------------------------------------------------------------------
/job_distribution.py:
--------------------------------------------------------------------------------
 1 | import numpy as np
 2 | 
 3 | 
 4 | class Dist:
 5 | 
 6 |     def __init__(self, num_res, max_nw_size, job_len):
 7 |         self.num_res = num_res
 8 |         self.max_nw_size = max_nw_size
 9 |         self.job_len = job_len
10 | 
11 |         self.job_small_chance = 0.8
12 | 
13 |         self.job_len_big_lower = job_len * 2 / 3
14 |         self.job_len_big_upper = job_len
15 | 
16 |         self.job_len_small_lower = 1
17 |         self.job_len_small_upper = job_len / 5
18 | 
19 |         self.dominant_res_lower = max_nw_size / 2
20 |         self.dominant_res_upper = max_nw_size
21 | 
22 |         self.other_res_lower = 1
23 |         self.other_res_upper = max_nw_size / 5
24 | 
25 |     def normal_dist(self):
26 | 
27 |         # new work duration
28 |         nw_len = np.random.randint(1, self.job_len + 1)  # same length in every dimension
29 | 
30 |         nw_size = np.zeros(self.num_res)
31 | 
32 |         for i in range(self.num_res):
33 |             nw_size[i] = np.random.randint(1, self.max_nw_size + 1)
34 | 
35 |         return nw_len, nw_size
36 | 
37 |     def bi_model_dist(self):
38 | 
39 |         # -- job length --
40 |         if np.random.rand() < self.job_small_chance:  # small job
41 |             nw_len = np.random.randint(self.job_len_small_lower,
42 |                                        self.job_len_small_upper + 1)
43 |         else:  # big job
44 |             nw_len = np.random.randint(self.job_len_big_lower,
45 |                                        self.job_len_big_upper + 1)
46 | 
47 |         nw_size = np.zeros(self.num_res)
48 | 
49 |         # -- job resource request --
50 |         dominant_res = np.random.randint(0, self.num_res)
51 |         for i in range(self.num_res):
52 |             if i == dominant_res:
53 |                 nw_size[i] = np.random.randint(self.dominant_res_lower,
54 |                                                self.dominant_res_upper + 1)
55 |             else:
56 |                 nw_size[i] = np.random.randint(self.other_res_lower,
57 |                                                self.other_res_upper + 1)
58 | 
59 |         return nw_len, nw_size
60 | 
61 | 
62 | def generate_sequence_work(pa, seed=42):
63 | 
64 |     np.random.seed(seed)
65 | 
66 |     simu_len = pa.simu_len * pa.num_ex
67 | 
68 |     nw_dist = pa.dist.bi_model_dist
69 | 
70 |     nw_len_seq = np.zeros(simu_len, dtype=int)
71 |     nw_size_seq = np.zeros((simu_len, pa.num_res), dtype=int)
72 | 
73 |     for i in range(simu_len):
74 | 
75 |         if np.random.rand() < pa.new_job_rate:  # a new job comes
76 | 
77 |             nw_len_seq[i], nw_size_seq[i, :] = nw_dist()
78 | 
79 |     nw_len_seq = np.reshape(nw_len_seq,
80 |                             [pa.num_ex, pa.simu_len])
81 |     nw_size_seq = np.reshape(nw_size_seq,
82 |                              [pa.num_ex, pa.simu_len, pa.num_res])
83 | 
84 |     return nw_len_seq, nw_size_seq
85 | 


--------------------------------------------------------------------------------
/parameters.py:
--------------------------------------------------------------------------------
 1 | import numpy as np
 2 | import math
 3 | 
 4 | import job_distribution
 5 | 
 6 | 
 7 | class Parameters:
 8 |     def __init__(self):
 9 | 
10 |         self.output_filename = 'data/tmp'
11 | 
12 |         self.num_epochs = 1000         # number of training epochs
13 |         self.simu_len = 10             # length of the busy cycle that repeats itself, length of each trajectory 10
14 |         self.num_ex = 1                # number of sequences, number of episodes
15 | 
16 |         self.output_freq = 10          # interval for output and store parameters
17 | 
18 |         self.num_seq_per_batch = 10    # number of sequences to compute baseline
19 |         self.episode_max_length = 200  # enforcing an artificial terminal
20 | 
21 |         self.num_res = 2               # number of resources in the system
22 |         self.num_nw = 5                # maximum allowed number of work in the queue
23 | 
24 |         self.time_horizon = 20         # number of time steps in the graph
25 |         self.max_job_len = 15          # maximum duration of new jobs
26 |         self.res_slot = 10             # maximum number of available resource slots
27 |         self.max_job_size = 10         # maximum resource request of new work
28 | 
29 |         self.backlog_size = 60         # backlog queue size
30 | 
31 |         self.max_track_since_new = 10  # track how many time steps since last new jobs
32 | 
33 |         self.job_num_cap = 40          # maximum number of distinct colors in current work graph
34 | 
35 |         self.new_job_rate = 0.7        # lambda in new job arrival Poisson Process
36 | 
37 |         self.discount = 1           # discount factor
38 | 
39 |         # distribution for new job arrival
40 |         self.dist = job_distribution.Dist(self.num_res, self.max_job_size, self.max_job_len)
41 | 
42 |         # graphical representation
43 |         assert self.backlog_size % self.time_horizon == 0  # such that it can be converted into an image
44 |         self.backlog_width = int(math.ceil(self.backlog_size / float(self.time_horizon * self.num_res)))
45 |         self.network_input_height = self.time_horizon * self.num_res
46 |         self.network_input_width = \
47 |             (self.res_slot +
48 |              self.max_job_size * self.num_nw) + \
49 |             self.backlog_width + \
50 |             1  # for extra info, 1) time since last new job
51 | 
52 |         # compact representation
53 |         self.network_compact_dim = (self.num_res + 1) * \
54 |             (self.time_horizon + self.num_nw) + 1  # + 1 for backlog indicator
55 | 
56 |         self.network_output_dim = self.num_nw + 1  # + 1 for void action
57 | 
58 |         self.delay_penalty = -1       # penalty for delaying things in the current work screen
59 |         self.hold_penalty = -1        # penalty for holding things in the new work screen
60 |         self.dismiss_penalty = -1     # penalty for missing a job because the queue is full
61 | 
62 |         self.num_frames = 1           # number of frames to combine and process
63 |         self.lr_rate = 0.001          # learning rate
64 |         self.rms_rho = 0.9            # for rms prop
65 |         self.rms_eps = 1e-9           # for rms prop
66 | 
67 |         self.unseen = False  # change random seed to generate unseen example
68 | 
69 |         # supervised learning mimic policy
70 |         self.batch_size = 10
71 |         self.evaluate_policy_name = "SJF"
72 | 
73 |     def compute_dependent_parameters(self):
74 |         assert self.backlog_size % self.time_horizon == 0  # such that it can be converted into an image
75 |         self.backlog_width = self.backlog_size / self.time_horizon
76 |         self.network_input_height = self.time_horizon
77 |         self.network_input_width = \
78 |             (self.res_slot +
79 |              self.max_job_size * self.num_nw) * self.num_res + \
80 |             self.backlog_width + \
81 |             1  # for extra info, 1) time since last new job
82 | 
83 |         # compact representation
84 |         self.network_compact_dim = (self.num_res + 1) * \
85 |             (self.time_horizon + self.num_nw) + 1  # + 1 for backlog indicator
86 | 
87 |         self.network_output_dim = self.num_nw + 1  # + 1 for void action
88 | 
89 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
  1 | # DeepRM+

  2 | 
Based on works of Hongzi Mao
HotNets'16 http://people.csail.mit.edu/hongzi/content/publications/DeepRM-HotNets16.pdf

  3 |  
  4 | 
Made improvements based on DeepRM http://github.com/hongzimao/deeprm

  5 |  
  6 | ## Improvement on algorithm structure:

  7 | 
  8 | #### Rebuild the network with a convolution neural network 
  9 | 
 10 | File: build_small_conv_pg_network in pg_network.py
 11 | 
Network structure:
Input: CNN with size 2*2, 16 filters

 12 | 
 13 | Output: Fully connected layer with # of actions output

 14 | 
Major improvement. Improved convergence rate (by ??? --> To Do)

 15 | #### Reshape state space. 
 16 | 
 17 | File: environment.py
 18 | 
 19 | 
In DeepRM, state space was generated by stacking vertically matrices in the following way:
 20 |  
 21 | 
State matrix for resource 1, job 1's request matrix for resource 1, job 2's request matrix for resource 1, ... , job n's request matrix for resource 1,\
 State matrix for resource 2, job 1's request matrix for resource 2, job 2's request matrix for resource 2, ... , job n's request matrix for resource 2.

 22 |  
 23 | 
I decide to put the related matrices closer, therefore stacking matrices in the following way:
 24 |  
 25 | 
Stacking vertically respectively: State matrix for resource 1, job 1's request matrix for resource 1, job 2's request matrix for resource 1, ... , job n's request matrix for resource 1,\
and State matrix for resource 2, job 1's request matrix for resource 2, job 2's request matrix for resource 2, ... , job n's request matrix for resource 2.
And then stack the above two long matrices vertically.

 26 |  
 27 | See picture below for better explanation:
 28 | 
 29 | 
 30 | 
 31 | <img src="https://github.com/BrightFeather/deeprm_conv/blob/master/deeprm%20state%20space.png" alt="Original state matrix" title="Original state matrix" height="175" />
 32 | Original state matrix
 33 | 
 34 | 
 35 | 
 36 | <img src="https://github.com/BrightFeather/deeprm_conv/blob/master/deeprm2%20state%20space.png" alt="Reshaped state matrix" title="Reshaped state matrix" height="305" />
 37 | Reshaped state matrix
 38 | 
 39 | 
 40 | 
 41 | 
Major improvement. Improved the average slowdown by 8.9% after 1000 epochs of training.
 42 | 
 43 | #### Rewrite penalty function. 
 44 | 
 45 | File: parameters.py 
 46 | 
 47 | 
I gave different weights of penalty for jobs already planned(in machine matrix), jobs in jobslot queue and jobs in backlog.

Minor improvement. Improved convergence rate.
 48 | 
 49 | ## Others


 50 | * Added log and save checkpoints to make record of slowdown and save models (pg_re_single_core.py and pg_re.py)
 51 | 

 52 | 
 53 | * Added launcher2 for convenient launching and debugging (launcher2.py)
 54 | 
 55 | ## Install prerequisites
 56 | 
 57 | ```
 58 | sudo apt-get update
 59 | sudo apt-get install python-numpy python-scipy python-dev python-pip python-nose g++ libopenblas-dev git
 60 | pip install --user Theano
 61 | pip install --user Lasagne==0.1
 62 | sudo apt-get install python-matplotlib
 63 | ```
 64 | 
 65 | ## Run code
 66 | In folder RL, create a data/ folder. 
 67 | 
 68 | Use `launcher.py` to launch experiments. 
 69 | 
 70 | 
 71 | ```
 72 | --exp_type <type of experiment> 
 73 | --num_res <number of resources> 
 74 | --num_nw <number of visible new work> 
 75 | --simu_len <simulation length> 
 76 | --num_ex <number of examples> 
 77 | --num_seq_per_batch <rough number of samples in one batch update> 
 78 | --eps_max_len <episode maximum length (terminated at the end)>
 79 | --num_epochs <number of epoch to do the training>
 80 | --time_horizon <time step into future, screen height> 
 81 | --res_slot <total number of resource slots, screen width> 
 82 | --max_job_len <maximum new job length> 
 83 | --max_job_size <maximum new job resource request> 
 84 | --new_job_rate <new job arrival rate> 
 85 | --dist <discount factor> 
 86 | --lr_rate <learning rate> 
 87 | --ba_size <batch size> 
 88 | --pg_re <parameter file for pg network> 
 89 | --v_re <parameter file for v network> 
 90 | --q_re <parameter file for q network> 
 91 | --out_freq <network output frequency> 
 92 | --ofile <output file name> 
 93 | --log <log file name> 
 94 | --render <plot dynamics> 
 95 | --unseen <generate unseen example> 
 96 | ```
 97 | 
 98 | 
 99 | The default variables are defined in `parameters.py`.
100 | 
101 | 
102 | Example: 
103 |   - launch supervised learning for policy estimation 
104 |   
105 |   ```
106 |   python launcher.py --exp_type=pg_su --simu_len=50 --num_ex=1000 --ofile=data/pg_su --out_freq=10 
107 |   ```
108 |   - launch policy gradient using network parameter just obtained
109 |   
110 |   ```
111 |   python launcher.py --exp_type=pg_re --pg_re=data/pg_su_net_file_20.pkl --simu_len=50 --num_ex=10 --ofile=data/pg_re
112 |   ```
113 |   - launch testing and comparing experiemnt on unseen examples with pg agent just trained
114 |   
115 |   ```
116 |   python launcher.py --exp_type=test --simu_len=50 --num_ex=10 --pg_re=data/pg_re_1600.pkl --unseen=True
117 |   ```
118 | 


--------------------------------------------------------------------------------
/launcher.py:
--------------------------------------------------------------------------------
  1 | import os
  2 | os.environ["THEANO_FLAGS"] = "device=cpu,floatX=float32"
  3 | import sys
  4 | import getopt
  5 | import matplotlib
  6 | matplotlib.use('Agg')
  7 | 
  8 | import parameters
  9 | import pg_re
 10 | import pg_su
 11 | import slow_down_cdf
 12 | 
 13 | 
 14 | def script_usage():
 15 |     print('--exp_type <type of experiment> \n'
 16 |           '--num_res <number of resources> \n'
 17 |           '--num_nw <number of visible new work> \n'
 18 |           '--simu_len <simulation length> \n'
 19 |           '--num_ex <number of examples> \n'
 20 |           '--num_seq_per_batch <rough number of samples in one batch update> \n'
 21 |           '--eps_max_len <episode maximum length (terminated at the end)> \n'
 22 |           '--num_epochs <number of epoch to do the training>\n'
 23 |           '--time_horizon <time step into future, screen height> \n'
 24 |           '--res_slot <total number of resource slots, screen width> \n'
 25 |           '--max_job_len <maximum new job length> \n'
 26 |           '--max_job_size <maximum new job resource request> \n'
 27 |           '--new_job_rate <new job arrival rate> \n'
 28 |           '--dist <discount factor> \n'
 29 |           '--lr_rate <learning rate> \n'
 30 |           '--ba_size <batch size> \n'
 31 |           '--pg_re <parameter file for pg network> \n'
 32 |           '--v_re <parameter file for v network> \n'
 33 |           '--q_re <parameter file for q network> \n'
 34 |           '--out_freq <network output frequency> \n'
 35 |           '--ofile <output file name> \n'
 36 |           '--log <log file name> \n'
 37 |           '--render <plot dynamics> \n'
 38 |           '--unseen <generate unseen example> \n')
 39 | 
 40 | 
 41 | def main():
 42 | 
 43 |     pa = parameters.Parameters()
 44 | 
 45 |     type_exp = 'pg_re'  # 'pg_su' 'pg_su_compact' 'v_su', 'pg_v_re', 'pg_re', q_re', 'test'
 46 | 
 47 |     pg_resume = None
 48 |     v_resume = None
 49 |     q_resume = None
 50 |     log = None
 51 | 
 52 |     render = False
 53 | 
 54 |     try:
 55 |         opts, args = getopt.getopt(
 56 |             sys.argv[1:],
 57 |             "hi:o:", ["exp_type=",
 58 |                       "num_res=",
 59 |                       "num_nw=",
 60 |                       "simu_len=",
 61 |                       "num_ex=",
 62 |                       "num_seq_per_batch=",
 63 |                       "eps_max_len=",
 64 |                       "num_epochs=",
 65 |                       "time_horizon=",
 66 |                       "res_slot=",
 67 |                       "max_job_len=",
 68 |                       "max_job_size=",
 69 |                       "new_job_rate=",
 70 |                       "dist=",
 71 |                       "lr_rate=",
 72 |                       "ba_size=",
 73 |                       "pg_re=",
 74 |                       "v_re=",
 75 |                       "q_re=",
 76 |                       "out_freq=",
 77 |                       "ofile=",
 78 |                       "log=",
 79 |                       "render=",
 80 |                       "unseen="])
 81 | 
 82 |     except getopt.GetoptError:
 83 |         script_usage()
 84 |         sys.exit(2)
 85 | 
 86 |     for opt, arg in opts:
 87 |         if opt == '-h':
 88 |             script_usage()
 89 |             sys.exit()
 90 |         elif opt in ("-e", "--exp_type"):
 91 |             type_exp = arg
 92 |         elif opt in ("-n", "--num_res"):
 93 |             pa.num_res = int(arg)
 94 |         elif opt in ("-w", "--num_nw"):
 95 |             pa.num_nw = int(arg)
 96 |         elif opt in ("-s", "--simu_len"):
 97 |             pa.simu_len = int(arg)
 98 |         elif opt in ("-n", "--num_ex"):
 99 |             pa.num_ex = int(arg)
100 |         elif opt in ("-sp", "--num_seq_per_batch"):
101 |             pa.num_seq_per_batch = int(arg)
102 |         elif opt in ("-el", "--eps_max_len"):
103 |             pa.episode_max_length = int(arg)
104 |         elif opt in ("-ne", "--num_epochs"):
105 |             pa.num_epochs = int(arg)
106 |         elif opt in ("-t", "--time_horizon"):
107 |             pa.time_horizon = int(arg)
108 |         elif opt in ("-rs", "--res_slot"):
109 |             pa.res_slot = int(arg)
110 |         elif opt in ("-ml", "--max_job_len"):
111 |             pa.max_job_len = int(arg)
112 |         elif opt in ("-ms", "--max_job_size"):
113 |             pa.max_job_size = int(arg)
114 |         elif opt in ("-nr", "--new_job_rate"):
115 |             pa.new_job_rate = float(arg)
116 |         elif opt in ("-d", "--dist"):
117 |             pa.discount = float(arg)
118 |         elif opt in ("-l", "--lr_rate"):
119 |             pa.lr_rate = float(arg)
120 |         elif opt in ("-b", "--ba_size"):
121 |             pa.batch_size = int(arg)
122 |         elif opt in ("-p", "--pg_re"):
123 |             pg_resume = arg
124 |         elif opt in ("-v", "--v_re"):
125 |             v_resume = arg
126 |         elif opt in ("-q", "--q_re"):
127 |             q_resume = arg
128 |         elif opt in ("-f", "--out_freq"):
129 |             pa.output_freq = int(arg)
130 |         elif opt in ("-o", "--ofile"):
131 |             pa.output_filename = arg
132 |         elif opt in ("-lg", "--log"):
133 |             log = arg
134 |         elif opt in ("-r", "--render"):
135 |             render = (arg == 'True')
136 |         elif opt in ("-u", "--unseen"):
137 |             pa.generate_unseen = (arg == 'True')
138 |         else:
139 |             script_usage()
140 |             sys.exit()
141 | 
142 |     pa.compute_dependent_parameters()
143 | 
144 |     if type_exp == 'pg_su':
145 |         pg_su.launch(pa, pg_resume, render, repre='image', end='all_done')
146 |     elif type_exp == 'v_su':
147 |         v_su.launch(pa, v_resume, render)
148 |     elif type_exp == 'pg_re':
149 |         pg_re.launch(pa, pg_resume, render, repre='image', end='all_done')
150 |     elif type_exp == 'pg_v_re':
151 |         pg_v_re.launch(pa, pg_resume, v_resume, render)
152 |     elif type_exp == 'test':
153 |         # quick_test.launch(pa, pg_resume, render)
154 |         slow_down_cdf.launch(pa, pg_resume, render, True)
155 |     # elif type_exp == 'q_re':
156 |     #     q_re.launch(pa, q_resume, render)
157 |     else:
158 |         print("Error: unkown experiment type " + str(type_exp))
159 |         exit(1)
160 | 
161 | 
162 | if __name__ == '__main__':
163 |     main()


--------------------------------------------------------------------------------
/pg_su.py:
--------------------------------------------------------------------------------
  1 | import numpy as np
  2 | import theano
  3 | import time
  4 | import sys
  5 | import cPickle
  6 | 
  7 | import environment
  8 | import pg_network
  9 | import other_agents
 10 | import job_distribution
 11 | 
 12 | np.set_printoptions(threshold='nan')
 13 | 
 14 | 
 15 | def add_sample(X, y, idx, X_to_add, y_to_add):
 16 |     X[idx, 0, :, :] = X_to_add
 17 |     y[idx] = y_to_add
 18 | 
 19 | 
 20 | def iterate_minibatches(inputs, targets, batchsize, shuffle=False):
 21 |     assert len(inputs) == len(targets)
 22 |     if shuffle:
 23 |         indices = np.arange(len(inputs))
 24 |         np.random.shuffle(indices)
 25 |     for start_idx in range(0, len(inputs) - batchsize + 1, batchsize):
 26 |         if shuffle:
 27 |             excerpt = indices[start_idx:start_idx + batchsize]
 28 |         else:
 29 |             excerpt = slice(start_idx, start_idx + batchsize)
 30 |         yield inputs[excerpt], targets[excerpt]
 31 | 
 32 | 
 33 | def launch(pa, pg_resume=None, render=False, repre='image', end='no_new_job'):
 34 | 
 35 |     env = environment.Env(pa, render=False, repre=repre, end=end)
 36 | 
 37 |     pg_learner = pg_network.PGLearner(pa)
 38 | 
 39 |     if pg_resume is not None:
 40 |         net_handle = open(pg_resume, 'r')
 41 |         net_params = cPickle.load(net_handle)
 42 |         pg_learner.set_net_params(net_params)
 43 | 
 44 |     if pa.evaluate_policy_name == "SJF":
 45 |         evaluate_policy = other_agents.get_sjf_action
 46 |     elif pa.evaluate_policy_name == "PACKER":
 47 |         evaluate_policy = other_agents.get_packer_action
 48 |     else:
 49 |         print("Panic: no policy known to evaluate.")
 50 |         exit(1)
 51 | 
 52 |     # ----------------------------
 53 |     print("Preparing for data...")
 54 |     # ----------------------------
 55 | 
 56 |     nw_len_seqs, nw_size_seqs = job_distribution.generate_sequence_work(pa, seed=42)
 57 | 
 58 |     # print 'nw_time_seqs=', nw_len_seqs
 59 |     # print 'nw_size_seqs=', nw_size_seqs
 60 | 
 61 |     mem_alloc = 4
 62 | 
 63 |     X = np.zeros([pa.simu_len * pa.num_ex * mem_alloc, 1,
 64 |                   pa.network_input_height, pa.network_input_width],
 65 |                  dtype=theano.config.floatX)
 66 |     y = np.zeros(pa.simu_len * pa.num_ex * mem_alloc,
 67 |                  dtype='int32')
 68 | 
 69 |     print 'network_input_height=', pa.network_input_height
 70 |     print 'network_input_width=', pa.network_input_width
 71 | 
 72 |     counter = 0
 73 | 
 74 |     for train_ex in range(pa.num_ex):
 75 | 
 76 |         env.reset()
 77 | 
 78 |         for _ in xrange(pa.episode_max_length):
 79 | 
 80 |             # ---- get current state ----
 81 |             ob = env.observe()
 82 | 
 83 |             a = evaluate_policy(env.machine, env.job_slot)
 84 | 
 85 |             if counter < pa.simu_len * pa.num_ex * mem_alloc:
 86 | 
 87 |                 add_sample(X, y, counter, ob, a)
 88 |                 counter += 1
 89 | 
 90 |             ob, rew, done, info = env.step(a, repeat=True)
 91 | 
 92 |             if done:  # hit void action, exit
 93 |                 break
 94 | 
 95 |         # roll to next example
 96 |         env.seq_no = (env.seq_no + 1) % env.pa.num_ex
 97 | 
 98 |     num_train = int(0.8 * counter)
 99 |     num_test = int(0.2 * counter)
100 | 
101 |     X_train, X_test = X[:num_train], X[num_train: num_train + num_test]
102 |     y_train, y_test = y[:num_train], y[num_train: num_train + num_test]
103 | 
104 |     # Normalization, make sure nothing becomes NaN
105 | 
106 |     # X_mean = np.average(X[:num_train + num_test], axis=0)
107 |     # X_std = np.std(X[:num_train + num_test], axis=0)
108 |     #
109 |     # X_train = (X_train - X_mean) / X_std
110 |     # X_test = (X_test - X_mean) / X_std
111 | 
112 |     # ----------------------------
113 |     print("Start training...")
114 |     # ----------------------------
115 | 
116 |     for epoch in xrange(pa.num_epochs):
117 | 
118 |         # In each epoch, we do a full pass over the training data:
119 |         train_err = 0
120 |         train_acc = 0
121 |         train_batches = 0
122 |         start_time = time.time()
123 |         for batch in iterate_minibatches(X_train, y_train, pa.batch_size, shuffle=True):
124 |             inputs, targets = batch
125 |             err, prob_act = pg_learner.su_train(inputs, targets)
126 |             pg_act = np.argmax(prob_act, axis=1)
127 |             train_err += err
128 |             train_acc += np.sum(pg_act == targets)
129 |             train_batches += 1
130 | 
131 |         # # And a full pass over the test data:
132 |         test_err = 0
133 |         test_acc = 0
134 |         test_batches = 0
135 |         for batch in iterate_minibatches(X_test, y_test, pa.batch_size, shuffle=False):
136 |             inputs, targets = batch
137 |             err, prob_act = pg_learner.su_test(inputs, targets)
138 |             pg_act = np.argmax(prob_act, axis=1)
139 |             test_err += err
140 |             test_acc += np.sum(pg_act == targets)
141 |             test_batches += 1
142 | 
143 |         # Then we print the results for this epoch:
144 |         print("Epoch {} of {} took {:.3f}s".format(
145 |             epoch + 1, pa.num_epochs, time.time() - start_time))
146 |         print("  training loss:    \t\t{:.6f}".format(train_err / train_batches))
147 |         print("  training accuracy:\t\t{:.2f} %".format(
148 |             train_acc / float(num_train) * 100))
149 |         print("  test loss:        \t\t{:.6f}".format(test_err / test_batches))
150 |         print("  test accuracy:    \t\t{:.2f} %".format(
151 |             test_acc / float(num_test) * 100))
152 | 
153 |         sys.stdout.flush()
154 | 
155 |         if epoch % pa.output_freq == 0:
156 | 
157 |             net_file = open(pa.output_filename + '_net_file_' + str(epoch) + '.pkl', 'wb')
158 |             cPickle.dump(pg_learner.return_net_params(), net_file, -1)
159 |             net_file.close()
160 | 
161 |     print("done")
162 | 
163 | 
164 | def main():
165 | 
166 |     import parameters
167 | 
168 |     pa = parameters.Parameters()
169 | 
170 |     pa.simu_len = 1000  # 1000
171 |     pa.num_ex = 100  # 100
172 |     pa.num_nw = 10
173 |     pa.num_seq_per_batch = 20
174 |     pa.output_freq = 50
175 | 
176 |     # pa.max_nw_size = 5
177 |     # pa.job_len = 5
178 |     pa.new_job_rate = 0.3
179 | 
180 |     pa.episode_max_length = 10000  # 2000
181 | 
182 |     pa.compute_dependent_parameters()
183 | 
184 |     pg_resume = None
185 |     # pg_resume = 'data/tmp_450.pkl'
186 | 
187 |     render = False
188 | 
189 |     launch(pa, pg_resume, render, repre='image', end='all_done')
190 | 
191 | 
192 | if __name__ == '__main__':
193 |     main()
194 | 


--------------------------------------------------------------------------------
/slow_down_cdf.py:
--------------------------------------------------------------------------------
  1 | import numpy as np
  2 | import cPickle
  3 | import matplotlib.pyplot as plt
  4 | 
  5 | import environment
  6 | import parameters
  7 | import pg_network
  8 | import other_agents
  9 | 
 10 | 
 11 | def discount(x, gamma):
 12 |     """
 13 |     Given vector x, computes a vector y such that
 14 |     y[i] = x[i] + gamma * x[i+1] + gamma^2 x[i+2] + ...
 15 |     """
 16 |     out = np.zeros(len(x))
 17 |     out[-1] = x[-1]
 18 |     for i in reversed(xrange(len(x)-1)):
 19 |         out[i] = x[i] + gamma*out[i+1]
 20 |     assert x.ndim >= 1
 21 |     # More efficient version:
 22 |     # scipy.signal.lfilter([1],[1,-gamma],x[::-1], axis=0)[::-1]
 23 |     return out
 24 | 
 25 | 
 26 | def categorical_sample(prob_n):
 27 |     """
 28 |     Sample from categorical distribution,
 29 |     specified by a vector of class probabilities
 30 |     """
 31 |     prob_n = np.asarray(prob_n)
 32 |     csprob_n = np.cumsum(prob_n)
 33 |     return (csprob_n > np.random.rand()).argmax()
 34 | 
 35 | 
 36 | def get_traj(test_type, pa, env, episode_max_length, pg_resume=None, render=False):
 37 |     """
 38 |     Run agent-environment loop for one whole episode (trajectory)
 39 |     Return dictionary of results
 40 |     """
 41 | 
 42 |     if test_type == 'PG':  # load trained parameters
 43 | 
 44 |         pg_learner = pg_network.PGLearner(pa)
 45 | 
 46 |         net_handle = open(pg_resume, 'rb')
 47 |         net_params = cPickle.load(net_handle)
 48 |         pg_learner.set_net_params(net_params)
 49 | 
 50 |     env.reset()
 51 |     rews = []
 52 | 
 53 |     ob = env.observe()
 54 | 
 55 |     for _ in xrange(episode_max_length):
 56 | 
 57 |         if test_type == 'PG':
 58 |             a = pg_learner.choose_action(ob)
 59 | 
 60 |         elif test_type == 'Tetris':
 61 |             a = other_agents.get_packer_action(env.machine, env.job_slot)
 62 | 
 63 |         elif test_type == 'SJF':
 64 |             a = other_agents.get_sjf_action(env.machine, env.job_slot)
 65 | 
 66 |         elif test_type == 'Random':
 67 |             a = other_agents.get_random_action(env.job_slot)
 68 | 
 69 |         ob, rew, done, info = env.step(a, repeat=True)
 70 | 
 71 |         rews.append(rew)
 72 | 
 73 |         if done: break
 74 |         if render: env.render()
 75 |         # env.render()
 76 | 
 77 |     return np.array(rews), info
 78 | 
 79 | 
 80 | def launch(pa, pg_resume=None, render=False, plot=False, repre='image', end='no_new_job'):
 81 | 
 82 |     # ---- Parameters ----
 83 | 
 84 |     test_types = ['Tetris', 'SJF', 'Random']
 85 | 
 86 |     if pg_resume is not None:
 87 |         test_types = ['PG'] + test_types
 88 | 
 89 |     env = environment.Env(pa, render, repre=repre, end=end)
 90 | 
 91 |     all_discount_rews = {}
 92 |     jobs_slow_down = {}
 93 |     work_complete = {}
 94 |     work_remain = {}
 95 |     job_len_remain = {}
 96 |     num_job_remain = {}
 97 |     job_remain_delay = {}
 98 | 
 99 |     for test_type in test_types:
100 |         all_discount_rews[test_type] = []
101 |         jobs_slow_down[test_type] = []
102 |         work_complete[test_type] = []
103 |         work_remain[test_type] = []
104 |         job_len_remain[test_type] = []
105 |         num_job_remain[test_type] = []
106 |         job_remain_delay[test_type] = []
107 | 
108 |     for seq_idx in xrange(pa.num_ex):
109 |         print('\n\n')
110 |         print("=============== " + str(seq_idx) + " ===============")
111 | 
112 |         for test_type in test_types:
113 | 
114 |             rews, info = get_traj(test_type, pa, env, pa.episode_max_length, pg_resume)
115 | 
116 |             print "---------- " + test_type + " -----------"
117 | 
118 |             print "total discount reward : \t %s" % (discount(rews, pa.discount)[0])
119 | 
120 |             all_discount_rews[test_type].append(
121 |                 discount(rews, pa.discount)[0]
122 |             )
123 | 
124 |             # ------------------------
125 |             # ---- per job stat ----
126 |             # ------------------------
127 | 
128 |             enter_time = np.array([info.record[i].enter_time for i in xrange(len(info.record))])
129 |             finish_time = np.array([info.record[i].finish_time for i in xrange(len(info.record))])
130 |             job_len = np.array([info.record[i].len for i in xrange(len(info.record))])
131 |             job_total_size = np.array([np.sum(info.record[i].res_vec) for i in xrange(len(info.record))])
132 | 
133 |             finished_idx = (finish_time >= 0)
134 |             unfinished_idx = (finish_time < 0)
135 | 
136 |             jobs_slow_down[test_type].append(
137 |                 (finish_time[finished_idx] - enter_time[finished_idx]) / job_len[finished_idx]
138 |             )
139 |             work_complete[test_type].append(
140 |                 np.sum(job_len[finished_idx] * job_total_size[finished_idx])
141 |             )
142 |             work_remain[test_type].append(
143 |                 np.sum(job_len[unfinished_idx] * job_total_size[unfinished_idx])
144 |             )
145 |             job_len_remain[test_type].append(
146 |                 np.sum(job_len[unfinished_idx])
147 |             )
148 |             num_job_remain[test_type].append(
149 |                 len(job_len[unfinished_idx])
150 |             )
151 |             job_remain_delay[test_type].append(
152 |                 np.sum(pa.episode_max_length - enter_time[unfinished_idx])
153 |             )
154 | 
155 |         env.seq_no = (env.seq_no + 1) % env.pa.num_ex
156 | 
157 |     # -- matplotlib colormap no overlap --
158 |     if plot:
159 |         num_colors = len(test_types)
160 |         cm = plt.get_cmap('gist_rainbow')
161 |         fig = plt.figure()
162 |         ax = fig.add_subplot(111)
163 |         ax.set_color_cycle([cm(1. * i / num_colors) for i in range(num_colors)])
164 | 
165 |         for test_type in test_types:
166 |             slow_down_cdf = np.sort(np.concatenate(jobs_slow_down[test_type]))
167 |             slow_down_yvals = np.arange(len(slow_down_cdf))/float(len(slow_down_cdf))
168 |             ax.plot(slow_down_cdf, slow_down_yvals, linewidth=2, label=test_type)
169 | 
170 |         plt.legend(loc=4)
171 |         plt.xlabel("job slowdown", fontsize=20)
172 |         plt.ylabel("CDF", fontsize=20)
173 |         # plt.show()
174 |         plt.savefig(pg_resume + "_slowdown_fig" + ".pdf")
175 | 
176 |     return all_discount_rews, jobs_slow_down
177 | 
178 | 
179 | def main():
180 |     pa = parameters.Parameters()
181 | 
182 |     pa.simu_len = 200  # 5000  # 1000
183 |     pa.num_ex = 10  # 100
184 |     pa.num_nw = 10
185 |     pa.num_seq_per_batch = 20
186 |     # pa.max_nw_size = 5
187 |     # pa.job_len = 5
188 |     pa.new_job_rate = 0.3
189 |     pa.discount = 1
190 | 
191 |     pa.episode_max_length = 20000  # 2000
192 | 
193 |     pa.compute_dependent_parameters()
194 | 
195 |     render = False
196 | 
197 |     plot = True  # plot slowdown cdf
198 | 
199 |     pg_resume = None
200 |     pg_resume = 'data/pg_re_discount_1_rate_0.3_simu_len_200_num_seq_per_batch_20_ex_10_nw_10_1450.pkl'
201 |     # pg_resume = 'data/pg_re_1000_discount_1_5990.pkl'
202 | 
203 |     pa.unseen = True
204 | 
205 |     launch(pa, pg_resume, render, plot, repre='image', end='all_done')
206 | 
207 | 
208 | if __name__ == '__main__':
209 |     main()
210 | 


--------------------------------------------------------------------------------
/pg_re_single_core_o.py:
--------------------------------------------------------------------------------
  1 | import numpy as np
  2 | import time
  3 | import theano
  4 | import cPickle
  5 | import matplotlib.pyplot as plt
  6 | 
  7 | import environment
  8 | import pg_network
  9 | import slow_down_cdf
 10 | 
 11 | 
 12 | def discount(x, gamma):
 13 |     """
 14 |     Given vector x, computes a vector y such that
 15 |     y[i] = x[i] + gamma * x[i+1] + gamma^2 x[i+2] + ...
 16 |     """
 17 |     out = np.zeros(len(x))
 18 |     out[-1] = x[-1]
 19 |     for i in reversed(xrange(len(x)-1)):
 20 |         out[i] = x[i] + gamma*out[i+1]
 21 |     assert x.ndim >= 1
 22 |     # More efficient version:
 23 |     # scipy.signal.lfilter([1],[1,-gamma],x[::-1], axis=0)[::-1]
 24 |     return out
 25 | 
 26 | 
 27 | def get_entropy(vec):
 28 |     entropy = - np.sum(vec * np.log(vec))
 29 |     if np.isnan(entropy):
 30 |         entropy = 0
 31 |     return entropy
 32 | 
 33 | 
 34 | def get_traj(agent, env, episode_max_length, render=False):
 35 |     """
 36 |     Run agent-environment loop for one whole episode (trajectory)
 37 |     Return dictionary of results
 38 |     """
 39 |     env.reset()
 40 |     obs = []
 41 |     acts = []
 42 |     rews = []
 43 |     entropy = []
 44 |     info = []
 45 | 
 46 |     ob = env.observe()
 47 | 
 48 |     for _ in xrange(episode_max_length):
 49 |         act_prob = agent.get_one_act_prob(ob)
 50 |         csprob_n = np.cumsum(act_prob)
 51 |         a = (csprob_n > np.random.rand()).argmax()
 52 | 
 53 |         obs.append(ob)  # store the ob at current decision making step
 54 |         acts.append(a)
 55 | 
 56 |         ob, rew, done, info = env.step(a, repeat=True)
 57 | 
 58 |         rews.append(rew)
 59 |         entropy.append(get_entropy(act_prob))
 60 | 
 61 |         if done: break
 62 |         if render: env.render()
 63 | 
 64 |     return {'reward': np.array(rews),
 65 |             'ob': np.array(obs),
 66 |             'action': np.array(acts),
 67 |             'entropy': entropy,
 68 |             'info': info
 69 |             }
 70 | 
 71 | 
 72 | def concatenate_all_ob(trajs, pa):
 73 | 
 74 |     timesteps_total = 0
 75 |     for i in xrange(len(trajs)):
 76 |         timesteps_total += len(trajs[i]['reward'])
 77 | 
 78 |     all_ob = np.zeros(
 79 |         (timesteps_total, 1, pa.network_input_height, pa.network_input_width),
 80 |         dtype=theano.config.floatX)
 81 | 
 82 |     timesteps = 0
 83 |     for i in xrange(len(trajs)):
 84 |         for j in xrange(len(trajs[i]['reward'])):
 85 |             all_ob[timesteps, 0, :, :] = trajs[i]['ob'][j]
 86 |             timesteps += 1
 87 | 
 88 |     return all_ob
 89 | 
 90 | 
 91 | def concatenate_all_ob_across_examples(all_ob, pa):
 92 |     num_ex = len(all_ob)
 93 |     total_samp = 0
 94 |     for i in xrange(num_ex):
 95 |         total_samp += all_ob[i].shape[0]
 96 | 
 97 |     all_ob_contact = np.zeros(
 98 |         (total_samp, 1, pa.network_input_height, pa.network_input_width),
 99 |         dtype=theano.config.floatX)
100 | 
101 |     total_samp = 0
102 | 
103 |     for i in xrange(num_ex):
104 |         prev_samp = total_samp
105 |         total_samp += all_ob[i].shape[0]
106 |         all_ob_contact[prev_samp : total_samp, :, :, :] = all_ob[i]
107 | 
108 |     return all_ob_contact
109 | 
110 | 
111 | def process_all_info(trajs):
112 |     enter_time = []
113 |     finish_time = []
114 |     job_len = []
115 | 
116 |     for traj in trajs:
117 |         enter_time.append(np.array([traj['info'].record[i].enter_time for i in xrange(len(traj['info'].record))]))
118 |         finish_time.append(np.array([traj['info'].record[i].finish_time for i in xrange(len(traj['info'].record))]))
119 |         job_len.append(np.array([traj['info'].record[i].len for i in xrange(len(traj['info'].record))]))
120 | 
121 |     enter_time = np.concatenate(enter_time)
122 |     finish_time = np.concatenate(finish_time)
123 |     job_len = np.concatenate(job_len)
124 | 
125 |     return enter_time, finish_time, job_len
126 | 
127 | 
128 | def plot_lr_curve(output_file_prefix, max_rew_lr_curve, mean_rew_lr_curve, slow_down_lr_curve,
129 |                   ref_discount_rews, ref_slow_down):
130 |     num_colors = len(ref_discount_rews) + 2
131 |     cm = plt.get_cmap('gist_rainbow')
132 | 
133 |     fig = plt.figure(figsize=(12, 5))
134 | 
135 |     ax = fig.add_subplot(121)
136 |     ax.set_color_cycle([cm(1. * i / num_colors) for i in range(num_colors)])
137 | 
138 |     ax.plot(mean_rew_lr_curve, linewidth=2, label='PG mean')
139 |     for k in ref_discount_rews:
140 |         ax.plot(np.tile(np.average(ref_discount_rews[k]), len(mean_rew_lr_curve)), linewidth=2, label=k)
141 |     ax.plot(max_rew_lr_curve, linewidth=2, label='PG max')
142 | 
143 |     plt.legend(loc=4)
144 |     plt.xlabel("Iteration", fontsize=20)
145 |     plt.ylabel("Discounted Total Reward", fontsize=20)
146 | 
147 |     ax = fig.add_subplot(122)
148 |     ax.set_color_cycle([cm(1. * i / num_colors) for i in range(num_colors)])
149 | 
150 |     ax.plot(slow_down_lr_curve, linewidth=2, label='PG mean')
151 |     for k in ref_discount_rews:
152 |         ax.plot(np.tile(np.average(np.concatenate(ref_slow_down[k])), len(slow_down_lr_curve)), linewidth=2, label=k)
153 | 
154 |     plt.legend(loc=1)
155 |     plt.xlabel("Iteration", fontsize=20)
156 |     plt.ylabel("Slowdown", fontsize=20)
157 | 
158 |     plt.savefig(output_file_prefix + "_lr_curve" + ".pdf")
159 | 
160 | 
161 | def launch(pa, pg_resume=None, render=False, repre='image', end='no_new_job'):
162 | 
163 |     env = environment.Env(pa, render=render, repre=repre, end=end)
164 | 
165 |     pg_learner = pg_network.PGLearner(pa)
166 | 
167 |     if pg_resume is not None:
168 |         net_handle = open(pg_resume, 'rb')
169 |         net_params = cPickle.load(net_handle)
170 |         pg_learner.set_net_params(net_params)
171 | 
172 |     # ----------------------------
173 |     print("Preparing for data...")
174 |     # ----------------------------
175 | 
176 |     ref_discount_rews, ref_slow_down = slow_down_cdf.launch(pa, pg_resume=None, render=False, plot=False, repre=repre, end=end)
177 | 
178 |     mean_rew_lr_curve = []
179 |     max_rew_lr_curve = []
180 |     slow_down_lr_curve = []
181 | 
182 |     timer_start = time.time()
183 | 
184 |     for iteration in xrange(pa.num_epochs):
185 | 
186 |         all_ob = []
187 |         all_action = []
188 |         all_adv = []
189 |         all_eprews = []
190 |         all_eplens = []
191 |         all_slowdown = []
192 |         all_entropy = []
193 | 
194 |         # go through all examples
195 |         for ex in xrange(pa.num_ex):
196 | 
197 |             # Collect trajectories until we get timesteps_per_batch total timesteps
198 |             trajs = []
199 | 
200 |             for i in xrange(pa.num_seq_per_batch):
201 |                 traj = get_traj(pg_learner, env, pa.episode_max_length)
202 |                 trajs.append(traj)
203 | 
204 |             # roll to next example
205 |             env.seq_no = (env.seq_no + 1) % env.pa.num_ex
206 | 
207 |             all_ob.append(concatenate_all_ob(trajs, pa))
208 | 
209 |             # Compute discounted sums of rewards
210 |             rets = [discount(traj["reward"], pa.discount) for traj in trajs]
211 |             maxlen = max(len(ret) for ret in rets)
212 |             padded_rets = [np.concatenate([ret, np.zeros(maxlen - len(ret))]) for ret in rets]
213 | 
214 |             # Compute time-dependent baseline
215 |             baseline = np.mean(padded_rets, axis=0)
216 | 
217 |             # Compute advantage function
218 |             advs = [ret - baseline[:len(ret)] for ret in rets]
219 |             all_action.append(np.concatenate([traj["action"] for traj in trajs]))
220 |             all_adv.append(np.concatenate(advs))
221 | 
222 |             all_eprews.append(np.array([discount(traj["reward"], pa.discount)[0] for traj in trajs]))  # episode total rewards
223 |             all_eplens.append(np.array([len(traj["reward"]) for traj in trajs]))  # episode lengths
224 | 
225 |             # All Job Stat
226 |             enter_time, finish_time, job_len = process_all_info(trajs)
227 |             finished_idx = (finish_time >= 0)
228 |             all_slowdown.append(
229 |                 (finish_time[finished_idx] - enter_time[finished_idx]) / job_len[finished_idx]
230 |             )
231 | 
232 |             # Action prob entropy
233 |             all_entropy.append(np.concatenate([traj["entropy"]]))
234 | 
235 |         all_ob = concatenate_all_ob_across_examples(all_ob, pa)
236 |         all_action = np.concatenate(all_action)
237 |         all_adv = np.concatenate(all_adv)
238 | 
239 |         # Do policy gradient update step
240 |         loss = pg_learner.train(all_ob, all_action, all_adv)
241 |         eprews = np.concatenate(all_eprews)  # episode total rewards
242 |         eplens = np.concatenate(all_eplens)  # episode lengths
243 | 
244 |         all_slowdown = np.concatenate(all_slowdown)
245 | 
246 |         all_entropy = np.concatenate(all_entropy)
247 | 
248 |         timer_end = time.time()
249 | 
250 |         print "-----------------"
251 |         print "Iteration: \t %i" % iteration
252 |         print "NumTrajs: \t %i" % len(eprews)
253 |         print "NumTimesteps: \t %i" % np.sum(eplens)
254 |         print "Loss:     \t %s" % loss
255 |         print "MaxRew: \t %s" % np.average([np.max(rew) for rew in all_eprews])
256 |         print "MeanRew: \t %s +- %s" % (eprews.mean(), eprews.std())
257 |         print "MeanSlowdown: \t %s" % np.mean(all_slowdown)
258 |         print "MeanLen: \t %s +- %s" % (eplens.mean(), eplens.std())
259 |         print "MeanEntropy \t %s" % (np.mean(all_entropy))
260 |         print "Elapsed time\t %s" % (timer_end - timer_start), "seconds"
261 |         print "-----------------"
262 | 
263 |         timer_start = time.time()
264 | 
265 |         max_rew_lr_curve.append(np.average([np.max(rew) for rew in all_eprews]))
266 |         mean_rew_lr_curve.append(eprews.mean())
267 |         slow_down_lr_curve.append(np.mean(all_slowdown))
268 | 
269 |         if iteration % pa.output_freq == 0:
270 |             param_file = open(pa.output_filename + '_' + str(iteration) + '.pkl', 'wb')
271 |             cPickle.dump(pg_learner.get_params(), param_file, -1)
272 |             param_file.close()
273 | 
274 |             slow_down_cdf.launch(pa, pa.output_filename + '_' + str(iteration) + '.pkl',
275 |                                  render=False, plot=True, repre=repre, end=end)
276 | 
277 |             plot_lr_curve(pa.output_filename,
278 |                           max_rew_lr_curve, mean_rew_lr_curve, slow_down_lr_curve,
279 |                           ref_discount_rews, ref_slow_down)
280 | 
281 | 
282 | def main():
283 | 
284 |     import parameters
285 | 
286 |     pa = parameters.Parameters()
287 | 
288 |     pa.simu_len = 200  # 1000
289 |     pa.num_ex = 10  # 100
290 |     pa.num_nw = 10
291 |     pa.num_seq_per_batch = 20
292 |     pa.output_freq = 50
293 | 
294 |     # pa.max_nw_size = 5
295 |     # pa.job_len = 5
296 |     pa.new_job_rate = 0.3
297 | 
298 |     pa.episode_max_length = 2000  # 2000
299 | 
300 |     pa.compute_dependent_parameters()
301 | 
302 |     pg_resume = None
303 |     # pg_resume = 'data/tmp_0.pkl'
304 | 
305 |     render = False
306 | 
307 |     launch(pa, pg_resume, render, repre='image', end='all_done')
308 | 
309 | 
310 | if __name__ == '__main__':
311 |     main()
312 | 


--------------------------------------------------------------------------------
/pg_re_single_core.py:
--------------------------------------------------------------------------------
  1 | import time
  2 | # import threading
  3 | import numpy as np
  4 | import theano
  5 | import cPickle
  6 | import matplotlib.pyplot as plt
  7 | import h5py
  8 | from datetime import datetime
  9 | import re
 10 | 
 11 | import environment
 12 | import pg_network
 13 | import slow_down_cdf
 14 | 
 15 | 
 16 | def discount(x, gamma):
 17 |     """
 18 |     Given vector x, computes a vector y such that
 19 |     y[i] = x[i] + gamma * x[i+1] + gamma^2 x[i+2] + ...
 20 |     """
 21 |     out = np.zeros(len(x))
 22 |     out[-1] = x[-1]
 23 |     for i in reversed(xrange(len(x)-1)):
 24 |         out[i] = x[i] + gamma*out[i+1]
 25 |     assert x.ndim >= 1
 26 |     # More efficient version:
 27 |     # scipy.signal.lfilter([1],[1,-gamma],x[::-1], axis=0)[::-1]
 28 |     return out
 29 | 
 30 | 
 31 | def get_entropy(vec):
 32 |     entropy = - np.sum(vec * np.log(vec))
 33 |     if np.isnan(entropy):
 34 |         entropy = 0
 35 |     return entropy
 36 | 
 37 | 
 38 | def get_traj(agent, env, episode_max_length, render=True):
 39 |     """
 40 |     Run agent-environment loop for one whole episode (trajectory)
 41 |     Return dictionary of results
 42 |     """
 43 |     env.reset()
 44 |     obs = []
 45 |     acts = []
 46 |     rews = []
 47 |     entropy = []
 48 |     info = []
 49 | 
 50 |     ob = env.observe()
 51 | 
 52 |     for _ in xrange(episode_max_length):
 53 |         act_prob = agent.get_one_act_prob(ob)
 54 |         csprob_n = np.cumsum(act_prob)
 55 |         a = (csprob_n > np.random.rand()).argmax() # select on prob distribution
 56 | 
 57 |         obs.append(ob)  # store the ob at current decision making step
 58 |         acts.append(a)
 59 | 
 60 |         ob, rew, done, info = env.step(a, repeat=True)
 61 | 
 62 |         rews.append(rew)
 63 |         entropy.append(get_entropy(act_prob))
 64 | 
 65 |         if done: break
 66 |         # if render: env.render()
 67 | 
 68 |     return {'reward': np.array(rews),
 69 |             'ob': np.array(obs),
 70 |             'action': np.array(acts),
 71 |             'entropy': entropy,
 72 |             'info': info
 73 |             }
 74 | 
 75 | 
 76 | def concatenate_all_ob(trajs, pa):
 77 | 
 78 |     timesteps_total = 0
 79 |     for i in xrange(len(trajs)):
 80 |         timesteps_total += len(trajs[i]['reward'])
 81 | 
 82 |     all_ob = np.zeros(
 83 |         (timesteps_total, 1, pa.network_input_height, pa.network_input_width),
 84 |         dtype=theano.config.floatX)
 85 | 
 86 |     timesteps = 0
 87 |     for i in xrange(len(trajs)):
 88 |         for j in xrange(len(trajs[i]['reward'])):
 89 |             all_ob[timesteps, 0, :, :] = trajs[i]['ob'][j]
 90 |             timesteps += 1
 91 | 
 92 |     return all_ob
 93 | 
 94 | 
 95 | def concatenate_all_ob_across_examples(all_ob, pa):
 96 |     num_ex = len(all_ob)
 97 |     total_samp = 0
 98 |     for i in xrange(num_ex):
 99 |         total_samp += all_ob[i].shape[0]
100 | 
101 |     all_ob_contact = np.zeros(
102 |         (total_samp, 1, pa.network_input_height, pa.network_input_width),
103 |         dtype=theano.config.floatX)
104 | 
105 |     total_samp = 0
106 | 
107 |     for i in xrange(num_ex):
108 |         prev_samp = total_samp
109 |         total_samp += all_ob[i].shape[0]
110 |         all_ob_contact[prev_samp : total_samp, :, :, :] = all_ob[i]
111 | 
112 |     return all_ob_contact
113 | 
114 | 
115 | def process_all_info(trajs):
116 |     enter_time = []
117 |     finish_time = []
118 |     job_len = []
119 | 
120 |     for traj in trajs:
121 |         enter_time.append(np.array([traj['info'].record[i].enter_time for i in xrange(len(traj['info'].record))]))
122 |         finish_time.append(np.array([traj['info'].record[i].finish_time for i in xrange(len(traj['info'].record))]))
123 |         job_len.append(np.array([traj['info'].record[i].len for i in xrange(len(traj['info'].record))]))
124 | 
125 |     enter_time = np.concatenate(enter_time)
126 |     finish_time = np.concatenate(finish_time)
127 |     job_len = np.concatenate(job_len)
128 | 
129 |     return enter_time, finish_time, job_len
130 | 
131 | 
132 | def plot_lr_curve(output_file_prefix, max_rew_lr_curve, mean_rew_lr_curve, slow_down_lr_curve,
133 |                   ref_discount_rews, ref_slow_down):
134 |     num_colors = len(ref_discount_rews) + 2
135 |     cm = plt.get_cmap('gist_rainbow')
136 | 
137 |     fig = plt.figure(figsize=(12, 5))
138 | 
139 |     ax = fig.add_subplot(121)
140 |     ax.set_color_cycle([cm(1. * i / num_colors) for i in range(num_colors)])
141 | 
142 |     ax.plot(mean_rew_lr_curve, linewidth=2, label='PG mean')
143 |     for k in ref_discount_rews:
144 |         ax.plot(np.tile(np.average(ref_discount_rews[k]), len(mean_rew_lr_curve)), linewidth=2, label=k)
145 |     ax.plot(max_rew_lr_curve, linewidth=2, label='PG max')
146 | 
147 |     plt.legend(loc=4)
148 |     plt.xlabel("Iteration", fontsize=20)
149 |     plt.ylabel("Discounted Total Reward", fontsize=20)
150 | 
151 |     ax = fig.add_subplot(122)
152 |     ax.set_color_cycle([cm(1. * i / num_colors) for i in range(num_colors)])
153 | 
154 |     ax.plot(slow_down_lr_curve, linewidth=2, label='PG mean')
155 |     for k in ref_discount_rews:
156 |         ax.plot(np.tile(np.average(np.concatenate(ref_slow_down[k])), len(slow_down_lr_curve)), linewidth=2, label=k)
157 | 
158 |     plt.legend(loc=1)
159 |     plt.xlabel("Iteration", fontsize=20)
160 |     plt.ylabel("Slowdown", fontsize=20)
161 | 
162 |     plt.savefig(output_file_prefix + "_lr_curve" + ".pdf")
163 | 
164 | 
165 | def launch(pa, pg_resume=None, render=True, repre='image', end='no_new_job'):
166 | 
167 |     f = open('log/re_log_' + datetime.now().strftime('%Y-%m-%d_%H:%M:%S'), 'a')
168 | 
169 |     env = environment.Env(pa, render=render, repre=repre, end=end)
170 | 
171 |     pg_learner = pg_network.PGLearner(pa)
172 | 
173 |     startIdx = 0
174 |     if pg_resume is not None:# and 're' in pg_resume:
175 |         net_handle = open(pg_resume, 'rb')
176 |         net_params = cPickle.load(net_handle)
177 |         pg_learner.set_net_params(net_params)
178 |         tmp = re.match('.+?(\d+).+',pg_resume)
179 |         startIdx = int(tmp.group(1))
180 | 
181 |     # ----------------------------
182 |     print("\nPreparing for data...")
183 |     # ----------------------------
184 | 
185 |     ref_discount_rews, ref_slow_down = slow_down_cdf.launch(pa, pg_resume=None, render=True, plot=False, repre=repre, end=end)
186 | 
187 |     mean_rew_lr_curve = []
188 |     max_rew_lr_curve = []
189 |     slow_down_lr_curve = []
190 | 
191 |     timer_start = time.time()
192 | 
193 |     print("\nStart reinforcement learning...")
194 | 
195 |     for iteration in xrange(startIdx, pa.num_epochs):
196 | 
197 |         all_ob = []
198 |         all_action = []
199 |         all_adv = []
200 |         all_eprews = []
201 |         all_eplens = []
202 |         all_slowdown = []
203 |         all_entropy = []
204 | 
205 |         # go through all examples
206 |         for ex in xrange(pa.num_ex):
207 | 
208 |             # Collect trajectories until we get timesteps_per_batch total timesteps
209 |             trajs = []
210 | 
211 |             for i in xrange(pa.num_seq_per_batch):
212 |                 traj = get_traj(pg_learner, env, pa.episode_max_length)
213 |                 trajs.append(traj)
214 | 
215 |             # roll to next example
216 |             env.seq_no = (env.seq_no + 1) % env.pa.num_ex
217 | 
218 |             all_ob.append(concatenate_all_ob(trajs, pa))
219 | 
220 |             # Compute discounted sums of rewards
221 |             rets = [discount(traj["reward"], pa.discount) for traj in trajs]
222 |             maxlen = max(len(ret) for ret in rets)
223 |             padded_rets = [np.concatenate([ret, np.zeros(maxlen - len(ret))]) for ret in rets]
224 | 
225 |             # Compute time-dependent baseline
226 |             baseline = np.mean(padded_rets, axis=0)
227 | 
228 |             # Compute advantage function
229 |             advs = [ret - baseline[:len(ret)] for ret in rets]
230 |             all_action.append(np.concatenate([traj["action"] for traj in trajs]))
231 |             all_adv.append(np.concatenate(advs))
232 | 
233 |             all_eprews.append(np.array([discount(traj["reward"], pa.discount)[0] for traj in trajs]))  # episode total rewards
234 |             all_eplens.append(np.array([len(traj["reward"]) for traj in trajs]))  # episode lengths
235 | 
236 |             # All Job Stat
237 |             enter_time, finish_time, job_len = process_all_info(trajs)
238 |             finished_idx = (finish_time >= 0)
239 |             all_slowdown.append(
240 |                 (finish_time[finished_idx] - enter_time[finished_idx]) / job_len[finished_idx]
241 |             )
242 | 
243 |             # Action prob entropy
244 |             all_entropy.append(np.concatenate([traj["entropy"]]))
245 | 
246 |         all_ob = concatenate_all_ob_across_examples(all_ob, pa)
247 |         all_action = np.concatenate(all_action)
248 |         all_adv = np.concatenate(all_adv)
249 | 
250 |         # Do policy gradient update step
251 |         loss = pg_learner.train(all_ob, all_action, all_adv)
252 |         eprews = np.concatenate(all_eprews)  # episode total rewards
253 |         eplens = np.concatenate(all_eplens)  # episode lengths
254 | 
255 |         all_slowdown = np.concatenate(all_slowdown)
256 | 
257 |         all_entropy = np.concatenate(all_entropy)
258 | 
259 |         timer_end = time.time()
260 | 
261 |         print "-----------------"
262 |         print "Iteration: \t %i" % iteration
263 |         print "NumTrajs: \t %i" % len(eprews)
264 |         print "NumTimesteps: \t %i" % np.sum(eplens)
265 |         print "Loss:     \t %s" % loss
266 |         print "MaxRew: \t %s" % np.average([np.max(rew) for rew in all_eprews])
267 |         print "MeanRew: \t %s +- %s" % (eprews.mean(), eprews.std())
268 |         print "MeanSlowdown: \t %s" % np.mean(all_slowdown)
269 |         print "MeanLen: \t %s +- %s" % (eplens.mean(), eplens.std())
270 |         print "MeanEntropy \t %s" % (np.mean(all_entropy))
271 |         print "Elapsed time\t %s" % (timer_end - timer_start), "seconds"
272 |         print "-----------------"
273 | 
274 | 
275 |         f.write("-----------------\n")
276 |         f.write("Iteration: \t %i\n" % (iteration))
277 |         f.write("NumTrajs: \t %i\n" % (len(eprews)))
278 |         f.write("NumTimesteps: \t %i\n" % (np.sum(eplens)))
279 |         f.write("Loss:     \t %s\n".format(loss))
280 |         f.write("MaxRew: \t %s\n" % (np.average([np.max(rew) for rew in all_eprews])))
281 |         f.write("MeanRew: \t %s +- %s\n" % (np.mean(eprews), np.std(eprews)))
282 |         f.write("MeanSlowdown: \t %s\n" % (np.mean(all_slowdown)))
283 |         f.write("MeanLen: \t %s +- %s\n" % (np.mean(eplens), np.std(eplens)))
284 |         f.write("MeanEntropy \t %s\n" % ((np.mean(all_entropy))))
285 |         f.write("Elapsed time\t %s seconds\n" % ((timer_end - timer_start)))
286 |         f.write("-----------------\n")
287 |         f.close()
288 | 
289 |         timer_start = time.time()
290 | 
291 |         max_rew_lr_curve.append(np.average([np.max(rew) for rew in all_eprews]))
292 |         mean_rew_lr_curve.append(eprews.mean())
293 |         slow_down_lr_curve.append(np.mean(all_slowdown))
294 | 
295 |         if iteration % pa.output_freq == 0:
296 |             param_file = open(pa.output_filename + '_' + str(iteration) + '.pkl', 'wb')
297 |             cPickle.dump(pg_learner.get_params(), param_file, -1)
298 |             param_file.close()
299 | 
300 |             # added by wjchen, to record accuracy and rewards
301 |             sample_file = h5py.File('log/re_record_iter'+str(len(slow_down_lr_curve))\
302 |                                     + datetime.now().strftime('%Y-%m-%d_%H:%M')+'.h5', 'w+')
303 |             sample_file.create_dataset('max_rew_lr_curve', data=max_rew_lr_curve)
304 |             sample_file.create_dataset('mean_rew_lr_curve', data=mean_rew_lr_curve)
305 |             sample_file.create_dataset('slow_down_lr_curve', data=slow_down_lr_curve)
306 |             #
307 |             ref_dr = sample_file.create_group('ref_discount_rews')
308 |             for k, v in ref_discount_rews.items():
309 |                 ref_dr[k] = np.average(v)
310 |             #
311 |             ref_sd = sample_file.create_group('ref_slow_down')
312 |             for k, v in ref_slow_down.items():
313 |                 ref_sd[k] = np.average(np.concatenate(v))
314 |             sample_file.close()
315 | 
316 |             slow_down_cdf.launch(pa, pa.output_filename + '_' + str(iteration) + '.pkl',
317 |                                  render=True, plot=True, repre=repre, end=end)
318 | 
319 |             plot_lr_curve(pa.output_filename,
320 |                          max_rew_lr_curve, mean_rew_lr_curve, slow_down_lr_curve,
321 |                          ref_discount_rews, ref_slow_down)
322 | 
323 | 
324 | def main():
325 | 
326 |     import parameters
327 | 
328 |     pa = parameters.Parameters()
329 | 
330 |     pa.simu_len = 200  # 1000
331 |     pa.num_ex = 10  # 100
332 |     pa.num_nw = 10
333 |     pa.num_seq_per_batch = 20
334 |     pa.output_freq = 50
335 | 
336 |     # pa.max_nw_size = 5
337 |     # pa.job_len = 5
338 |     pa.new_job_rate = 0.3
339 | 
340 |     pa.episode_max_length = 2000  # 2000
341 | 
342 |     pa.compute_dependent_parameters()
343 | 
344 |     pg_resume = None
345 |     # pg_resume = 'data/tmp_0.pkl'
346 | 
347 |     render = True
348 | 
349 |     launch(pa, pg_resume, render, repre='image', end='all_done')
350 | 
351 | 
352 | if __name__ == '__main__':
353 |     main()
354 | 


--------------------------------------------------------------------------------
/pg_network.py:
--------------------------------------------------------------------------------
  1 | import numpy as np
  2 | import theano, theano.tensor as T
  3 | import lasagne
  4 | from collections import OrderedDict
  5 | 
  6 | 
  7 | def rmsprop_updates(grads, params, stepsize, rho=0.9, epsilon=1e-9):
  8 | 
  9 |     updates = []
 10 | 
 11 |     for param, grad in zip(params, grads):
 12 |         accum = theano.shared(np.zeros(param.get_value(borrow=True).shape, dtype=param.dtype))
 13 |         accum_new = rho * accum + (1 - rho) * grad ** 2
 14 |         updates.append((accum, accum_new))
 15 |         updates.append((param, param + (stepsize * grad / T.sqrt(accum_new + epsilon))))
 16 |         # lasagne has '-' after param
 17 |     return updates
 18 | 
 19 | 
 20 | def utils_floatX(arr):
 21 |     return np.asarray(arr, dtype=theano.config.floatX)
 22 | 
 23 | 
 24 | def adam_update(grads, params, learning_rate=0.001, beta1=0.9,
 25 |                 beta2=0.999, epsilon=1e-8):
 26 | 
 27 |     t_prev = theano.shared(utils_floatX(0.))
 28 |     updates = OrderedDict()
 29 | 
 30 |     # Using theano constant to prevent upcasting of float32
 31 |     one = T.constant(1)
 32 | 
 33 |     t = t_prev + 1
 34 |     a_t = learning_rate*T.sqrt(one-beta2**t)/(one-beta1**t)
 35 | 
 36 |     for param, g_t in zip(params, grads):
 37 |         value = param.get_value(borrow=True)
 38 |         m_prev = theano.shared(np.zeros(value.shape, dtype=value.dtype),
 39 |                                broadcastable=param.broadcastable)
 40 |         v_prev = theano.shared(np.zeros(value.shape, dtype=value.dtype),
 41 |                                broadcastable=param.broadcastable)
 42 | 
 43 |         m_t = beta1*m_prev + (one-beta1)*g_t
 44 |         v_t = beta2*v_prev + (one-beta2)*g_t**2
 45 |         step = a_t*m_t/(T.sqrt(v_t) + epsilon)
 46 | 
 47 |         updates[m_prev] = m_t
 48 |         updates[v_prev] = v_t
 49 |         updates[param] = param + step
 50 | 
 51 |     updates[t_prev] = t
 52 |     return updates
 53 | 
 54 | 
 55 | class PGLearner:
 56 |     def __init__(self, pa):
 57 | 
 58 |         self.input_height = pa.network_input_height
 59 |         self.input_width = pa.network_input_width
 60 |         self.output_height = pa.network_output_dim
 61 | 
 62 |         self.num_frames = pa.num_frames
 63 | 
 64 |         self.update_counter = 0
 65 | 
 66 |         states = T.tensor4('states')
 67 |         actions = T.ivector('actions')
 68 |         values = T.vector('values')
 69 | 
 70 |         print 'network_input_height=', pa.network_input_height
 71 |         print 'network_input_width=', pa.network_input_width
 72 |         print 'network_output_dim=', pa.network_output_dim
 73 | 
 74 |         # image representation
 75 |         self.l_out = \
 76 |             build_small_conv_pg_network(pa.network_input_height, pa.network_input_width, pa.network_output_dim)
 77 | 
 78 |         # compact representation
 79 |         # self.l_out = \
 80 |         #     build_compact_pg_network(pa.network_input_height, pa.network_input_width, pa.network_output_dim)
 81 | 
 82 |         self.lr_rate = pa.lr_rate
 83 |         self.rms_rho = pa.rms_rho
 84 |         self.rms_eps = pa.rms_eps
 85 | 
 86 |         params = lasagne.layers.helper.get_all_params(self.l_out)
 87 | 
 88 |         print ' params=', params, ' count=', lasagne.layers.count_params(self.l_out)
 89 | 
 90 |         self._get_param = theano.function([], params)
 91 | 
 92 |         # ===================================
 93 |         # training function part
 94 |         # ===================================
 95 | 
 96 |         prob_act = lasagne.layers.get_output(self.l_out, states)
 97 | 
 98 |         self._get_act_prob = theano.function([states], prob_act, allow_input_downcast=True)
 99 | 
100 |         # --------  Policy Gradient  --------
101 | 
102 |         N = states.shape[0]
103 | 
104 |         loss = T.log(prob_act[T.arange(N), actions]).dot(values) / N  # call it "loss"
105 | 
106 |         grads = T.grad(loss, params)
107 | 
108 |         updates = rmsprop_updates(
109 |             grads, params, self.lr_rate, self.rms_rho, self.rms_eps)
110 | 
111 |         # updates = adam_update(
112 |         #     grads, params, self.lr_rate)
113 | 
114 |         self._train_fn = theano.function([states, actions, values], loss,
115 |                                          updates=updates, allow_input_downcast=True)
116 | 
117 |         self._get_loss = theano.function([states, actions, values], loss, allow_input_downcast=True)
118 | 
119 |         self._get_grad = theano.function([states, actions, values], grads, allow_input_downcast=True)
120 | 
121 |         # --------  Supervised Learning  --------
122 | 
123 |         su_target = T.ivector('su_target')
124 | 
125 |         # su_diff = su_target - prob_act
126 |         # su_loss = 0.5 * su_diff ** 2
127 | 
128 |         su_loss = lasagne.objectives.categorical_crossentropy(prob_act, su_target)
129 |         su_loss = su_loss.mean()
130 | 
131 |         l2_penalty = lasagne.regularization.regularize_network_params(self.l_out, lasagne.regularization.l2)
132 |         # l1_penalty = lasagne.regularization.regularize_network_params(self.l_out, lasagne.regularization.l1)
133 | 
134 |         su_loss += 1e-3*l2_penalty
135 |         print 'lr_rate=', self.lr_rate
136 | 
137 |         su_updates = lasagne.updates.rmsprop(su_loss, params,
138 |                                              self.lr_rate, self.rms_rho, self.rms_eps)
139 |         #su_updates = lasagne.updates.nesterov_momentum(su_loss, params, self.lr_rate)
140 | 
141 |         self._su_train_fn = theano.function([states, su_target], [su_loss, prob_act], updates=su_updates)
142 | 
143 |         self._su_loss = theano.function([states, su_target], [su_loss, prob_act])
144 | 
145 |         self._debug = theano.function([states], [states.flatten(2)])
146 | 
147 |     # get the action based on the estimated value
148 |     def choose_action(self, state):
149 | 
150 |         act_prob = self.get_one_act_prob(state)
151 | 
152 |         csprob_n = np.cumsum(act_prob)
153 |         act = (csprob_n > np.random.rand()).argmax()
154 | 
155 |         # print(act_prob, act)
156 | 
157 |         return act
158 | 
159 |     def train(self, states, actions, values):
160 | 
161 |         loss = self._train_fn(states, actions, values)
162 |         return loss
163 | 
164 |     def get_params(self):
165 | 
166 |         return self._get_param()
167 | 
168 |     def get_grad(self, states, actions, values):
169 | 
170 |         return self._get_grad(states, actions, values)
171 | 
172 |     def get_one_act_prob(self, state):
173 | 
174 |         states = np.zeros((1, 1, self.input_height, self.input_width), dtype=theano.config.floatX)
175 |         states[0, :, :] = state
176 |         act_prob = self._get_act_prob(states)[0]
177 | 
178 |         return act_prob
179 | 
180 |     def get_act_probs(self, states):  # multiple states, assuming in floatX format
181 |         act_probs = self._get_act_prob(states)
182 |         return act_probs
183 | 
184 |     #  -------- Supervised Learning --------
185 |     def su_train(self, states, target):
186 |         loss, prob_act = self._su_train_fn(states, target)
187 |         return np.sqrt(loss), prob_act
188 | 
189 |     def su_test(self, states, target):
190 |         loss, prob_act = self._su_loss(states, target)
191 |         return np.sqrt(loss), prob_act
192 | 
193 |     #  -------- Save/Load network parameters --------
194 |     def return_net_params(self):
195 |         return lasagne.layers.helper.get_all_param_values(self.l_out)
196 | 
197 |     def set_net_params(self, net_params):
198 |         lasagne.layers.helper.set_all_param_values(self.l_out, net_params)
199 | 
200 | 
201 | # ===================================
202 | # build neural network
203 | # ===================================
204 | 
205 | 
206 | def build_pg_network(input_height, input_width, output_length):
207 | 
208 |     # l_in = lasagne.layers.InputLayer(
209 |     #     shape=(None, 1, input_height, input_width),
210 |     # )
211 |     #
212 |     # l_hid1 = lasagne.layers.DenseLayer(
213 |     #     l_in,
214 |     #     num_units=20,
215 |     #     # nonlinearity=lasagne.nonlinearities.tanh,
216 |     #     nonlinearity=lasagne.nonlinearities.rectify,
217 |     #     # W=lasagne.init.Normal(.0201),
218 |     #     #W=lasagne.init.Normal(.01),
219 |     #     W=lasagne.init.HeNormal('relu'),
220 |     #     b=lasagne.init.Constant(0.05)
221 |     # )
222 |     #
223 |     # #l_hid1_drop = lasagne.layers.DropoutLayer(l_hid1, p=0.5)
224 |     #
225 |     # l_hid2 = lasagne.layers.DenseLayer(
226 |     #     l_hid1,
227 |     #     num_units=20,
228 |     #     # nonlinearity=lasagne.nonlinearities.tanh,
229 |     #     nonlinearity=lasagne.nonlinearities.rectify,
230 |     #     # W=lasagne.init.Normal(.0201),
231 |     #     #W=lasagne.init.Normal(.01),
232 |     #     W=lasagne.init.HeNormal('relu'),
233 |     #     b=lasagne.init.Constant(0.05)
234 |     # )
235 |     #
236 |     # l_hid3 = lasagne.layers.DenseLayer(
237 |     #     l_hid2,
238 |     #     num_units=20,
239 |     #     # nonlinearity=lasagne.nonlinearities.tanh,
240 |     #     nonlinearity=lasagne.nonlinearities.rectify,
241 |     #     # W=lasagne.init.Normal(.0201),
242 |     #     #W=lasagne.init.Normal(.01),
243 |     #     W=lasagne.init.HeNormal('relu'),
244 |     #     b=lasagne.init.Constant(0.05)
245 |     # )
246 |     #
247 |     #
248 |     # #50% dropout again:
249 |     # #l_hid2_drop = lasagne.layers.DropoutLayer(l_hid2, p=0.5)
250 |     #
251 |     # l_out = lasagne.layers.DenseLayer(
252 |     #     l_hid3,
253 |     #     num_units=output_length,
254 |     #     nonlinearity=lasagne.nonlinearities.softmax,
255 |     #     # W=lasagne.init.Normal(.0001),
256 |     #     #W=lasagne.init.Normal(.01),
257 |     #     W=lasagne.init.HeNormal('relu'),
258 |     #     b=lasagne.init.Constant(0.05)
259 |     # )
260 |     #
261 |     # return l_out
262 | 
263 |     l_in = lasagne.layers.InputLayer(
264 |         shape=(None, 1, input_height, input_width),
265 |     )
266 | 
267 |     l_hid = lasagne.layers.DenseLayer(
268 |         l_in,
269 |         num_units=20,
270 |         # nonlinearity=lasagne.nonlinearities.tanh,
271 |         nonlinearity=lasagne.nonlinearities.rectify,
272 |         # W=lasagne.init.Normal(.0201),
273 |         W=lasagne.init.Normal(.01),
274 |         b=lasagne.init.Constant(0)
275 |     )
276 | 
277 |     l_out = lasagne.layers.DenseLayer(
278 |         l_hid,
279 |         num_units=output_length,
280 |         nonlinearity=lasagne.nonlinearities.softmax,
281 |         # W=lasagne.init.Normal(.0001),
282 |         W=lasagne.init.Normal(.01),
283 |         b=lasagne.init.Constant(0)
284 |     )
285 | 
286 |     return l_out
287 | 
288 | 
289 | def build_compact_pg_network(input_height, input_width, output_length):
290 |     l_in = lasagne.layers.InputLayer(
291 |         shape=(None, 1, input_height, input_width),
292 |         )
293 | 
294 |     l_hid1 = lasagne.layers.DenseLayer(
295 |         l_in,
296 |         num_units=520,
297 |         # nonlinearity=lasagne.nonlinearities.tanh,
298 |         nonlinearity=lasagne.nonlinearities.rectify,
299 |         W=lasagne.init.HeNormal('relu'),
300 |         b=lasagne.init.Constant(0.05)
301 |         )
302 | 
303 |     l_hid2 = lasagne.layers.DenseLayer(
304 |         l_hid1,
305 |         num_units=20,
306 |         # nonlinearity=lasagne.nonlinearities.tanh,
307 |         nonlinearity=lasagne.nonlinearities.rectify,
308 |         # W=lasagne.init.Normal(.0201),
309 |         #W=lasagne.init.Normal(.01),
310 |         W=lasagne.init.HeNormal('relu'),
311 |         b=lasagne.init.Constant(0.05)
312 |         )
313 | 
314 |     l_hid3 = lasagne.layers.DenseLayer(
315 |         l_hid2,
316 |         num_units=20,
317 |         # nonlinearity=lasagne.nonlinearities.tanh,
318 |         nonlinearity=lasagne.nonlinearities.rectify,
319 |         # W=lasagne.init.Normal(.0201),
320 |         #W=lasagne.init.Normal(.01),
321 |         W=lasagne.init.HeNormal('relu'),
322 |         b=lasagne.init.Constant(0.05)
323 |         )
324 | 
325 | 
326 |     #50% dropout again:
327 |     #l_hid2_drop = lasagne.layers.DropoutLayer(l_hid2, p=0.5)
328 | 
329 |     l_out = lasagne.layers.DenseLayer(
330 |         l_hid3,
331 |         num_units=output_length,
332 |         nonlinearity=lasagne.nonlinearities.softmax,
333 |         # W=lasagne.init.Normal(.0001),
334 |         #W=lasagne.init.Normal(.01),
335 |         W=lasagne.init.HeNormal('relu'),
336 |         b=lasagne.init.Constant(0.05)
337 |         )
338 | 
339 |     return l_out
340 | 
341 | def build_small_conv_pg_network(input_height, input_width, output_length):
342 | 
343 |     l_in = lasagne.layers.InputLayer(
344 |         shape=(None, 1, input_height, input_width),
345 |     )
346 | 
347 |     l_hid1 = lasagne.layers.Conv2DLayer(
348 |         l_in,
349 |         num_filters=16,
350 |         filter_size=2,
351 |         stride=(1, 1),
352 |         pad=1,
353 |         # untie_biases=False,
354 |         W=lasagne.init.GlorotUniform(.01),
355 |         b=lasagne.init.Constant(0.),
356 |         nonlinearity=lasagne.nonlinearities.rectify,
357 |         convolution=theano.tensor.nnet.conv2d
358 |     )
359 | 
360 |     l_out = lasagne.layers.DenseLayer(
361 |         l_hid1,
362 |         num_units=output_length,
363 |         nonlinearity=lasagne.nonlinearities.softmax,
364 |         # W=lasagne.init.Normal(.0001),
365 |         W=lasagne.init.Normal(.01),
366 |         b=lasagne.init.Constant(0)
367 |     )
368 | 
369 |     return l_out
370 | 
371 | def build_big_conv_pg_network(input_height, input_width, output_length):
372 | 
373 |     l_in = lasagne.layers.InputLayer(
374 |         shape=(None, 1, input_height, input_width),
375 |     )
376 | 
377 |     l_hid1 = lasagne.layers.Conv2DLayer(
378 |         l_in,
379 |         num_filters=8,
380 |         filter_size=4,
381 |         stride=(2, 2),
382 |         pad=2,
383 |         # untie_biases=False,
384 |         W=lasagne.init.GlorotUniform(.01),
385 |         b=lasagne.init.Constant(0.),
386 |         nonlinearity=lasagne.nonlinearities.rectify,
387 |         convolution=theano.tensor.nnet.conv2d
388 |     )
389 | 
390 |     l_hid2 = lasagne.layers.Conv2DLayer(
391 |         l_hid1,
392 |         num_filters=16,
393 |         filter_size=2,
394 |         stride=(1, 1),
395 |         pad=1,
396 |         # untie_biases=False,
397 |         W=lasagne.init.GlorotUniform(.01),
398 |         b=lasagne.init.Constant(0.),
399 |         nonlinearity=lasagne.nonlinearities.rectify,
400 |         convolution=theano.tensor.nnet.conv2d
401 |     )
402 | 
403 |     l_hid3 = lasagne.layers.DenseLayer(
404 |         l_hid2,
405 |         num_units=20,
406 |         # nonlinearity=lasagne.nonlinearities.tanh,
407 |         nonlinearity=lasagne.nonlinearities.rectify,
408 |         # W=lasagne.init.Normal(.0201),
409 |         W=lasagne.init.Normal(.01), # std = 0.01
410 |         b=lasagne.init.Constant(0)
411 |     )
412 | 
413 |     l_out = lasagne.layers.DenseLayer(
414 |         l_hid3,
415 |         num_units=output_length,
416 |         nonlinearity=lasagne.nonlinearities.softmax,
417 |         # W=lasagne.init.Normal(.0001),
418 |         W=lasagne.init.Normal(.01),
419 |         b=lasagne.init.Constant(0)
420 |     )
421 | 
422 |     return l_out


--------------------------------------------------------------------------------
/pg_re_o.py:
--------------------------------------------------------------------------------
  1 | import time
  2 | import threading
  3 | import numpy as np
  4 | import theano
  5 | import cPickle
  6 | import matplotlib.pyplot as plt
  7 | 
  8 | from multiprocessing import Process
  9 | from multiprocessing import Manager
 10 | 
 11 | import environment
 12 | import job_distribution
 13 | import pg_network
 14 | import slow_down_cdf
 15 | 
 16 | 
 17 | def init_accums(pg_learner):  # in rmsprop
 18 |     accums = []
 19 |     params = pg_learner.get_params()
 20 |     for param in params:
 21 |         accum = np.zeros(param.shape, dtype=param.dtype)
 22 |         accums.append(accum)
 23 |     return accums
 24 | 
 25 | 
 26 | def rmsprop_updates_outside(grads, params, accums, stepsize, rho=0.9, epsilon=1e-9):
 27 | 
 28 |     assert len(grads) == len(params)
 29 |     assert len(grads) == len(accums)
 30 |     for dim in xrange(len(grads)):
 31 |         accums[dim] = rho * accums[dim] + (1 - rho) * grads[dim] ** 2
 32 |         params[dim] += (stepsize * grads[dim] / np.sqrt(accums[dim] + epsilon))
 33 | 
 34 | 
 35 | def discount(x, gamma):
 36 |     """
 37 |     Given vector x, computes a vector y such that
 38 |     y[i] = x[i] + gamma * x[i+1] + gamma^2 x[i+2] + ...
 39 |     """
 40 |     out = np.zeros(len(x))
 41 |     out[-1] = x[-1]
 42 |     for i in reversed(xrange(len(x)-1)):
 43 |         out[i] = x[i] + gamma*out[i+1]
 44 |     assert x.ndim >= 1
 45 |     # More efficient version:
 46 |     # scipy.signal.lfilter([1],[1,-gamma],x[::-1], axis=0)[::-1]
 47 |     return out
 48 | 
 49 | 
 50 | def get_entropy(vec):
 51 |     entropy = - np.sum(vec * np.log(vec))
 52 |     if np.isnan(entropy):
 53 |         entropy = 0
 54 |     return entropy
 55 | 
 56 | 
 57 | def get_traj(agent, env, episode_max_length):
 58 |     """
 59 |     Run agent-environment loop for one whole episode (trajectory)
 60 |     Return dictionary of results
 61 |     """
 62 |     env.reset()
 63 |     obs = []
 64 |     acts = []
 65 |     rews = []
 66 |     entropy = []
 67 |     info = []
 68 | 
 69 |     ob = env.observe()
 70 | 
 71 |     for _ in xrange(episode_max_length):
 72 |         act_prob = agent.get_one_act_prob(ob)
 73 |         csprob_n = np.cumsum(act_prob)
 74 |         a = (csprob_n > np.random.rand()).argmax()
 75 | 
 76 |         obs.append(ob)  # store the ob at current decision making step
 77 |         acts.append(a)
 78 | 
 79 |         ob, rew, done, info = env.step(a, repeat=True)
 80 | 
 81 |         rews.append(rew)
 82 |         entropy.append(get_entropy(act_prob))
 83 | 
 84 |         if done: break
 85 | 
 86 |     return {'reward': np.array(rews),
 87 |             'ob': np.array(obs),
 88 |             'action': np.array(acts),
 89 |             'entropy': entropy,
 90 |             'info': info
 91 |             }
 92 | 
 93 | 
 94 | def concatenate_all_ob(trajs, pa):
 95 | 
 96 |     timesteps_total = 0
 97 |     for i in xrange(len(trajs)):
 98 |         timesteps_total += len(trajs[i]['reward'])
 99 | 
100 |     all_ob = np.zeros(
101 |         (timesteps_total, 1, pa.network_input_height, pa.network_input_width),
102 |         dtype=theano.config.floatX)
103 | 
104 |     timesteps = 0
105 |     for i in xrange(len(trajs)):
106 |         for j in xrange(len(trajs[i]['reward'])):
107 |             all_ob[timesteps, 0, :, :] = trajs[i]['ob'][j]
108 |             timesteps += 1
109 | 
110 |     return all_ob
111 | 
112 | 
113 | def concatenate_all_ob_across_examples(all_ob, pa):
114 |     num_ex = len(all_ob)
115 |     total_samp = 0
116 |     for i in xrange(num_ex):
117 |         total_samp += all_ob[i].shape[0]
118 | 
119 |     all_ob_contact = np.zeros(
120 |         (total_samp, 1, pa.network_input_height, pa.network_input_width),
121 |         dtype=theano.config.floatX)
122 | 
123 |     total_samp = 0
124 | 
125 |     for i in xrange(num_ex):
126 |         prev_samp = total_samp
127 |         total_samp += all_ob[i].shape[0]
128 |         all_ob_contact[prev_samp : total_samp, :, :, :] = all_ob[i]
129 | 
130 |     return all_ob_contact
131 | 
132 | 
133 | def process_all_info(trajs):
134 |     enter_time = []
135 |     finish_time = []
136 |     job_len = []
137 | 
138 |     for traj in trajs:
139 |         enter_time.append(np.array([traj['info'].record[i].enter_time for i in xrange(len(traj['info'].record))]))
140 |         finish_time.append(np.array([traj['info'].record[i].finish_time for i in xrange(len(traj['info'].record))]))
141 |         job_len.append(np.array([traj['info'].record[i].len for i in xrange(len(traj['info'].record))]))
142 | 
143 |     enter_time = np.concatenate(enter_time)
144 |     finish_time = np.concatenate(finish_time)
145 |     job_len = np.concatenate(job_len)
146 | 
147 |     return enter_time, finish_time, job_len
148 | 
149 | 
150 | def plot_lr_curve(output_file_prefix, max_rew_lr_curve, mean_rew_lr_curve, slow_down_lr_curve,
151 |                   ref_discount_rews, ref_slow_down):
152 |     num_colors = len(ref_discount_rews) + 2
153 |     cm = plt.get_cmap('gist_rainbow')
154 | 
155 |     fig = plt.figure(figsize=(12, 5))
156 | 
157 |     ax = fig.add_subplot(121)
158 |     ax.set_color_cycle([cm(1. * i / num_colors) for i in range(num_colors)])
159 | 
160 |     ax.plot(mean_rew_lr_curve, linewidth=2, label='PG mean')
161 |     for k in ref_discount_rews:
162 |         ax.plot(np.tile(np.average(ref_discount_rews[k]), len(mean_rew_lr_curve)), linewidth=2, label=k)
163 |     ax.plot(max_rew_lr_curve, linewidth=2, label='PG max')
164 | 
165 |     plt.legend(loc=4)
166 |     plt.xlabel("Iteration", fontsize=20)
167 |     plt.ylabel("Discounted Total Reward", fontsize=20)
168 | 
169 |     ax = fig.add_subplot(122)
170 |     ax.set_color_cycle([cm(1. * i / num_colors) for i in range(num_colors)])
171 | 
172 |     ax.plot(slow_down_lr_curve, linewidth=2, label='PG mean')
173 |     for k in ref_discount_rews:
174 |         ax.plot(np.tile(np.average(np.concatenate(ref_slow_down[k])), len(slow_down_lr_curve)), linewidth=2, label=k)
175 | 
176 |     plt.legend(loc=1)
177 |     plt.xlabel("Iteration", fontsize=20)
178 |     plt.ylabel("Slowdown", fontsize=20)
179 | 
180 |     plt.savefig(output_file_prefix + "_lr_curve" + ".pdf")
181 | 
182 | 
183 | def get_traj_worker(pg_learner, env, pa, result):
184 | 
185 |     trajs = []
186 | 
187 |     for i in xrange(pa.num_seq_per_batch):
188 |         traj = get_traj(pg_learner, env, pa.episode_max_length)
189 |         trajs.append(traj)
190 | 
191 |     all_ob = concatenate_all_ob(trajs, pa)
192 | 
193 |     # Compute discounted sums of rewards
194 |     rets = [discount(traj["reward"], pa.discount) for traj in trajs]
195 |     maxlen = max(len(ret) for ret in rets)
196 |     padded_rets = [np.concatenate([ret, np.zeros(maxlen - len(ret))]) for ret in rets]
197 | 
198 |     # Compute time-dependent baseline
199 |     baseline = np.mean(padded_rets, axis=0)
200 | 
201 |     # Compute advantage function
202 |     advs = [ret - baseline[:len(ret)] for ret in rets]
203 |     all_action = np.concatenate([traj["action"] for traj in trajs])
204 |     all_adv = np.concatenate(advs)
205 | 
206 |     all_eprews = np.array([discount(traj["reward"], pa.discount)[0] for traj in trajs])  # episode total rewards
207 |     all_eplens = np.array([len(traj["reward"]) for traj in trajs])  # episode lengths
208 | 
209 |     # All Job Stat
210 |     enter_time, finish_time, job_len = process_all_info(trajs)
211 |     finished_idx = (finish_time >= 0)
212 |     all_slowdown = (finish_time[finished_idx] - enter_time[finished_idx]) / job_len[finished_idx]
213 | 
214 |     all_entropy = np.concatenate([traj["entropy"] for traj in trajs])
215 | 
216 |     result.append({"all_ob": all_ob,
217 |                    "all_action": all_action,
218 |                    "all_adv": all_adv,
219 |                    "all_eprews": all_eprews,
220 |                    "all_eplens": all_eplens,
221 |                    "all_slowdown": all_slowdown,
222 |                    "all_entropy": all_entropy})
223 | 
224 | 
225 | def launch(pa, pg_resume=None, render=False, repre='image', end='no_new_job'):
226 | 
227 |     # ----------------------------
228 |     print("Preparing for workers...")
229 |     # ----------------------------
230 | 
231 |     pg_learners = []
232 |     envs = []
233 | 
234 |     nw_len_seqs, nw_size_seqs = job_distribution.generate_sequence_work(pa, seed=42)
235 | 
236 |     for ex in xrange(pa.num_ex):
237 | 
238 |         print "-prepare for env-", ex
239 | 
240 |         env = environment.Env(pa, nw_len_seqs=nw_len_seqs, nw_size_seqs=nw_size_seqs,
241 |                               render=False, repre=repre, end=end)
242 |         env.seq_no = ex
243 |         envs.append(env)
244 | 
245 |     for ex in xrange(pa.batch_size + 1):  # last worker for updating the parameters
246 | 
247 |         print "-prepare for worker-", ex
248 | 
249 |         pg_learner = pg_network.PGLearner(pa)
250 | 
251 |         if pg_resume is not None:
252 |             net_handle = open(pg_resume, 'rb')
253 |             net_params = cPickle.load(net_handle)
254 |             pg_learner.set_net_params(net_params)
255 | 
256 |         pg_learners.append(pg_learner)
257 | 
258 |     accums = init_accums(pg_learners[pa.batch_size])
259 | 
260 |     # --------------------------------------
261 |     print("Preparing for reference data...")
262 |     # --------------------------------------
263 | 
264 |     ref_discount_rews, ref_slow_down = slow_down_cdf.launch(pa, pg_resume=None, render=False, plot=False, repre=repre, end=end)
265 |     mean_rew_lr_curve = []
266 |     max_rew_lr_curve = []
267 |     slow_down_lr_curve = []
268 | 
269 |     # --------------------------------------
270 |     print("Start training...")
271 |     # --------------------------------------
272 | 
273 |     timer_start = time.time()
274 | 
275 |     for iteration in xrange(1, pa.num_epochs):
276 | 
277 |         ps = []  # threads
278 |         manager = Manager()  # managing return results
279 |         manager_result = manager.list([])
280 | 
281 |         ex_indices = range(pa.num_ex)
282 |         np.random.shuffle(ex_indices)
283 | 
284 |         all_eprews = []
285 |         grads_all = []
286 |         loss_all = []
287 |         eprews = []
288 |         eplens = []
289 |         all_slowdown = []
290 |         all_entropy = []
291 | 
292 |         ex_counter = 0
293 |         for ex in xrange(pa.num_ex):
294 | 
295 |             ex_idx = ex_indices[ex]
296 |             p = Process(target=get_traj_worker,
297 |                         args=(pg_learners[ex_counter], envs[ex_idx], pa, manager_result, ))
298 |             ps.append(p)
299 | 
300 |             ex_counter += 1
301 | 
302 |             if ex_counter >= pa.batch_size or ex == pa.num_ex - 1:
303 | 
304 |                 print ex, "out of", pa.num_ex
305 | 
306 |                 ex_counter = 0
307 | 
308 |                 for p in ps:
309 |                     p.start()
310 | 
311 |                 for p in ps:
312 |                     p.join()
313 | 
314 |                 result = []  # convert list from shared memory
315 |                 for r in manager_result:
316 |                     result.append(r)
317 | 
318 |                 ps = []
319 |                 manager_result = manager.list([])
320 | 
321 |                 all_ob = concatenate_all_ob_across_examples([r["all_ob"] for r in result], pa)
322 |                 all_action = np.concatenate([r["all_action"] for r in result])
323 |                 all_adv = np.concatenate([r["all_adv"] for r in result])
324 | 
325 |                 # Do policy gradient update step, using the first agent
326 |                 # put the new parameter in the last 'worker', then propagate the update at the end
327 |                 grads = pg_learners[pa.batch_size].get_grad(all_ob, all_action, all_adv)
328 | 
329 |                 grads_all.append(grads)
330 | 
331 |                 all_eprews.extend([r["all_eprews"] for r in result])
332 | 
333 |                 eprews.extend(np.concatenate([r["all_eprews"] for r in result]))  # episode total rewards
334 |                 eplens.extend(np.concatenate([r["all_eplens"] for r in result]))  # episode lengths
335 | 
336 |                 all_slowdown.extend(np.concatenate([r["all_slowdown"] for r in result]))
337 |                 all_entropy.extend(np.concatenate([r["all_entropy"] for r in result]))
338 | 
339 |         # assemble gradients
340 |         grads = grads_all[0]
341 |         for i in xrange(1, len(grads_all)):
342 |             for j in xrange(len(grads)):
343 |                 grads[j] += grads_all[i][j]
344 | 
345 |         # propagate network parameters to others
346 |         params = pg_learners[pa.batch_size].get_params()
347 | 
348 |         rmsprop_updates_outside(grads, params, accums, pa.lr_rate, pa.rms_rho, pa.rms_eps)
349 | 
350 |         for i in xrange(pa.batch_size + 1):
351 |             pg_learners[i].set_net_params(params)
352 | 
353 |         timer_end = time.time()
354 | 
355 |         print "-----------------"
356 |         print "Iteration: \t %i" % iteration
357 |         print "NumTrajs: \t %i" % len(eprews)
358 |         print "NumTimesteps: \t %i" % np.sum(eplens)
359 |         # print "Loss:     \t %s" % np.mean(loss_all)
360 |         print "MaxRew: \t %s" % np.average([np.max(rew) for rew in all_eprews])
361 |         print "MeanRew: \t %s +- %s" % (np.mean(eprews), np.std(eprews))
362 |         print "MeanSlowdown: \t %s" % np.mean(all_slowdown)
363 |         print "MeanLen: \t %s +- %s" % (np.mean(eplens), np.std(eplens))
364 |         print "MeanEntropy \t %s" % (np.mean(all_entropy))
365 |         print "Elapsed time\t %s" % (timer_end - timer_start), "seconds"
366 |         print "-----------------"
367 | 
368 |         timer_start = time.time()
369 | 
370 |         max_rew_lr_curve.append(np.average([np.max(rew) for rew in all_eprews]))
371 |         mean_rew_lr_curve.append(np.mean(eprews))
372 |         slow_down_lr_curve.append(np.mean(all_slowdown))
373 | 
374 |         if iteration % pa.output_freq == 0:
375 |             param_file = open(pa.output_filename + '_' + str(iteration) + '.pkl', 'wb')
376 |             cPickle.dump(pg_learners[pa.batch_size].get_params(), param_file, -1)
377 |             param_file.close()
378 | 
379 |             pa.unseen = True
380 |             slow_down_cdf.launch(pa, pa.output_filename + '_' + str(iteration) + '.pkl',
381 |                                  render=False, plot=True, repre=repre, end=end)
382 |             pa.unseen = False
383 |             # test on unseen examples
384 | 
385 |             plot_lr_curve(pa.output_filename,
386 |                           max_rew_lr_curve, mean_rew_lr_curve, slow_down_lr_curve,
387 |                           ref_discount_rews, ref_slow_down)
388 | 
389 | 
390 | def main():
391 | 
392 |     import parameters
393 | 
394 |     pa = parameters.Parameters()
395 | 
396 |     pa.simu_len = 50  # 1000
397 |     pa.num_ex = 50  # 100
398 |     pa.num_nw = 10
399 |     pa.num_seq_per_batch = 20
400 |     pa.output_freq = 50
401 |     pa.batch_size = 10
402 | 
403 |     # pa.max_nw_size = 5
404 |     # pa.job_len = 5
405 |     pa.new_job_rate = 0.3
406 | 
407 |     pa.episode_max_length = 2000  # 2000
408 | 
409 |     pa.compute_dependent_parameters()
410 | 
411 |     pg_resume = None
412 |     # pg_resume = 'data/tmp_450.pkl'
413 | 
414 |     render = False
415 | 
416 |     launch(pa, pg_resume, render, repre='image', end='all_done')
417 | 
418 | 
419 | if __name__ == '__main__':
420 |     main()
421 | 


--------------------------------------------------------------------------------
/pg_re.py:
--------------------------------------------------------------------------------
  1 | import time
  2 | import threading
  3 | import numpy as np
  4 | import theano
  5 | import cPickle
  6 | import matplotlib.pyplot as plt
  7 | import h5py
  8 | from datetime import datetime
  9 | 
 10 | from multiprocessing import Process
 11 | from multiprocessing import Manager
 12 | 
 13 | import environment
 14 | import job_distribution
 15 | import pg_network
 16 | import slow_down_cdf
 17 | import re
 18 | 
 19 | 
 20 | def init_accums(pg_learner):  # in rmsprop
 21 |     accums = []
 22 |     params = pg_learner.get_params()
 23 |     for param in params:
 24 |         accum = np.zeros(param.shape, dtype=param.dtype)
 25 |         accums.append(accum)
 26 |     return accums
 27 | 
 28 | 
 29 | def rmsprop_updates_outside(grads, params, accums, stepsize, rho=0.9, epsilon=1e-9):
 30 | 
 31 |     assert len(grads) == len(params)
 32 |     assert len(grads) == len(accums)
 33 |     for dim in xrange(len(grads)):
 34 |         accums[dim] = rho * accums[dim] + (1 - rho) * grads[dim] ** 2
 35 |         params[dim] += (stepsize * grads[dim] / np.sqrt(accums[dim] + epsilon))
 36 | 
 37 | 
 38 | def discount(x, gamma):
 39 |     """
 40 |     Given vector x, computes a vector y such that
 41 |     y[i] = x[i] + gamma * x[i+1] + gamma^2 x[i+2] + ...
 42 |     """
 43 |     out = np.zeros(len(x))
 44 |     out[-1] = x[-1]
 45 |     for i in reversed(xrange(len(x)-1)):
 46 |         out[i] = x[i] + gamma*out[i+1]
 47 |     assert x.ndim >= 1
 48 |     # More efficient version:
 49 |     # scipy.signal.lfilter([1],[1,-gamma],x[::-1], axis=0)[::-1]
 50 |     return out
 51 | 
 52 | 
 53 | def get_entropy(vec):
 54 |     entropy = - np.sum(vec * np.log(vec))
 55 |     if np.isnan(entropy):
 56 |         entropy = 0
 57 |     return entropy
 58 | 
 59 | 
 60 | def get_traj(agent, env, episode_max_length):
 61 |     """
 62 |     Run agent-environment loop for one whole episode (trajectory)
 63 |     Return dictionary of results
 64 |     """
 65 |     env.reset()
 66 |     obs = []
 67 |     acts = []
 68 |     rews = []
 69 |     entropy = []
 70 |     info = []
 71 | 
 72 |     ob = env.observe()
 73 | 
 74 |     for _ in xrange(episode_max_length):
 75 |         act_prob = agent.get_one_act_prob(ob)
 76 |         csprob_n = np.cumsum(act_prob) # cumulative sum of probability
 77 |         a = (csprob_n > np.random.rand()).argmax()
 78 | 
 79 |         obs.append(ob)  # store the ob at current decision making step
 80 |         acts.append(a)
 81 | 
 82 |         ob, rew, done, info = env.step(a, repeat=True)
 83 | 
 84 |         rews.append(rew)
 85 |         entropy.append(get_entropy(act_prob))
 86 | 
 87 |         if done: break
 88 | 
 89 |     return {'reward': np.array(rews),
 90 |             'ob': np.array(obs),
 91 |             'action': np.array(acts),
 92 |             'entropy': entropy,
 93 |             'info': info
 94 |             }
 95 | 
 96 | 
 97 | def concatenate_all_ob(trajs, pa):
 98 | 
 99 |     timesteps_total = 0
100 |     for i in xrange(len(trajs)):
101 |         timesteps_total += len(trajs[i]['reward'])
102 | 
103 |     all_ob = np.zeros(
104 |         (timesteps_total, 1, pa.network_input_height, pa.network_input_width),
105 |         dtype=theano.config.floatX)
106 | 
107 |     timesteps = 0
108 |     for i in xrange(len(trajs)):
109 |         for j in xrange(len(trajs[i]['reward'])):
110 |             all_ob[timesteps, 0, :, :] = trajs[i]['ob'][j]
111 |             timesteps += 1
112 | 
113 |     return all_ob
114 | 
115 | 
116 | def concatenate_all_ob_across_examples(all_ob, pa):
117 |     num_ex = len(all_ob)
118 |     total_samp = 0
119 |     for i in xrange(num_ex):
120 |         total_samp += all_ob[i].shape[0]
121 | 
122 |     all_ob_contact = np.zeros(
123 |         (total_samp, 1, pa.network_input_height, pa.network_input_width),
124 |         dtype=theano.config.floatX)
125 | 
126 |     total_samp = 0
127 | 
128 |     for i in xrange(num_ex):
129 |         prev_samp = total_samp
130 |         total_samp += all_ob[i].shape[0]
131 |         all_ob_contact[prev_samp : total_samp, :, :, :] = all_ob[i]
132 | 
133 |     return all_ob_contact
134 | 
135 | 
136 | def process_all_info(trajs):
137 |     enter_time = []
138 |     finish_time = []
139 |     job_len = []
140 | 
141 |     for traj in trajs:
142 |         enter_time.append(np.array([traj['info'].record[i].enter_time for i in xrange(len(traj['info'].record))]))
143 |         finish_time.append(np.array([traj['info'].record[i].finish_time for i in xrange(len(traj['info'].record))]))
144 |         job_len.append(np.array([traj['info'].record[i].len for i in xrange(len(traj['info'].record))]))
145 | 
146 |     enter_time = np.concatenate(enter_time)
147 |     finish_time = np.concatenate(finish_time)
148 |     job_len = np.concatenate(job_len)
149 | 
150 |     return enter_time, finish_time, job_len
151 | 
152 | 
153 | def plot_lr_curve(output_file_prefix, max_rew_lr_curve, mean_rew_lr_curve, slow_down_lr_curve,
154 |                   ref_discount_rews, ref_slow_down):
155 |     num_colors = len(ref_discount_rews) + 2
156 |     cm = plt.get_cmap('gist_rainbow')
157 | 
158 |     fig = plt.figure(figsize=(12, 5))
159 | 
160 |     ax = fig.add_subplot(121)
161 |     ax.set_color_cycle([cm(1. * i / num_colors) for i in range(num_colors)])
162 | 
163 |     ax.plot(mean_rew_lr_curve, linewidth=2, label='PG mean')
164 |     for k in ref_discount_rews:
165 |         ax.plot(np.tile(np.average(ref_discount_rews[k]), len(mean_rew_lr_curve)), linewidth=2, label=k)
166 |     ax.plot(max_rew_lr_curve, linewidth=2, label='PG max')
167 | 
168 |     plt.legend(loc=4)
169 |     plt.xlabel("Iteration", fontsize=20)
170 |     plt.ylabel("Discounted Total Reward", fontsize=20)
171 | 
172 |     ax = fig.add_subplot(122)
173 |     ax.set_color_cycle([cm(1. * i / num_colors) for i in range(num_colors)])
174 | 
175 |     ax.plot(slow_down_lr_curve, linewidth=2, label='PG mean')
176 |     for k in ref_discount_rews:
177 |         ax.plot(np.tile(np.average(np.concatenate(ref_slow_down[k])), len(slow_down_lr_curve)), linewidth=2, label=k)
178 | 
179 |     plt.legend(loc=1)
180 |     plt.xlabel("Iteration", fontsize=20)
181 |     plt.ylabel("Slowdown", fontsize=20)
182 | 
183 |     plt.savefig(output_file_prefix + "_lr_curve" + ".pdf")
184 | 
185 | 
186 | def get_traj_worker(pg_learner, env, pa, result):
187 | 
188 |     trajs = []
189 | 
190 |     for i in xrange(pa.num_seq_per_batch):
191 |         traj = get_traj(pg_learner, env, pa.episode_max_length)
192 |         trajs.append(traj)
193 | 
194 |     all_ob = concatenate_all_ob(trajs, pa)
195 | 
196 |     # Compute discounted sums of rewards
197 |     rets = [discount(traj["reward"], pa.discount) for traj in trajs]
198 |     maxlen = max(len(ret) for ret in rets)
199 |     padded_rets = [np.concatenate([ret, np.zeros(maxlen - len(ret))]) for ret in rets]
200 | 
201 |     # Compute time-dependent baseline
202 |     baseline = np.mean(padded_rets, axis=0)
203 | 
204 |     # Compute advantage function
205 |     advs = [ret - baseline[:len(ret)] for ret in rets]
206 |     all_action = np.concatenate([traj["action"] for traj in trajs])
207 |     all_adv = np.concatenate(advs)
208 | 
209 |     all_eprews = np.array([discount(traj["reward"], pa.discount)[0] for traj in trajs])  # episode total rewards
210 |     all_eplens = np.array([len(traj["reward"]) for traj in trajs])  # episode lengths
211 | 
212 |     # All Job Stat
213 |     enter_time, finish_time, job_len = process_all_info(trajs)
214 |     finished_idx = (finish_time >= 0)
215 |     all_slowdown = (finish_time[finished_idx] - enter_time[finished_idx]) / job_len[finished_idx]
216 | 
217 |     all_entropy = np.concatenate([traj["entropy"] for traj in trajs])
218 | 
219 |     result.append({"all_ob": all_ob,
220 |                    "all_action": all_action,
221 |                    "all_adv": all_adv,
222 |                    "all_eprews": all_eprews,
223 |                    "all_eplens": all_eplens,
224 |                    "all_slowdown": all_slowdown,
225 |                    "all_entropy": all_entropy})
226 | 
227 | 
228 | def launch(pa, pg_resume=None, render=False, repre='image', end='no_new_job'):
229 | 
230 |     # ----------------------------
231 |     print("Preparing for workers...")
232 |     # ----------------------------
233 | 
234 |     pg_learners = []
235 |     envs = []
236 | 
237 |     nw_len_seqs, nw_size_seqs = job_distribution.generate_sequence_work(pa, seed=42)
238 | 
239 |     for ex in xrange(pa.num_ex): # number of sequences
240 | 
241 |         print "-prepare for env-", ex
242 | 
243 |         env = environment.Env(pa, nw_len_seqs=nw_len_seqs, nw_size_seqs=nw_size_seqs,
244 |                               render=True, repre=repre, end=end)
245 |         env.seq_no = ex
246 |         envs.append(env)
247 | 
248 |     for ex in xrange(pa.batch_size + 1):  # last worker for updating the parameters
249 | 
250 |         print "-prepare for worker-", ex
251 | 
252 |         pg_learner = pg_network.PGLearner(pa)
253 | 
254 |         startIndex = 0
255 |         if pg_resume is not None:
256 |             net_handle = open(pg_resume, 'rb')
257 |             net_params = cPickle.load(net_handle)
258 |             pg_learner.set_net_params(net_params)
259 |             startIndex = re.match(pg_resume,'\d+').group()
260 |             startIndex = int(startIndex)
261 | 
262 |         pg_learners.append(pg_learner)
263 | 
264 |     accums = init_accums(pg_learners[pa.batch_size])
265 | 
266 |     # --------------------------------------
267 |     print("Preparing for reference data...")
268 |     # --------------------------------------
269 | 
270 |     # Reference examples, get reference discounted rewards and reference slowdown from random, SJF and Tetris algorithms
271 |     ref_discount_rews, ref_slow_down = slow_down_cdf.launch(pa, pg_resume=None, render=True, plot=False, repre=repre, end=end)
272 |     mean_rew_lr_curve = []
273 |     max_rew_lr_curve = []
274 |     slow_down_lr_curve = []
275 | 
276 |     # --------------------------------------
277 |     print("Start training...")
278 |     # --------------------------------------
279 | 
280 |     timer_start = time.time()
281 | 
282 | 
283 | 
284 |     for iteration in xrange(startIndex, pa.num_epochs):
285 | 
286 |         ps = []  # threads
287 |         manager = Manager()  # managing return results
288 |         manager_result = manager.list([])
289 | 
290 |         ex_indices = range(pa.num_ex)
291 |         np.random.shuffle(ex_indices)
292 | 
293 |         all_eprews = []
294 |         grads_all = []
295 |         loss_all = []
296 |         eprews = []
297 |         eplens = []
298 |         all_slowdown = []
299 |         all_entropy = []
300 | 
301 |         ex_counter = 0
302 |         for ex in xrange(pa.num_ex):
303 | 
304 |             ex_idx = ex_indices[ex]
305 |             p = Process(target=get_traj_worker,
306 |                         args=(pg_learners[ex_counter], envs[ex_idx], pa, manager_result, ))
307 |             ps.append(p)
308 | 
309 |             ex_counter += 1
310 |             # append pa.num_ex number of Processes in ps until going inside if
311 |             if ex_counter >= pa.batch_size or ex == pa.num_ex - 1:
312 | 
313 |                 print ex+1, "out of", pa.num_ex
314 | 
315 |                 ex_counter = 0
316 | 
317 |                 for p in ps:
318 |                     p.start()
319 | 
320 |                 for p in ps:
321 |                     p.join()
322 | 
323 |                 result = []  # convert list from shared memory
324 |                 for r in manager_result:
325 |                     result.append(r)
326 | 
327 |                 ps = []
328 |                 manager_result = manager.list([])
329 | 
330 |                 all_ob = concatenate_all_ob_across_examples([r["all_ob"] for r in result], pa)
331 |                 all_action = np.concatenate([r["all_action"] for r in result])
332 |                 all_adv = np.concatenate([r["all_adv"] for r in result])
333 | 
334 |                 # Do policy gradient update step, using the first agent
335 |                 # put the new parameter in the last 'worker', then propagate the update at the end
336 |                 grads = pg_learners[pa.batch_size].get_grad(all_ob, all_action, all_adv) #(states, actions, values)
337 | 
338 |                 grads_all.append(grads)
339 | 
340 |                 all_eprews.extend([r["all_eprews"] for r in result])
341 | 
342 |                 eprews.extend(np.concatenate([r["all_eprews"] for r in result]))  # episode total rewards
343 |                 eplens.extend(np.concatenate([r["all_eplens"] for r in result]))  # episode lengths
344 | 
345 |                 all_slowdown.extend(np.concatenate([r["all_slowdown"] for r in result]))
346 |                 all_entropy.extend(np.concatenate([r["all_entropy"] for r in result]))
347 | 
348 |         # assemble gradients
349 |         grads = grads_all[0]
350 |         for i in xrange(1, len(grads_all)):
351 |             for j in xrange(len(grads)):
352 |                 grads[j] += grads_all[i][j]
353 | 
354 |         # propagate network parameters to others
355 |         params = pg_learners[pa.batch_size].get_params()
356 | 
357 |         rmsprop_updates_outside(grads, params, accums, pa.lr_rate, pa.rms_rho, pa.rms_eps)
358 | 
359 |         for i in xrange(pa.batch_size + 1):
360 |             pg_learners[i].set_net_params(params)
361 | 
362 |         timer_end = time.time()
363 | 
364 |         print "-----------------"
365 |         print "Iteration: \t %i" % iteration
366 |         print "NumTrajs: \t %i" % len(eprews)
367 |         print "NumTimesteps: \t %i" % np.sum(eplens)
368 |         # print "Loss:     \t %s" % np.mean(loss_all)
369 |         print "MaxRew: \t %s" % np.average([np.max(rew) for rew in all_eprews])
370 |         print "MeanRew: \t %s +- %s" % (np.mean(eprews), np.std(eprews))
371 |         print "MeanSlowdown: \t %s" % np.mean(all_slowdown)
372 |         print "MeanLen: \t %s +- %s" % (np.mean(eplens), np.std(eplens))
373 |         print "MeanEntropy \t %s" % (np.mean(all_entropy))
374 |         print "Elapsed time\t %s" % (timer_end - timer_start), "seconds"
375 |         print "-----------------"
376 | 
377 |         f = open('log/re_log_' + datetime.now().strftime('%Y-%m-%d_%H:%M:%S'), 'w+')
378 |         f.write("-----------------\n")
379 |         f.write("Iteration: \t %i\n" % (iteration))
380 |         f.write("NumTrajs: \t %i\n" % (len(eprews)))
381 |         f.write("NumTimesteps: \t %i\n" % (np.sum(eplens)))
382 |         # f.write("Loss:     \t %s\n".format(loss))
383 |         f.write("MaxRew: \t %s\n" % (np.average([np.max(rew) for rew in all_eprews])))
384 |         f.write("MeanRew: \t %s +- %s\n" % (np.mean(eprews), np.std(eprews)))
385 |         f.write("MeanSlowdown: \t %s\n" % (np.mean(all_slowdown)))
386 |         f.write("MeanLen: \t %s +- %s\n" % (np.mean(eplens), np.std(eplens)))
387 |         f.write("MeanEntropy \t %s\n" % ((np.mean(all_entropy))))
388 |         f.write("Elapsed time\t %s seconds\n" % ((timer_end - timer_start)))
389 |         f.write("-----------------\n")
390 |         f.close()
391 | 
392 |         timer_start = time.time()
393 | 
394 |         max_rew_lr_curve.append(np.average([np.max(rew) for rew in all_eprews]))
395 |         mean_rew_lr_curve.append(np.mean(eprews))
396 |         slow_down_lr_curve.append(np.mean(all_slowdown))
397 | 
398 |         if iteration % pa.output_freq == 0:
399 |             param_file = open(pa.output_filename + '_' + str(iteration) + '.pkl', 'wb')
400 |             cPickle.dump(pg_learners[pa.batch_size].get_params(), param_file, -1)
401 |             param_file.close()
402 | 
403 |             # added by wjchen, to record accuracy and rewards
404 |             sample_file = h5py.File('log/re_record'+str(len(slow_down_lr_curve))\
405 |                                     + datetime.now().strftime('%Y-%m-%d_%H:%M')+'.h5', 'w')
406 |             sample_file.create_dataset('max_rew_lr_curve', data=max_rew_lr_curve)
407 |             sample_file.create_dataset('mean_rew_lr_curve', data=mean_rew_lr_curve)
408 |             sample_file.create_dataset('slow_down_lr_curve', data=slow_down_lr_curve)
409 | 
410 |             ref_dr = sample_file.create_group('ref_discount_rews')
411 |             for k, v in ref_discount_rews.items():
412 |                 ref_dr[k] = np.average(v)
413 | 
414 |             ref_sd = sample_file.create_group('ref_slow_down')
415 |             for k, v in ref_slow_down.items():
416 |                 ref_sd[k] = np.average(np.concatenate(v))
417 |             sample_file.close()
418 |             # print ref_slow_down
419 |             # print ref_discount_rews
420 |             #
421 |             print '\n----Reference Slowdown----'
422 |             for k, v in ref_slow_down.items():
423 |                 print "{}: {}".format(k, np.average(np.concatenate(v)))
424 | 
425 |             print '\n----Reference Discount Reward----'
426 |             for k, v in ref_discount_rews.items():
427 |                 print "{}: {}".format(k, np.average(v))
428 | 
429 |             pa.unseen = True
430 |             slow_down_cdf.launch(pa, pa.output_filename + '_' + str(iteration) + '.pkl',
431 |                                  render=True, plot=True, repre=repre, end=end)
432 |             pa.unseen = False
433 |             # test on unseen examples
434 | 
435 |             plot_lr_curve(pa.output_filename,
436 |                           max_rew_lr_curve, mean_rew_lr_curve, slow_down_lr_curve,
437 |                           ref_discount_rews, ref_slow_down)  # draw average of ref_discount_rews, ref_slow_down
438 | 
439 | 
440 | def main():
441 | 
442 |     import parameters
443 | 
444 |     pa = parameters.Parameters()
445 | 
446 |     pa.simu_len = 50  # 1000
447 |     pa.num_ex = 50  # 100
448 |     pa.num_nw = 10
449 |     pa.num_seq_per_batch = 20
450 |     pa.output_freq = 50
451 |     pa.batch_size = 10
452 | 
453 |     # pa.max_nw_size = 5
454 |     # pa.job_len = 5
455 |     pa.new_job_rate = 0.3
456 | 
457 |     pa.episode_max_length = 2000  # 2000
458 | 
459 |     pa.compute_dependent_parameters()
460 | 
461 |     pg_resume = None
462 |     # pg_resume = 'data/tmp_450.pkl'
463 | 
464 |     render = True
465 | 
466 |     launch(pa, pg_resume, render, repre='image', end='all_done')
467 | 
468 | 
469 | if __name__ == '__main__':
470 |     main()
471 | 


--------------------------------------------------------------------------------
/environment.py:
--------------------------------------------------------------------------------
  1 | import numpy as np
  2 | import math
  3 | import matplotlib.pyplot as plt
  4 | import theano
  5 | 
  6 | import parameters
  7 | 
  8 | plt.interactive(False)
  9 | 
 10 | 
 11 | class Env:
 12 |     def __init__(self, pa, nw_len_seqs=None, nw_size_seqs=None,
 13 |                  seed=42, render=True, repre='image', end='no_new_job'):
 14 | 
 15 |         self.pa = pa
 16 |         self.render = render
 17 |         self.repre = repre  # image or compact representation
 18 |         self.end = end  # termination type, 'no_new_job' or 'all_done'
 19 | 
 20 |         self.nw_dist = pa.dist.bi_model_dist
 21 | 
 22 |         self.curr_time = 0
 23 | 
 24 |         # set up random seed
 25 |         if self.pa.unseen:
 26 |             np.random.seed(314159)
 27 |         else:
 28 |             np.random.seed(seed)
 29 | 
 30 |         if nw_len_seqs is None or nw_size_seqs is None:
 31 |             # generate new work, two arrays representing time and size of #pa.simu_len * self.pa.num_ex of jobs.
 32 |             # length in time, size in number of requested units
 33 |             self.nw_len_seqs, self.nw_size_seqs = \
 34 |                 self.generate_sequence_work(self.pa.simu_len * self.pa.num_ex)  # 50 * 1000
 35 | 
 36 |             self.workload = np.zeros(pa.num_res)
 37 |             for i in xrange(pa.num_res):
 38 |                 self.workload[i] = \
 39 |                     np.sum(self.nw_size_seqs[:, i] * self.nw_len_seqs) / \
 40 |                     float(pa.res_slot) / \
 41 |                     float(len(self.nw_len_seqs))
 42 |                 print("Load on # " + str(i) + " resource dimension is " + str(self.workload[i]))
 43 |             self.nw_len_seqs = np.reshape(self.nw_len_seqs,
 44 |                                           [self.pa.num_ex, self.pa.simu_len])
 45 |             self.nw_size_seqs = np.reshape(self.nw_size_seqs,
 46 |                                            [self.pa.num_ex, self.pa.simu_len, self.pa.num_res])
 47 |         else:
 48 |             self.nw_len_seqs = nw_len_seqs
 49 |             self.nw_size_seqs = nw_size_seqs
 50 | 
 51 |         self.seq_no = 0  # which example sequence
 52 |         self.seq_idx = 0  # index in that sequence
 53 | 
 54 |         # initialize system
 55 |         self.machine = Machine(pa)
 56 |         self.job_slot = JobSlot(pa)
 57 |         self.job_backlog = JobBacklog(pa)
 58 |         self.job_record = JobRecord()
 59 |         self.extra_info = ExtraInfo(pa)
 60 | 
 61 |     def generate_sequence_work(self, simu_len):
 62 | 
 63 |         nw_len_seq = np.zeros(simu_len, dtype=int)
 64 |         nw_size_seq = np.zeros((simu_len, self.pa.num_res), dtype=int)
 65 | 
 66 |         for i in range(simu_len):
 67 | 
 68 |             if np.random.rand() < self.pa.new_job_rate:  # a new job comes
 69 | 
 70 |                 nw_len_seq[i], nw_size_seq[i, :] = self.nw_dist()
 71 | 
 72 |         return nw_len_seq, nw_size_seq
 73 | 
 74 |     def get_new_job_from_seq(self, seq_no, seq_idx):
 75 |         new_job = Job(res_vec=self.nw_size_seqs[seq_no, seq_idx, :],
 76 |                       job_len=self.nw_len_seqs[seq_no, seq_idx],
 77 |                       job_id=len(self.job_record.record),
 78 |                       enter_time=self.curr_time)
 79 |         return new_job
 80 | 
 81 |     def observe(self):
 82 |         if self.repre == 'image':
 83 | 
 84 |             backlog_width = int(math.ceil(self.pa.backlog_size / float(self.pa.time_horizon * self.pa.num_res)))
 85 | 
 86 |             image_repr = np.zeros((self.pa.network_input_height, self.pa.network_input_width))
 87 | 
 88 |             cl_pt = 0
 89 | 
 90 |             for i in xrange(self.pa.num_res):
 91 |                 ir_pt = 0
 92 | 
 93 |                 # print image_repr[cl_pt: cl_pt+self.pa.time_horizon, ir_pt: ir_pt + self.pa.res_slot].shape
 94 |                 # print self.machine.canvas[i, :, :].shape
 95 |                 # print '======================'
 96 |                 image_repr[cl_pt: cl_pt + self.pa.time_horizon, ir_pt: ir_pt + self.pa.res_slot] = self.machine.canvas[
 97 |                                                                                                    i, :, :]
 98 |                 ir_pt += self.pa.res_slot
 99 | 
100 |                 for j in xrange(self.pa.num_nw):
101 | 
102 |                     if self.job_slot.slot[j] is not None:  # fill in a block of work
103 |                         image_repr[cl_pt: cl_pt + self.job_slot.slot[j].len,
104 |                         ir_pt: ir_pt + self.job_slot.slot[j].res_vec[i]] = 1
105 | 
106 |                     ir_pt += self.pa.max_job_size
107 | 
108 |                 cl_pt += self.pa.time_horizon
109 | 
110 |             image_repr[: self.job_backlog.curr_size / backlog_width,
111 |             ir_pt: ir_pt + backlog_width] = 1
112 |             if self.job_backlog.curr_size % backlog_width > 0:
113 |                 image_repr[self.job_backlog.curr_size / backlog_width,
114 |                 ir_pt: ir_pt + self.job_backlog.curr_size % backlog_width] = 1
115 |             ir_pt += backlog_width
116 | 
117 |             image_repr[:, ir_pt: ir_pt + 1] = self.extra_info.time_since_last_new_job / \
118 |                                               float(self.extra_info.max_tracking_time_since_last_job)
119 |             ir_pt += 1
120 | 
121 |             assert ir_pt == image_repr.shape[1]
122 | 
123 |             return image_repr
124 | 
125 |         elif self.repre == 'compact':
126 | 
127 |             compact_repr = np.zeros(self.pa.time_horizon * (self.pa.num_res + 1) +  # current work
128 |                                     self.pa.num_nw * (self.pa.num_res + 1) +  # new work
129 |                                     1,  # backlog indicator
130 |                                     dtype=theano.config.floatX)
131 | 
132 |             cr_pt = 0
133 | 
134 |             # current work reward, after each time step, how many jobs left in the machine
135 |             job_allocated = np.ones(self.pa.time_horizon) * len(self.machine.running_job)
136 |             for j in self.machine.running_job:
137 |                 job_allocated[j.finish_time - self.curr_time:] -= 1
138 | 
139 |             compact_repr[cr_pt: cr_pt + self.pa.time_horizon] = job_allocated
140 |             cr_pt += self.pa.time_horizon
141 | 
142 |             # current work available slots
143 |             for i in range(self.pa.num_res):
144 |                 compact_repr[cr_pt: cr_pt + self.pa.time_horizon] = self.machine.avbl_slot[:, i]
145 |                 cr_pt += self.pa.time_horizon
146 | 
147 |             # new work duration and size
148 |             for i in range(self.pa.num_nw):
149 | 
150 |                 if self.job_slot.slot[i] is None:
151 |                     compact_repr[cr_pt: cr_pt + self.pa.num_res + 1] = 0
152 |                     cr_pt += self.pa.num_res + 1
153 |                 else:
154 |                     compact_repr[cr_pt] = self.job_slot.slot[i].len
155 |                     cr_pt += 1
156 | 
157 |                     for j in range(self.pa.num_res):
158 |                         compact_repr[cr_pt] = self.job_slot.slot[i].res_vec[j]
159 |                         cr_pt += 1
160 | 
161 |             # backlog queue
162 |             compact_repr[cr_pt] = self.job_backlog.curr_size
163 |             cr_pt += 1
164 | 
165 |             assert cr_pt == len(compact_repr)  # fill up the compact representation vector
166 | 
167 |             return compact_repr
168 | 
169 |     def plot_state(self):
170 |         plt.figure("screen", figsize=(20, 5))
171 | 
172 |         skip_row = 0
173 | 
174 |         for i in xrange(self.pa.num_res):
175 | 
176 |             plt.subplot(self.pa.num_res,
177 |                         1 + self.pa.num_nw + 1,  # first +1 for current work, last +1 for backlog queue
178 |                         i * (self.pa.num_nw + 1) + skip_row + 1)  # plot the backlog at the end, +1 to avoid 0
179 | 
180 |             plt.imshow(self.machine.canvas[i, :, :], interpolation='nearest', vmax=1)
181 |             # plt.show()
182 |             for j in xrange(self.pa.num_nw):
183 | 
184 |                 job_slot = np.zeros((self.pa.time_horizon, self.pa.max_job_size))
185 |                 if self.job_slot.slot[j] is not None:  # fill in a block of work
186 |                     job_slot[: self.job_slot.slot[j].len, :self.job_slot.slot[j].res_vec[i]] = 1
187 | 
188 |                 plt.subplot(self.pa.num_res,
189 |                             1 + self.pa.num_nw + 1,  # first +1 for current work, last +1 for backlog queue
190 |                             1 + i * (
191 |                             self.pa.num_nw + 1) + j + skip_row + 1)  # plot the backlog at the end, +1 to avoid 0
192 | 
193 |                 plt.imshow(job_slot, interpolation='nearest', vmax=1)
194 | 
195 |                 if j == self.pa.num_nw - 1:
196 |                     skip_row += 1
197 | 
198 |         skip_row -= 1
199 |         backlog_width = int(math.ceil(self.pa.backlog_size / float(self.pa.time_horizon)))
200 |         backlog = np.zeros((self.pa.time_horizon, backlog_width))
201 | 
202 |         backlog[: self.job_backlog.curr_size / backlog_width, : backlog_width] = 1
203 |         backlog[self.job_backlog.curr_size / backlog_width, : self.job_backlog.curr_size % backlog_width] = 1
204 | 
205 |         plt.subplot(self.pa.num_res,
206 |                     1 + self.pa.num_nw + 1,  # first +1 for current work, last +1 for backlog queue
207 |                     self.pa.num_nw + 1 + 1)
208 | 
209 |         plt.imshow(backlog, interpolation='nearest', vmax=1)
210 | 
211 |         plt.subplot(self.pa.num_res,
212 |                     1 + self.pa.num_nw + 1,  # first +1 for current work, last +1 for backlog queue
213 |                     self.pa.num_res * (self.pa.num_nw + 1) + skip_row + 1)  # plot the backlog at the end, +1 to avoid 0
214 | 
215 |         extra_info = np.ones((self.pa.time_horizon, 1)) * \
216 |                      self.extra_info.time_since_last_new_job / \
217 |                      float(self.extra_info.max_tracking_time_since_last_job)
218 | 
219 |         plt.imshow(extra_info, interpolation='nearest', vmax=1)
220 | 
221 |         # plt.show()     # manual
222 |         # plt.pause(0.01)  # automatic
223 | 
224 |     def get_reward(self):
225 | 
226 |         reward = 0
227 |         for j in self.machine.running_job:
228 |             reward += self.pa.delay_penalty / float(j.len)
229 | 
230 |         for j in self.job_slot.slot:
231 |             if j is not None:
232 |                 reward += self.pa.hold_penalty / float(j.len)
233 | 
234 |         for j in self.job_backlog.backlog:
235 |             if j is not None:
236 |                 reward += self.pa.dismiss_penalty / float(j.len)
237 | 
238 |         return reward
239 | 
240 |     def step(self, a, repeat=False):
241 | 
242 |         status = None
243 | 
244 |         done = False
245 |         reward = 0
246 |         info = None
247 | 
248 |         if a == self.pa.num_nw:  # explicit void action
249 |             status = 'MoveOn'
250 |         elif self.job_slot.slot[a] is None:  # implicit void action
251 |             status = 'MoveOn'
252 |         else:
253 |             allocated = self.machine.allocate_job(self.job_slot.slot[a], self.curr_time)
254 |             if not allocated:  # implicit void action
255 |                 status = 'MoveOn'
256 |             else:
257 |                 status = 'Allocate'
258 | 
259 |         if status == 'MoveOn':
260 |             self.curr_time += 1
261 |             self.machine.time_proceed(self.curr_time)
262 |             self.extra_info.time_proceed()
263 | 
264 |             # add new jobs
265 |             self.seq_idx += 1
266 | 
267 |             if self.end == "no_new_job":  # end of new job sequence
268 |                 if self.seq_idx >= self.pa.simu_len:
269 |                     done = True
270 |             elif self.end == "all_done":  # everything has to be finished
271 |                 if self.seq_idx >= self.pa.simu_len and \
272 |                                 len(self.machine.running_job) == 0 and \
273 |                         all(s is None for s in self.job_slot.slot) and \
274 |                         all(s is None for s in self.job_backlog.backlog):
275 |                     done = True
276 |                 elif self.curr_time > self.pa.episode_max_length:  # run too long, force termination
277 |                     done = True
278 | 
279 |             if not done:
280 | 
281 |                 if self.seq_idx < self.pa.simu_len:  # otherwise, end of new job sequence, i.e. no new jobs
282 |                     new_job = self.get_new_job_from_seq(self.seq_no, self.seq_idx)
283 | 
284 |                     if new_job.len > 0:  # a new job comes
285 | 
286 |                         to_backlog = True
287 |                         for i in xrange(self.pa.num_nw):
288 |                             if self.job_slot.slot[i] is None:  # put in new visible job slots
289 |                                 self.job_slot.slot[i] = new_job
290 |                                 self.job_record.record[new_job.id] = new_job
291 |                                 to_backlog = False
292 |                                 break
293 | 
294 |                         if to_backlog:
295 |                             if self.job_backlog.curr_size < self.pa.backlog_size:
296 |                                 self.job_backlog.backlog[self.job_backlog.curr_size] = new_job
297 |                                 self.job_backlog.curr_size += 1
298 |                                 self.job_record.record[new_job.id] = new_job
299 |                             else:  # abort, backlog full
300 |                                 print("Backlog is full.")
301 |                                 # exit(1)
302 | 
303 |                         self.extra_info.new_job_comes()
304 | 
305 |             reward = self.get_reward()
306 | 
307 |         elif status == 'Allocate':
308 |             self.job_record.record[self.job_slot.slot[a].id] = self.job_slot.slot[a]
309 |             self.job_slot.slot[a] = None
310 | 
311 |             # dequeue backlog
312 |             if self.job_backlog.curr_size > 0:
313 |                 self.job_slot.slot[a] = self.job_backlog.backlog[0]  # if backlog empty, it will be 0
314 |                 self.job_backlog.backlog[: -1] = self.job_backlog.backlog[1:]
315 |                 self.job_backlog.backlog[-1] = None
316 |                 self.job_backlog.curr_size -= 1
317 | 
318 |         ob = self.observe()
319 | 
320 |         info = self.job_record
321 | 
322 |         if done:
323 |             self.seq_idx = 0
324 | 
325 |             if not repeat:
326 |                 self.seq_no = (self.seq_no + 1) % self.pa.num_ex
327 | 
328 |             self.reset()
329 | 
330 |         if self.render:
331 |             self.plot_state()
332 | 
333 |         return ob, reward, done, info
334 | 
335 |     def reset(self):
336 |         self.seq_idx = 0
337 |         self.curr_time = 0
338 | 
339 |         # initialize system
340 |         self.machine = Machine(self.pa)
341 |         self.job_slot = JobSlot(self.pa)
342 |         self.job_backlog = JobBacklog(self.pa)
343 |         self.job_record = JobRecord()
344 |         self.extra_info = ExtraInfo(self.pa)
345 | 
346 | 
347 | class Job:
348 |     def __init__(self, res_vec, job_len, job_id, enter_time):
349 |         self.id = job_id
350 |         self.res_vec = res_vec
351 |         self.len = job_len
352 |         self.enter_time = enter_time
353 |         self.start_time = -1  # not being allocated
354 |         self.finish_time = -1
355 | 
356 | 
357 | class JobSlot:
358 |     def __init__(self, pa):
359 |         self.slot = [None] * pa.num_nw
360 | 
361 | 
362 | class JobBacklog:
363 |     def __init__(self, pa):
364 |         self.backlog = [None] * pa.backlog_size
365 |         self.curr_size = 0
366 | 
367 | 
368 | class JobRecord:
369 |     def __init__(self):
370 |         self.record = {}
371 | 
372 | 
373 | class Machine:
374 |     def __init__(self, pa):
375 |         self.num_res = pa.num_res
376 |         self.time_horizon = pa.time_horizon
377 |         self.res_slot = pa.res_slot
378 | 
379 |         self.avbl_slot = np.ones((self.time_horizon, self.num_res)) * self.res_slot
380 | 
381 |         self.running_job = []
382 | 
383 |         # colormap for graphical representation
384 |         self.colormap = np.arange(1 / float(pa.job_num_cap), 1, 1 / float(pa.job_num_cap))
385 |         np.random.shuffle(self.colormap)
386 | 
387 |         # graphical representation
388 |         self.canvas = np.zeros((pa.num_res, pa.time_horizon, pa.res_slot))
389 | 
390 |     def allocate_job(self, job, curr_time):
391 | 
392 |         allocated = False
393 | 
394 |         for t in xrange(0, self.time_horizon - job.len):
395 | 
396 |             new_avbl_res = self.avbl_slot[t: t + job.len, :] - job.res_vec
397 | 
398 |             if np.all(new_avbl_res[:] >= 0):
399 | 
400 |                 allocated = True
401 | 
402 |                 self.avbl_slot[t: t + job.len, :] = new_avbl_res
403 |                 job.start_time = curr_time + t
404 |                 job.finish_time = job.start_time + job.len
405 | 
406 |                 self.running_job.append(job)
407 | 
408 |                 # update graphical representation
409 | 
410 |                 used_color = np.unique(self.canvas[:])
411 |                 # WARNING: there should be enough colors in the color map
412 |                 for color in self.colormap:
413 |                     if color not in used_color:
414 |                         new_color = color
415 |                         break
416 | 
417 |                 assert job.start_time != -1
418 |                 assert job.finish_time != -1
419 |                 assert job.finish_time > job.start_time
420 |                 canvas_start_time = job.start_time - curr_time
421 |                 canvas_end_time = job.finish_time - curr_time
422 | 
423 |                 for res in xrange(self.num_res):
424 |                     for i in range(canvas_start_time, canvas_end_time):
425 |                         avbl_slot = np.where(self.canvas[res, i, :] == 0)[0]
426 |                         self.canvas[res, i, avbl_slot[: job.res_vec[res]]] = new_color
427 | 
428 |                 break
429 | 
430 |         return allocated
431 | 
432 |     def time_proceed(self, curr_time):
433 | 
434 |         self.avbl_slot[:-1, :] = self.avbl_slot[1:, :]
435 |         self.avbl_slot[-1, :] = self.res_slot
436 | 
437 |         for job in self.running_job:
438 | 
439 |             if job.finish_time <= curr_time:
440 |                 self.running_job.remove(job)
441 | 
442 |         # update graphical representation
443 | 
444 |         self.canvas[:, :-1, :] = self.canvas[:, 1:, :]
445 |         self.canvas[:, -1, :] = 0
446 | 
447 | 
448 | class ExtraInfo:
449 |     def __init__(self, pa):
450 |         self.time_since_last_new_job = 0
451 |         self.max_tracking_time_since_last_job = pa.max_track_since_new
452 | 
453 |     def new_job_comes(self):
454 |         self.time_since_last_new_job = 0
455 | 
456 |     def time_proceed(self):
457 |         if self.time_since_last_new_job < self.max_tracking_time_since_last_job:
458 |             self.time_since_last_new_job += 1
459 | 
460 | 
461 | # ==========================================================================
462 | # ------------------------------- Unit Tests -------------------------------
463 | # ==========================================================================
464 | 
465 | 
466 | def test_backlog():
467 |     pa = parameters.Parameters()
468 |     pa.num_nw = 5
469 |     pa.simu_len = 50
470 |     pa.num_ex = 10
471 |     pa.new_job_rate = 1
472 |     pa.compute_dependent_parameters()
473 | 
474 |     env = Env(pa, render=True, repre='image')
475 | 
476 |     env.step(5)
477 |     env.step(5)
478 |     env.step(5)
479 |     env.step(5)
480 |     env.step(5)
481 | 
482 |     env.step(5)
483 |     assert env.job_backlog.backlog[0] is not None
484 |     assert env.job_backlog.backlog[1] is None
485 |     print "New job is backlogged."
486 | 
487 |     env.step(5)
488 |     env.step(5)
489 |     env.step(5)
490 |     env.step(5)
491 | 
492 |     job = env.job_backlog.backlog[0]
493 |     env.step(0)
494 |     assert env.job_slot.slot[0] == job
495 | 
496 |     job = env.job_backlog.backlog[0]
497 |     env.step(0)
498 |     assert env.job_slot.slot[0] == job
499 | 
500 |     job = env.job_backlog.backlog[0]
501 |     env.step(1)
502 |     assert env.job_slot.slot[1] == job
503 | 
504 |     job = env.job_backlog.backlog[0]
505 |     env.step(1)
506 |     assert env.job_slot.slot[1] == job
507 | 
508 |     env.step(5)
509 | 
510 |     job = env.job_backlog.backlog[0]
511 |     env.step(3)
512 |     assert env.job_slot.slot[3] == job
513 | 
514 |     print "- Backlog test passed -"
515 | 
516 | 
517 | def test_compact_speed():
518 |     pa = parameters.Parameters()
519 |     pa.simu_len = 50
520 |     pa.num_ex = 10
521 |     pa.new_job_rate = 0.3
522 |     pa.compute_dependent_parameters()
523 | 
524 |     env = Env(pa, render=True, repre='compact')
525 | 
526 |     import other_agents
527 |     import time
528 | 
529 |     start_time = time.time()
530 |     for i in xrange(100000):
531 |         a = other_agents.get_sjf_action(env.machine, env.job_slot)
532 |         env.step(a)
533 |     end_time = time.time()
534 |     print "- Elapsed time: ", end_time - start_time, "sec -"
535 | 
536 | 
537 | def test_image_speed():
538 |     pa = parameters.Parameters()
539 |     pa.simu_len = 50
540 |     pa.num_ex = 10
541 |     pa.new_job_rate = 0.3
542 |     pa.compute_dependent_parameters()
543 | 
544 |     env = Env(pa, render=True, repre='image')
545 | 
546 |     import other_agents
547 |     import time
548 | 
549 |     start_time = time.time()
550 |     for i in xrange(100000):
551 |         a = other_agents.get_sjf_action(env.machine, env.job_slot)
552 |         env.step(a)
553 |     end_time = time.time()
554 |     print "- Elapsed time: ", end_time - start_time, "sec -"
555 | 
556 | 
557 | if __name__ == '__main__':
558 |     test_backlog()
559 |     test_compact_speed()
560 |     test_image_speed()
561 | 


--------------------------------------------------------------------------------