├── __init__.py ├── dagger ├── __init__.py ├── model │ ├── model.index │ ├── model.meta │ └── model.data-00000-of-00001 ├── project_root.py ├── experts.py ├── models.py ├── run_sender.py ├── best_cwnds.yml ├── train.py ├── worker.py └── dagger.py ├── env ├── __init__.py ├── 12mbps.trace ├── .gitignore ├── project_root.py ├── datagram.proto ├── run_receiver.py ├── environment.py ├── receiver.py ├── datagram_pb2.py ├── sender.py └── 0.57mbps-poisson.trace ├── tests ├── __init__.py ├── project_root.py ├── test_sender.py ├── test_helpers.py └── test_environment.py ├── helpers ├── __init__.py ├── .gitignore ├── pkill.py ├── generate_trace.py ├── shift_cut_trace.py ├── my_gce_helper.py ├── train_dagger.py ├── setup.py ├── assistant.py └── helpers.py ├── README.md ├── .gitignore └── COPYING /__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /dagger/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /env/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /env/12mbps.trace: -------------------------------------------------------------------------------- 1 | 1 2 | -------------------------------------------------------------------------------- /helpers/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /helpers/.gitignore: -------------------------------------------------------------------------------- 1 | *.trace 2 | TABLE 3 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Indigo: Empirically learned congestion control 2 | -------------------------------------------------------------------------------- /env/.gitignore: -------------------------------------------------------------------------------- 1 | *.trace 2 | !12mbps.trace 3 | sampling_time 4 | -------------------------------------------------------------------------------- /dagger/model/model.index: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/StanfordSNR/indigo/HEAD/dagger/model/model.index -------------------------------------------------------------------------------- /dagger/model/model.meta: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/StanfordSNR/indigo/HEAD/dagger/model/model.meta -------------------------------------------------------------------------------- /dagger/model/model.data-00000-of-00001: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/StanfordSNR/indigo/HEAD/dagger/model/model.data-00000-of-00001 -------------------------------------------------------------------------------- /dagger/project_root.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | from os import path 4 | DIR = path.abspath(path.join(path.dirname(path.abspath(__file__)), os.pardir)) 5 | sys.path.append(DIR) 6 | -------------------------------------------------------------------------------- /env/project_root.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | from os import path 4 | DIR = path.abspath(path.join(path.dirname(path.abspath(__file__)), os.pardir)) 5 | sys.path.append(DIR) 6 | -------------------------------------------------------------------------------- /tests/project_root.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | from os import path 4 | DIR = path.abspath(path.join(path.dirname(path.abspath(__file__)), os.pardir)) 5 | sys.path.append(DIR) 6 | -------------------------------------------------------------------------------- /env/datagram.proto: -------------------------------------------------------------------------------- 1 | syntax = "proto3"; 2 | 3 | message Data { 4 | fixed32 seq_num = 1; 5 | fixed32 send_ts = 2; 6 | fixed64 sent_bytes = 3; 7 | fixed32 delivered_time = 4; 8 | fixed64 delivered = 5; 9 | string payload = 6; 10 | } 11 | 12 | message Ack { 13 | fixed32 seq_num = 1; 14 | fixed32 send_ts = 2; 15 | fixed64 sent_bytes = 3; 16 | fixed32 delivered_time = 4; 17 | fixed64 delivered = 5; 18 | fixed32 ack_bytes = 6; 19 | } 20 | -------------------------------------------------------------------------------- /env/run_receiver.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | # Copyright 2018 Francis Y. Yan, Jestin Ma 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | 18 | import argparse 19 | from receiver import Receiver 20 | 21 | 22 | def main(): 23 | parser = argparse.ArgumentParser() 24 | parser.add_argument('ip', metavar='IP') 25 | parser.add_argument('port', type=int) 26 | args = parser.parse_args() 27 | 28 | receiver = Receiver(args.ip, args.port) 29 | 30 | try: 31 | receiver.handshake() 32 | receiver.run() 33 | except KeyboardInterrupt: 34 | pass 35 | finally: 36 | receiver.cleanup() 37 | 38 | 39 | if __name__ == '__main__': 40 | main() 41 | -------------------------------------------------------------------------------- /helpers/pkill.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | # Copyright 2018 Francis Y. Yan, Jestin Ma 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | 18 | import sys 19 | import signal 20 | from subprocess import call 21 | 22 | 23 | def signal_handler(signum, frame): 24 | pass 25 | 26 | signal.signal(signal.SIGTERM, signal_handler) 27 | signal.signal(signal.SIGINT, signal_handler) 28 | 29 | 30 | # kill mahimahi shells 31 | pkill_cmds = ['pkill -f mm-delay', 'pkill -f mm-link'] 32 | # kill all scripts in the directory specified by the first argument 33 | if len(sys.argv) == 2: 34 | pkill_cmds.append('pkill -f %s' % sys.argv[1]) 35 | 36 | for cmd in pkill_cmds: 37 | sys.stderr.write('$ %s\n' % cmd) 38 | call(cmd, shell=True) 39 | -------------------------------------------------------------------------------- /tests/test_sender.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | # Copyright 2018 Francis Y. Yan, Jestin Ma 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | 18 | import sys 19 | import argparse 20 | import numpy as np 21 | import project_root 22 | from env.sender import Sender 23 | 24 | 25 | class Policy(object): 26 | def __init__(self, state_dim, action_cnt): 27 | self.state_dim = state_dim 28 | self.action_cnt = action_cnt 29 | 30 | def sample_action(self, state): 31 | return np.random.randint(0, self.action_cnt) 32 | 33 | 34 | def main(): 35 | parser = argparse.ArgumentParser() 36 | parser.add_argument('port', type=int) 37 | args = parser.parse_args() 38 | 39 | sender = Sender(args.port, train=False) 40 | policy = Policy(sender.state_dim, sender.action_cnt) 41 | sender.set_sample_action(policy.sample_action) 42 | 43 | try: 44 | sender.handshake() 45 | sender.run() 46 | except KeyboardInterrupt: 47 | pass 48 | finally: 49 | sender.cleanup() 50 | 51 | 52 | if __name__ == '__main__': 53 | main() 54 | -------------------------------------------------------------------------------- /helpers/generate_trace.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | # Copyright 2018 Francis Y. Yan, Jestin Ma 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | 18 | import argparse 19 | import numpy as np 20 | from os import path 21 | from helpers import make_sure_path_exists 22 | 23 | 24 | def main(): 25 | parser = argparse.ArgumentParser() 26 | parser.add_argument('--bandwidth', metavar='Mbps', required=True, 27 | help='constant bandwidth (Mbps)') 28 | parser.add_argument('--output-dir', metavar='DIR', required=True, 29 | help='directory to output trace') 30 | args = parser.parse_args() 31 | 32 | # number of packets in 60 seconds 33 | num_packets = int(float(args.bandwidth) * 5000) 34 | ts_list = np.linspace(0, 60000, num=num_packets, endpoint=False) 35 | 36 | # trace path 37 | make_sure_path_exists(args.output_dir) 38 | trace_path = path.join(args.output_dir, '%smbps.trace' % args.bandwidth) 39 | 40 | # write timestamps to trace 41 | with open(trace_path, 'w') as trace: 42 | for ts in ts_list: 43 | trace.write('%d\n' % ts) 44 | 45 | 46 | if __name__ == '__main__': 47 | main() 48 | -------------------------------------------------------------------------------- /helpers/shift_cut_trace.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | # Copyright 2018 Francis Y. Yan, Jestin Ma 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | 18 | import argparse 19 | 20 | 21 | def main(): 22 | parser = argparse.ArgumentParser() 23 | parser.add_argument( 24 | 'input_trace', metavar='INPUT-TRACE', 25 | help='input trace file that needs to shift to start from 0 ' 26 | 'and cut into 60 seconds long') 27 | parser.add_argument( 28 | 'output_trace', metavar='OUTPUT-TRACE', 29 | help='output trace file after shifting and cutting') 30 | args = parser.parse_args() 31 | 32 | input_trace = open(args.input_trace) 33 | output_trace = open(args.output_trace, 'w') 34 | 35 | starter_ts = None 36 | while True: 37 | line = input_trace.readline() 38 | if not line: 39 | break 40 | 41 | ts = int(line) 42 | if ts < 10000: 43 | continue 44 | 45 | if starter_ts is None: 46 | starter_ts = ts 47 | 48 | if ts <= 70000: 49 | output_trace.write('%d\n' % (ts - starter_ts)) 50 | else: 51 | break 52 | 53 | input_trace.close() 54 | output_trace.close() 55 | 56 | 57 | if __name__ == '__main__': 58 | main() 59 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | ### Python 2 | 3 | # Byte-compiled / optimized / DLL files 4 | __pycache__/ 5 | *.py[cod] 6 | *$py.class 7 | 8 | # C extensions 9 | *.so 10 | 11 | # Distribution / packaging 12 | .Python 13 | build/ 14 | develop-eggs/ 15 | dist/ 16 | downloads/ 17 | eggs/ 18 | .eggs/ 19 | lib/ 20 | lib64/ 21 | parts/ 22 | sdist/ 23 | var/ 24 | wheels/ 25 | *.egg-info/ 26 | .installed.cfg 27 | *.egg 28 | MANIFEST 29 | 30 | # PyInstaller 31 | # Usually these files are written by a python script from a template 32 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 33 | *.manifest 34 | *.spec 35 | 36 | # Installer logs 37 | pip-log.txt 38 | pip-delete-this-directory.txt 39 | 40 | # Unit test / coverage reports 41 | htmlcov/ 42 | .tox/ 43 | .coverage 44 | .coverage.* 45 | .cache 46 | nosetests.xml 47 | coverage.xml 48 | *.cover 49 | .hypothesis/ 50 | .pytest_cache/ 51 | 52 | # Translations 53 | *.mo 54 | *.pot 55 | 56 | # Django stuff: 57 | *.log 58 | local_settings.py 59 | db.sqlite3 60 | 61 | # Flask stuff: 62 | instance/ 63 | .webassets-cache 64 | 65 | # Scrapy stuff: 66 | .scrapy 67 | 68 | # Sphinx documentation 69 | docs/_build/ 70 | 71 | # PyBuilder 72 | target/ 73 | 74 | # Jupyter Notebook 75 | .ipynb_checkpoints 76 | 77 | # pyenv 78 | .python-version 79 | 80 | # celery beat schedule file 81 | celerybeat-schedule 82 | 83 | # SageMath parsed files 84 | *.sage.py 85 | 86 | # Environments 87 | .env 88 | .venv 89 | env/ 90 | venv/ 91 | ENV/ 92 | env.bak/ 93 | venv.bak/ 94 | 95 | # Spyder project settings 96 | .spyderproject 97 | .spyproject 98 | 99 | # Rope project settings 100 | .ropeproject 101 | 102 | # mkdocs documentation 103 | /site 104 | 105 | # mypy 106 | .mypy_cache/ 107 | 108 | 109 | ### Vim 110 | # Swap 111 | [._]*.s[a-v][a-z] 112 | [._]*.sw[a-p] 113 | [._]s[a-rt-v][a-z] 114 | [._]ss[a-gi-z] 115 | [._]sw[a-p] 116 | 117 | # Session 118 | Session.vim 119 | 120 | # Temporary 121 | .netrwhist 122 | *~ 123 | # Auto-generated tag files 124 | tags 125 | # Persistent undo 126 | [._]*.un~ 127 | -------------------------------------------------------------------------------- /tests/test_helpers.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | # Copyright 2018 Francis Y. Yan, Jestin Ma 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | 18 | import numpy as np 19 | import project_root 20 | from helpers.helpers import RingBuffer, MeanVarHistory 21 | 22 | 23 | def test_ring_buffer(): 24 | buf = RingBuffer(5) 25 | 26 | buf.append(1) 27 | buf.append(2) 28 | assert np.array_equal(buf.get(), np.array([1, 2])) 29 | assert buf.get().dtype == np.float 30 | buf.reset() 31 | 32 | for i in xrange(1, 7): 33 | buf.append(i) 34 | assert np.array_equal(buf.get(), np.array([2, 3, 4, 5, 6])) 35 | buf.reset() 36 | 37 | assert np.array_equal(buf.get(), np.array([])) 38 | 39 | print 'test_ring_buffer: success' 40 | 41 | 42 | def test_mean_var_history(): 43 | h = MeanVarHistory() 44 | 45 | h.append([1, 2, 3]) 46 | h.append(np.array([4, 5])) 47 | 48 | correct = np.array([1, 2, 3, 4, 5]) 49 | assert np.isclose(h.get_mean(), np.mean(correct)) 50 | assert np.isclose(h.get_var(), np.var(correct)) 51 | assert np.isclose(h.get_std(), np.std(correct)) 52 | 53 | h.reset() 54 | 55 | h.append([1, 1, 1]) 56 | assert np.allclose(h.normalize_copy([2, 3, 4]), np.array([1e5, 2e5, 3e5])) 57 | 58 | h.append([3, 3, 3]) 59 | x = np.array([2.0, 4.0]) 60 | h.normalize_inplace(x) 61 | assert np.allclose(x, np.array([0.0, 2.0])) 62 | 63 | print 'test_mean_var_history: success' 64 | 65 | 66 | def main(): 67 | test_ring_buffer() 68 | test_mean_var_history() 69 | 70 | 71 | if __name__ == '__main__': 72 | main() 73 | -------------------------------------------------------------------------------- /helpers/my_gce_helper.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | # Copyright 2018 Francis Y. Yan, Jestin Ma 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | 18 | import sys 19 | import argparse 20 | from subprocess import check_output 21 | 22 | 23 | def main(): 24 | parser = argparse.ArgumentParser() 25 | parser.add_argument( 26 | '--username', default='francisyyan', 27 | help='username used for train cmd (default: francisyyan)') 28 | 29 | parser.add_argument( 30 | '--table', metavar='TABLE', nargs='?', default='TABLE', 31 | help='(messy) table of VM instances copied from Google Cloud Platform') 32 | args = parser.parse_args() 33 | 34 | cmd = 'grep -E -o "([0-9]{1,3}[\.]){3}[0-9]{1,3}" ' + args.table 35 | ip_list = check_output(cmd, shell=True).split() 36 | 37 | ret_cmd = ('~/RLCC/dagger/train.py --username %s --rlcc-dir ' 38 | '/home/%s/RLCC --ps-hosts ') % (args.username, args.username) 39 | ret_int_ip_list = '' 40 | ret_ext_ip_list = '' 41 | 42 | worker_port = 16000 43 | for i in xrange(0, len(ip_list), 2): 44 | internal_ip = ip_list[i] 45 | external_ip = ip_list[i + 1] 46 | 47 | if i == 0: 48 | ret_cmd += internal_ip + ':15000 --worker-hosts ' 49 | else: 50 | ret_cmd += internal_ip + ':%d,' % worker_port 51 | worker_port += 1 52 | 53 | ret_int_ip_list += '%s,' % internal_ip 54 | ret_ext_ip_list += '%s,' % external_ip 55 | 56 | print ret_cmd[:-1] 57 | print ret_int_ip_list[:-1] 58 | print ret_ext_ip_list[:-1] 59 | 60 | 61 | if __name__ == '__main__': 62 | main() 63 | -------------------------------------------------------------------------------- /tests/test_environment.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | # Copyright 2018 Francis Y. Yan, Jestin Ma 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | 18 | from os import path 19 | import sys 20 | import numpy as np 21 | import project_root 22 | from env.environment import Environment 23 | 24 | 25 | def create_env(): 26 | uplink_trace = path.join(project_root.DIR, 'env', '12mbps.trace') 27 | downlink_trace = uplink_trace 28 | mahimahi_cmd = ( 29 | 'mm-delay 20 mm-link %s %s ' 30 | '--downlink-queue=droptail --downlink-queue-args=packets=200' % 31 | (uplink_trace, downlink_trace)) 32 | 33 | env = Environment(mahimahi_cmd) 34 | return env 35 | 36 | 37 | class Learner(object): 38 | def __init__(self, env): 39 | self.env = env 40 | 41 | self.state_dim = env.state_dim 42 | self.action_cnt = env.action_cnt 43 | 44 | env.set_sample_action(self.sample_action) 45 | 46 | def sample_action(self, state): 47 | return np.random.randint(0, self.action_cnt) 48 | 49 | def cleanup(self): 50 | self.env.cleanup() 51 | 52 | def run(self): 53 | for episode_i in xrange(1, 3): 54 | sys.stderr.write('--- Episode %d\n' % episode_i) 55 | self.env.reset() 56 | 57 | # get an episode of experience 58 | self.env.rollout() 59 | 60 | # update model 61 | self.update_model() 62 | 63 | def update_model(self): 64 | sys.stderr.write('Updating model...\n') 65 | 66 | 67 | def main(): 68 | env = create_env() 69 | learner = Learner(env) 70 | 71 | try: 72 | learner.run() 73 | except KeyboardInterrupt: 74 | pass 75 | finally: 76 | learner.cleanup() 77 | 78 | 79 | if __name__ == '__main__': 80 | main() 81 | -------------------------------------------------------------------------------- /dagger/experts.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018 Francis Y. Yan, Jestin Ma 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | 16 | from env.sender import Sender 17 | from helpers.helpers import apply_op 18 | 19 | 20 | def action_error(actions, idx, cwnd, target): 21 | """ Returns the absolute difference between the target and an action 22 | applied to the cwnd. 23 | The action is [op, val] located at actions[idx]. 24 | """ 25 | op = actions[idx][0] 26 | val = actions[idx][1] 27 | return abs(apply_op(op, cwnd, val) - target) 28 | 29 | 30 | def get_best_action(actions, cwnd, target): 31 | """ Returns the best action by finding the action that leads to the 32 | closest resulting cwnd to target. 33 | """ 34 | return min(actions, 35 | key=lambda idx: action_error(actions, idx, cwnd, target)) 36 | 37 | 38 | class NaiveDaggerExpert(object): 39 | """ Naive modified LEDBAT implementation """ 40 | 41 | def __init__(self): 42 | self.base_delay = float("inf") 43 | self.target = 100.0 44 | self.gain = 1.0 45 | 46 | def sample_action(self, state, cwnd): 47 | ewma_delay = state # assume this is the state 48 | self.base_delay = min(self.base_delay, ewma_delay) 49 | queuing_delay = ewma_delay - self.base_delay 50 | off_target = self.target - queuing_delay 51 | cwnd_inc = self.gain * off_target / cwnd 52 | target_cwnd = cwnd + cwnd_inc 53 | 54 | # Gets the action that gives the resulting cwnd closest to the 55 | # expert target cwnd. 56 | action = get_best_action(Sender.action_mapping, cwnd, target_cwnd) 57 | return action 58 | 59 | class TrueDaggerExpert(object): 60 | """ Ground truth expert policy """ 61 | 62 | def __init__(self, env): 63 | assert hasattr(env, 'best_cwnd'), ('Using true dagger expert but not ' 64 | 'given a best cwnd when creating ' 65 | 'the environment in worker.py.') 66 | self.best_cwnd = env.best_cwnd 67 | 68 | def sample_action(self, cwnd): 69 | # Gets the action that gives the resulting cwnd closest to the 70 | # best cwnd. 71 | action = get_best_action(Sender.action_mapping, cwnd, self.best_cwnd) 72 | return action 73 | -------------------------------------------------------------------------------- /helpers/train_dagger.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | # Copyright 2018 Francis Y. Yan, Jestin Ma 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | 18 | import sys 19 | import argparse 20 | from subprocess import check_call, check_output 21 | 22 | 23 | def main(): 24 | """ Runs a sequence of commands to perform DAgger training. """ 25 | parser = argparse.ArgumentParser() 26 | 27 | parser.add_argument( 28 | '--username', default='francisyyan', 29 | help='username used in ssh (default: francisyyan)') 30 | parser.add_argument( 31 | '--rlcc-dir', default='~/RLCC', 32 | help='path to RLCC/ (default: ~/RLCC)') 33 | parser.add_argument( 34 | '--table', default='TABLE', 35 | help='table for my_gce_helper.py (default: TABLE)') 36 | parser.add_argument( 37 | '--git-push', action='store_true', 38 | help='git force push and amend latest commit (default: False)') 39 | parser.add_argument( 40 | '--git-pull', action='store_true', 41 | help='whether to do a git pull from all workers (default: False)') 42 | parser.add_argument( 43 | '--commit', default='master', 44 | help='commit for git-pull (default: master)') 45 | 46 | args = parser.parse_args() 47 | 48 | gce_helper_cmd = ('%s/helpers/my_gce_helper.py --table %s --username %s' 49 | % (args.rlcc_dir, args.table, args.username)) 50 | gce_helper_out = check_output(gce_helper_cmd, shell=True).split('\n') 51 | train_cmd = gce_helper_out[0] 52 | remote_ip = gce_helper_out[1] 53 | 54 | assistant_cmd = ('%s/helpers/assistant.py --remote=%s --username=%s ' 55 | '--rlcc-dir=%s ' 56 | % (args.rlcc_dir, remote_ip, 57 | args.username, args.rlcc_dir)) 58 | 59 | if args.git_push: 60 | check_call('git add -A && ' 61 | 'git commit --amend --no-edit && ' 62 | 'git push -f', shell=True) 63 | 64 | check_call(assistant_cmd + '--commit=%s git_checkout' % (args.commit), shell=True) 65 | 66 | if args.git_pull: 67 | check_call(assistant_cmd + 'git_pull', shell=True) 68 | 69 | check_call(train_cmd, shell=True) 70 | 71 | 72 | if __name__ == '__main__': 73 | main() 74 | -------------------------------------------------------------------------------- /env/environment.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018 Francis Y. Yan, Jestin Ma 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | 16 | import os 17 | from os import path 18 | import sys 19 | import signal 20 | from subprocess import Popen 21 | from sender import Sender 22 | import project_root 23 | from helpers.helpers import get_open_udp_port 24 | 25 | 26 | class Environment(object): 27 | def __init__(self, mahimahi_cmd): 28 | self.mahimahi_cmd = mahimahi_cmd 29 | self.state_dim = Sender.state_dim 30 | self.action_cnt = Sender.action_cnt 31 | 32 | # variables below will be filled in during setup 33 | self.sender = None 34 | self.receiver = None 35 | 36 | def set_sample_action(self, sample_action): 37 | """Set the sender's policy. Must be called before calling reset().""" 38 | 39 | self.sample_action = sample_action 40 | 41 | def reset(self): 42 | """Must be called before running rollout().""" 43 | 44 | self.cleanup() 45 | 46 | self.port = get_open_udp_port() 47 | 48 | # start sender as an instance of Sender class 49 | sys.stderr.write('Starting sender...\n') 50 | self.sender = Sender(self.port, train=True) 51 | self.sender.set_sample_action(self.sample_action) 52 | 53 | # start receiver in a subprocess 54 | sys.stderr.write('Starting receiver...\n') 55 | receiver_src = path.join( 56 | project_root.DIR, 'env', 'run_receiver.py') 57 | recv_cmd = 'python %s $MAHIMAHI_BASE %s' % (receiver_src, self.port) 58 | cmd = "%s -- sh -c '%s'" % (self.mahimahi_cmd, recv_cmd) 59 | sys.stderr.write('$ %s\n' % cmd) 60 | self.receiver = Popen(cmd, preexec_fn=os.setsid, shell=True) 61 | 62 | # sender completes the handshake sent from receiver 63 | self.sender.handshake() 64 | 65 | def rollout(self): 66 | """Run sender in env, get final reward of an episode, reset sender.""" 67 | 68 | sys.stderr.write('Obtaining an episode from environment...\n') 69 | self.sender.run() 70 | 71 | def cleanup(self): 72 | if self.sender: 73 | self.sender.cleanup() 74 | self.sender = None 75 | 76 | if self.receiver: 77 | try: 78 | os.killpg(os.getpgid(self.receiver.pid), signal.SIGTERM) 79 | except OSError as e: 80 | sys.stderr.write('%s\n' % e) 81 | finally: 82 | self.receiver = None 83 | -------------------------------------------------------------------------------- /helpers/setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | # Copyright 2018 Francis Y. Yan, Jestin Ma 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | 18 | import sys 19 | import argparse 20 | from os import path 21 | from subprocess import check_call, Popen 22 | 23 | 24 | def setup_local(args): 25 | if args.install_deps: 26 | cmd = 'sudo add-apt-repository -y ppa:keithw/mahimahi' 27 | sys.stderr.write('$ %s\n' % cmd) 28 | check_call(cmd, shell=True) 29 | 30 | cmd = 'sudo apt-get -y update' 31 | sys.stderr.write('$ %s\n' % cmd) 32 | check_call(cmd, shell=True) 33 | 34 | deps = 'mahimahi python-pip python-dev' 35 | cmd = 'sudo apt-get -yq --force-yes install %s' % deps 36 | sys.stderr.write('$ %s\n' % cmd) 37 | check_call(cmd, shell=True) 38 | 39 | cmd = 'sudo pip install --upgrade pip' 40 | sys.stderr.write('$ %s\n' % cmd) 41 | check_call(cmd, shell=True) 42 | 43 | cmd = 'sudo pip install numpy tensorflow' 44 | sys.stderr.write('$ %s\n' % cmd) 45 | check_call(cmd, shell=True) 46 | 47 | cmd = 'sudo sysctl -w net.ipv4.ip_forward=1' 48 | sys.stderr.write('$ %s\n' % cmd) 49 | check_call(cmd, shell=True) 50 | 51 | 52 | def setup(args): 53 | if args.local: 54 | setup_local(args) 55 | else: 56 | procs = [] 57 | ip_list = args.remote.split(',') 58 | 59 | for ip in ip_list: 60 | host = args.username + '@' + ip 61 | ssh_cmd = ['ssh', host, '-o', 'StrictHostKeyChecking=no'] 62 | 63 | setup_src = path.join(args.rlcc_dir, 'helpers', 'setup.py') 64 | cmd = ssh_cmd + ['python', setup_src, '--local'] 65 | 66 | if args.install_deps: 67 | cmd += ['--install-deps'] 68 | 69 | sys.stderr.write('$ %s\n' % ' '.join(cmd)) 70 | procs.append(Popen(cmd)) 71 | 72 | for proc in procs: 73 | proc.communicate() 74 | 75 | 76 | def main(): 77 | parser = argparse.ArgumentParser() 78 | 79 | group = parser.add_mutually_exclusive_group(required=True) 80 | group.add_argument( 81 | '--local', action='store_true', help='setup on local machine') 82 | group.add_argument( 83 | '--remote', metavar='IP,...', 84 | help='comma-separated list of IP addresses of remote hosts') 85 | 86 | parser.add_argument( 87 | '--install-deps', action='store_true', 88 | help='install dependencies: tensorflow, mahimahi, etc.') 89 | parser.add_argument( 90 | '--username', default='francisyyan', 91 | help='username used in ssh connection (default: francisyyan)') 92 | parser.add_argument( 93 | '--rlcc-dir', metavar='DIR', default='/home/francisyyan/RLCC', 94 | help='absolute path to RLCC/ (default: /home/francisyyan/RLCC)') 95 | args = parser.parse_args() 96 | 97 | setup(args) 98 | 99 | 100 | if __name__ == '__main__': 101 | main() 102 | -------------------------------------------------------------------------------- /dagger/models.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018 Francis Y. Yan, Jestin Ma 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | 16 | import numpy as np 17 | import tensorflow as tf 18 | from tensorflow.contrib import layers, rnn 19 | 20 | 21 | class DaggerNetwork(object): 22 | def __init__(self, state_dim, action_cnt): 23 | self.states = tf.placeholder(tf.float32, [None, state_dim]) 24 | 25 | actor_h1 = layers.relu(self.states, 8) 26 | actor_h2 = layers.relu(actor_h1, 8) 27 | self.action_scores = layers.linear(actor_h2, action_cnt) 28 | self.action_probs = tf.nn.softmax(self.action_scores, 29 | name='action_probs') 30 | 31 | self.trainable_vars = tf.get_collection( 32 | tf.GraphKeys.TRAINABLE_VARIABLES, tf.get_variable_scope().name) 33 | 34 | 35 | class DaggerLSTM(object): 36 | def __init__(self, state_dim, action_cnt): 37 | # dummy variable used to verify that sharing variables is working 38 | self.cnt = tf.get_variable( 39 | 'cnt', [], tf.float32, 40 | initializer=tf.constant_initializer(0.0)) 41 | self.add_one = self.cnt.assign_add(1.0) 42 | 43 | # self.input: [batch_size, max_time, state_dim] 44 | self.input = tf.placeholder(tf.float32, [None, None, state_dim]) 45 | 46 | self.num_layers = 1 47 | self.lstm_dim = 32 48 | stacked_lstm = rnn.MultiRNNCell([rnn.BasicLSTMCell(self.lstm_dim) 49 | for _ in xrange(self.num_layers)]) 50 | 51 | self.state_in = [] 52 | state_tuple_in = [] 53 | for _ in xrange(self.num_layers): 54 | c_in = tf.placeholder(tf.float32, [None, self.lstm_dim]) 55 | h_in = tf.placeholder(tf.float32, [None, self.lstm_dim]) 56 | self.state_in.append((c_in, h_in)) 57 | state_tuple_in.append(rnn.LSTMStateTuple(c_in, h_in)) 58 | 59 | self.state_in = tuple(self.state_in) 60 | state_tuple_in = tuple(state_tuple_in) 61 | 62 | # self.output: [batch_size, max_time, lstm_dim] 63 | output, state_tuple_out = tf.nn.dynamic_rnn( 64 | stacked_lstm, self.input, initial_state=state_tuple_in) 65 | 66 | self.state_out = self.convert_state_out(state_tuple_out) 67 | 68 | # map output to scores 69 | self.action_scores = layers.linear(output, action_cnt) 70 | self.action_probs = tf.nn.softmax(self.action_scores) 71 | 72 | self.trainable_vars = tf.get_collection( 73 | tf.GraphKeys.TRAINABLE_VARIABLES, tf.get_variable_scope().name) 74 | 75 | def convert_state_out(self, state_tuple_out): 76 | state_out = [] 77 | for lstm_state_tuple in state_tuple_out: 78 | state_out.append((lstm_state_tuple.c, lstm_state_tuple.h)) 79 | 80 | return tuple(state_out) 81 | 82 | def zero_init_state(self, batch_size): 83 | init_state = [] 84 | for _ in xrange(self.num_layers): 85 | c_init = np.zeros([batch_size, self.lstm_dim], np.float32) 86 | h_init = np.zeros([batch_size, self.lstm_dim], np.float32) 87 | init_state.append((c_init, h_init)) 88 | 89 | return init_state 90 | -------------------------------------------------------------------------------- /dagger/run_sender.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | # Copyright 2018 Francis Y. Yan, Jestin Ma 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | 18 | import argparse 19 | import project_root 20 | import numpy as np 21 | import tensorflow as tf 22 | from os import path 23 | from env.sender import Sender 24 | from models import DaggerLSTM 25 | from helpers.helpers import normalize, one_hot, softmax 26 | 27 | 28 | class Learner(object): 29 | def __init__(self, state_dim, action_cnt, restore_vars): 30 | self.aug_state_dim = state_dim + action_cnt 31 | self.action_cnt = action_cnt 32 | self.prev_action = action_cnt - 1 33 | 34 | with tf.variable_scope('global'): 35 | self.model = DaggerLSTM( 36 | state_dim=self.aug_state_dim, action_cnt=action_cnt) 37 | 38 | self.lstm_state = self.model.zero_init_state(1) 39 | 40 | self.sess = tf.Session() 41 | 42 | # restore saved variables 43 | saver = tf.train.Saver(self.model.trainable_vars) 44 | saver.restore(self.sess, restore_vars) 45 | 46 | # init the remaining vars, especially those created by optimizer 47 | uninit_vars = set(tf.global_variables()) 48 | uninit_vars -= set(self.model.trainable_vars) 49 | self.sess.run(tf.variables_initializer(uninit_vars)) 50 | 51 | def sample_action(self, state): 52 | norm_state = normalize(state) 53 | 54 | one_hot_action = one_hot(self.prev_action, self.action_cnt) 55 | aug_state = norm_state + one_hot_action 56 | 57 | # Get probability of each action from the local network. 58 | pi = self.model 59 | feed_dict = { 60 | pi.input: [[aug_state]], 61 | pi.state_in: self.lstm_state, 62 | } 63 | ops_to_run = [pi.action_probs, pi.state_out] 64 | action_probs, self.lstm_state = self.sess.run(ops_to_run, feed_dict) 65 | 66 | # Choose an action to take 67 | action = np.argmax(action_probs[0][0]) 68 | self.prev_action = action 69 | 70 | # action = np.argmax(np.random.multinomial(1, action_probs[0] - 1e-5)) 71 | # temperature = 1.0 72 | # temp_probs = softmax(action_probs[0] / temperature) 73 | # action = np.argmax(np.random.multinomial(1, temp_probs - 1e-5)) 74 | return action 75 | 76 | 77 | def main(): 78 | parser = argparse.ArgumentParser() 79 | parser.add_argument('port', type=int) 80 | parser.add_argument('--debug', action='store_true') 81 | args = parser.parse_args() 82 | 83 | sender = Sender(args.port, debug=args.debug) 84 | 85 | model_path = path.join(project_root.DIR, 'dagger', 'model', 'model') 86 | 87 | learner = Learner( 88 | state_dim=Sender.state_dim, 89 | action_cnt=Sender.action_cnt, 90 | restore_vars=model_path) 91 | 92 | sender.set_sample_action(learner.sample_action) 93 | 94 | try: 95 | sender.handshake() 96 | sender.run() 97 | except KeyboardInterrupt: 98 | pass 99 | finally: 100 | sender.cleanup() 101 | 102 | 103 | if __name__ == '__main__': 104 | main() 105 | -------------------------------------------------------------------------------- /helpers/assistant.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | # Copyright 2018 Francis Y. Yan, Jestin Ma 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | 18 | import sys 19 | import argparse 20 | from subprocess import Popen, check_call, check_output, call 21 | 22 | 23 | def run_cmd(args, host, procs): 24 | cmd = args.cmd 25 | cmd_in_ssh = None 26 | 27 | if cmd == 'copy_key': 28 | cmd_to_run = ('KEY=$(cat ~/.ssh/id_rsa.pub); ' 29 | 'ssh -o StrictHostKeyChecking=no %s ' 30 | '"grep -qF \'$KEY\' .ssh/authorized_keys || ' 31 | 'echo \'$KEY\' >> .ssh/authorized_keys"' % host) 32 | check_call(cmd_to_run, shell=True) 33 | 34 | elif cmd == 'git_clone': 35 | cmd_in_ssh = 'git clone https://github.com/StanfordSNR/indigo.git' 36 | 37 | elif cmd == 'git_checkout': 38 | cmd_in_ssh = ('cd %s && git fetch --all && ' 39 | 'git checkout %s' % (args.rlcc_dir, args.commit)) 40 | 41 | elif cmd == 'git_pull': 42 | cmd_in_ssh = ('cd %s && git fetch --all && ' 43 | 'git reset --hard @~1 && git pull' % args.rlcc_dir) 44 | 45 | elif cmd == 'rm_history': 46 | cmd_in_ssh = ('rm -f %s/history' % args.rlcc_dir) 47 | 48 | elif cmd == 'cp_history': 49 | cmd_to_run = ('rsync --ignore-missing-args %s:%s/history %s/%s_history' 50 | % (host, args.rlcc_dir, args.local_rlcc_dir, host)) 51 | check_call(cmd_to_run, shell=True) 52 | 53 | else: 54 | cmd_in_ssh = cmd 55 | 56 | if cmd_in_ssh: 57 | cmd = ['ssh', '-o', 'StrictHostKeyChecking=no', host, cmd_in_ssh] 58 | procs.append(Popen(cmd)) 59 | 60 | 61 | def main(): 62 | parser = argparse.ArgumentParser() 63 | 64 | parser.add_argument( 65 | '--remote', required=True, metavar='IP,...', 66 | help='comma-separated list of IP addresses of remote hosts') 67 | parser.add_argument( 68 | '--username', default='francisyyan', 69 | help='username used in ssh (default: francisyyan)') 70 | parser.add_argument( 71 | '--rlcc-dir', metavar='DIR', default='~/RLCC', 72 | help='path to RLCC/ (default: ~/RLCC)') 73 | parser.add_argument( 74 | '--local-rlcc-dir', metavar='DIR', default='~/RLCC', 75 | help='path to RLCC/ (default: ~/RLCC)') 76 | parser.add_argument( 77 | '--commit', metavar='COMMIT', default='master', 78 | help='Commit to use when checking out (default: master)') 79 | parser.add_argument('cmd') 80 | args = parser.parse_args() 81 | 82 | ip_list = args.remote.split(',') 83 | procs = [] 84 | 85 | sys.stderr.write('%d IPs in total\n' % len(ip_list)) 86 | 87 | for ip in ip_list: 88 | host = args.username + '@' + ip 89 | 90 | if args.cmd == 'remove_key': 91 | call('ssh-keygen -f "/home/%s/.ssh/known_hosts" -R %s' % (args.username, ip), shell=True) 92 | elif args.cmd == 'test_ssh': 93 | call(['ssh', '-o', 'StrictHostKeyChecking=no', '-o', 'ConnectTimeout=4', host, 'echo $HOSTNAME']) 94 | else: 95 | run_cmd(args, host, procs) 96 | 97 | for proc in procs: 98 | proc.communicate() 99 | 100 | 101 | if __name__ == '__main__': 102 | main() 103 | -------------------------------------------------------------------------------- /env/receiver.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018 Francis Y. Yan, Jestin Ma 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | 16 | import sys 17 | import json 18 | import socket 19 | import select 20 | import datagram_pb2 21 | import project_root 22 | from helpers.helpers import READ_FLAGS, ERR_FLAGS, READ_ERR_FLAGS, ALL_FLAGS 23 | 24 | 25 | class Receiver(object): 26 | def __init__(self, ip, port): 27 | self.peer_addr = (ip, port) 28 | 29 | # UDP socket and poller 30 | self.sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) 31 | self.sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) 32 | 33 | self.poller = select.poll() 34 | self.poller.register(self.sock, ALL_FLAGS) 35 | 36 | def cleanup(self): 37 | self.sock.close() 38 | 39 | def construct_ack_from_data(self, serialized_data): 40 | """Construct a serialized ACK that acks a serialized datagram.""" 41 | 42 | data = datagram_pb2.Data() 43 | data.ParseFromString(serialized_data) 44 | 45 | ack = datagram_pb2.Ack() 46 | ack.seq_num = data.seq_num 47 | ack.send_ts = data.send_ts 48 | ack.sent_bytes = data.sent_bytes 49 | ack.delivered_time = data.delivered_time 50 | ack.delivered = data.delivered 51 | ack.ack_bytes = len(serialized_data) 52 | 53 | return ack.SerializeToString() 54 | 55 | def handshake(self): 56 | """Handshake with peer sender. Must be called before run().""" 57 | 58 | self.sock.setblocking(0) # non-blocking UDP socket 59 | 60 | TIMEOUT = 1000 # ms 61 | 62 | retry_times = 0 63 | self.poller.modify(self.sock, READ_ERR_FLAGS) 64 | 65 | while True: 66 | self.sock.sendto('Hello from receiver', self.peer_addr) 67 | events = self.poller.poll(TIMEOUT) 68 | 69 | if not events: # timed out 70 | retry_times += 1 71 | if retry_times > 10: 72 | sys.stderr.write( 73 | '[receiver] Handshake failed after 10 retries\n') 74 | return 75 | else: 76 | sys.stderr.write( 77 | '[receiver] Handshake timed out and retrying...\n') 78 | continue 79 | 80 | for fd, flag in events: 81 | assert self.sock.fileno() == fd 82 | 83 | if flag & ERR_FLAGS: 84 | sys.exit('Channel closed or error occurred') 85 | 86 | if flag & READ_FLAGS: 87 | msg, addr = self.sock.recvfrom(1600) 88 | 89 | if addr == self.peer_addr: 90 | if msg != 'Hello from sender': 91 | # 'Hello from sender' was presumably lost 92 | # received subsequent data from peer sender 93 | ack = self.construct_ack_from_data(msg) 94 | if ack is not None: 95 | self.sock.sendto(ack, self.peer_addr) 96 | return 97 | 98 | def run(self): 99 | self.sock.setblocking(1) # blocking UDP socket 100 | 101 | while True: 102 | serialized_data, addr = self.sock.recvfrom(1600) 103 | 104 | if addr == self.peer_addr: 105 | ack = self.construct_ack_from_data(serialized_data) 106 | if ack is not None: 107 | self.sock.sendto(ack, self.peer_addr) 108 | -------------------------------------------------------------------------------- /dagger/best_cwnds.yml: -------------------------------------------------------------------------------- 1 | 5: 2 | 1: 5 3 | 10: 15 4 | 20: 20 5 | 30: 30 6 | 40: 35 7 | 50: 45 8 | 60: 55 9 | 70: 65 10 | 80: 70 11 | 90: 80 12 | 100: 90 13 | 110: 95 14 | 120: 105 15 | 130: 115 16 | 140: 120 17 | 150: 130 18 | 160: 140 19 | 10: 20 | 1: 10 21 | 10: 20 22 | 20: 40 23 | 30: 60 24 | 40: 70 25 | 50: 90 26 | 60: 110 27 | 70: 120 28 | 80: 140 29 | 90: 160 30 | 100: 170 31 | 110: 190 32 | 120: 210 33 | 130: 220 34 | 140: 240 35 | 150: 260 36 | 160: 270 37 | 20: 38 | 1: 20 39 | 10: 50 40 | 20: 80 41 | 30: 110 42 | 40: 140 43 | 50: 180 44 | 60: 210 45 | 70: 240 46 | 80: 280 47 | 90: 310 48 | 100: 340 49 | 110: 380 50 | 120: 410 51 | 130: 440 52 | 140: 480 53 | 150: 510 54 | 160: 540 55 | 30: 56 | 1: 20 57 | 10: 70 58 | 20: 110 59 | 30: 160 60 | 40: 210 61 | 50: 260 62 | 60: 310 63 | 70: 360 64 | 80: 410 65 | 90: 460 66 | 100: 510 67 | 110: 560 68 | 120: 610 69 | 130: 660 70 | 140: 710 71 | 150: 770 72 | 160: 810 73 | 40: 74 | 1: 30 75 | 10: 90 76 | 20: 150 77 | 30: 220 78 | 40: 280 79 | 50: 350 80 | 60: 420 81 | 70: 480 82 | 80: 550 83 | 90: 620 84 | 100: 680 85 | 110: 750 86 | 120: 820 87 | 130: 880 88 | 140: 950 89 | 150: 1020 90 | 160: 1090 91 | 50: 92 | 1: 30 93 | 10: 110 94 | 20: 190 95 | 30: 270 96 | 40: 350 97 | 50: 440 98 | 60: 520 99 | 70: 600 100 | 80: 690 101 | 90: 770 102 | 100: 860 103 | 110: 940 104 | 120: 1020 105 | 130: 1110 106 | 140: 1190 107 | 150: 1270 108 | 160: 1360 109 | 60: 110 | 1: 40 111 | 10: 130 112 | 20: 220 113 | 30: 320 114 | 40: 420 115 | 50: 520 116 | 60: 630 117 | 70: 720 118 | 80: 830 119 | 90: 920 120 | 100: 1030 121 | 110: 1130 122 | 120: 1220 123 | 130: 1330 124 | 140: 1430 125 | 150: 1530 126 | 160: 1630 127 | 70: 128 | 1: 40 129 | 10: 150 130 | 20: 270 131 | 30: 380 132 | 40: 490 133 | 50: 610 134 | 60: 730 135 | 70: 850 136 | 80: 960 137 | 90: 1080 138 | 100: 1190 139 | 110: 1310 140 | 120: 1430 141 | 130: 1550 142 | 140: 1660 143 | 150: 1780 144 | 160: 1900 145 | 80: 146 | 1: 50 147 | 10: 170 148 | 20: 300 149 | 30: 430 150 | 40: 570 151 | 50: 700 152 | 60: 830 153 | 70: 970 154 | 80: 1100 155 | 90: 1230 156 | 100: 1360 157 | 110: 1510 158 | 120: 1640 159 | 130: 1770 160 | 140: 1900 161 | 150: 2040 162 | 160: 2160 163 | 90: 164 | 1: 50 165 | 10: 190 166 | 20: 340 167 | 30: 490 168 | 40: 640 169 | 50: 780 170 | 60: 940 171 | 70: 1080 172 | 80: 1240 173 | 90: 1380 174 | 100: 1540 175 | 110: 1690 176 | 120: 1830 177 | 130: 1980 178 | 140: 2140 179 | 150: 2290 180 | 160: 2440 181 | 100: 182 | 1: 60 183 | 10: 210 184 | 20: 380 185 | 30: 540 186 | 40: 710 187 | 50: 870 188 | 60: 1040 189 | 70: 1210 190 | 80: 1370 191 | 90: 1540 192 | 100: 1720 193 | 110: 1870 194 | 120: 2050 195 | 130: 2210 196 | 140: 2370 197 | 150: 2540 198 | 160: 2700 199 | 120: 200 | 1: 60 201 | 10: 240 202 | 20: 440 203 | 30: 640 204 | 40: 830 205 | 50: 1050 206 | 60: 1230 207 | 70: 1440 208 | 80: 1640 209 | 90: 1840 210 | 100: 2040 211 | 110: 2250 212 | 120: 2450 213 | 130: 2650 214 | 140: 2840 215 | 150: 3040 216 | 160: 3240 217 | 150: 218 | 1: 80 219 | 10: 300 220 | 20: 550 221 | 30: 810 222 | 40: 1050 223 | 50: 1300 224 | 60: 1560 225 | 70: 1820 226 | 80: 2050 227 | 90: 2320 228 | 100: 2550 229 | 110: 2800 230 | 120: 3050 231 | 130: 3300 232 | 140: 3560 233 | 150: 3800 234 | 160: 4050 235 | 160: 236 | 1: 90 237 | 10: 330 238 | 20: 590 239 | 30: 860 240 | 40: 1120 241 | 50: 1390 242 | 60: 1660 243 | 70: 1920 244 | 80: 2180 245 | 90: 2460 246 | 100: 2710 247 | 110: 2990 248 | 120: 3230 249 | 130: 3510 250 | 140: 3790 251 | 150: 4040 252 | 160: 4330 253 | 200: 254 | 1: 110 255 | 10: 410 256 | 20: 760 257 | 30: 1060 258 | 40: 1410 259 | 50: 1740 260 | 60: 2060 261 | 70: 2410 262 | 80: 2760 263 | 90: 3060 264 | 100: 3390 265 | 110: 3720 266 | 120: 4080 267 | 130: 4410 268 | 140: 4720 269 | 150: 5080 270 | 160: 5430 271 | -------------------------------------------------------------------------------- /dagger/train.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | # Copyright 2018 Francis Y. Yan, Jestin Ma 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | 18 | import os 19 | import sys 20 | import time 21 | import signal 22 | import argparse 23 | import project_root 24 | from os import path 25 | from subprocess import Popen, call 26 | from helpers.helpers import get_open_udp_port 27 | 28 | 29 | def run(args): 30 | # run worker.py on ps and worker hosts 31 | for job_name in ['ps', 'worker']: 32 | host_list = args[job_name + '_list'] 33 | procs = args[job_name + '_procs'] 34 | 35 | for i in xrange(len(host_list)): 36 | ssh_cmd = ['ssh', host_list[i]] 37 | 38 | cmd = ['python', args['worker_src'], 39 | '--ps-hosts', args['ps_hosts'], 40 | '--worker-hosts', args['worker_hosts'], 41 | '--job-name', job_name, 42 | '--task-index', str(i)] 43 | 44 | cmd = ssh_cmd + cmd 45 | 46 | sys.stderr.write('$ %s\n' % ' '.join(cmd)) 47 | procs.append(Popen(cmd, preexec_fn=os.setsid)) 48 | 49 | # ps will block forever 50 | for ps_proc in args['ps_procs']: 51 | ps_proc.communicate() 52 | 53 | 54 | def cleanup(args): 55 | all_procs = args['ps_procs'] + args['worker_procs'] 56 | for proc in all_procs: 57 | try: 58 | os.killpg(os.getpgid(proc.pid), signal.SIGTERM) 59 | except OSError as e: 60 | sys.stderr.write('%s\n' % e) 61 | 62 | host_set = set(args['ps_list'] + args['worker_list']) 63 | pkill_script = path.join(args['rlcc_dir'], 'helpers', 'pkill.py') 64 | 65 | for host in host_set: 66 | kill_cmd = ['ssh', host, 'python', pkill_script, args['rlcc_dir']] 67 | sys.stderr.write('$ %s\n' % ' '.join(kill_cmd)) 68 | call(kill_cmd) 69 | 70 | sys.stderr.write('\nAll cleaned up.\n') 71 | 72 | 73 | def construct_args(prog_args): 74 | # construct a dictionary of arguments 75 | args = {} 76 | 77 | # file paths 78 | args['rlcc_dir'] = prog_args.rlcc_dir 79 | args['worker_src'] = path.join(args['rlcc_dir'], 'dagger', 'worker.py') 80 | 81 | # hostnames and processes 82 | args['ps_hosts'] = prog_args.ps_hosts 83 | args['worker_hosts'] = prog_args.worker_hosts 84 | 85 | args['ps_list'] = prog_args.ps_hosts.split(',') 86 | args['worker_list'] = prog_args.worker_hosts.split(',') 87 | args['username'] = prog_args.username 88 | 89 | for i, host in enumerate(args['ps_list']): 90 | args['ps_list'][i] = args['username'] + '@' + host.split(':')[0] 91 | 92 | for i, host in enumerate(args['worker_list']): 93 | args['worker_list'][i] = args['username'] + '@' + host.split(':')[0] 94 | 95 | args['ps_procs'] = [] 96 | args['worker_procs'] = [] 97 | 98 | return args 99 | 100 | 101 | def main(): 102 | parser = argparse.ArgumentParser() 103 | parser.add_argument( 104 | '--ps-hosts', required=True, metavar='[HOSTNAME:PORT, ...]', 105 | help='comma-separated list of hostname:port of parameter servers') 106 | parser.add_argument( 107 | '--worker-hosts', required=True, metavar='[HOSTNAME:PORT, ...]', 108 | help='comma-separated list of hostname:port of workers') 109 | parser.add_argument( 110 | '--username', default='ubuntu', 111 | help='username used in ssh connection (default: ubuntu)') 112 | parser.add_argument( 113 | '--rlcc-dir', metavar='DIR', default='/home/ubuntu/RLCC', 114 | help='absolute path to RLCC/ (default: /home/ubuntu/RLCC)') 115 | prog_args = parser.parse_args() 116 | args = construct_args(prog_args) 117 | 118 | # run worker.py on ps and worker hosts 119 | try: 120 | run(args) 121 | except KeyboardInterrupt: 122 | pass 123 | finally: 124 | cleanup(args) 125 | 126 | 127 | if __name__ == '__main__': 128 | main() 129 | -------------------------------------------------------------------------------- /helpers/helpers.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018 Francis Y. Yan, Jestin Ma 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | 16 | import os 17 | import time 18 | import errno 19 | import select 20 | import socket 21 | import numpy as np 22 | import operator 23 | 24 | 25 | READ_FLAGS = select.POLLIN | select.POLLPRI 26 | WRITE_FLAGS = select.POLLOUT 27 | ERR_FLAGS = select.POLLERR | select.POLLHUP | select.POLLNVAL 28 | READ_ERR_FLAGS = READ_FLAGS | ERR_FLAGS 29 | ALL_FLAGS = READ_FLAGS | WRITE_FLAGS | ERR_FLAGS 30 | 31 | math_ops = { 32 | '+': operator.add, 33 | '-': operator.sub, 34 | '*': operator.mul, 35 | '/': operator.div, 36 | } 37 | 38 | 39 | def apply_op(op, op1, op2): 40 | return math_ops[op](op1, op2) 41 | 42 | 43 | def curr_ts_ms(): 44 | if not hasattr(curr_ts_ms, 'epoch'): 45 | curr_ts_ms.epoch = time.time() 46 | 47 | return int((time.time() - curr_ts_ms.epoch) * 1000) 48 | 49 | 50 | def make_sure_path_exists(path): 51 | try: 52 | os.makedirs(path) 53 | except OSError as e: 54 | if e.errno != errno.EEXIST: 55 | raise 56 | 57 | 58 | def get_open_udp_port(): 59 | s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) 60 | s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) 61 | 62 | s.bind(('', 0)) 63 | port = s.getsockname()[1] 64 | s.close() 65 | return port 66 | 67 | 68 | def normalize(state): 69 | return [state[0] / 200.0, state[1] / 200.0, 70 | state[2] / 200.0, state[3] / 5000.0] 71 | 72 | 73 | def one_hot(action, action_cnt): 74 | ret = [0.0] * action_cnt 75 | ret[action] = 1.0 76 | return ret 77 | 78 | 79 | def softmax(x): 80 | e_x = np.exp(x - np.max(x)) 81 | return e_x / e_x.sum(axis=0) 82 | 83 | 84 | class RingBuffer(object): 85 | def __init__(self, length): 86 | self.full_len = length 87 | self.real_len = 0 88 | self.index = 0 89 | self.data = np.zeros(length) 90 | 91 | def append(self, x): 92 | self.data[self.index] = x 93 | self.index = (self.index + 1) % self.full_len 94 | if self.real_len < self.full_len: 95 | self.real_len += 1 96 | 97 | def get(self): 98 | idx = (self.index - self.real_len + 99 | np.arange(self.real_len)) % self.full_len 100 | return self.data[idx] 101 | 102 | def reset(self): 103 | self.real_len = 0 104 | self.index = 0 105 | self.data.fill(0) 106 | 107 | 108 | class MeanVarHistory(object): 109 | def __init__(self): 110 | self.length = 0 111 | self.mean = 0.0 112 | self.square_mean = 0.0 113 | self.var = 0.0 114 | 115 | def append(self, x): 116 | """Append x to history. 117 | 118 | Args: 119 | x: a list or numpy array. 120 | """ 121 | # x: a list or numpy array 122 | length_new = self.length + len(x) 123 | ratio_old = float(self.length) / length_new 124 | ratio_new = float(len(x)) / length_new 125 | 126 | self.length = length_new 127 | self.mean = self.mean * ratio_old + np.mean(x) * ratio_new 128 | self.square_mean = (self.square_mean * ratio_old + 129 | np.mean(np.square(x)) * ratio_new) 130 | self.var = self.square_mean - np.square(self.mean) 131 | 132 | def get_mean(self): 133 | return self.mean 134 | 135 | def get_var(self): 136 | return self.var if self.var > 0 else 1e-10 137 | 138 | def get_std(self): 139 | return np.sqrt(self.get_var()) 140 | 141 | def normalize_copy(self, x): 142 | """Normalize x and returns a copy. 143 | 144 | Args: 145 | x: a list or numpy array. 146 | """ 147 | return [(v - self.mean) / self.get_std() for v in x] 148 | 149 | def normalize_inplace(self, x): 150 | """Normalize x in place. 151 | 152 | Args: 153 | x: a numpy array with float dtype. 154 | """ 155 | x -= self.mean 156 | x /= self.get_std() 157 | 158 | def reset(self): 159 | self.length = 0 160 | self.mean = 0.0 161 | self.square_mean = 0.0 162 | self.var = 0.0 163 | -------------------------------------------------------------------------------- /env/datagram_pb2.py: -------------------------------------------------------------------------------- 1 | # Generated by the protocol buffer compiler. DO NOT EDIT! 2 | # source: datagram.proto 3 | 4 | import sys 5 | _b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1')) 6 | from google.protobuf import descriptor as _descriptor 7 | from google.protobuf import message as _message 8 | from google.protobuf import reflection as _reflection 9 | from google.protobuf import symbol_database as _symbol_database 10 | from google.protobuf import descriptor_pb2 11 | # @@protoc_insertion_point(imports) 12 | 13 | _sym_db = _symbol_database.Default() 14 | 15 | 16 | 17 | 18 | DESCRIPTOR = _descriptor.FileDescriptor( 19 | name='datagram.proto', 20 | package='', 21 | syntax='proto3', 22 | serialized_pb=_b('\n\x0e\x64\x61tagram.proto\"x\n\x04\x44\x61ta\x12\x0f\n\x07seq_num\x18\x01 \x01(\x07\x12\x0f\n\x07send_ts\x18\x02 \x01(\x07\x12\x12\n\nsent_bytes\x18\x03 \x01(\x06\x12\x16\n\x0e\x64\x65livered_time\x18\x04 \x01(\x07\x12\x11\n\tdelivered\x18\x05 \x01(\x06\x12\x0f\n\x07payload\x18\x06 \x01(\t\"y\n\x03\x41\x63k\x12\x0f\n\x07seq_num\x18\x01 \x01(\x07\x12\x0f\n\x07send_ts\x18\x02 \x01(\x07\x12\x12\n\nsent_bytes\x18\x03 \x01(\x06\x12\x16\n\x0e\x64\x65livered_time\x18\x04 \x01(\x07\x12\x11\n\tdelivered\x18\x05 \x01(\x06\x12\x11\n\tack_bytes\x18\x06 \x01(\x07\x62\x06proto3') 23 | ) 24 | _sym_db.RegisterFileDescriptor(DESCRIPTOR) 25 | 26 | 27 | 28 | 29 | _DATA = _descriptor.Descriptor( 30 | name='Data', 31 | full_name='Data', 32 | filename=None, 33 | file=DESCRIPTOR, 34 | containing_type=None, 35 | fields=[ 36 | _descriptor.FieldDescriptor( 37 | name='seq_num', full_name='Data.seq_num', index=0, 38 | number=1, type=7, cpp_type=3, label=1, 39 | has_default_value=False, default_value=0, 40 | message_type=None, enum_type=None, containing_type=None, 41 | is_extension=False, extension_scope=None, 42 | options=None), 43 | _descriptor.FieldDescriptor( 44 | name='send_ts', full_name='Data.send_ts', index=1, 45 | number=2, type=7, cpp_type=3, label=1, 46 | has_default_value=False, default_value=0, 47 | message_type=None, enum_type=None, containing_type=None, 48 | is_extension=False, extension_scope=None, 49 | options=None), 50 | _descriptor.FieldDescriptor( 51 | name='sent_bytes', full_name='Data.sent_bytes', index=2, 52 | number=3, type=6, cpp_type=4, label=1, 53 | has_default_value=False, default_value=0, 54 | message_type=None, enum_type=None, containing_type=None, 55 | is_extension=False, extension_scope=None, 56 | options=None), 57 | _descriptor.FieldDescriptor( 58 | name='delivered_time', full_name='Data.delivered_time', index=3, 59 | number=4, type=7, cpp_type=3, label=1, 60 | has_default_value=False, default_value=0, 61 | message_type=None, enum_type=None, containing_type=None, 62 | is_extension=False, extension_scope=None, 63 | options=None), 64 | _descriptor.FieldDescriptor( 65 | name='delivered', full_name='Data.delivered', index=4, 66 | number=5, type=6, cpp_type=4, label=1, 67 | has_default_value=False, default_value=0, 68 | message_type=None, enum_type=None, containing_type=None, 69 | is_extension=False, extension_scope=None, 70 | options=None), 71 | _descriptor.FieldDescriptor( 72 | name='payload', full_name='Data.payload', index=5, 73 | number=6, type=9, cpp_type=9, label=1, 74 | has_default_value=False, default_value=_b("").decode('utf-8'), 75 | message_type=None, enum_type=None, containing_type=None, 76 | is_extension=False, extension_scope=None, 77 | options=None), 78 | ], 79 | extensions=[ 80 | ], 81 | nested_types=[], 82 | enum_types=[ 83 | ], 84 | options=None, 85 | is_extendable=False, 86 | syntax='proto3', 87 | extension_ranges=[], 88 | oneofs=[ 89 | ], 90 | serialized_start=18, 91 | serialized_end=138, 92 | ) 93 | 94 | 95 | _ACK = _descriptor.Descriptor( 96 | name='Ack', 97 | full_name='Ack', 98 | filename=None, 99 | file=DESCRIPTOR, 100 | containing_type=None, 101 | fields=[ 102 | _descriptor.FieldDescriptor( 103 | name='seq_num', full_name='Ack.seq_num', index=0, 104 | number=1, type=7, cpp_type=3, label=1, 105 | has_default_value=False, default_value=0, 106 | message_type=None, enum_type=None, containing_type=None, 107 | is_extension=False, extension_scope=None, 108 | options=None), 109 | _descriptor.FieldDescriptor( 110 | name='send_ts', full_name='Ack.send_ts', index=1, 111 | number=2, type=7, cpp_type=3, label=1, 112 | has_default_value=False, default_value=0, 113 | message_type=None, enum_type=None, containing_type=None, 114 | is_extension=False, extension_scope=None, 115 | options=None), 116 | _descriptor.FieldDescriptor( 117 | name='sent_bytes', full_name='Ack.sent_bytes', index=2, 118 | number=3, type=6, cpp_type=4, label=1, 119 | has_default_value=False, default_value=0, 120 | message_type=None, enum_type=None, containing_type=None, 121 | is_extension=False, extension_scope=None, 122 | options=None), 123 | _descriptor.FieldDescriptor( 124 | name='delivered_time', full_name='Ack.delivered_time', index=3, 125 | number=4, type=7, cpp_type=3, label=1, 126 | has_default_value=False, default_value=0, 127 | message_type=None, enum_type=None, containing_type=None, 128 | is_extension=False, extension_scope=None, 129 | options=None), 130 | _descriptor.FieldDescriptor( 131 | name='delivered', full_name='Ack.delivered', index=4, 132 | number=5, type=6, cpp_type=4, label=1, 133 | has_default_value=False, default_value=0, 134 | message_type=None, enum_type=None, containing_type=None, 135 | is_extension=False, extension_scope=None, 136 | options=None), 137 | _descriptor.FieldDescriptor( 138 | name='ack_bytes', full_name='Ack.ack_bytes', index=5, 139 | number=6, type=7, cpp_type=3, label=1, 140 | has_default_value=False, default_value=0, 141 | message_type=None, enum_type=None, containing_type=None, 142 | is_extension=False, extension_scope=None, 143 | options=None), 144 | ], 145 | extensions=[ 146 | ], 147 | nested_types=[], 148 | enum_types=[ 149 | ], 150 | options=None, 151 | is_extendable=False, 152 | syntax='proto3', 153 | extension_ranges=[], 154 | oneofs=[ 155 | ], 156 | serialized_start=140, 157 | serialized_end=261, 158 | ) 159 | 160 | DESCRIPTOR.message_types_by_name['Data'] = _DATA 161 | DESCRIPTOR.message_types_by_name['Ack'] = _ACK 162 | 163 | Data = _reflection.GeneratedProtocolMessageType('Data', (_message.Message,), dict( 164 | DESCRIPTOR = _DATA, 165 | __module__ = 'datagram_pb2' 166 | # @@protoc_insertion_point(class_scope:Data) 167 | )) 168 | _sym_db.RegisterMessage(Data) 169 | 170 | Ack = _reflection.GeneratedProtocolMessageType('Ack', (_message.Message,), dict( 171 | DESCRIPTOR = _ACK, 172 | __module__ = 'datagram_pb2' 173 | # @@protoc_insertion_point(class_scope:Ack) 174 | )) 175 | _sym_db.RegisterMessage(Ack) 176 | 177 | 178 | # @@protoc_insertion_point(module_scope) 179 | -------------------------------------------------------------------------------- /dagger/worker.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | # Copyright 2018 Francis Y. Yan, Jestin Ma 4 | # 5 | # Licensed under the Apache License, Version 2.0 (the "License"); 6 | # you may not use this file except in compliance with the License. 7 | # You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | 18 | import sys 19 | import yaml 20 | import argparse 21 | import project_root 22 | import numpy as np 23 | import tensorflow as tf 24 | from subprocess import check_call 25 | from os import path 26 | from dagger import DaggerLeader, DaggerWorker 27 | from env.environment import Environment 28 | from env.sender import Sender 29 | 30 | 31 | def prepare_traces(bandwidth): 32 | trace_dir = path.join(project_root.DIR, 'env') 33 | 34 | if type(bandwidth) == int: 35 | trace_path = path.join(trace_dir, '%dmbps.trace' % bandwidth) 36 | 37 | if not path.exists(trace_path): 38 | gen_trace = path.join(project_root.DIR, 'helpers', 39 | 'generate_trace.py') 40 | cmd = ['python', gen_trace, '--output-dir', trace_dir, 41 | '--bandwidth', str(bandwidth)] 42 | sys.stderr.write('$ %s\n' % ' '.join(cmd)) 43 | check_call(cmd) 44 | 45 | uplink_trace = trace_path 46 | downlink_trace = uplink_trace 47 | else: 48 | trace_path = path.join(trace_dir, bandwidth) 49 | # intentionally switch uplink and downlink traces due to sender first 50 | uplink_trace = trace_path + '.down' 51 | downlink_trace = trace_path + '.up' 52 | 53 | return uplink_trace, downlink_trace 54 | 55 | 56 | def create_env(task_index): 57 | """ Creates and returns an Environment which contains a single 58 | sender-receiver connection. The environment is run inside mahimahi 59 | shells. The environment knows the best cwnd to pass to the expert policy. 60 | """ 61 | 62 | best_cwnds_file = path.join(project_root.DIR, 'dagger', 'best_cwnds.yml') 63 | best_cwnd_map = yaml.load(open(best_cwnds_file)) 64 | 65 | if task_index == 0: 66 | trace_path = path.join(project_root.DIR, 'env', '0.57mbps-poisson.trace') 67 | mm_cmd = 'mm-delay 28 mm-loss uplink 0.0477 mm-link %s %s --uplink-queue=droptail --uplink-queue-args=packets=14' % (trace_path, trace_path) 68 | best_cwnd = 5 69 | elif task_index == 1: 70 | trace_path = path.join(project_root.DIR, 'env', '2.64mbps-poisson.trace') 71 | mm_cmd = 'mm-delay 88 mm-link %s %s --uplink-queue=droptail --uplink-queue-args=packets=130' % (trace_path, trace_path) 72 | best_cwnd = 40 73 | elif task_index == 2: 74 | trace_path = path.join(project_root.DIR, 'env', '3.04mbps-poisson.trace') 75 | mm_cmd = 'mm-delay 130 mm-link %s %s --uplink-queue=droptail --uplink-queue-args=packets=426' % (trace_path, trace_path) 76 | best_cwnd = 70 77 | elif task_index <= 18: 78 | bandwidth = [5, 10, 20, 50] 79 | delay = [10, 20, 40, 80] 80 | 81 | cartesian = [(b, d) for b in bandwidth for d in delay] 82 | bandwidth, delay = cartesian[task_index - 3] 83 | 84 | uplink_trace, downlink_trace = prepare_traces(bandwidth) 85 | mm_cmd = 'mm-delay %d mm-link %s %s' % (delay, uplink_trace, downlink_trace) 86 | best_cwnd = best_cwnd_map[bandwidth][delay] 87 | elif task_index == 19: 88 | trace_path = path.join(project_root.DIR, 'env', '100.42mbps.trace') 89 | mm_cmd = 'mm-delay 27 mm-link %s %s --uplink-queue=droptail --uplink-queue-args=packets=173' % (trace_path, trace_path) 90 | best_cwnd = 500 91 | elif task_index == 20: 92 | trace_path = path.join(project_root.DIR, 'env', '77.72mbps.trace') 93 | mm_cmd = 'mm-delay 51 mm-loss uplink 0.0006 mm-link %s %s --uplink-queue=droptail --uplink-queue-args=packets=94' % (trace_path, trace_path) 94 | best_cwnd = 690 95 | elif task_index == 21: 96 | trace_path = path.join(project_root.DIR, 'env', '114.68mbps.trace') 97 | mm_cmd = 'mm-delay 45 mm-link %s %s --uplink-queue=droptail --uplink-queue-args=packets=450' % (trace_path, trace_path) 98 | best_cwnd = 870 99 | elif task_index <= 29: 100 | bandwidth = [100, 200] 101 | delay = [10, 20, 40, 80] 102 | 103 | cartesian = [(b, d) for b in bandwidth for d in delay] 104 | bandwidth, delay = cartesian[task_index - 26] 105 | 106 | uplink_trace, downlink_trace = prepare_traces(bandwidth) 107 | mm_cmd = 'mm-delay %d mm-link %s %s' % (delay, uplink_trace, downlink_trace) 108 | best_cwnd = best_cwnd_map[bandwidth][delay] 109 | 110 | env = Environment(mm_cmd) 111 | env.best_cwnd = best_cwnd 112 | 113 | return env 114 | 115 | 116 | def run(args): 117 | """ For each worker/parameter server, starts the appropriate job 118 | associated with the cluster and server. 119 | """ 120 | 121 | job_name = args.job_name 122 | task_index = args.task_index 123 | sys.stderr.write('Starting job %s task %d\n' % (job_name, task_index)) 124 | 125 | ps_hosts = args.ps_hosts.split(',') 126 | worker_hosts = args.worker_hosts.split(',') 127 | num_workers = len(worker_hosts) 128 | 129 | cluster = tf.train.ClusterSpec({'ps': ps_hosts, 'worker': worker_hosts}) 130 | server = tf.train.Server(cluster, job_name=job_name, task_index=task_index) 131 | 132 | if job_name == 'ps': 133 | # Sets up the queue, shared variables, and global classifier. 134 | worker_tasks = set([idx for idx in xrange(num_workers)]) 135 | leader = DaggerLeader(cluster, server, worker_tasks) 136 | try: 137 | leader.run(debug=True) 138 | except KeyboardInterrupt: 139 | pass 140 | finally: 141 | leader.cleanup() 142 | 143 | elif job_name == 'worker': 144 | # Sets up the env, shared variables (sync, classifier, queue, etc) 145 | env = create_env(task_index) 146 | learner = DaggerWorker(cluster, server, task_index, env) 147 | try: 148 | learner.run(debug=True) 149 | except KeyboardInterrupt: 150 | pass 151 | finally: 152 | learner.cleanup() 153 | 154 | 155 | def main(): 156 | parser = argparse.ArgumentParser() 157 | parser.add_argument( 158 | '--ps-hosts', required=True, metavar='[HOSTNAME:PORT, ...]', 159 | help='comma-separated list of hostname:port of parameter servers') 160 | parser.add_argument( 161 | '--worker-hosts', required=True, metavar='[HOSTNAME:PORT, ...]', 162 | help='comma-separated list of hostname:port of workers') 163 | parser.add_argument('--job-name', choices=['ps', 'worker'], 164 | required=True, help='ps or worker') 165 | parser.add_argument('--task-index', metavar='N', type=int, required=True, 166 | help='index of task') 167 | args = parser.parse_args() 168 | 169 | # run parameter servers and workers 170 | run(args) 171 | 172 | 173 | if __name__ == '__main__': 174 | main() 175 | -------------------------------------------------------------------------------- /env/sender.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018 Francis Y. Yan, Jestin Ma 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | 16 | import time 17 | import sys 18 | import json 19 | import socket 20 | import select 21 | from os import path 22 | import numpy as np 23 | import datagram_pb2 24 | import project_root 25 | from helpers.helpers import ( 26 | curr_ts_ms, apply_op, 27 | READ_FLAGS, ERR_FLAGS, READ_ERR_FLAGS, WRITE_FLAGS, ALL_FLAGS) 28 | 29 | 30 | def format_actions(action_list): 31 | """ Returns the action list, initially a list with elements "[op][val]" 32 | like /2.0, -3.0, +1.0, formatted as a dictionary. 33 | 34 | The dictionary keys are the unique indices (to retrieve the action) and 35 | the values are lists ['op', val], such as ['+', '2.0']. 36 | """ 37 | return {idx: [action[0], float(action[1:])] 38 | for idx, action in enumerate(action_list)} 39 | 40 | 41 | class Sender(object): 42 | # RL exposed class/static variables 43 | max_steps = 1000 44 | state_dim = 4 45 | action_mapping = format_actions(["/2.0", "-10.0", "+0.0", "+10.0", "*2.0"]) 46 | action_cnt = len(action_mapping) 47 | 48 | def __init__(self, port=0, train=False, debug=False): 49 | self.train = train 50 | self.debug = debug 51 | 52 | # UDP socket and poller 53 | self.peer_addr = None 54 | 55 | self.sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) 56 | self.sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) 57 | self.sock.bind(('0.0.0.0', port)) 58 | sys.stderr.write('[sender] Listening on port %s\n' % 59 | self.sock.getsockname()[1]) 60 | 61 | self.poller = select.poll() 62 | self.poller.register(self.sock, ALL_FLAGS) 63 | 64 | self.dummy_payload = 'x' * 1400 65 | 66 | if self.debug: 67 | self.sampling_file = open(path.join(project_root.DIR, 'env', 'sampling_time'), 'w', 0) 68 | 69 | # congestion control related 70 | self.seq_num = 0 71 | self.next_ack = 0 72 | self.cwnd = 10.0 73 | self.step_len_ms = 10 74 | 75 | # state variables for RLCC 76 | self.delivered_time = 0 77 | self.delivered = 0 78 | self.sent_bytes = 0 79 | 80 | self.min_rtt = float('inf') 81 | self.delay_ewma = None 82 | self.send_rate_ewma = None 83 | self.delivery_rate_ewma = None 84 | 85 | self.step_start_ms = None 86 | self.running = True 87 | 88 | if self.train: 89 | self.step_cnt = 0 90 | 91 | self.ts_first = None 92 | self.rtt_buf = [] 93 | 94 | def cleanup(self): 95 | if self.debug and self.sampling_file: 96 | self.sampling_file.close() 97 | self.sock.close() 98 | 99 | def handshake(self): 100 | """Handshake with peer receiver. Must be called before run().""" 101 | 102 | while True: 103 | msg, addr = self.sock.recvfrom(1600) 104 | 105 | if msg == 'Hello from receiver' and self.peer_addr is None: 106 | self.peer_addr = addr 107 | self.sock.sendto('Hello from sender', self.peer_addr) 108 | sys.stderr.write('[sender] Handshake success! ' 109 | 'Receiver\'s address is %s:%s\n' % addr) 110 | break 111 | 112 | self.sock.setblocking(0) # non-blocking UDP socket 113 | 114 | def set_sample_action(self, sample_action): 115 | """Set the policy. Must be called before run().""" 116 | 117 | self.sample_action = sample_action 118 | 119 | def update_state(self, ack): 120 | """ Update the state variables listed in __init__() """ 121 | self.next_ack = max(self.next_ack, ack.seq_num + 1) 122 | curr_time_ms = curr_ts_ms() 123 | 124 | # Update RTT 125 | rtt = float(curr_time_ms - ack.send_ts) 126 | self.min_rtt = min(self.min_rtt, rtt) 127 | 128 | if self.train: 129 | if self.ts_first is None: 130 | self.ts_first = curr_time_ms 131 | self.rtt_buf.append(rtt) 132 | 133 | delay = rtt - self.min_rtt 134 | if self.delay_ewma is None: 135 | self.delay_ewma = delay 136 | else: 137 | self.delay_ewma = 0.875 * self.delay_ewma + 0.125 * delay 138 | 139 | # Update BBR's delivery rate 140 | self.delivered += ack.ack_bytes 141 | self.delivered_time = curr_time_ms 142 | delivery_rate = (0.008 * (self.delivered - ack.delivered) / 143 | max(1, self.delivered_time - ack.delivered_time)) 144 | 145 | if self.delivery_rate_ewma is None: 146 | self.delivery_rate_ewma = delivery_rate 147 | else: 148 | self.delivery_rate_ewma = ( 149 | 0.875 * self.delivery_rate_ewma + 0.125 * delivery_rate) 150 | 151 | # Update Vegas sending rate 152 | send_rate = 0.008 * (self.sent_bytes - ack.sent_bytes) / max(1, rtt) 153 | 154 | if self.send_rate_ewma is None: 155 | self.send_rate_ewma = send_rate 156 | else: 157 | self.send_rate_ewma = ( 158 | 0.875 * self.send_rate_ewma + 0.125 * send_rate) 159 | 160 | def take_action(self, action_idx): 161 | old_cwnd = self.cwnd 162 | op, val = self.action_mapping[action_idx] 163 | 164 | self.cwnd = apply_op(op, self.cwnd, val) 165 | self.cwnd = max(2.0, self.cwnd) 166 | 167 | def window_is_open(self): 168 | return self.seq_num - self.next_ack < self.cwnd 169 | 170 | def send(self): 171 | data = datagram_pb2.Data() 172 | data.seq_num = self.seq_num 173 | data.send_ts = curr_ts_ms() 174 | data.sent_bytes = self.sent_bytes 175 | data.delivered_time = self.delivered_time 176 | data.delivered = self.delivered 177 | data.payload = self.dummy_payload 178 | 179 | serialized_data = data.SerializeToString() 180 | self.sock.sendto(serialized_data, self.peer_addr) 181 | 182 | self.seq_num += 1 183 | self.sent_bytes += len(serialized_data) 184 | 185 | def recv(self): 186 | serialized_ack, addr = self.sock.recvfrom(1600) 187 | 188 | if addr != self.peer_addr: 189 | return 190 | 191 | ack = datagram_pb2.Ack() 192 | ack.ParseFromString(serialized_ack) 193 | 194 | self.update_state(ack) 195 | 196 | if self.step_start_ms is None: 197 | self.step_start_ms = curr_ts_ms() 198 | 199 | # At each step end, feed the state: 200 | if curr_ts_ms() - self.step_start_ms > self.step_len_ms: # step's end 201 | state = [self.delay_ewma, 202 | self.delivery_rate_ewma, 203 | self.send_rate_ewma, 204 | self.cwnd] 205 | 206 | # time how long it takes to get an action from the NN 207 | if self.debug: 208 | start_sample = time.time() 209 | 210 | action = self.sample_action(state) 211 | 212 | if self.debug: 213 | self.sampling_file.write('%.2f ms\n' % ((time.time() - start_sample) * 1000)) 214 | 215 | self.take_action(action) 216 | 217 | self.delay_ewma = None 218 | self.delivery_rate_ewma = None 219 | self.send_rate_ewma = None 220 | 221 | self.step_start_ms = curr_ts_ms() 222 | 223 | if self.train: 224 | self.step_cnt += 1 225 | if self.step_cnt >= Sender.max_steps: 226 | self.step_cnt = 0 227 | self.running = False 228 | 229 | self.compute_performance() 230 | 231 | def run(self): 232 | TIMEOUT = 1000 # ms 233 | 234 | self.poller.modify(self.sock, ALL_FLAGS) 235 | curr_flags = ALL_FLAGS 236 | 237 | while self.running: 238 | if self.window_is_open(): 239 | if curr_flags != ALL_FLAGS: 240 | self.poller.modify(self.sock, ALL_FLAGS) 241 | curr_flags = ALL_FLAGS 242 | else: 243 | if curr_flags != READ_ERR_FLAGS: 244 | self.poller.modify(self.sock, READ_ERR_FLAGS) 245 | curr_flags = READ_ERR_FLAGS 246 | 247 | events = self.poller.poll(TIMEOUT) 248 | 249 | if not events: # timed out 250 | self.send() 251 | 252 | for fd, flag in events: 253 | assert self.sock.fileno() == fd 254 | 255 | if flag & ERR_FLAGS: 256 | sys.exit('Error occurred to the channel') 257 | 258 | if flag & READ_FLAGS: 259 | self.recv() 260 | 261 | if flag & WRITE_FLAGS: 262 | if self.window_is_open(): 263 | self.send() 264 | 265 | def compute_performance(self): 266 | duration = curr_ts_ms() - self.ts_first 267 | tput = 0.008 * self.delivered / duration 268 | perc_delay = np.percentile(self.rtt_buf, 95) 269 | 270 | with open(path.join(project_root.DIR, 'env', 'perf'), 'a', 0) as perf: 271 | perf.write('%.2f %d\n' % (tput, perc_delay)) 272 | -------------------------------------------------------------------------------- /COPYING: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | APPENDIX: How to apply the Apache License to your work. 180 | 181 | To apply the Apache License to your work, attach the following 182 | boilerplate notice, with the fields enclosed by brackets "[]" 183 | replaced with your own identifying information. (Don't include 184 | the brackets!) The text should be enclosed in the appropriate 185 | comment syntax for the file format. We also recommend that a 186 | file or class name and description of purpose be included on the 187 | same "printed page" as the copyright notice for easier 188 | identification within third-party archives. 189 | 190 | Copyright [yyyy] [name of copyright owner] 191 | 192 | Licensed under the Apache License, Version 2.0 (the "License"); 193 | you may not use this file except in compliance with the License. 194 | You may obtain a copy of the License at 195 | 196 | http://www.apache.org/licenses/LICENSE-2.0 197 | 198 | Unless required by applicable law or agreed to in writing, software 199 | distributed under the License is distributed on an "AS IS" BASIS, 200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 201 | See the License for the specific language governing permissions and 202 | limitations under the License. 203 | -------------------------------------------------------------------------------- /dagger/dagger.py: -------------------------------------------------------------------------------- 1 | # Copyright 2018 Francis Y. Yan, Jestin Ma 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | 16 | import sys 17 | import time 18 | import project_root 19 | import numpy as np 20 | import tensorflow as tf 21 | import datetime 22 | from tensorflow import contrib 23 | from os import path 24 | from models import DaggerLSTM 25 | from experts import TrueDaggerExpert 26 | from env.sender import Sender 27 | from helpers.helpers import ( 28 | make_sure_path_exists, normalize, one_hot, curr_ts_ms) 29 | from subprocess import check_output 30 | 31 | 32 | class Status: 33 | EP_DONE = 0 34 | WORKER_DONE = 1 35 | WORKER_START = 2 36 | PS_DONE = 3 37 | 38 | 39 | class DaggerLeader(object): 40 | def __init__(self, cluster, server, worker_tasks): 41 | self.cluster = cluster 42 | self.server = server 43 | self.worker_tasks = worker_tasks 44 | self.num_workers = len(worker_tasks) 45 | self.aggregated_states = [] 46 | self.aggregated_actions = [] 47 | self.max_eps = 1000 48 | self.checkpoint_delta = 10 49 | self.checkpoint = self.checkpoint_delta 50 | self.learn_rate = 0.01 51 | self.regularization_lambda = 1e-4 52 | self.train_step = 0 53 | 54 | self.state_dim = Sender.state_dim 55 | self.action_cnt = Sender.action_cnt 56 | self.aug_state_dim = self.state_dim + self.action_cnt 57 | 58 | # Create the master network and training/sync queues 59 | with tf.variable_scope('global'): 60 | self.global_network = DaggerLSTM( 61 | state_dim=self.aug_state_dim, action_cnt=self.action_cnt) 62 | 63 | self.leader_device_cpu = '/job:ps/task:0/cpu:0' 64 | with tf.device(self.leader_device_cpu): 65 | with tf.variable_scope('global_cpu'): 66 | self.global_network_cpu = DaggerLSTM( 67 | state_dim=self.aug_state_dim, action_cnt=self.action_cnt) 68 | 69 | cpu_vars = self.global_network_cpu.trainable_vars 70 | gpu_vars = self.global_network.trainable_vars 71 | self.sync_op = tf.group(*[v1.assign(v2) for v1, v2 in zip( 72 | cpu_vars, gpu_vars)]) 73 | 74 | self.default_batch_size = 300 75 | self.default_init_state = self.global_network.zero_init_state( 76 | self.default_batch_size) 77 | 78 | # Each element is [[aug_state]], [action] 79 | self.train_q = tf.FIFOQueue( 80 | self.num_workers, [tf.float32, tf.int32], 81 | shared_name='training_feed') 82 | 83 | # Keys: worker indices, values: Tensorflow messaging queues 84 | # Queue Elements: Status message 85 | self.sync_queues = {} 86 | for idx in worker_tasks: 87 | queue_name = 'sync_q_%d' % idx 88 | self.sync_queues[idx] = tf.FIFOQueue(3, [tf.int16], 89 | shared_name=queue_name) 90 | 91 | self.setup_tf_ops(server) 92 | 93 | self.sess = tf.Session( 94 | server.target, config=tf.ConfigProto(allow_soft_placement=True)) 95 | self.sess.run(tf.global_variables_initializer()) 96 | 97 | def cleanup(self): 98 | """ Sends messages to workers to stop and saves the model. """ 99 | for idx in self.worker_tasks: 100 | self.sess.run(self.sync_queues[idx].enqueue(Status.PS_DONE)) 101 | self.save_model() 102 | 103 | def save_model(self, checkpoint=None): 104 | """ Takes care of saving/checkpointing the model. """ 105 | if checkpoint is None: 106 | model_path = path.join(self.logdir, 'model') 107 | else: 108 | model_path = path.join(self.logdir, 'checkpoint-%d' % checkpoint) 109 | 110 | # save parameters to parameter server 111 | saver = tf.train.Saver(self.global_network.trainable_vars) 112 | saver.save(self.sess, model_path) 113 | sys.stderr.write('\nModel saved to param. server at %s\n' % model_path) 114 | 115 | def setup_tf_ops(self, server): 116 | """ Sets up Tensorboard operators and tools, such as the optimizer, 117 | summary values, Tensorboard, and Session. 118 | """ 119 | 120 | self.actions = tf.placeholder(tf.int32, [None, None]) 121 | 122 | reg_loss = 0.0 123 | for x in self.global_network.trainable_vars: 124 | if x.name == 'global/cnt:0': 125 | continue 126 | reg_loss += tf.nn.l2_loss(x) 127 | reg_loss *= self.regularization_lambda 128 | 129 | cross_entropy_loss = tf.reduce_mean( 130 | tf.nn.sparse_softmax_cross_entropy_with_logits( 131 | labels=self.actions, 132 | logits=self.global_network.action_scores)) 133 | 134 | self.total_loss = cross_entropy_loss + reg_loss 135 | 136 | optimizer = tf.train.AdamOptimizer(self.learn_rate) 137 | self.train_op = optimizer.minimize(self.total_loss) 138 | 139 | tf.summary.scalar('reduced_ce_loss', cross_entropy_loss) 140 | tf.summary.scalar('reg_loss', reg_loss) 141 | tf.summary.scalar('total_loss', self.total_loss) 142 | self.summary_op = tf.summary.merge_all() 143 | 144 | git_commit = check_output( 145 | 'cd %s && git rev-parse @' % project_root.DIR, shell=True) 146 | date_time = datetime.datetime.now().strftime('%Y-%m-%d-%H-%M-%S') 147 | log_name = date_time + '-%s' % git_commit.strip() 148 | self.logdir = path.join(project_root.DIR, 'dagger', 'logs', log_name) 149 | make_sure_path_exists(self.logdir) 150 | self.summary_writer = tf.summary.FileWriter(self.logdir) 151 | 152 | def wait_on_workers(self): 153 | """ Update which workers are done or dead. Stale tokens will 154 | eventually be cleaned out. 155 | Returns the number of workers that finished their episode. 156 | """ 157 | workers_ep_done = 0 158 | while workers_ep_done < len(self.worker_tasks): 159 | # Let the workers dequeue their start tokens 160 | time.sleep(0.5) 161 | 162 | # check in each queue for worker messages and update workers 163 | workers_done = [] 164 | for idx in self.worker_tasks: 165 | worker_queue = self.sync_queues[idx] 166 | msg = self.sess.run(worker_queue.dequeue()) 167 | 168 | if msg == Status.EP_DONE: 169 | workers_ep_done += 1 170 | elif msg == Status.WORKER_DONE: 171 | workers_done.append(idx) 172 | self.sess.run(worker_queue.close()) 173 | else: 174 | self.sess.run(worker_queue.enqueue(msg)) 175 | 176 | for worker in workers_done: 177 | self.worker_tasks.remove(worker) 178 | 179 | return workers_ep_done 180 | 181 | def run_one_train_step(self, batch_states, batch_actions): 182 | """ Runs one step of the training operator on the given data. 183 | At times will update Tensorboard and save a checkpointed model. 184 | Returns the total loss calculated. 185 | """ 186 | 187 | summary = True if self.train_step % 10 == 0 else False 188 | 189 | ops_to_run = [self.train_op, self.total_loss] 190 | 191 | if summary: 192 | ops_to_run.append(self.summary_op) 193 | 194 | pi = self.global_network 195 | 196 | start_ts = curr_ts_ms() 197 | ret = self.sess.run(ops_to_run, feed_dict={ 198 | pi.input: batch_states, 199 | self.actions: batch_actions, 200 | pi.state_in: self.init_state}) 201 | 202 | elapsed = (curr_ts_ms() - start_ts) / 1000.0 203 | sys.stderr.write('train step %d: time %.2f\n' % 204 | (self.train_step, elapsed)) 205 | 206 | if summary: 207 | self.summary_writer.add_summary(ret[2], self.train_step) 208 | 209 | return ret[1] 210 | 211 | def train(self): 212 | """ Runs the training operator until the loss converges. 213 | """ 214 | curr_iter = 0 215 | 216 | min_loss = float('inf') 217 | iters_since_min_loss = 0 218 | 219 | batch_size = min(len(self.aggregated_states), self.default_batch_size) 220 | num_batches = len(self.aggregated_states) / batch_size 221 | 222 | if batch_size != self.default_batch_size: 223 | self.init_state = self.global_network.zero_init_state(batch_size) 224 | else: 225 | self.init_state = self.default_init_state 226 | 227 | while True: 228 | curr_iter += 1 229 | 230 | mean_loss = 0.0 231 | max_loss = 0.0 232 | 233 | for batch_num in xrange(num_batches): 234 | self.train_step += 1 235 | 236 | start = batch_num * batch_size 237 | end = start + batch_size 238 | 239 | batch_states = self.aggregated_states[start:end] 240 | batch_actions = self.aggregated_actions[start:end] 241 | 242 | loss = self.run_one_train_step(batch_states, batch_actions) 243 | 244 | mean_loss += loss 245 | max_loss = max(loss, max_loss) 246 | 247 | mean_loss /= num_batches 248 | 249 | sys.stderr.write('--- iter %d: max loss %.4f, mean loss %.4f\n' % 250 | (curr_iter, max_loss, mean_loss)) 251 | 252 | if max_loss < min_loss - 0.001: 253 | min_loss = max_loss 254 | iters_since_min_loss = 0 255 | else: 256 | iters_since_min_loss += 1 257 | 258 | if curr_iter > 50: 259 | break 260 | 261 | if iters_since_min_loss >= max(0.2 * curr_iter, 10): 262 | break 263 | 264 | self.sess.run(self.global_network.add_one) 265 | 266 | # copy trained variables from GPU to CPU 267 | self.sess.run(self.sync_op) 268 | 269 | print 'DaggerLeader:global_network:cnt', self.sess.run(self.global_network.cnt) 270 | print 'DaggerLeader:global_network_cpu:cnt', self.sess.run(self.global_network_cpu.cnt) 271 | sys.stdout.flush() 272 | 273 | def run(self, debug=False): 274 | for curr_ep in xrange(self.max_eps): 275 | if debug: 276 | sys.stderr.write('[PSERVER EP %d]: waiting for workers %s\n' % 277 | (curr_ep, self.worker_tasks)) 278 | 279 | workers_ep_done = self.wait_on_workers() 280 | 281 | # If workers had data, dequeue ALL the samples and train 282 | if workers_ep_done > 0: 283 | while True: 284 | num_samples = self.sess.run(self.train_q.size()) 285 | if num_samples == 0: 286 | break 287 | 288 | data = self.sess.run(self.train_q.dequeue()) 289 | self.aggregated_states.append(data[0]) 290 | self.aggregated_actions.append(data[1]) 291 | 292 | if debug: 293 | sys.stderr.write('[PSERVER]: start training\n') 294 | 295 | self.train() 296 | else: 297 | if debug: 298 | sys.stderr.write('[PSERVER]: quitting...\n') 299 | break 300 | 301 | # Save the network model for testing every so often 302 | if curr_ep == self.checkpoint: 303 | self.save_model(curr_ep) 304 | self.checkpoint += self.checkpoint_delta 305 | 306 | # After training, tell workers to start another episode 307 | for idx in self.worker_tasks: 308 | worker_queue = self.sync_queues[idx] 309 | self.sess.run(worker_queue.enqueue(Status.WORKER_START)) 310 | 311 | 312 | class DaggerWorker(object): 313 | def __init__(self, cluster, server, task_idx, env): 314 | # Distributed tensorflow and logging related 315 | self.cluster = cluster 316 | self.env = env 317 | self.task_idx = task_idx 318 | self.leader_device = '/job:ps/task:0' 319 | self.worker_device = '/job:worker/task:%d' % task_idx 320 | self.num_workers = cluster.num_tasks('worker') 321 | 322 | # Buffers and parameters required to train 323 | self.curr_ep = 0 324 | self.state_buf = [] 325 | self.action_buf = [] 326 | self.state_dim = env.state_dim 327 | self.action_cnt = env.action_cnt 328 | 329 | self.aug_state_dim = self.state_dim + self.action_cnt 330 | self.prev_action = self.action_cnt - 1 331 | 332 | self.expert = TrueDaggerExpert(env) 333 | # Must call env.set_sample_action() before env.rollout() 334 | env.set_sample_action(self.sample_action) 335 | 336 | # Set up Tensorflow for synchronization, training 337 | self.setup_tf_ops() 338 | self.sess = tf.Session( 339 | server.target, config=tf.ConfigProto(allow_soft_placement=True)) 340 | self.sess.run(tf.global_variables_initializer()) 341 | 342 | def cleanup(self): 343 | self.env.cleanup() 344 | self.sess.run(self.sync_q.enqueue(Status.WORKER_DONE)) 345 | 346 | def setup_tf_ops(self): 347 | """ Sets up the shared Tensorflow operators and structures 348 | Refer to DaggerLeader for more information 349 | """ 350 | 351 | # Set up the shared global network and local network. 352 | with tf.device(self.leader_device): 353 | with tf.variable_scope('global_cpu'): 354 | self.global_network_cpu = DaggerLSTM( 355 | state_dim=self.aug_state_dim, action_cnt=self.action_cnt) 356 | 357 | with tf.device(self.worker_device): 358 | with tf.variable_scope('local'): 359 | self.local_network = DaggerLSTM( 360 | state_dim=self.aug_state_dim, action_cnt=self.action_cnt) 361 | 362 | self.init_state = self.local_network.zero_init_state(1) 363 | self.lstm_state = self.init_state 364 | 365 | # Build shared queues for training data and synchronization 366 | self.train_q = tf.FIFOQueue( 367 | self.num_workers, [tf.float32, tf.int32], 368 | shared_name='training_feed') 369 | 370 | self.sync_q = tf.FIFOQueue(3, [tf.int16], 371 | shared_name=('sync_q_%d' % self.task_idx)) 372 | 373 | # Training data is [[aug_state]], [action] 374 | self.state_data = tf.placeholder( 375 | tf.float32, shape=(None, self.aug_state_dim)) 376 | self.action_data = tf.placeholder(tf.int32, shape=(None)) 377 | self.enqueue_train_op = self.train_q.enqueue( 378 | [self.state_data, self.action_data]) 379 | 380 | # Sync local network to global network (CPU) 381 | local_vars = self.local_network.trainable_vars 382 | global_vars = self.global_network_cpu.trainable_vars 383 | self.sync_op = tf.group(*[v1.assign(v2) for v1, v2 in zip( 384 | local_vars, global_vars)]) 385 | 386 | def sample_action(self, state): 387 | """ Given a state buffer in the past step, returns an action 388 | to perform. 389 | 390 | Appends to the state/action buffers the state and the 391 | "correct" action to take according to the expert. 392 | """ 393 | cwnd = state[self.state_dim - 1] 394 | expert_action = self.expert.sample_action(cwnd) 395 | 396 | # For decision-making, normalize. 397 | norm_state = normalize(state) 398 | 399 | one_hot_action = one_hot(self.prev_action, self.action_cnt) 400 | aug_state = norm_state + one_hot_action 401 | 402 | # Fill in state_buf, action_buf 403 | self.state_buf.append(aug_state) 404 | self.action_buf.append(expert_action) 405 | 406 | # Always use the expert on the first episode to get our bearings. 407 | if self.curr_ep == 0: 408 | self.prev_action = expert_action 409 | return expert_action 410 | 411 | # Get probability of each action from the local network. 412 | pi = self.local_network 413 | feed_dict = { 414 | pi.input: [[aug_state]], 415 | pi.state_in: self.lstm_state, 416 | } 417 | ops_to_run = [pi.action_probs, pi.state_out] 418 | action_probs, self.lstm_state = self.sess.run(ops_to_run, feed_dict) 419 | 420 | # Choose an action to take and update current LSTM state 421 | # action = np.argmax(np.random.multinomial(1, action_probs[0][0] - 1e-5)) 422 | action = np.argmax(action_probs[0][0]) 423 | self.prev_action = action 424 | 425 | return action 426 | 427 | def rollout(self): 428 | """ Start an episode/flow with an empty dataset/environment. """ 429 | self.state_buf = [] 430 | self.action_buf = [] 431 | self.prev_action = self.action_cnt - 1 432 | self.lstm_state = self.init_state 433 | 434 | self.env.reset() 435 | self.env.rollout() 436 | 437 | def run(self, debug=False): 438 | """Runs for max_ep episodes, each time sending data to the leader.""" 439 | 440 | pi = self.local_network 441 | while True: 442 | if debug: 443 | sys.stderr.write('[WORKER %d Ep %d] Starting...\n' % 444 | (self.task_idx, self.curr_ep)) 445 | 446 | # Reset local parameters to global 447 | self.sess.run(self.sync_op) 448 | 449 | print 'DaggerWorker:global_network_cpu:cnt', self.sess.run(self.global_network_cpu.cnt) 450 | print 'DaggerWorker:local_network:cnt', self.sess.run(self.local_network.cnt) 451 | sys.stdout.flush() 452 | 453 | # Start a single episode, populating state-action buffers. 454 | self.rollout() 455 | 456 | if debug: 457 | queue_size = self.sess.run(self.train_q.size()) 458 | sys.stderr.write( 459 | '[WORKER %d Ep %d]: enqueueing a sequence of data ' 460 | 'into queue of size %d\n' % 461 | (self.task_idx, self.curr_ep, queue_size)) 462 | 463 | # Enqueue a sequence of data into the training queue. 464 | self.sess.run(self.enqueue_train_op, feed_dict={ 465 | self.state_data: self.state_buf, 466 | self.action_data: self.action_buf}) 467 | self.sess.run(self.sync_q.enqueue(Status.EP_DONE)) 468 | 469 | if debug: 470 | queue_size = self.sess.run(self.train_q.size()) 471 | sys.stderr.write( 472 | '[WORKER %d Ep %d]: finished queueing data. ' 473 | 'queue size now %d\n' % 474 | (self.task_idx, self.curr_ep, queue_size)) 475 | 476 | if debug: 477 | sys.stderr.write('[WORKER %d Ep %d]: waiting for server\n' % 478 | (self.task_idx, self.curr_ep)) 479 | 480 | # Let the leader dequeue EP_DONE 481 | time.sleep(0.5) 482 | 483 | # Wait until pserver finishes training by blocking on sync_q 484 | # Only proceeds when it finds a message from the pserver. 485 | msg = self.sess.run(self.sync_q.dequeue()) 486 | while (msg != Status.WORKER_START and msg != Status.PS_DONE): 487 | self.sess.run(self.sync_q.enqueue(msg)) 488 | time.sleep(0.5) 489 | msg = self.sess.run(self.sync_q.dequeue()) 490 | 491 | if msg == Status.PS_DONE: 492 | break 493 | 494 | self.curr_ep += 1 495 | -------------------------------------------------------------------------------- /env/0.57mbps-poisson.trace: -------------------------------------------------------------------------------- 1 | 100 2 | 200 3 | 300 4 | 400 5 | 500 6 | 600 7 | 700 8 | 800 9 | 900 10 | 1000 11 | 1009 12 | 1015 13 | 1023 14 | 1054 15 | 1103 16 | 1115 17 | 1141 18 | 1161 19 | 1169 20 | 1207 21 | 1224 22 | 1250 23 | 1286 24 | 1287 25 | 1288 26 | 1302 27 | 1309 28 | 1311 29 | 1353 30 | 1362 31 | 1369 32 | 1392 33 | 1400 34 | 1426 35 | 1431 36 | 1431 37 | 1431 38 | 1440 39 | 1445 40 | 1488 41 | 1513 42 | 1529 43 | 1536 44 | 1546 45 | 1549 46 | 1564 47 | 1566 48 | 1570 49 | 1577 50 | 1616 51 | 1682 52 | 1685 53 | 1686 54 | 1697 55 | 1740 56 | 1747 57 | 1777 58 | 1783 59 | 1791 60 | 1822 61 | 1871 62 | 1879 63 | 1908 64 | 1931 65 | 1972 66 | 1973 67 | 1991 68 | 2077 69 | 2082 70 | 2091 71 | 2106 72 | 2115 73 | 2136 74 | 2138 75 | 2140 76 | 2158 77 | 2160 78 | 2182 79 | 2182 80 | 2189 81 | 2214 82 | 2223 83 | 2238 84 | 2258 85 | 2267 86 | 2295 87 | 2297 88 | 2315 89 | 2338 90 | 2341 91 | 2342 92 | 2354 93 | 2373 94 | 2415 95 | 2427 96 | 2435 97 | 2467 98 | 2481 99 | 2505 100 | 2521 101 | 2523 102 | 2524 103 | 2525 104 | 2548 105 | 2557 106 | 2563 107 | 2569 108 | 2576 109 | 2577 110 | 2605 111 | 2637 112 | 2648 113 | 2649 114 | 2651 115 | 2663 116 | 2666 117 | 2675 118 | 2681 119 | 2693 120 | 2695 121 | 2706 122 | 2787 123 | 2789 124 | 2848 125 | 2866 126 | 2874 127 | 2901 128 | 2907 129 | 2912 130 | 2968 131 | 2976 132 | 3001 133 | 3032 134 | 3034 135 | 3043 136 | 3064 137 | 3082 138 | 3118 139 | 3134 140 | 3151 141 | 3174 142 | 3206 143 | 3218 144 | 3231 145 | 3240 146 | 3259 147 | 3277 148 | 3286 149 | 3316 150 | 3320 151 | 3334 152 | 3374 153 | 3384 154 | 3388 155 | 3392 156 | 3409 157 | 3429 158 | 3435 159 | 3452 160 | 3487 161 | 3499 162 | 3512 163 | 3526 164 | 3576 165 | 3614 166 | 3633 167 | 3657 168 | 3717 169 | 3735 170 | 3790 171 | 3793 172 | 3793 173 | 3821 174 | 3839 175 | 3854 176 | 3898 177 | 3898 178 | 3917 179 | 3918 180 | 3921 181 | 3924 182 | 3951 183 | 4055 184 | 4106 185 | 4113 186 | 4117 187 | 4147 188 | 4151 189 | 4152 190 | 4172 191 | 4202 192 | 4277 193 | 4347 194 | 4374 195 | 4388 196 | 4399 197 | 4461 198 | 4469 199 | 4476 200 | 4477 201 | 4528 202 | 4568 203 | 4600 204 | 4643 205 | 4682 206 | 4686 207 | 4722 208 | 4779 209 | 4787 210 | 4855 211 | 4880 212 | 4887 213 | 4890 214 | 4902 215 | 4941 216 | 5039 217 | 5043 218 | 5092 219 | 5103 220 | 5129 221 | 5132 222 | 5167 223 | 5197 224 | 5199 225 | 5199 226 | 5254 227 | 5262 228 | 5292 229 | 5299 230 | 5318 231 | 5323 232 | 5326 233 | 5343 234 | 5362 235 | 5366 236 | 5366 237 | 5370 238 | 5372 239 | 5383 240 | 5417 241 | 5458 242 | 5463 243 | 5466 244 | 5471 245 | 5476 246 | 5476 247 | 5495 248 | 5521 249 | 5565 250 | 5577 251 | 5583 252 | 5598 253 | 5624 254 | 5631 255 | 5641 256 | 5665 257 | 5677 258 | 5679 259 | 5685 260 | 5721 261 | 5746 262 | 5773 263 | 5813 264 | 5858 265 | 5900 266 | 5914 267 | 5954 268 | 5980 269 | 5981 270 | 5999 271 | 6072 272 | 6088 273 | 6156 274 | 6177 275 | 6196 276 | 6199 277 | 6201 278 | 6211 279 | 6215 280 | 6218 281 | 6237 282 | 6239 283 | 6266 284 | 6277 285 | 6306 286 | 6374 287 | 6388 288 | 6392 289 | 6399 290 | 6419 291 | 6445 292 | 6476 293 | 6500 294 | 6507 295 | 6515 296 | 6520 297 | 6520 298 | 6544 299 | 6555 300 | 6565 301 | 6576 302 | 6577 303 | 6590 304 | 6633 305 | 6658 306 | 6681 307 | 6747 308 | 6779 309 | 6781 310 | 6812 311 | 6816 312 | 6819 313 | 6845 314 | 6871 315 | 6885 316 | 6896 317 | 6930 318 | 6947 319 | 6959 320 | 6966 321 | 6997 322 | 7009 323 | 7033 324 | 7046 325 | 7049 326 | 7054 327 | 7062 328 | 7121 329 | 7139 330 | 7159 331 | 7221 332 | 7221 333 | 7236 334 | 7257 335 | 7260 336 | 7265 337 | 7270 338 | 7346 339 | 7386 340 | 7392 341 | 7397 342 | 7430 343 | 7434 344 | 7461 345 | 7606 346 | 7607 347 | 7631 348 | 7636 349 | 7643 350 | 7691 351 | 7726 352 | 7736 353 | 7765 354 | 7792 355 | 7816 356 | 7828 357 | 7899 358 | 7915 359 | 7928 360 | 7928 361 | 7975 362 | 8012 363 | 8035 364 | 8085 365 | 8094 366 | 8109 367 | 8122 368 | 8122 369 | 8124 370 | 8129 371 | 8135 372 | 8140 373 | 8192 374 | 8192 375 | 8204 376 | 8229 377 | 8280 378 | 8287 379 | 8429 380 | 8509 381 | 8540 382 | 8555 383 | 8593 384 | 8625 385 | 8637 386 | 8660 387 | 8674 388 | 8676 389 | 8687 390 | 8691 391 | 8697 392 | 8715 393 | 8734 394 | 8745 395 | 8755 396 | 8776 397 | 8804 398 | 8851 399 | 8873 400 | 8887 401 | 8926 402 | 8943 403 | 8955 404 | 8987 405 | 8989 406 | 9026 407 | 9083 408 | 9109 409 | 9129 410 | 9141 411 | 9147 412 | 9163 413 | 9168 414 | 9168 415 | 9181 416 | 9190 417 | 9197 418 | 9208 419 | 9237 420 | 9250 421 | 9301 422 | 9342 423 | 9357 424 | 9368 425 | 9398 426 | 9401 427 | 9426 428 | 9437 429 | 9444 430 | 9450 431 | 9459 432 | 9481 433 | 9491 434 | 9503 435 | 9535 436 | 9573 437 | 9574 438 | 9576 439 | 9584 440 | 9584 441 | 9604 442 | 9662 443 | 9663 444 | 9682 445 | 9697 446 | 9741 447 | 9795 448 | 9805 449 | 9809 450 | 9835 451 | 9858 452 | 9877 453 | 9891 454 | 9905 455 | 9905 456 | 9905 457 | 9940 458 | 9954 459 | 9998 460 | 10021 461 | 10026 462 | 10076 463 | 10085 464 | 10088 465 | 10111 466 | 10127 467 | 10134 468 | 10139 469 | 10157 470 | 10179 471 | 10197 472 | 10242 473 | 10257 474 | 10319 475 | 10374 476 | 10395 477 | 10396 478 | 10420 479 | 10440 480 | 10457 481 | 10457 482 | 10479 483 | 10494 484 | 10498 485 | 10522 486 | 10592 487 | 10647 488 | 10670 489 | 10700 490 | 10705 491 | 10721 492 | 10765 493 | 10771 494 | 10788 495 | 10795 496 | 10799 497 | 10822 498 | 10840 499 | 10899 500 | 10932 501 | 10934 502 | 11000 503 | 11062 504 | 11072 505 | 11100 506 | 11145 507 | 11154 508 | 11207 509 | 11225 510 | 11230 511 | 11240 512 | 11250 513 | 11311 514 | 11352 515 | 11372 516 | 11378 517 | 11380 518 | 11388 519 | 11394 520 | 11402 521 | 11424 522 | 11436 523 | 11516 524 | 11517 525 | 11544 526 | 11546 527 | 11550 528 | 11562 529 | 11576 530 | 11630 531 | 11635 532 | 11670 533 | 11698 534 | 11780 535 | 11790 536 | 11807 537 | 11815 538 | 11837 539 | 11874 540 | 11892 541 | 11957 542 | 11978 543 | 11979 544 | 11980 545 | 11981 546 | 12015 547 | 12017 548 | 12045 549 | 12050 550 | 12080 551 | 12099 552 | 12109 553 | 12153 554 | 12187 555 | 12191 556 | 12219 557 | 12229 558 | 12242 559 | 12250 560 | 12250 561 | 12258 562 | 12296 563 | 12348 564 | 12473 565 | 12475 566 | 12480 567 | 12485 568 | 12507 569 | 12516 570 | 12528 571 | 12562 572 | 12600 573 | 12621 574 | 12631 575 | 12655 576 | 12677 577 | 12680 578 | 12693 579 | 12725 580 | 12733 581 | 12737 582 | 12738 583 | 12790 584 | 12791 585 | 12855 586 | 12869 587 | 12893 588 | 12927 589 | 12936 590 | 12946 591 | 12967 592 | 12995 593 | 13000 594 | 13027 595 | 13057 596 | 13091 597 | 13110 598 | 13144 599 | 13153 600 | 13166 601 | 13177 602 | 13190 603 | 13203 604 | 13241 605 | 13285 606 | 13289 607 | 13306 608 | 13309 609 | 13309 610 | 13335 611 | 13353 612 | 13408 613 | 13447 614 | 13465 615 | 13491 616 | 13540 617 | 13602 618 | 13606 619 | 13607 620 | 13630 621 | 13663 622 | 13683 623 | 13692 624 | 13694 625 | 13699 626 | 13724 627 | 13734 628 | 13763 629 | 13765 630 | 13820 631 | 13821 632 | 13838 633 | 13888 634 | 13896 635 | 13900 636 | 13910 637 | 13952 638 | 13971 639 | 14018 640 | 14021 641 | 14031 642 | 14042 643 | 14093 644 | 14096 645 | 14143 646 | 14146 647 | 14181 648 | 14188 649 | 14198 650 | 14220 651 | 14253 652 | 14256 653 | 14268 654 | 14307 655 | 14324 656 | 14348 657 | 14370 658 | 14380 659 | 14468 660 | 14508 661 | 14510 662 | 14536 663 | 14558 664 | 14558 665 | 14563 666 | 14574 667 | 14628 668 | 14644 669 | 14655 670 | 14680 671 | 14689 672 | 14699 673 | 14726 674 | 14731 675 | 14734 676 | 14779 677 | 14802 678 | 14881 679 | 14884 680 | 14896 681 | 14899 682 | 14915 683 | 14934 684 | 14973 685 | 14977 686 | 14979 687 | 15016 688 | 15018 689 | 15035 690 | 15108 691 | 15195 692 | 15217 693 | 15233 694 | 15239 695 | 15243 696 | 15250 697 | 15266 698 | 15313 699 | 15364 700 | 15369 701 | 15402 702 | 15403 703 | 15409 704 | 15415 705 | 15489 706 | 15501 707 | 15503 708 | 15507 709 | 15534 710 | 15545 711 | 15558 712 | 15591 713 | 15622 714 | 15623 715 | 15632 716 | 15674 717 | 15693 718 | 15695 719 | 15717 720 | 15782 721 | 15802 722 | 15817 723 | 15842 724 | 15899 725 | 16021 726 | 16032 727 | 16093 728 | 16102 729 | 16111 730 | 16167 731 | 16199 732 | 16209 733 | 16213 734 | 16267 735 | 16289 736 | 16290 737 | 16309 738 | 16333 739 | 16355 740 | 16364 741 | 16366 742 | 16387 743 | 16389 744 | 16396 745 | 16467 746 | 16564 747 | 16582 748 | 16605 749 | 16635 750 | 16639 751 | 16700 752 | 16727 753 | 16746 754 | 16764 755 | 16793 756 | 16799 757 | 16819 758 | 16822 759 | 16827 760 | 16861 761 | 16865 762 | 16884 763 | 16901 764 | 16935 765 | 16941 766 | 16948 767 | 16953 768 | 16958 769 | 16991 770 | 17030 771 | 17033 772 | 17051 773 | 17062 774 | 17075 775 | 17090 776 | 17144 777 | 17154 778 | 17157 779 | 17185 780 | 17195 781 | 17202 782 | 17221 783 | 17239 784 | 17247 785 | 17316 786 | 17341 787 | 17349 788 | 17363 789 | 17446 790 | 17475 791 | 17480 792 | 17522 793 | 17535 794 | 17560 795 | 17569 796 | 17645 797 | 17649 798 | 17668 799 | 17669 800 | 17703 801 | 17706 802 | 17747 803 | 17763 804 | 17770 805 | 17798 806 | 17819 807 | 17827 808 | 17837 809 | 17850 810 | 17856 811 | 17959 812 | 17992 813 | 18020 814 | 18026 815 | 18059 816 | 18063 817 | 18073 818 | 18116 819 | 18130 820 | 18137 821 | 18139 822 | 18140 823 | 18160 824 | 18223 825 | 18273 826 | 18297 827 | 18319 828 | 18337 829 | 18342 830 | 18344 831 | 18360 832 | 18390 833 | 18436 834 | 18440 835 | 18496 836 | 18505 837 | 18551 838 | 18610 839 | 18659 840 | 18662 841 | 18664 842 | 18671 843 | 18688 844 | 18710 845 | 18735 846 | 18754 847 | 18772 848 | 18814 849 | 18824 850 | 18940 851 | 18968 852 | 18972 853 | 18977 854 | 18998 855 | 19026 856 | 19047 857 | 19081 858 | 19090 859 | 19105 860 | 19132 861 | 19149 862 | 19181 863 | 19228 864 | 19246 865 | 19268 866 | 19270 867 | 19282 868 | 19288 869 | 19323 870 | 19345 871 | 19411 872 | 19437 873 | 19440 874 | 19459 875 | 19468 876 | 19479 877 | 19489 878 | 19502 879 | 19520 880 | 19534 881 | 19577 882 | 19579 883 | 19602 884 | 19619 885 | 19690 886 | 19717 887 | 19727 888 | 19732 889 | 19759 890 | 19789 891 | 19803 892 | 19864 893 | 19873 894 | 20007 895 | 20016 896 | 20059 897 | 20066 898 | 20101 899 | 20148 900 | 20153 901 | 20255 902 | 20262 903 | 20275 904 | 20297 905 | 20302 906 | 20307 907 | 20309 908 | 20324 909 | 20347 910 | 20406 911 | 20406 912 | 20412 913 | 20421 914 | 20433 915 | 20504 916 | 20545 917 | 20573 918 | 20589 919 | 20607 920 | 20638 921 | 20640 922 | 20659 923 | 20679 924 | 20682 925 | 20707 926 | 20711 927 | 20714 928 | 20728 929 | 20761 930 | 20791 931 | 20812 932 | 20862 933 | 20862 934 | 20890 935 | 20912 936 | 20949 937 | 20960 938 | 20961 939 | 20974 940 | 20979 941 | 20990 942 | 21022 943 | 21040 944 | 21078 945 | 21084 946 | 21122 947 | 21171 948 | 21178 949 | 21215 950 | 21258 951 | 21278 952 | 21295 953 | 21299 954 | 21321 955 | 21331 956 | 21342 957 | 21390 958 | 21415 959 | 21434 960 | 21454 961 | 21474 962 | 21477 963 | 21496 964 | 21503 965 | 21526 966 | 21606 967 | 21657 968 | 21659 969 | 21661 970 | 21700 971 | 21706 972 | 21707 973 | 21715 974 | 21720 975 | 21721 976 | 21727 977 | 21788 978 | 21833 979 | 21928 980 | 21955 981 | 21972 982 | 21973 983 | 21984 984 | 22003 985 | 22068 986 | 22112 987 | 22124 988 | 22138 989 | 22139 990 | 22173 991 | 22183 992 | 22185 993 | 22196 994 | 22198 995 | 22218 996 | 22241 997 | 22242 998 | 22252 999 | 22293 1000 | 22312 1001 | 22332 1002 | 22373 1003 | 22388 1004 | 22416 1005 | 22426 1006 | 22470 1007 | 22484 1008 | 22494 1009 | 22504 1010 | 22530 1011 | 22542 1012 | 22565 1013 | 22566 1014 | 22567 1015 | 22567 1016 | 22576 1017 | 22581 1018 | 22608 1019 | 22614 1020 | 22618 1021 | 22624 1022 | 22642 1023 | 22700 1024 | 22713 1025 | 22724 1026 | 22728 1027 | 22737 1028 | 22781 1029 | 22783 1030 | 22876 1031 | 22893 1032 | 22896 1033 | 22902 1034 | 22908 1035 | 22926 1036 | 22935 1037 | 22940 1038 | 22945 1039 | 22975 1040 | 23013 1041 | 23077 1042 | 23081 1043 | 23081 1044 | 23097 1045 | 23146 1046 | 23159 1047 | 23187 1048 | 23250 1049 | 23273 1050 | 23279 1051 | 23281 1052 | 23314 1053 | 23342 1054 | 23344 1055 | 23347 1056 | 23352 1057 | 23355 1058 | 23359 1059 | 23362 1060 | 23362 1061 | 23402 1062 | 23419 1063 | 23439 1064 | 23440 1065 | 23479 1066 | 23520 1067 | 23521 1068 | 23532 1069 | 23533 1070 | 23548 1071 | 23562 1072 | 23592 1073 | 23610 1074 | 23629 1075 | 23645 1076 | 23649 1077 | 23652 1078 | 23676 1079 | 23703 1080 | 23733 1081 | 23758 1082 | 23836 1083 | 23866 1084 | 23885 1085 | 23897 1086 | 23918 1087 | 23926 1088 | 23941 1089 | 23942 1090 | 23945 1091 | 23974 1092 | 23982 1093 | 24048 1094 | 24066 1095 | 24080 1096 | 24082 1097 | 24102 1098 | 24104 1099 | 24106 1100 | 24112 1101 | 24125 1102 | 24127 1103 | 24145 1104 | 24168 1105 | 24199 1106 | 24215 1107 | 24262 1108 | 24264 1109 | 24266 1110 | 24283 1111 | 24289 1112 | 24290 1113 | 24345 1114 | 24364 1115 | 24377 1116 | 24399 1117 | 24418 1118 | 24428 1119 | 24464 1120 | 24478 1121 | 24555 1122 | 24576 1123 | 24594 1124 | 24601 1125 | 24604 1126 | 24609 1127 | 24610 1128 | 24618 1129 | 24629 1130 | 24629 1131 | 24676 1132 | 24687 1133 | 24719 1134 | 24726 1135 | 24741 1136 | 24757 1137 | 24790 1138 | 24796 1139 | 24894 1140 | 24896 1141 | 24902 1142 | 24924 1143 | 24977 1144 | 24996 1145 | 25006 1146 | 25132 1147 | 25157 1148 | 25231 1149 | 25232 1150 | 25238 1151 | 25240 1152 | 25258 1153 | 25264 1154 | 25296 1155 | 25306 1156 | 25315 1157 | 25321 1158 | 25329 1159 | 25345 1160 | 25352 1161 | 25390 1162 | 25406 1163 | 25407 1164 | 25444 1165 | 25449 1166 | 25477 1167 | 25549 1168 | 25549 1169 | 25579 1170 | 25579 1171 | 25586 1172 | 25589 1173 | 25608 1174 | 25621 1175 | 25632 1176 | 25637 1177 | 25641 1178 | 25645 1179 | 25653 1180 | 25661 1181 | 25666 1182 | 25677 1183 | 25719 1184 | 25724 1185 | 25732 1186 | 25750 1187 | 25821 1188 | 25860 1189 | 25896 1190 | 25909 1191 | 25926 1192 | 25944 1193 | 25949 1194 | 25955 1195 | 26004 1196 | 26054 1197 | 26062 1198 | 26112 1199 | 26113 1200 | 26150 1201 | 26190 1202 | 26217 1203 | 26243 1204 | 26256 1205 | 26268 1206 | 26277 1207 | 26301 1208 | 26349 1209 | 26433 1210 | 26510 1211 | 26512 1212 | 26549 1213 | 26557 1214 | 26568 1215 | 26570 1216 | 26589 1217 | 26598 1218 | 26616 1219 | 26630 1220 | 26638 1221 | 26655 1222 | 26659 1223 | 26672 1224 | 26681 1225 | 26698 1226 | 26719 1227 | 26751 1228 | 26803 1229 | 26814 1230 | 26815 1231 | 26821 1232 | 26832 1233 | 26843 1234 | 26859 1235 | 26905 1236 | 26918 1237 | 26943 1238 | 26952 1239 | 27054 1240 | 27059 1241 | 27075 1242 | 27085 1243 | 27089 1244 | 27109 1245 | 27129 1246 | 27138 1247 | 27160 1248 | 27175 1249 | 27180 1250 | 27220 1251 | 27245 1252 | 27338 1253 | 27364 1254 | 27371 1255 | 27381 1256 | 27400 1257 | 27521 1258 | 27524 1259 | 27536 1260 | 27548 1261 | 27552 1262 | 27554 1263 | 27570 1264 | 27586 1265 | 27612 1266 | 27615 1267 | 27654 1268 | 27656 1269 | 27678 1270 | 27706 1271 | 27734 1272 | 27744 1273 | 27751 1274 | 27788 1275 | 27832 1276 | 27849 1277 | 27859 1278 | 27862 1279 | 27901 1280 | 27919 1281 | 27946 1282 | 27965 1283 | 27983 1284 | 27988 1285 | 28082 1286 | 28089 1287 | 28141 1288 | 28229 1289 | 28231 1290 | 28257 1291 | 28271 1292 | 28279 1293 | 28314 1294 | 28314 1295 | 28365 1296 | 28369 1297 | 28403 1298 | 28429 1299 | 28439 1300 | 28439 1301 | 28505 1302 | 28521 1303 | 28583 1304 | 28598 1305 | 28615 1306 | 28624 1307 | 28628 1308 | 28637 1309 | 28656 1310 | 28676 1311 | 28689 1312 | 28695 1313 | 28695 1314 | 28717 1315 | 28718 1316 | 28726 1317 | 28756 1318 | 28758 1319 | 28803 1320 | 28807 1321 | 28808 1322 | 28878 1323 | 28941 1324 | 28946 1325 | 28955 1326 | 28966 1327 | 28981 1328 | 29021 1329 | 29068 1330 | 29161 1331 | 29198 1332 | 29246 1333 | 29265 1334 | 29295 1335 | 29298 1336 | 29312 1337 | 29336 1338 | 29379 1339 | 29409 1340 | 29423 1341 | 29445 1342 | 29459 1343 | 29464 1344 | 29486 1345 | 29487 1346 | 29489 1347 | 29513 1348 | 29532 1349 | 29581 1350 | 29587 1351 | 29591 1352 | 29603 1353 | 29614 1354 | 29615 1355 | 29644 1356 | 29644 1357 | 29711 1358 | 29714 1359 | 29718 1360 | 29744 1361 | 29744 1362 | 29767 1363 | 29771 1364 | 29783 1365 | 29787 1366 | 29790 1367 | 29792 1368 | 29819 1369 | 29838 1370 | 29860 1371 | 29916 1372 | 29963 1373 | 29965 1374 | 29986 1375 | 30006 1376 | 30012 1377 | 30040 1378 | 30043 1379 | 30079 1380 | 30089 1381 | 30122 1382 | 30126 1383 | 30126 1384 | 30159 1385 | 30172 1386 | 30173 1387 | 30185 1388 | 30200 1389 | 30255 1390 | 30270 1391 | 30304 1392 | 30307 1393 | 30309 1394 | 30374 1395 | 30391 1396 | 30394 1397 | 30414 1398 | 30425 1399 | 30431 1400 | 30496 1401 | 30518 1402 | 30533 1403 | 30547 1404 | 30548 1405 | 30565 1406 | 30575 1407 | 30581 1408 | 30587 1409 | 30627 1410 | 30636 1411 | 30659 1412 | 30747 1413 | 30768 1414 | 30787 1415 | 30800 1416 | 30841 1417 | 30852 1418 | 30895 1419 | 30940 1420 | 30953 1421 | 30970 1422 | 30974 1423 | 31016 1424 | 31056 1425 | 31078 1426 | 31086 1427 | 31097 1428 | 31098 1429 | 31105 1430 | 31143 1431 | 31181 1432 | 31212 1433 | 31219 1434 | 31281 1435 | 31288 1436 | 31319 1437 | 31342 1438 | 31363 1439 | 31382 1440 | 31393 1441 | 31407 1442 | 31423 1443 | 31438 1444 | 31488 1445 | 31502 1446 | 31606 1447 | 31613 1448 | 31643 1449 | 31689 1450 | 31739 1451 | 31778 1452 | 31794 1453 | 31797 1454 | 31846 1455 | 31909 1456 | 31944 1457 | 31976 1458 | 31994 1459 | 32008 1460 | 32015 1461 | 32019 1462 | 32020 1463 | 32076 1464 | 32083 1465 | 32100 1466 | 32130 1467 | 32131 1468 | 32139 1469 | 32139 1470 | 32140 1471 | 32170 1472 | 32188 1473 | 32190 1474 | 32206 1475 | 32247 1476 | 32269 1477 | 32287 1478 | 32296 1479 | 32300 1480 | 32312 1481 | 32320 1482 | 32336 1483 | 32398 1484 | 32401 1485 | 32439 1486 | 32441 1487 | 32478 1488 | 32512 1489 | 32520 1490 | 32535 1491 | 32546 1492 | 32593 1493 | 32594 1494 | 32615 1495 | 32625 1496 | 32625 1497 | 32626 1498 | 32628 1499 | 32663 1500 | 32731 1501 | 32747 1502 | 32752 1503 | 32764 1504 | 32814 1505 | 32824 1506 | 32845 1507 | 32862 1508 | 32878 1509 | 32893 1510 | 32895 1511 | 32920 1512 | 32948 1513 | 32981 1514 | 32986 1515 | 33012 1516 | 33048 1517 | 33079 1518 | 33108 1519 | 33128 1520 | 33207 1521 | 33236 1522 | 33260 1523 | 33292 1524 | 33295 1525 | 33302 1526 | 33340 1527 | 33343 1528 | 33347 1529 | 33356 1530 | 33415 1531 | 33421 1532 | 33425 1533 | 33434 1534 | 33452 1535 | 33458 1536 | 33460 1537 | 33465 1538 | 33578 1539 | 33579 1540 | 33656 1541 | 33658 1542 | 33662 1543 | 33693 1544 | 33711 1545 | 33728 1546 | 33729 1547 | 33732 1548 | 33749 1549 | 33758 1550 | 33764 1551 | 33772 1552 | 33786 1553 | 33789 1554 | 33802 1555 | 33820 1556 | 33824 1557 | 33824 1558 | 33861 1559 | 33875 1560 | 33898 1561 | 33901 1562 | 33925 1563 | 33957 1564 | 33958 1565 | 33964 1566 | 33969 1567 | 34028 1568 | 34030 1569 | 34032 1570 | 34051 1571 | 34063 1572 | 34087 1573 | 34107 1574 | 34128 1575 | 34164 1576 | 34190 1577 | 34203 1578 | 34259 1579 | 34259 1580 | 34267 1581 | 34269 1582 | 34322 1583 | 34359 1584 | 34376 1585 | 34459 1586 | 34526 1587 | 34535 1588 | 34557 1589 | 34568 1590 | 34579 1591 | 34637 1592 | 34639 1593 | 34686 1594 | 34686 1595 | 34689 1596 | 34692 1597 | 34723 1598 | 34754 1599 | 34795 1600 | 34813 1601 | 34819 1602 | 34822 1603 | 34826 1604 | 34851 1605 | 34855 1606 | 34893 1607 | 34965 1608 | 35030 1609 | 35039 1610 | 35039 1611 | 35064 1612 | 35091 1613 | 35096 1614 | 35099 1615 | 35153 1616 | 35160 1617 | 35208 1618 | 35212 1619 | 35223 1620 | 35291 1621 | 35304 1622 | 35343 1623 | 35365 1624 | 35397 1625 | 35401 1626 | 35431 1627 | 35433 1628 | 35450 1629 | 35487 1630 | 35490 1631 | 35504 1632 | 35512 1633 | 35556 1634 | 35569 1635 | 35575 1636 | 35599 1637 | 35629 1638 | 35640 1639 | 35666 1640 | 35714 1641 | 35717 1642 | 35721 1643 | 35738 1644 | 35741 1645 | 35754 1646 | 35772 1647 | 35814 1648 | 35831 1649 | 35848 1650 | 35850 1651 | 35874 1652 | 35895 1653 | 35903 1654 | 35935 1655 | 35949 1656 | 35963 1657 | 35983 1658 | 35984 1659 | 35996 1660 | 36009 1661 | 36027 1662 | 36033 1663 | 36064 1664 | 36065 1665 | 36070 1666 | 36089 1667 | 36091 1668 | 36091 1669 | 36148 1670 | 36163 1671 | 36223 1672 | 36234 1673 | 36312 1674 | 36316 1675 | 36371 1676 | 36386 1677 | 36431 1678 | 36481 1679 | 36512 1680 | 36538 1681 | 36578 1682 | 36629 1683 | 36629 1684 | 36639 1685 | 36640 1686 | 36641 1687 | 36654 1688 | 36712 1689 | 36747 1690 | 36755 1691 | 36769 1692 | 36817 1693 | 36861 1694 | 36874 1695 | 36916 1696 | 36981 1697 | 36992 1698 | 37001 1699 | 37016 1700 | 37046 1701 | 37117 1702 | 37117 1703 | 37118 1704 | 37138 1705 | 37156 1706 | 37161 1707 | 37161 1708 | 37189 1709 | 37232 1710 | 37286 1711 | 37308 1712 | 37316 1713 | 37341 1714 | 37400 1715 | 37409 1716 | 37417 1717 | 37468 1718 | 37537 1719 | 37544 1720 | 37547 1721 | 37549 1722 | 37560 1723 | 37577 1724 | 37597 1725 | 37606 1726 | 37607 1727 | 37618 1728 | 37619 1729 | 37644 1730 | 37656 1731 | 37657 1732 | 37696 1733 | 37717 1734 | 37765 1735 | 37796 1736 | 37806 1737 | 37820 1738 | 37859 1739 | 37888 1740 | 37890 1741 | 37890 1742 | 37918 1743 | 37933 1744 | 37951 1745 | 37993 1746 | 38002 1747 | 38003 1748 | 38047 1749 | 38098 1750 | 38112 1751 | 38121 1752 | 38138 1753 | 38173 1754 | 38186 1755 | 38203 1756 | 38208 1757 | 38301 1758 | 38305 1759 | 38316 1760 | 38358 1761 | 38390 1762 | 38407 1763 | 38416 1764 | 38430 1765 | 38435 1766 | 38467 1767 | 38515 1768 | 38529 1769 | 38550 1770 | 38558 1771 | 38592 1772 | 38635 1773 | 38636 1774 | 38677 1775 | 38698 1776 | 38715 1777 | 38736 1778 | 38773 1779 | 38798 1780 | 38814 1781 | 38822 1782 | 38867 1783 | 38909 1784 | 38914 1785 | 38945 1786 | 38991 1787 | 39018 1788 | 39022 1789 | 39146 1790 | 39171 1791 | 39227 1792 | 39227 1793 | 39259 1794 | 39308 1795 | 39350 1796 | 39381 1797 | 39408 1798 | 39412 1799 | 39505 1800 | 39539 1801 | 39549 1802 | 39556 1803 | 39564 1804 | 39568 1805 | 39608 1806 | 39649 1807 | 39650 1808 | 39662 1809 | 39680 1810 | 39692 1811 | 39699 1812 | 39722 1813 | 39766 1814 | 39786 1815 | 39791 1816 | 39804 1817 | 39837 1818 | 39919 1819 | 39960 1820 | 39963 1821 | 39965 1822 | 40010 1823 | 40012 1824 | 40016 1825 | 40027 1826 | 40037 1827 | 40049 1828 | 40050 1829 | 40065 1830 | 40104 1831 | 40106 1832 | 40169 1833 | 40172 1834 | 40218 1835 | 40223 1836 | 40333 1837 | 40333 1838 | 40338 1839 | 40355 1840 | 40386 1841 | 40395 1842 | 40402 1843 | 40408 1844 | 40430 1845 | 40502 1846 | 40548 1847 | 40550 1848 | 40571 1849 | 40583 1850 | 40607 1851 | 40680 1852 | 40692 1853 | 40697 1854 | 40729 1855 | 40743 1856 | 40763 1857 | 40768 1858 | 40813 1859 | 40817 1860 | 40839 1861 | 40846 1862 | 40855 1863 | 40859 1864 | 40901 1865 | 40907 1866 | 40917 1867 | 40928 1868 | 40949 1869 | 40959 1870 | 40974 1871 | 40997 1872 | 41005 1873 | 41014 1874 | 41016 1875 | 41054 1876 | 41070 1877 | 41093 1878 | 41108 1879 | 41108 1880 | 41123 1881 | 41124 1882 | 41128 1883 | 41143 1884 | 41173 1885 | 41212 1886 | 41248 1887 | 41289 1888 | 41308 1889 | 41319 1890 | 41322 1891 | 41350 1892 | 41361 1893 | 41366 1894 | 41404 1895 | 41414 1896 | 41423 1897 | 41426 1898 | 41434 1899 | 41442 1900 | 41447 1901 | 41447 1902 | 41450 1903 | 41484 1904 | 41541 1905 | 41542 1906 | 41570 1907 | 41622 1908 | 41626 1909 | 41635 1910 | 41711 1911 | 41745 1912 | 41747 1913 | 41816 1914 | 41834 1915 | 41834 1916 | 41863 1917 | 41867 1918 | 41885 1919 | 41911 1920 | 41916 1921 | 41929 1922 | 41958 1923 | 41970 1924 | 42000 1925 | 42007 1926 | 42038 1927 | 42107 1928 | 42126 1929 | 42138 1930 | 42159 1931 | 42170 1932 | 42171 1933 | 42176 1934 | 42215 1935 | 42226 1936 | 42266 1937 | 42282 1938 | 42288 1939 | 42292 1940 | 42320 1941 | 42322 1942 | 42353 1943 | 42370 1944 | 42374 1945 | 42435 1946 | 42448 1947 | 42463 1948 | 42500 1949 | 42516 1950 | 42560 1951 | 42625 1952 | 42629 1953 | 42642 1954 | 42652 1955 | 42658 1956 | 42681 1957 | 42689 1958 | 42788 1959 | 42808 1960 | 42831 1961 | 42832 1962 | 42837 1963 | 42854 1964 | 42863 1965 | 42896 1966 | 42905 1967 | 42928 1968 | 42940 1969 | 42942 1970 | 42964 1971 | 42966 1972 | 43007 1973 | 43034 1974 | 43053 1975 | 43082 1976 | 43088 1977 | 43092 1978 | 43112 1979 | 43123 1980 | 43146 1981 | 43158 1982 | 43175 1983 | 43187 1984 | 43214 1985 | 43215 1986 | 43224 1987 | 43231 1988 | 43254 1989 | 43259 1990 | 43268 1991 | 43326 1992 | 43330 1993 | 43342 1994 | 43373 1995 | 43396 1996 | 43401 1997 | 43404 1998 | 43414 1999 | 43423 2000 | 43445 2001 | 43450 2002 | 43481 2003 | 43496 2004 | 43537 2005 | 43560 2006 | 43566 2007 | 43592 2008 | 43612 2009 | 43626 2010 | 43671 2011 | 43679 2012 | 43686 2013 | 43712 2014 | 43724 2015 | 43748 2016 | 43758 2017 | 43830 2018 | 43843 2019 | 43852 2020 | 43854 2021 | 43872 2022 | 43949 2023 | 43981 2024 | 43998 2025 | 44000 2026 | 44030 2027 | 44031 2028 | 44060 2029 | 44060 2030 | 44087 2031 | 44141 2032 | 44157 2033 | 44173 2034 | 44191 2035 | 44220 2036 | 44220 2037 | 44232 2038 | 44245 2039 | 44255 2040 | 44273 2041 | 44283 2042 | 44295 2043 | 44300 2044 | 44320 2045 | 44346 2046 | 44352 2047 | 44364 2048 | 44398 2049 | 44414 2050 | 44432 2051 | 44440 2052 | 44446 2053 | 44482 2054 | 44491 2055 | 44506 2056 | 44517 2057 | 44526 2058 | 44599 2059 | 44662 2060 | 44694 2061 | 44709 2062 | 44852 2063 | 44892 2064 | 44893 2065 | 44913 2066 | 44934 2067 | 44998 2068 | 45011 2069 | 45021 2070 | 45028 2071 | 45038 2072 | 45082 2073 | 45092 2074 | 45156 2075 | 45194 2076 | 45195 2077 | 45216 2078 | 45241 2079 | 45260 2080 | 45267 2081 | 45289 2082 | 45313 2083 | 45326 2084 | 45332 2085 | 45334 2086 | 45346 2087 | 45366 2088 | 45412 2089 | 45431 2090 | 45433 2091 | 45442 2092 | 45468 2093 | 45478 2094 | 45519 2095 | 45540 2096 | 45548 2097 | 45553 2098 | 45579 2099 | 45607 2100 | 45617 2101 | 45706 2102 | 45728 2103 | 45729 2104 | 45759 2105 | 45783 2106 | 45834 2107 | 45864 2108 | 45865 2109 | 45901 2110 | 45931 2111 | 46018 2112 | 46026 2113 | 46107 2114 | 46131 2115 | 46185 2116 | 46186 2117 | 46197 2118 | 46214 2119 | 46227 2120 | 46249 2121 | 46252 2122 | 46299 2123 | 46304 2124 | 46305 2125 | 46311 2126 | 46360 2127 | 46366 2128 | 46380 2129 | 46395 2130 | 46401 2131 | 46403 2132 | 46410 2133 | 46432 2134 | 46500 2135 | 46507 2136 | 46536 2137 | 46563 2138 | 46575 2139 | 46641 2140 | 46661 2141 | 46670 2142 | 46678 2143 | 46691 2144 | 46704 2145 | 46726 2146 | 46742 2147 | 46749 2148 | 46757 2149 | 46767 2150 | 46791 2151 | 46797 2152 | 46800 2153 | 46812 2154 | 46887 2155 | 46936 2156 | 46947 2157 | 46949 2158 | 46960 2159 | 46962 2160 | 47026 2161 | 47031 2162 | 47041 2163 | 47092 2164 | 47127 2165 | 47137 2166 | 47139 2167 | 47167 2168 | 47172 2169 | 47195 2170 | 47212 2171 | 47215 2172 | 47290 2173 | 47292 2174 | 47305 2175 | 47309 2176 | 47344 2177 | 47346 2178 | 47350 2179 | 47365 2180 | 47376 2181 | 47381 2182 | 47398 2183 | 47401 2184 | 47429 2185 | 47463 2186 | 47485 2187 | 47491 2188 | 47509 2189 | 47531 2190 | 47569 2191 | 47581 2192 | 47586 2193 | 47610 2194 | 47612 2195 | 47627 2196 | 47648 2197 | 47687 2198 | 47689 2199 | 47690 2200 | 47706 2201 | 47710 2202 | 47729 2203 | 47773 2204 | 47781 2205 | 47804 2206 | 47824 2207 | 47898 2208 | 47904 2209 | 47905 2210 | 47913 2211 | 47940 2212 | 47958 2213 | 47985 2214 | 47985 2215 | 47989 2216 | 47990 2217 | 48006 2218 | 48048 2219 | 48073 2220 | 48084 2221 | 48092 2222 | 48129 2223 | 48192 2224 | 48193 2225 | 48200 2226 | 48234 2227 | 48251 2228 | 48257 2229 | 48266 2230 | 48272 2231 | 48280 2232 | 48286 2233 | 48386 2234 | 48392 2235 | 48393 2236 | 48407 2237 | 48420 2238 | 48461 2239 | 48464 2240 | 48464 2241 | 48466 2242 | 48498 2243 | 48526 2244 | 48558 2245 | 48560 2246 | 48562 2247 | 48573 2248 | 48646 2249 | 48655 2250 | 48717 2251 | 48727 2252 | 48733 2253 | 48747 2254 | 48767 2255 | 48783 2256 | 48812 2257 | 48837 2258 | 48841 2259 | 48847 2260 | 48871 2261 | 48884 2262 | 48900 2263 | 48947 2264 | 48948 2265 | 48988 2266 | 49000 2267 | 49015 2268 | 49018 2269 | 49022 2270 | 49028 2271 | 49038 2272 | 49049 2273 | 49099 2274 | 49102 2275 | 49140 2276 | 49206 2277 | 49279 2278 | 49303 2279 | 49305 2280 | 49341 2281 | 49374 2282 | 49403 2283 | 49405 2284 | 49447 2285 | 49553 2286 | 49578 2287 | 49606 2288 | 49607 2289 | 49629 2290 | 49653 2291 | 49656 2292 | 49656 2293 | 49694 2294 | 49699 2295 | 49701 2296 | 49720 2297 | 49747 2298 | 49756 2299 | 49795 2300 | 49798 2301 | 49800 2302 | 49801 2303 | 49860 2304 | 49903 2305 | 49922 2306 | 49943 2307 | 49951 2308 | 49965 2309 | 49966 2310 | 49968 2311 | 49973 2312 | 50016 2313 | 50029 2314 | 50070 2315 | 50112 2316 | 50127 2317 | 50154 2318 | 50169 2319 | 50172 2320 | 50184 2321 | 50206 2322 | 50222 2323 | 50260 2324 | 50263 2325 | 50280 2326 | 50324 2327 | 50332 2328 | 50334 2329 | 50335 2330 | 50363 2331 | 50370 2332 | 50382 2333 | 50382 2334 | 50384 2335 | 50397 2336 | 50424 2337 | 50444 2338 | 50475 2339 | 50482 2340 | 50482 2341 | 50555 2342 | 50555 2343 | 50622 2344 | 50682 2345 | 50714 2346 | 50745 2347 | 50791 2348 | 50797 2349 | 50816 2350 | 50887 2351 | 50923 2352 | 50936 2353 | 50951 2354 | 50952 2355 | 50986 2356 | 50989 2357 | 51006 2358 | 51008 2359 | 51060 2360 | 51080 2361 | 51087 2362 | 51108 2363 | 51135 2364 | 51156 2365 | 51201 2366 | 51205 2367 | 51209 2368 | 51269 2369 | 51274 2370 | 51278 2371 | 51294 2372 | 51325 2373 | 51350 2374 | 51380 2375 | 51381 2376 | 51394 2377 | 51419 2378 | 51426 2379 | 51432 2380 | 51432 2381 | 51433 2382 | 51450 2383 | 51482 2384 | 51491 2385 | 51505 2386 | 51508 2387 | 51515 2388 | 51516 2389 | 51524 2390 | 51536 2391 | 51547 2392 | 51548 2393 | 51565 2394 | 51599 2395 | 51602 2396 | 51603 2397 | 51613 2398 | 51618 2399 | 51632 2400 | 51665 2401 | 51779 2402 | 51779 2403 | 51797 2404 | 51811 2405 | 51823 2406 | 51829 2407 | 51855 2408 | 51881 2409 | 51885 2410 | 51893 2411 | 51933 2412 | 51934 2413 | 51954 2414 | 51967 2415 | 51995 2416 | 52000 2417 | 52012 2418 | 52018 2419 | 52034 2420 | 52059 2421 | 52062 2422 | 52079 2423 | 52087 2424 | 52096 2425 | 52109 2426 | 52139 2427 | 52140 2428 | 52165 2429 | 52213 2430 | 52229 2431 | 52269 2432 | 52291 2433 | 52329 2434 | 52350 2435 | 52365 2436 | 52367 2437 | 52370 2438 | 52399 2439 | 52480 2440 | 52483 2441 | 52523 2442 | 52525 2443 | 52545 2444 | 52553 2445 | 52569 2446 | 52654 2447 | 52659 2448 | 52673 2449 | 52679 2450 | 52695 2451 | 52707 2452 | 52785 2453 | 52804 2454 | 52812 2455 | 52822 2456 | 52826 2457 | 52828 2458 | 52831 2459 | 52841 2460 | 52848 2461 | 52880 2462 | 52886 2463 | 52893 2464 | 52920 2465 | 52926 2466 | 52978 2467 | 52986 2468 | 52996 2469 | 53002 2470 | 53036 2471 | 53042 2472 | 53061 2473 | 53112 2474 | 53120 2475 | 53157 2476 | 53167 2477 | 53176 2478 | 53191 2479 | 53220 2480 | 53247 2481 | 53255 2482 | 53278 2483 | 53284 2484 | 53294 2485 | 53357 2486 | 53371 2487 | 53447 2488 | 53463 2489 | 53479 2490 | 53504 2491 | 53543 2492 | 53559 2493 | 53594 2494 | 53676 2495 | 53695 2496 | 53716 2497 | 53748 2498 | 53751 2499 | 53757 2500 | 53779 2501 | 53812 2502 | 53823 2503 | 53834 2504 | 53837 2505 | 53851 2506 | 53893 2507 | 53905 2508 | 53984 2509 | 54051 2510 | 54073 2511 | 54122 2512 | 54122 2513 | 54130 2514 | 54137 2515 | 54164 2516 | 54167 2517 | 54174 2518 | 54189 2519 | 54207 2520 | 54224 2521 | 54240 2522 | 54259 2523 | 54260 2524 | 54357 2525 | 54358 2526 | 54376 2527 | 54378 2528 | 54395 2529 | 54414 2530 | 54444 2531 | 54449 2532 | 54475 2533 | 54567 2534 | 54713 2535 | 54734 2536 | 54745 2537 | 54858 2538 | 54919 2539 | 54935 2540 | 54967 2541 | 54974 2542 | 54982 2543 | 55028 2544 | 55046 2545 | 55080 2546 | 55083 2547 | 55084 2548 | 55087 2549 | 55087 2550 | 55100 2551 | 55111 2552 | 55147 2553 | 55203 2554 | 55215 2555 | 55233 2556 | 55281 2557 | 55297 2558 | 55328 2559 | 55347 2560 | 55441 2561 | 55451 2562 | 55458 2563 | 55487 2564 | 55532 2565 | 55597 2566 | 55640 2567 | 55690 2568 | 55721 2569 | 55749 2570 | 55798 2571 | 55802 2572 | 55810 2573 | 55814 2574 | 55846 2575 | 55922 2576 | 55927 2577 | 55934 2578 | 55991 2579 | 56045 2580 | 56064 2581 | 56077 2582 | 56077 2583 | 56194 2584 | 56201 2585 | 56250 2586 | 56253 2587 | 56259 2588 | 56276 2589 | 56281 2590 | 56300 2591 | 56319 2592 | 56330 2593 | 56347 2594 | 56355 2595 | 56455 2596 | 56459 2597 | 56470 2598 | 56494 2599 | 56520 2600 | 56533 2601 | 56578 2602 | 56608 2603 | 56661 2604 | 56669 2605 | 56687 2606 | 56689 2607 | 56731 2608 | 56767 2609 | 56788 2610 | 56789 2611 | 56795 2612 | 56809 2613 | 56847 2614 | 56868 2615 | 56876 2616 | 56881 2617 | 56886 2618 | 56894 2619 | 56910 2620 | 56921 2621 | 56947 2622 | 56961 2623 | 56966 2624 | 56980 2625 | 56997 2626 | 57007 2627 | 57037 2628 | 57043 2629 | 57053 2630 | 57067 2631 | 57068 2632 | 57068 2633 | 57090 2634 | 57098 2635 | 57099 2636 | 57118 2637 | 57152 2638 | 57158 2639 | 57195 2640 | 57199 2641 | 57230 2642 | 57301 2643 | 57383 2644 | 57414 2645 | 57444 2646 | 57464 2647 | 57468 2648 | 57484 2649 | 57499 2650 | 57506 2651 | 57508 2652 | 57524 2653 | 57533 2654 | 57575 2655 | 57608 2656 | 57639 2657 | 57666 2658 | 57676 2659 | 57697 2660 | 57734 2661 | 57774 2662 | 57776 2663 | 57777 2664 | 57797 2665 | 57799 2666 | 57848 2667 | 57855 2668 | 57857 2669 | 57932 2670 | 57946 2671 | 58052 2672 | 58071 2673 | 58082 2674 | 58089 2675 | 58096 2676 | 58129 2677 | 58160 2678 | 58162 2679 | 58167 2680 | 58171 2681 | 58178 2682 | 58196 2683 | 58202 2684 | 58211 2685 | 58213 2686 | 58213 2687 | 58224 2688 | 58298 2689 | 58305 2690 | 58319 2691 | 58320 2692 | 58388 2693 | 58397 2694 | 58398 2695 | 58399 2696 | 58402 2697 | 58454 2698 | 58472 2699 | 58494 2700 | 58499 2701 | 58535 2702 | 58539 2703 | 58555 2704 | 58556 2705 | 58589 2706 | 58632 2707 | 58644 2708 | 58671 2709 | 58674 2710 | 58698 2711 | 58709 2712 | 58722 2713 | 58723 2714 | 58769 2715 | 58772 2716 | 58774 2717 | 58776 2718 | 58780 2719 | 58909 2720 | 58941 2721 | 58964 2722 | 58968 2723 | 58991 2724 | 58995 2725 | 59000 2726 | 59017 2727 | 59021 2728 | 59021 2729 | 59045 2730 | 59059 2731 | 59112 2732 | 59138 2733 | 59146 2734 | 59150 2735 | 59173 2736 | 59175 2737 | 59176 2738 | 59187 2739 | 59202 2740 | 59221 2741 | 59226 2742 | 59233 2743 | 59263 2744 | 59264 2745 | 59270 2746 | 59300 2747 | 59318 2748 | 59364 2749 | 59373 2750 | 59425 2751 | 59430 2752 | 59436 2753 | 59444 2754 | 59461 2755 | 59469 2756 | 59486 2757 | 59497 2758 | 59500 2759 | 59515 2760 | 59523 2761 | 59536 2762 | 59536 2763 | 59555 2764 | 59560 2765 | 59581 2766 | 59601 2767 | 59611 2768 | 59627 2769 | 59641 2770 | 59651 2771 | 59653 2772 | 59676 2773 | 59680 2774 | 59682 2775 | 59684 2776 | 59790 2777 | 59846 2778 | 59860 2779 | 59875 2780 | 59890 2781 | 59893 2782 | 59903 2783 | 59944 2784 | 59948 2785 | 59953 2786 | 60027 2787 | --------------------------------------------------------------------------------