├── fastec2 ├── __init__.py ├── get_insttypes.py ├── scripts.py ├── spot.py ├── insttypes.txt ├── prices.csv └── ec2.py ├── examples └── summary.py ├── README.md ├── setup.py └── .gitignore /fastec2/__init__.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import fire,sys 4 | from .ec2 import * 5 | from .spot import * 6 | 7 | def main(): 8 | if len(sys.argv)>=2 and sys.argv[1]=='i': interact(*sys.argv[2:]) 9 | else: fire.Fire(EC2) 10 | 11 | if __name__ == '__main__': main() 12 | 13 | -------------------------------------------------------------------------------- /examples/summary.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | from fastec2 import * 3 | 4 | e = EC2('Oregon') 5 | print('-- instances --') 6 | e.instances() 7 | print('-- volumes --') 8 | e.print_resources('volumes') 9 | print('-- snapshots --') 10 | e.print_resources('snapshots', owned=True) 11 | print('-- amis --') 12 | e.amis() 13 | 14 | -------------------------------------------------------------------------------- /fastec2/get_insttypes.py: -------------------------------------------------------------------------------- 1 | import pandas as pd, re 2 | 3 | df = pd.read_csv('https://pricing.us-east-1.amazonaws.com/offers/v1.0/aws/AmazonEC2/current/us-east-1/index.csv', 4 | skiprows=5, low_memory=False) 5 | 6 | s = [o for o in df['Instance Type'].unique() if isinstance(o, str) and '.' in o] 7 | 8 | def _sortkey(o): 9 | a,o = o.split('.') 10 | num,o = re.sub(r'^(\d*)(.*?)$', r'0\1 \2', o).split() 11 | return a,int(num),o 12 | 13 | s = sorted(list(s), key=_sortkey) 14 | 15 | with open('insttypes.txt', 'w') as f: 16 | for o in s: f.write(o+'\n') 17 | 18 | prices = df[(df['TermType']=='OnDemand') & (df['PricePerUnit']>0) & (df['Tenancy']=='Shared') 19 | & pd.isna(df['Pre Installed S/W']) & (df['Operating System']=='Linux') & (df['CapacityStatus']=='Used')] 20 | 21 | prices[['Instance Type', 'PricePerUnit']].to_csv('prices.csv', index=False) 22 | 23 | -------------------------------------------------------------------------------- /fastec2/scripts.py: -------------------------------------------------------------------------------- 1 | sync_tmpl = """settings {{ 2 | logfile = "/tmp/lsyncd.log", 3 | statusFile = "/tmp/lsyncd.status", 4 | }} 5 | sync {{ 6 | default.rsync, 7 | delete = false, 8 | source = "/home/{user}/fastec2/{name}/", 9 | target = "{ip}:fastec2/{name}" 10 | }}""" 11 | 12 | lsync_cfg = """[Unit] 13 | Description=lsyncd 14 | After=network.target 15 | 16 | [Service] 17 | Type=simple 18 | Restart=on-failure 19 | RestartSec=3 20 | User={user} 21 | ExecStart=/usr/bin/lsyncd -nodaemon -pidfile /tmp/lsyncd.pid /home/{user}/fastec2/sync.conf 22 | ExecReload=/bin/kill -HUP $MAINPID 23 | PIDFile=/tmp/lsyncd.pid 24 | 25 | [Install] 26 | WantedBy=multi-user.target 27 | """ 28 | 29 | script_svc_tmpl = """[Unit] 30 | Description={script} 31 | After=network.target 32 | 33 | [Service] 34 | Type=simple 35 | User={user} 36 | ExecStart={path}/{script}.sh 37 | 38 | [Install] 39 | WantedBy=multi-user.target 40 | """ 41 | 42 | 43 | script_tmpl = """#!/usr/bin/env bash 44 | 45 | source ~/anaconda3/etc/profile.d/conda.sh 46 | conda activate 47 | python -m fastai.launch {path}/{script}.py |& tee -a ~/fastec2/{name}/{script}.log 48 | """ 49 | 50 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ## fastec2 2 | 3 | AWS EC2 computer management for regular folks... 4 | 5 | ## Installation 6 | 7 | ```bash 8 | $ pip install git+https://github.com/fastai/fastec2.git 9 | ``` 10 | 11 | To add tab completion for your shell (replace *bash* with *fish* if you use the fish shell, although note as at Feb-2019 there are reports fish completions may be broken in the Google Fire library that this relies on): 12 | 13 | ```bash 14 | $ fe2 -- --completion bash > ~/.fe2-completion 15 | $ echo 'source ~/.fe2-completion' >> ~/.bashrc 16 | $ source ~/.bashrc 17 | ``` 18 | 19 | ## Usage 20 | 21 | For a list of commands, type: 22 | 23 | ```bash 24 | $ fe2 25 | 26 | Usage: fe2 - 27 | fe2 - change-type 28 | fe2 - connect 29 | fe2 - get-ami 30 | fe2 - get-price-hist 31 | ... 32 | ``` 33 | 34 | Each command can provide help, as follows: 35 | 36 | ```bash 37 | $ fe2 change-type -- --help 38 | 39 | Usage: fe2 change-type NAME INSTTYPE 40 | fe2 change-type --name NAME --insttype INSTTYPE 41 | ``` 42 | 43 | An [introduction and guide](https://www.fast.ai/2019/02/15/fastec2/) is available for the command line API and REPL. See the `examples` directory for examples of the Python API. 44 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup, find_packages 2 | from os import path 3 | from io import open 4 | 5 | here = path.abspath(path.dirname(__file__)) + '/' 6 | 7 | with open(path.join(here, 'README.md'), encoding='utf-8') as f: long_description = f.read() 8 | 9 | setup( 10 | name='fastec2', 11 | version='1.0.0', 12 | description="AWS EC2 computer management for regular folks", 13 | long_description=long_description, 14 | long_description_content_type='text/markdown', 15 | url='https://github.com/fastai/fastec2', 16 | author='Jeremy Howard', 17 | classifiers=[ 18 | # 3 - Alpha 19 | # 4 - Beta 20 | # 5 - Production/Stable 21 | 'Development Status :: 3 - Alpha', 22 | 'Intended Audience :: Developers', 23 | 'Topic :: System :: Networking', 24 | 'License :: OSI Approved :: Apache Software License', 25 | 'Programming Language :: Python :: 3.6', 26 | 'Programming Language :: Python :: 3.7', 27 | ], 28 | 29 | keywords='development', 30 | packages=["fastec2"], 31 | package_data={'fastec2': [ 'insttypes.txt', 'prices.csv' ]}, 32 | entry_points={ 'console_scripts': [ 'fe2=fastec2:main'] }, 33 | install_requires=[ 'boto3', 'awscli', 'fire', 'numpy', 'pandas', 'paramiko', 'pysftp' ], 34 | ) 35 | 36 | -------------------------------------------------------------------------------- /fastec2/spot.py: -------------------------------------------------------------------------------- 1 | from . import ec2 2 | from .ec2 import * 3 | 4 | class SpotRequest(): 5 | def __init__(self, e, d): 6 | self.e,self.d = e,d 7 | self._load() 8 | 9 | @classmethod 10 | def get(cls, e, srid): 11 | d = e._get_request(srid) 12 | if d is None: return None 13 | return cls(e, d) 14 | 15 | @classmethod 16 | def from_instance(cls, e, inst): 17 | inst = e.get_instance(inst) 18 | d = e._describe('spot_instance_requests',{'instance-id':inst.id}) 19 | if not d: return None 20 | return cls(e, d[0]) 21 | 22 | def load(self): 23 | if self.d is None: return 24 | self.d = e._get_request(self.d['SpotInstanceRequestId']) 25 | self._load() 26 | 27 | def _load(self): 28 | for s in '''spot_instance_request_id create_time instance_id instance_interruption_behavior 29 | launched_availability_zone spot_price state status tags type launch_specification'''.split(): 30 | setattr(self, s, self.d[ec2.snake2camel(s)]) 31 | self.id = self.spot_instance_request_id 32 | 33 | def __repr__(self): 34 | return f'{self.name} ({self.id} {self.state}): {self.instance_type}' 35 | 36 | def cancel(self): 37 | res = ec2.result(self.e._ec2.cancel_spot_instance_requests(SpotInstanceRequestIds=[self.id])) 38 | self.e.remove_name(self.id) 39 | 40 | @property 41 | def instance_type(self): return self.launch_specification['InstanceType'] 42 | 43 | @property 44 | def name(self): return ec2._boto3_name(self) 45 | 46 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | script.sh 2 | *~ 3 | .pypirc 4 | ~* 5 | tmp* 6 | tags 7 | 8 | # Byte-compiled / optimized / DLL files 9 | __pycache__/ 10 | *.py[cod] 11 | *$py.class 12 | 13 | # C extensions 14 | *.so 15 | 16 | # Distribution / packaging 17 | .Python 18 | env/ 19 | build/ 20 | develop-eggs/ 21 | dist/ 22 | downloads/ 23 | eggs/ 24 | .eggs/ 25 | lib/ 26 | lib64/ 27 | parts/ 28 | sdist/ 29 | var/ 30 | wheels/ 31 | *.egg-info/ 32 | .installed.cfg 33 | *.egg 34 | 35 | # PyInstaller 36 | # Usually these files are written by a python script from a template 37 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 38 | *.manifest 39 | *.spec 40 | 41 | # Installer logs 42 | pip-log.txt 43 | pip-delete-this-directory.txt 44 | 45 | # Unit test / coverage reports 46 | htmlcov/ 47 | .tox/ 48 | .coverage 49 | .coverage.* 50 | .cache 51 | nosetests.xml 52 | coverage.xml 53 | *.cover 54 | .hypothesis/ 55 | 56 | # Translations 57 | *.mo 58 | *.pot 59 | 60 | # Django stuff: 61 | *.log 62 | local_settings.py 63 | 64 | # Flask stuff: 65 | instance/ 66 | .webassets-cache 67 | 68 | # Scrapy stuff: 69 | .scrapy 70 | 71 | # Sphinx documentation 72 | docs/_build/ 73 | 74 | # PyBuilder 75 | target/ 76 | 77 | # Jupyter Notebook 78 | .ipynb_checkpoints 79 | 80 | # pyenv 81 | .python-version 82 | 83 | # celery beat schedule file 84 | celerybeat-schedule 85 | 86 | # SageMath parsed files 87 | *.sage.py 88 | 89 | # dotenv 90 | .env 91 | 92 | # virtualenv 93 | .venv 94 | venv/ 95 | ENV/ 96 | 97 | # Spyder project settings 98 | .spyderproject 99 | .spyproject 100 | 101 | # Rope project settings 102 | .ropeproject 103 | 104 | # mkdocs documentation 105 | /site 106 | 107 | # mypy 108 | .mypy_cache/ 109 | 110 | .vscode 111 | *.swp 112 | 113 | # osx generated files 114 | .DS_Store 115 | .DS_Store? 116 | .Trashes 117 | ehthumbs.db 118 | Thumbs.db 119 | .idea 120 | 121 | # pytest 122 | .pytest_cache 123 | -------------------------------------------------------------------------------- /fastec2/insttypes.txt: -------------------------------------------------------------------------------- 1 | a1.large 2 | a1.medium 3 | a1.xlarge 4 | a1.2xlarge 5 | a1.4xlarge 6 | c1.medium 7 | c1.xlarge 8 | c3.large 9 | c3.xlarge 10 | c3.2xlarge 11 | c3.4xlarge 12 | c3.8xlarge 13 | c4.large 14 | c4.xlarge 15 | c4.2xlarge 16 | c4.4xlarge 17 | c4.8xlarge 18 | c5.large 19 | c5.xlarge 20 | c5.2xlarge 21 | c5.4xlarge 22 | c5.9xlarge 23 | c5.18xlarge 24 | c5d.large 25 | c5d.xlarge 26 | c5d.2xlarge 27 | c5d.4xlarge 28 | c5d.9xlarge 29 | c5d.18xlarge 30 | c5n.large 31 | c5n.xlarge 32 | c5n.2xlarge 33 | c5n.4xlarge 34 | c5n.9xlarge 35 | c5n.18xlarge 36 | cc2.8xlarge 37 | cr1.8xlarge 38 | d2.xlarge 39 | d2.2xlarge 40 | d2.4xlarge 41 | d2.8xlarge 42 | f1.2xlarge 43 | f1.4xlarge 44 | f1.16xlarge 45 | g2.2xlarge 46 | g2.8xlarge 47 | g3.4xlarge 48 | g3.8xlarge 49 | g3.16xlarge 50 | g3s.xlarge 51 | h1.2xlarge 52 | h1.4xlarge 53 | h1.8xlarge 54 | h1.16xlarge 55 | hs1.8xlarge 56 | i2.xlarge 57 | i2.2xlarge 58 | i2.4xlarge 59 | i2.8xlarge 60 | i3.large 61 | i3.metal 62 | i3.xlarge 63 | i3.2xlarge 64 | i3.4xlarge 65 | i3.8xlarge 66 | i3.16xlarge 67 | m1.large 68 | m1.medium 69 | m1.small 70 | m1.xlarge 71 | m2.xlarge 72 | m2.2xlarge 73 | m2.4xlarge 74 | m3.large 75 | m3.medium 76 | m3.xlarge 77 | m3.2xlarge 78 | m4.large 79 | m4.xlarge 80 | m4.2xlarge 81 | m4.4xlarge 82 | m4.10xlarge 83 | m4.16xlarge 84 | m5.large 85 | m5.metal 86 | m5.xlarge 87 | m5.2xlarge 88 | m5.4xlarge 89 | m5.12xlarge 90 | m5.24xlarge 91 | m5a.large 92 | m5a.xlarge 93 | m5a.2xlarge 94 | m5a.4xlarge 95 | m5a.12xlarge 96 | m5a.24xlarge 97 | m5d.large 98 | m5d.metal 99 | m5d.xlarge 100 | m5d.2xlarge 101 | m5d.4xlarge 102 | m5d.12xlarge 103 | m5d.24xlarge 104 | p2.xlarge 105 | p2.8xlarge 106 | p2.16xlarge 107 | p3.2xlarge 108 | p3.8xlarge 109 | p3.16xlarge 110 | p3dn.24xlarge 111 | r3.large 112 | r3.xlarge 113 | r3.2xlarge 114 | r3.4xlarge 115 | r3.8xlarge 116 | r4.large 117 | r4.xlarge 118 | r4.2xlarge 119 | r4.4xlarge 120 | r4.8xlarge 121 | r4.16xlarge 122 | r5.large 123 | r5.metal 124 | r5.xlarge 125 | r5.2xlarge 126 | r5.4xlarge 127 | r5.12xlarge 128 | r5.24xlarge 129 | r5a.large 130 | r5a.xlarge 131 | r5a.2xlarge 132 | r5a.4xlarge 133 | r5a.12xlarge 134 | r5a.24xlarge 135 | r5d.large 136 | r5d.metal 137 | r5d.xlarge 138 | r5d.2xlarge 139 | r5d.4xlarge 140 | r5d.12xlarge 141 | r5d.24xlarge 142 | t1.micro 143 | t2.large 144 | t2.medium 145 | t2.micro 146 | t2.nano 147 | t2.small 148 | t2.xlarge 149 | t2.2xlarge 150 | t3.large 151 | t3.medium 152 | t3.micro 153 | t3.nano 154 | t3.small 155 | t3.xlarge 156 | t3.2xlarge 157 | x1.16xlarge 158 | x1.32xlarge 159 | x1e.xlarge 160 | x1e.2xlarge 161 | x1e.4xlarge 162 | x1e.8xlarge 163 | x1e.16xlarge 164 | x1e.32xlarge 165 | z1d.large 166 | z1d.metal 167 | z1d.xlarge 168 | z1d.2xlarge 169 | z1d.3xlarge 170 | z1d.6xlarge 171 | z1d.12xlarge 172 | -------------------------------------------------------------------------------- /fastec2/prices.csv: -------------------------------------------------------------------------------- 1 | Instance Type,PricePerUnit 2 | t2.small,0.023 3 | t2.large,0.0928 4 | r5.2xlarge,0.504 5 | r3.4xlarge,1.33 6 | z1d.12xlarge,4.464 7 | m5a.large,0.086 8 | m5.12xlarge,2.304 9 | m4.xlarge,0.2 10 | r3.2xlarge,0.665 11 | r5d.24xlarge,6.912 12 | m5d.large,0.113 13 | m5d.2xlarge,0.452 14 | c5n.4xlarge,0.864 15 | m1.small,0.044 16 | r3.8xlarge,2.66 17 | g2.2xlarge,0.65 18 | i3.4xlarge,1.248 19 | r5a.2xlarge,0.452 20 | h1.4xlarge,0.936 21 | m3.medium,0.067 22 | a1.medium,0.0255 23 | t3.large,0.0832 24 | r4.8xlarge,2.128 25 | r5a.24xlarge,5.424 26 | h1.2xlarge,0.468 27 | r5a.large,0.113 28 | c3.4xlarge,0.84 29 | r5.24xlarge,6.048 30 | c4.xlarge,0.199 31 | i3.large,0.156 32 | c5n.xlarge,0.216 33 | m3.large,0.133 34 | r5.12xlarge,3.024 35 | r4.2xlarge,0.532 36 | i3.xlarge,0.312 37 | r4.large,0.133 38 | g3.4xlarge,1.14 39 | z1d.2xlarge,0.744 40 | i2.8xlarge,6.82 41 | h1.16xlarge,3.744 42 | t2.medium,0.0464 43 | x1e.2xlarge,1.668 44 | m5a.2xlarge,0.344 45 | r5.large,0.126 46 | f1.16xlarge,13.2 47 | c5d.xlarge,0.192 48 | c1.medium,0.13 49 | c3.8xlarge,1.68 50 | m5.metal,4.608 51 | c5d.9xlarge,1.728 52 | z1d.xlarge,0.372 53 | m1.large,0.175 54 | m5a.4xlarge,0.688 55 | m5.large,0.096 56 | hs1.8xlarge,4.6 57 | i2.4xlarge,3.41 58 | m5d.xlarge,0.226 59 | r4.4xlarge,1.064 60 | x1e.8xlarge,6.672 61 | g3.8xlarge,2.28 62 | d2.xlarge,0.69 63 | m5.2xlarge,0.384 64 | c5n.18xlarge,3.888 65 | m4.2xlarge,0.4 66 | m5.24xlarge,4.608 67 | c5n.2xlarge,0.432 68 | f1.4xlarge,3.3 69 | c5d.18xlarge,3.456 70 | c4.large,0.1 71 | r5d.large,0.144 72 | x1e.32xlarge,26.688 73 | t2.xlarge,0.1856 74 | r5.4xlarge,1.008 75 | c5n.9xlarge,1.944 76 | a1.2xlarge,0.204 77 | c5.4xlarge,0.68 78 | z1d.metal,4.464 79 | c5d.2xlarge,0.384 80 | m5a.24xlarge,4.128 81 | t3.small,0.0208 82 | m5d.metal,5.424 83 | i3.8xlarge,2.496 84 | c3.2xlarge,0.42 85 | t1.micro,0.02 86 | m2.4xlarge,0.98 87 | x1.32xlarge,13.338 88 | r5d.2xlarge,0.576 89 | r4.xlarge,0.266 90 | i3.metal,4.992 91 | m2.2xlarge,0.49 92 | m5.4xlarge,0.768 93 | t3.xlarge,0.1664 94 | m1.medium,0.087 95 | c5n.large,0.108 96 | i3.2xlarge,0.624 97 | c4.4xlarge,0.796 98 | d2.4xlarge,2.76 99 | x1.16xlarge,6.669 100 | r5d.xlarge,0.288 101 | t3.medium,0.0416 102 | x1e.4xlarge,3.336 103 | g3.16xlarge,4.56 104 | g2.8xlarge,2.6 105 | m5.xlarge,0.192 106 | r3.xlarge,0.333 107 | t3.2xlarge,0.3328 108 | m5d.24xlarge,5.424 109 | c5.18xlarge,3.06 110 | g3s.xlarge,0.75 111 | r5a.12xlarge,2.712 112 | z1d.6xlarge,2.232 113 | x1e.xlarge,0.834 114 | p2.xlarge,0.9 115 | m4.10xlarge,2.0 116 | c1.xlarge,0.52 117 | m3.2xlarge,0.532 118 | m3.xlarge,0.266 119 | m4.16xlarge,3.2 120 | c3.large,0.105 121 | x1e.16xlarge,13.344 122 | d2.8xlarge,5.52 123 | t3.nano,0.0052 124 | m5a.12xlarge,2.064 125 | m5a.xlarge,0.172 126 | t2.nano,0.0058 127 | c4.2xlarge,0.398 128 | a1.4xlarge,0.408 129 | t2.micro,0.0116 130 | r4.16xlarge,4.256 131 | i2.xlarge,0.853 132 | p3dn.24xlarge,31.212 133 | cc2.8xlarge,2.0 134 | h1.8xlarge,1.872 135 | r5d.4xlarge,1.152 136 | c5d.4xlarge,0.768 137 | cr1.8xlarge,3.5 138 | r5a.xlarge,0.226 139 | z1d.3xlarge,1.116 140 | m4.4xlarge,0.8 141 | r5a.4xlarge,0.904 142 | r5d.metal,6.912 143 | c3.xlarge,0.21 144 | d2.2xlarge,1.38 145 | p3.16xlarge,24.48 146 | i2.2xlarge,1.705 147 | r3.large,0.166 148 | t2.2xlarge,0.3712 149 | p2.16xlarge,14.4 150 | m1.xlarge,0.35 151 | c5.xlarge,0.17 152 | i3.16xlarge,4.992 153 | c4.8xlarge,1.591 154 | m4.large,0.1 155 | c5d.large,0.096 156 | c5.2xlarge,0.34 157 | m5d.12xlarge,2.712 158 | c5.9xlarge,1.53 159 | m5d.4xlarge,0.904 160 | a1.xlarge,0.102 161 | c5.large,0.085 162 | r5.xlarge,0.252 163 | r5d.12xlarge,3.456 164 | p2.8xlarge,7.2 165 | p3.8xlarge,12.24 166 | f1.2xlarge,1.65 167 | r5.metal,6.048 168 | t3.micro,0.0104 169 | m2.xlarge,0.245 170 | p3.2xlarge,3.06 171 | a1.large,0.051 172 | z1d.large,0.186 173 | -------------------------------------------------------------------------------- /fastec2/ec2.py: -------------------------------------------------------------------------------- 1 | import numpy as np, pandas as pd 2 | import boto3, re, time, typing, socket, paramiko, os, pysftp, collections, json, shlex, sys 3 | import inspect, subprocess, shutil 4 | from typing import Callable,List,Dict,Tuple,Union,Optional,Iterable 5 | from types import SimpleNamespace 6 | from pathlib import Path 7 | from dateutil.parser import parse 8 | from pkg_resources import resource_filename 9 | from pdb import set_trace 10 | from .spot import * 11 | from .scripts import * 12 | 13 | __all__ = 'EC2 result results snake2camel make_filter listify'.split() 14 | 15 | here = os.path.abspath(os.path.dirname(__file__)) + '/' 16 | 17 | def snake2camel(s, split='_'): return ''.join([w.title() for w in s.split(split)]) 18 | def _make_dict(d:Dict): return [{'Key':k, 'Value': v } for k,v in (d or {}).items()] 19 | def _get_dict(l): 20 | if l is None: return None 21 | return collections.defaultdict(str, {o['Key']:o['Value'] for o in l}) 22 | 23 | def _boto3_name(self): 24 | d = _get_dict(self.tags) 25 | return None if d is None else d['Name'] 26 | boto3.resources.base.ServiceResource.name = property(_boto3_name) 27 | 28 | def _boto3_repr(self): 29 | clname = self.__class__.__name__ 30 | if clname == 'ec2.Instance': 31 | return f'{self.name} ({self.id} {self.instance_type} {self.state["Name"]}): {self.public_ip_address or "No public IP"}' 32 | elif clname == 'ec2.Volume': 33 | return f'{self.name} ({self.id} {self.state}): {self.size}GB' 34 | elif clname == 'ec2.Snapshot': 35 | return f'{self.name} ({self.id} {self.state}): {self.volume_size}GB' 36 | elif clname == 'ec2.Image': 37 | root_dev = [o for o in self.block_device_mappings if self.root_device_name == o['DeviceName']] 38 | return f'{self.name} ({self.id} {self.state}): {root_dev[0]["Ebs"]["VolumeSize"]}GB' 39 | else: 40 | identifiers = [f'{ident}={repr(getattr(self, ident))}' for ident in self.meta.identifiers] 41 | return f"{self.__class__.__name__}({', '.join(identifiers)})" 42 | boto3.resources.base.ServiceResource.__repr__ = _boto3_repr 43 | 44 | _in_notebook = False 45 | try: 46 | from ipykernel.kernelapp import IPKernelApp 47 | _in_notebook = IPKernelApp.initialized() 48 | except: pass 49 | 50 | def listify(p=None, q=None): 51 | "Make `p` listy and the same length as `q`." 52 | if p is None: p=[] 53 | elif isinstance(p, str): p=[p] 54 | elif not isinstance(p, Iterable): p=[p] 55 | n = q if type(q)==int else len(p) if q is None else len(q) 56 | if len(p)==1: p = p * n 57 | assert len(p)==n, f'List len mismatch ({len(p)} vs {n})' 58 | return list(p) 59 | 60 | def make_filter(d:Dict=None): 61 | if d is None: d={} 62 | d = {k.replace('_','-'):v for k,v in d.items()} 63 | return {'Filters': [{'Name':k, 'Values':listify(v)} for k,v in (d or {}).items()]} 64 | 65 | def results(r): 66 | if isinstance(r, typing.List): r = r[0] 67 | return {o:r[o] for o in r.keys() if o !='ResponseMetadata'} 68 | if not k: return None 69 | return r[k[0]] 70 | 71 | def result(r): 72 | if isinstance(r, typing.List): r = r[0] 73 | k = [o for o in r.keys() if o !='ResponseMetadata'] 74 | if not k: return None 75 | return r[k[0]] 76 | 77 | def _get_regions(): 78 | endpoint_file = resource_filename('botocore', 'data/endpoints.json') 79 | with open(endpoint_file, 'r') as f: a = json.load(f) 80 | return {k:v['description'] for k,v in a['partitions'][0]['regions'].items()} 81 | 82 | def _get_insttypes(): 83 | "Dict of instance types (eg 'p3.8xlarge') for each instance category (eg 'p3')" 84 | s = [o.strip() for o in open(here+'insttypes.txt').readlines()] 85 | d = collections.defaultdict(list) 86 | for l in s: d[l[:2]].append(l.strip()) 87 | return d 88 | 89 | 90 | class EC2(): 91 | def __init__(self, region:str=None): 92 | self.regions = _get_regions() 93 | self.curr_region = '' 94 | if region: 95 | self.curr_region = self.region(region) 96 | boto3.setup_default_session(region_name=self.curr_region) 97 | self._ec2 = boto3.client('ec2') 98 | self._ec2r = boto3.resource('ec2') 99 | self.insttypes = _get_insttypes() 100 | self.typenames = SimpleNamespace(**{ 101 | o.replace('.','_'):o for o in sum(self.insttypes.values(), [])}) 102 | 103 | def _resources(self, coll_name, owned=None, **filters): 104 | coll = getattr(self._ec2r,coll_name) 105 | filt = make_filter(filters) 106 | if owned: filt['OwnerIds']=['self'] 107 | return coll.filter(**filt) 108 | 109 | def print_resources(self, coll_name, owned=None, **filters): 110 | for o in self._resources(coll_name, owned=owned, **filters): print(o) 111 | 112 | def resource(self, coll_name, **filters): 113 | "The first resource from collection `coll_name` matching `filters`" 114 | try: return next(iter(self._resources(coll_name, **filters))) 115 | except StopIteration: raise KeyError(f'Resource not found: {coll_name}; {filters}') from None 116 | 117 | def region(self, region:str): 118 | "Get first region containing substring `region`" 119 | if region in self.regions: return region 120 | return next(r for r,n in self.regions.items() if region in n) 121 | 122 | def _describe(self, f:str, d:Dict=None, **kwargs): 123 | "Calls `describe_{f}` with filter `d` and `kwargs`" 124 | return result(getattr(self._ec2, 'describe_'+f)(**make_filter(d), **kwargs)) 125 | 126 | def get_instances(self): 127 | "Get all non-terminated instances" 128 | states = ['pending', 'running', 'stopping', 'stopped'] 129 | return SimpleNamespace(**{ 130 | (o.name or "##NONE##"):o for o in self._resources('instances', instance_state_name=states)}) 131 | 132 | def instances(self): 133 | "Print all non-terminated instances" 134 | for n,o in self.get_instances().__dict__.items(): print(o) 135 | 136 | def _price_hist(self, insttype): 137 | types = self.insttypes[insttype] 138 | prices = self._ec2.describe_spot_price_history(InstanceTypes=types, ProductDescriptions=["Linux/UNIX"]) 139 | df = pd.DataFrame(prices['SpotPriceHistory']) 140 | df["SpotPrice"] = df.SpotPrice.astype(float) 141 | return df.pivot_table(values='SpotPrice', index='Timestamp', columns='InstanceType', aggfunc='min' 142 | ).resample('1D').min().reindex(columns=types).tail(50) 143 | 144 | def price_hist(self, insttype): 145 | pv = self._price_hist(insttype) 146 | res = pv.tail(3).T 147 | if _in_notebook: 148 | pv.plot() 149 | return res 150 | print(res) 151 | 152 | def price_demand(self, insttype): 153 | "On demand prices for `insttype` (currently only shows us-east-1 prices)" 154 | prices = dict(pd.read_csv(here+'prices.csv').values) 155 | return [(o,round(prices[o],3)) for o in self.insttypes[insttype]] 156 | 157 | def waitfor(self, resource, event, ident, timeout=180): 158 | waiter = self._ec2.get_waiter(f'{resource}_{event}') 159 | waiter.config.max_attempts = timeout//15 160 | filt = {f'{snake2camel(resource)}Ids': [ident]} 161 | waiter.wait(**filt) 162 | time.sleep(5) 163 | 164 | def get_secgroup(self, secgroupname): 165 | "Get security group from `secgroupname`, creating it if needed (with just port 22 ingress)" 166 | try: secgroup = self.resource('security_groups', group_name=secgroupname) 167 | except KeyError: 168 | vpc = self.resource('vpcs', isDefault='true') 169 | secgroup = self._ec2r.create_security_group(GroupName=secgroupname, Description=secgroupname, VpcId=vpc.id) 170 | secgroup.authorize_ingress(IpPermissions=[{ 171 | 'IpRanges': [{'CidrIp': '0.0.0.0/0'}], 'FromPort': 22, 'ToPort': 22, 172 | 'IpProtocol': 'tcp'}] ) 173 | return secgroup 174 | 175 | def get_amis(self, description=None, owner=None, filt_func=None): 176 | """Return all AMIs with `owner` (or private AMIs if None), optionally matching `description` and `filt_func`. 177 | Sorted by `creation_date` descending""" 178 | if filt_func is None: filt_func=lambda o:True 179 | filt = dict(architecture='x86_64', virtualization_type='hvm', state='available', root_device_type='ebs') 180 | if owner is None: filt['is_public'] = 'false' 181 | else: filt['owner-id'] = owner 182 | if description is not None: filt['description'] = description 183 | amis = self._resources('images', **filt) 184 | amis = [o for o in amis if filt_func(o)] 185 | if owner is None: amis = [o for o in amis if o.product_codes is None] 186 | return sorted(amis, key=lambda o: parse(o.creation_date), reverse=True) 187 | 188 | def amis(self, description=None, owner=None, filt_func=None): 189 | """Return all AMIs with `owner` (or private AMIs if None), optionally matching `description` and `filt_func`. 190 | Sorted by `creation_date` descending""" 191 | for ami in self.get_amis(description, owner, filt_func): print(ami) 192 | 193 | def get_ami(self, ami=None): 194 | "Look up `ami` if provided, otherwise find latest Ubuntu 18.04 image" 195 | if ami is None: 196 | amis = self.get_amis('Canonical, Ubuntu, 18.04 LTS*',owner='099720109477', 197 | filt_func=lambda o: not re.search(r'UNSUPPORTED|minimal', o.description)) 198 | assert amis, 'AMI not found' 199 | return amis[0] 200 | 201 | if ami.__class__.__name__ == f'ec2.Image': return ami 202 | # If passed a valid AMI id, just return it 203 | try: return self.resource('images', image_id=ami) 204 | except KeyError: pass 205 | if ami: return self.resource('images', name=ami, is_public='false') 206 | 207 | def ami(self, aminame=None): print(self.get_ami(aminame)) 208 | 209 | def create_volume(self, ssh, size=None, name=None, snapshot=None, iops=None): 210 | inst = ssh.inst 211 | if name is None: name=inst.name 212 | if snapshot is None: 213 | if size is None: raise Exception('Must pass snapshot or size') 214 | else: 215 | snapshot = self.get_snapshot(snapshot) 216 | if size is None: size = snapshot.volume_size 217 | az = inst.placement['AvailabilityZone'] 218 | xtra = {'SnapshotId':snapshot.id} if snapshot else {} 219 | if iops: 220 | xtra['Iops']=iops 221 | xtra['VolumeType']='io1' 222 | else: xtra['VolumeType']='gp2' 223 | vol = self._ec2r.create_volume(AvailabilityZone=az, Size=size, **xtra) 224 | self.create_name(vol.id, name) 225 | self.waitfor('volume','available', vol.id) 226 | self.attach_volume(inst, vol) 227 | if snapshot is None: ssh.setup_vol(vol) 228 | else: ssh.mount(vol) 229 | return vol 230 | 231 | def create_snapshot(self, vol, name=None, wait=False): 232 | if name is None: name=vol.name 233 | snap = vol.create_snapshot() 234 | self.create_name(snap.id, name) 235 | if wait: self.waitfor('snapshot', 'completed', snap.id) 236 | return snap 237 | 238 | def _get_resource(self, o, cname, pref): 239 | if o.__class__.__name__ == f'ec2.{cname}': return o 240 | coll_name = f'{cname.lower()}s' 241 | if o.startswith(f'{pref}-'): 242 | return self.resource(coll_name, **{f'{cname.lower()}_id': o}) 243 | return self.resource(coll_name, **{'tag:Name':o}) 244 | 245 | def get_snapshot(self, snap): return self._get_resource(snap, 'Snapshot', 'snap') 246 | def get_volume(self, vol): return self._get_resource(vol, 'Volume', 'vol') 247 | def get_instance(self, inst): return self._get_resource(inst, 'Instance', 'i') 248 | def get_request(self, srid): return SpotRequest.get(self, srid) 249 | def get_request_from_instance(self, inst): return SpotRequest.from_instance(self, inst) 250 | def get_requests(self): 251 | return [SpotRequest(self, o) for o in 252 | self._describe('spot_instance_requests', {'state':['open','active']})] 253 | def requests(self): 254 | for o in self.get_requests(): print(o) 255 | 256 | def mount_volume(self, ssh, vol, attach=True, perm=False): 257 | vol = self.get_volume(vol) 258 | inst = ssh.inst 259 | if attach: self.attach_volume(inst, vol) 260 | ssh.mount(vol, perm=perm) 261 | 262 | def attach_volume(self, inst, vol): 263 | inst = self.get_instance(inst) 264 | vol.attach_to_instance(Device='/dev/sdh',InstanceId=inst.id) 265 | self.waitfor('volume', 'in_use', vol.id) 266 | 267 | def detach_volume(self, ssh, vol, wait=True): 268 | ssh.umount() 269 | vol.detach_from_instance() 270 | if wait: self.waitfor('volume', 'available', vol.id) 271 | 272 | def change_type(self, inst, insttype): 273 | inst = self.get_instance(inst) 274 | inst.modify_attribute(Attribute='instanceType', Value=insttype) 275 | 276 | def freeze(self, inst, name=None): 277 | inst = self.get_instance(inst) 278 | if name is None: name=inst.name 279 | amiid = self._ec2.create_image(InstanceId=inst.id, Name=name)['ImageId'] 280 | ami = self.get_ami(amiid) 281 | snid = ami.block_device_mappings[0]['Ebs']['SnapshotId'] 282 | self.create_name(snid, name) 283 | return ami 284 | 285 | def _launch_spec(self, ami, keyname, disksize, instancetype, secgroupid, iops=None): 286 | assert self._describe('key_pairs', {'key-name':keyname}), 'key not found' 287 | ami = self.get_ami(ami) 288 | ebs = ({'VolumeSize': disksize, 'VolumeType': 'io1', 'Iops': iops } 289 | if iops else {'VolumeSize': disksize, 'VolumeType': 'gp2'}) 290 | return { 'ImageId': ami.id, 'InstanceType': instancetype, 291 | 'SecurityGroupIds': [secgroupid], 'KeyName': keyname, 292 | "BlockDeviceMappings": [{ "DeviceName": "/dev/sda1", "Ebs": ebs, }] } 293 | 294 | def _get_request(self, srid): 295 | srs = self._describe('spot_instance_requests', {'spot-instance-request-id':srid}) 296 | if not srs: return None 297 | return srs[0] 298 | 299 | def create_tag(self, resource_id, key, val): 300 | self._ec2.create_tags(Resources=[resource_id], Tags=_make_dict({key: val})) 301 | 302 | def create_name(self, resource_id, name): 303 | self.create_tag(resource_id, 'Name', name) 304 | 305 | def remove_name(self, resource_id): 306 | self._ec2.delete_tags(Resources=[resource_id],Tags=[{"Key": 'Name'}]) 307 | 308 | def request_spot(self, name, ami, keyname, disksize, instancetype, secgroupid, iops=None): 309 | spec = self._launch_spec(ami, keyname, disksize, instancetype, secgroupid, iops) 310 | sr = result(self._ec2.request_spot_instances( 311 | LaunchSpecification=spec, InstanceInterruptionBehavior='stop', Type='persistent')) 312 | assert len(sr)==1, 'spot request failed' 313 | srid = sr[0]['SpotInstanceRequestId'] 314 | try: self.waitfor('spot_instance_request', 'fulfilled', srid) 315 | except: raise Exception(self._get_request(srid)['Fault']['Message']) from None 316 | for _ in range(20): 317 | time.sleep(1) 318 | try: sr = SpotRequest.get(self, srid) 319 | except: sr = None 320 | if sr is not None: break 321 | self.create_name(sr.id, name) 322 | return sr 323 | 324 | def request_demand(self, ami, keyname, disksize, instancetype, secgroupid, iops=None): 325 | spec = self._launch_spec(ami, keyname, disksize, instancetype, secgroupid, iops) 326 | return self._ec2r.create_instances(MinCount=1, MaxCount=1, **spec)[0] 327 | 328 | def _wait_ssh(self, inst): 329 | self.waitfor('instance', 'running', inst.id) 330 | for i in range(720//5): 331 | try: 332 | with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: 333 | s.connect((inst.public_ip_address, 22)) 334 | time.sleep(1) 335 | return inst 336 | except (ConnectionRefusedError,BlockingIOError): time.sleep(5) 337 | 338 | def get_launch(self, name, ami, disksize, instancetype, keyname:str='default', secgroupname:str='ssh', 339 | iops:int=None, spot:bool=False): 340 | "Creates new instance `name` and returns `Instance` object" 341 | insts = self._describe('instances', {'tag:Name':name}) 342 | assert not insts, 'name already exists' 343 | secgroupid = self.get_secgroup(secgroupname).id 344 | if spot: 345 | sr = self.request_spot(name, ami, keyname, disksize, instancetype, secgroupid, iops) 346 | inst = self._ec2r.Instance(sr.instance_id) 347 | else: 348 | inst = self.request_demand(ami, keyname, disksize, instancetype, secgroupid, iops) 349 | self.waitfor('instance','running', inst.id) 350 | inst.load() 351 | self.create_name(inst.id, name) 352 | self._wait_ssh(inst) 353 | inst.load() 354 | return inst 355 | 356 | def ip(self, inst): return self.get_instance(inst).public_ip_address 357 | 358 | def launch(self, name, ami, disksize, instancetype, keyname:str='default', 359 | secgroupname:str='ssh', iops:int=None, spot:bool=False): 360 | print(self.get_launch(name, ami, disksize, instancetype, keyname, secgroupname, iops, spot)) 361 | 362 | def instance(self, inst:str): 363 | "Show `Instance` details for `inst`" 364 | print(self.get_instance(inst)) 365 | 366 | def start(self, inst, show=True): 367 | "Starts instance `inst`" 368 | inst = self.get_instance(inst) 369 | inst.start() 370 | self._wait_ssh(inst) 371 | if show: print(inst) 372 | else: return inst 373 | 374 | def terminate(self, inst): 375 | "Starts instance `inst`" 376 | inst = self.get_instance(inst) 377 | sr = SpotRequest.from_instance(self, inst) 378 | if sr is not None: sr.cancel() 379 | inst.terminate() 380 | self.remove_name(inst.id) 381 | 382 | def stop(self, inst): 383 | "Stops instance `inst`" 384 | self.get_instance(inst).stop() 385 | 386 | def connect(self, inst, ports=None, user=None, keyfile='~/.ssh/id_rsa'): 387 | """Replace python process with an ssh process connected to instance `inst`; 388 | use `user@name` otherwise defaults to user 'ubuntu'. `ports` (int or list) creates tunnels""" 389 | if user is None: 390 | if isinstance(inst,str) and '@' in inst: user,inst = inst.split('@') 391 | else: user = 'ubuntu' 392 | inst = self.get_instance(inst) 393 | #tunnel = [] 394 | tunnel = [f'-L {o}:localhost:{o}' for o in listify(ports)] 395 | os.execvp('ssh', ['ssh', f'{user}@{inst.public_ip_address}', 396 | '-i', os.path.expanduser(keyfile), *tunnel]) 397 | 398 | def sshs(self, inst, user='ubuntu', keyfile='~/.ssh/id_rsa'): 399 | inst = self.get_instance(inst) 400 | ssh = self.ssh(inst, user=user, keyfile=keyfile) 401 | ftp = pysftp.Connection(ssh) 402 | return inst,ssh,ftp 403 | 404 | def ssh(self, inst, user='ubuntu', keyfile='~/.ssh/id_rsa'): 405 | "Return a paramiko ssh connection objected connected to instance `inst`" 406 | inst = self.get_instance(inst) 407 | keyfile = os.path.expanduser(keyfile) 408 | key = paramiko.RSAKey.from_private_key_file(keyfile) 409 | client = paramiko.SSHClient() 410 | client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) 411 | client.connect(hostname=inst.public_ip_address, username=user, pkey=key) 412 | client.raise_stderr = True 413 | client.inst = inst 414 | client.user = user 415 | client.launch_tmux() 416 | return client 417 | 418 | def setup_files(self, ssh, name, keyfile): 419 | fpath = Path.home()/'fastec2' 420 | (fpath/name).mkdir(parents=True, exist_ok=True) 421 | (fpath/'files').mkdir(parents=True, exist_ok=True) 422 | os.system(f"touch {fpath}/files/update.sh") 423 | os.system(f"chmod u+x {fpath}/files/update.sh") 424 | os.system(f"touch {fpath}/files/setup.sh") 425 | os.system(f"chmod u+x {fpath}/files/setup.sh") 426 | 427 | ssh.send('mkdir -p ~/fastec2') 428 | ssh.send(f'export FE2_DIR=~/fastec2/{name}') 429 | ssh.send(f'echo {name} > ~/fastec2/current') 430 | ip = ssh.inst.public_ip_address 431 | os.system(f"rsync -e 'ssh -o StrictHostKeyChecking=no -i {keyfile}' -az {fpath/'files'}/ {ssh.user}@{ip}:fastec2/{name}/") 432 | os.system(f"rsync -e 'ssh -o StrictHostKeyChecking=no -i {keyfile}' -az {fpath/name} {ssh.user}@{ip}:fastec2/") 433 | 434 | def setup_lsync(self, ssh, name, myip, conf_fn='sync.conf'): 435 | if myip is None: 436 | myip = subprocess.check_output(['curl', '-s', 'http://169.254.169.254/latest/meta-data/public-ipv4']).decode().strip() 437 | ssh.send(f'ssh-keyscan {myip} >> ~/.ssh/known_hosts') 438 | ssh.write(f'fastec2/{conf_fn}', sync_tmpl.format(user=ssh.user, name=name, ip=myip)) 439 | ssh.write('lsync.service', lsync_cfg.format(user=ssh.user)) 440 | ssh.send('cd') 441 | ssh.send('sudo mv lsync.service /etc/systemd/system/') 442 | ssh.send('sudo systemctl start lsync') 443 | ssh.send('sudo systemctl enable lsync') 444 | 445 | def setup_script(self, ssh, script, path): 446 | name = ssh.inst.name 447 | ssh.write(f'{script}.sh', script_tmpl.format(script=script, path=path, name=name)) 448 | ssh.send(f'chmod u+x {script}.sh') 449 | ssh.send(f'mv {script}.sh {path}/') 450 | ssh.write(f'{script}.service', script_svc_tmpl.format( 451 | script=script, path=path, name=name, user=ssh.user)) 452 | ssh.send(f'sudo mv {script}.service /etc/systemd/system/') 453 | ssh.send(f'sudo systemctl enable {script}') 454 | ssh.send(f'echo To run: sudo systemctl start {script}') 455 | ssh.send(f'echo To monitor: journalctl -f -u {script}') 456 | 457 | def script(self, scriptname, inst, user='ubuntu', keyfile='~/.ssh/id_rsa'): 458 | inst = self.get_instance(inst) 459 | name = inst.name 460 | ssh = self.ssh(inst, user, keyfile) 461 | shutil.copy(scriptname, Path.home()/'fastec2'/name/scriptname) 462 | self.setup_files(ssh, name, keyfile) 463 | ssh.send(f'cd fastec2/{name}') 464 | ssh.send(f'chmod u+x {scriptname}') 465 | ssh.send('./'+scriptname) 466 | 467 | def _run_ssh(ssh, cmd, pty=False): 468 | stdin, stdout, stderr = ssh.exec_command(cmd, get_pty=pty) 469 | stdout_str = stdout.read().decode() 470 | stderr_str = stderr.read().decode() 471 | if stdout.channel.recv_exit_status() != 0: raise Exception(stderr_str) 472 | if ssh.raise_stderr: 473 | if stderr_str: raise Exception(stderr_str) 474 | return stdout_str 475 | return stdout_str,stderr_str 476 | 477 | def _check_ssh(ssh): assert ssh.run('echo hi')[0] == 'hi\n' 478 | 479 | def _write_ssh(ssh, fn, s): ssh.open_sftp().open(fn, 'w').write(s) 480 | 481 | def _volid_to_dev(ssh, vol): 482 | volid = vol.id.split('-')[1] 483 | try: 484 | res = ssh.run(f'readlink -f /dev/disk/by-id/nvme-Amazon_Elastic_Block_Store_vol{volid}').strip() 485 | except: return '/dev/xvdh' #XXX 486 | assert '/dev/disk/by-id/' not in res, 'Failed to find volume link; is it attached?' 487 | return res 488 | 489 | def _setup_vol(ssh, vol): 490 | dev = _volid_to_dev(ssh, vol) 491 | cmds = [ 492 | f'sudo mkfs -q -t ext4 {dev}', 493 | f'sudo mkdir -p /mnt/fe2_disk', 494 | f'sudo mount {dev} /mnt/fe2_disk', 495 | f'sudo chown -R ubuntu /mnt/fe2_disk', 496 | ] 497 | for c in cmds: ssh.run(c) 498 | ssh.write('/mnt/fe2_disk/chk', 'ok') 499 | 500 | def _mount(ssh, vol, perm=False): 501 | dev = _volid_to_dev(ssh, vol) 502 | ssh.run(f'sudo mkdir -p /mnt/fe2_disk') 503 | if perm: 504 | ssh.run(f"echo '{dev} /mnt/fe2_disk ext4 defaults 0 0' | sudo tee -a /etc/fstab") 505 | ssh.run(f'sudo mount -a') 506 | else: 507 | ssh.run(f'sudo mount -t ext4 {dev} /mnt/fe2_disk') 508 | 509 | def _umount(ssh): ssh.run('sudo umount /mnt/fe2_disk') 510 | 511 | def _launch_tmux(ssh, name=None): 512 | if name is None: name=ssh.inst.name 513 | try: 514 | r = ssh.run(f'tmux ls | grep {name}') 515 | if r: return ssh 516 | except: pass 517 | ssh.run(f'tmux new -s {name} -n {name} -d', pty=True) 518 | return ssh 519 | 520 | def _send_tmux(ssh, cmd, name=None): 521 | if name is None: name=ssh.inst.name 522 | ssh.run(f'tmux send-keys -t {name} -l {shlex.quote(cmd)}') 523 | ssh.run(f'tmux send-keys Enter') 524 | 525 | def _ssh_runscript(ssh, script): 526 | ssh.write('/tmp/tmp.sh', script) 527 | ssh.run('chmod u+x /tmp/tmp.sh') 528 | res = ssh.run('/tmp/tmp.sh') 529 | ssh.run('rm /tmp/tmp.sh') 530 | return res 531 | 532 | paramiko.SSHClient.run = _run_ssh 533 | paramiko.SSHClient.check = _check_ssh 534 | paramiko.SSHClient.send = _send_tmux 535 | paramiko.SSHClient.write = _write_ssh 536 | paramiko.SSHClient.launch_tmux = _launch_tmux 537 | paramiko.SSHClient.mount = _mount 538 | paramiko.SSHClient.umount = _umount 539 | paramiko.SSHClient.setup_vol = _setup_vol 540 | paramiko.SSHClient.runscript = _ssh_runscript 541 | 542 | def _pysftp_init(self, ssh): 543 | self._sftp_live = True 544 | self._transport = ssh.get_transport() 545 | self._sftp = paramiko.SFTPClient.from_transport(self._transport) 546 | 547 | def _put_dir(sftp, fr, to): 548 | sftp.makedirs(to) 549 | sftp.put_d(os.path.expanduser(fr), to) 550 | 551 | def _put_key(sftp, name): 552 | sftp.put(os.path.expanduser(f'~/.ssh/{name}'), f'.ssh/{name}') 553 | sftp.chmod(f'.ssh/{name}', 600) 554 | 555 | pysftp.Connection.__init__ = _pysftp_init 556 | pysftp.Connection.put_dir = _put_dir 557 | pysftp.Connection.put_key = _put_key 558 | 559 | def interact(region=''): 560 | os.execvp('ipython', ['ipython', '--autocall=2', '-ic', 561 | f'import fastec2; e=fastec2.EC2("{region}")']) 562 | 563 | --------------------------------------------------------------------------------