├── .circleci └── config.yml ├── .gitignore ├── Makefile ├── README.md ├── docs ├── api.md ├── cli.md └── tasks.md ├── pitcrew.spec ├── pitcrew ├── __init__.py ├── __version__.py ├── app.py ├── cli.py ├── context.py ├── docs.py ├── executor.py ├── file.py ├── loader.py ├── logger.py ├── passwords.py ├── task.py ├── tasks │ ├── __init__.py │ ├── apt_get │ │ ├── install.py │ │ └── update.py │ ├── crew │ │ ├── install.py │ │ └── release │ │ │ ├── __init__.py │ │ │ ├── darwin.py │ │ │ └── linux.py │ ├── docker │ │ ├── run.py │ │ └── stop.py │ ├── ensure │ │ └── aws │ │ │ └── route53 │ │ │ └── has_records.py │ ├── examples │ │ └── deploy_pitcrew │ │ │ ├── __init__.py │ │ │ ├── build.py │ │ │ ├── doc.html.j2 │ │ │ └── water.css │ ├── facts │ │ └── system │ │ │ └── uname.py │ ├── fs │ │ ├── chmod.py │ │ ├── chown.py │ │ ├── digests │ │ │ ├── md5.py │ │ │ └── sha256.py │ │ ├── is_directory.py │ │ ├── is_file.py │ │ ├── list.py │ │ ├── read.py │ │ ├── stat.py │ │ ├── touch.py │ │ └── write.py │ ├── git │ │ └── clone.py │ ├── homebrew │ │ └── install.py │ ├── install │ │ ├── __init__.py │ │ ├── homebrew.py │ │ └── xcode_cli.py │ └── providers │ │ ├── docker.py │ │ ├── local.py │ │ └── ssh.py ├── template.py ├── templates │ ├── new_task.py.j2 │ ├── task.md.j2 │ └── tasks.md.j2 ├── test │ ├── __init__.py │ └── util.py └── util.py ├── requirements-build.txt ├── requirements-development.txt ├── requirements.txt ├── setup.py └── tests ├── __init__.py ├── test_cli.py └── test_task.py /.circleci/config.yml: -------------------------------------------------------------------------------- 1 | # Python CircleCI 2.0 configuration file 2 | # 3 | # Check https://circleci.com/docs/2.0/language-python/ for more details 4 | # 5 | version: 2 6 | jobs: 7 | build: 8 | docker: 9 | - image: circleci/python:3.6.1 10 | 11 | working_directory: ~/repo 12 | 13 | steps: 14 | - checkout 15 | - setup_remote_docker 16 | 17 | # Download and cache dependencies 18 | - restore_cache: 19 | keys: 20 | - v1-dependencies-{{ checksum "requirements-development.txt" }} 21 | # fallback to using the latest cache if no exact match is found 22 | - v1-dependencies- 23 | 24 | - run: 25 | name: install dependencies 26 | command: | 27 | python3 -m venv env 28 | . env/bin/activate 29 | pip install -e . 30 | 31 | - save_cache: 32 | paths: 33 | - ./env 34 | key: v1-dependencies-{{ checksum "requirements-development.txt" }} 35 | 36 | - run: 37 | name: check that docs are up-to-date 38 | command: | 39 | . env/bin/activate 40 | PYTHONPATH=. python pitcrew/cli.py docs 41 | if ! git diff --no-ext-diff --quiet --exit-code; then 42 | exit 1 43 | fi 44 | 45 | - run: 46 | name: run task tests 47 | command: | 48 | . env/bin/activate 49 | PYTHONPATH=. python pitcrew/cli.py test 50 | 51 | - run: 52 | name: run unit tests 53 | command: | 54 | . env/bin/activate 55 | python -m unittest 56 | 57 | - run: 58 | name: check formatting 59 | command: | 60 | . env/bin/activate 61 | black --check pitcrew tests 62 | 63 | - run: 64 | name: check docs 65 | command: | 66 | . env/bin/activate 67 | crew docs --check 68 | 69 | - run: 70 | name: check flake 71 | command: | 72 | . env/bin/activate 73 | flake8 tests/**/*.py pit crew/**/*.py --ignore=E501,E203,W503 74 | 75 | - store_artifacts: 76 | path: test-reports 77 | destination: test-reports 78 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /env 2 | *.pyc 3 | __pycache__ 4 | /state.yml 5 | .DS_Store 6 | 7 | /build 8 | /out 9 | /dist 10 | /pkg 11 | /pitcrew.egg-info 12 | /pitcrew/tasks/private 13 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: docs 2 | 3 | docs: 4 | PYTHONPATH=. pydocmd simple pitcrew.task++ pitcrew.context++ pitcrew.file++ > docs/api.md 5 | 6 | .PHONY: build 7 | 8 | build: 9 | ./env/bin/pyinstaller pitcrew.spec 10 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # 🔧 Pitcrew 2 | 3 | AsyncIO-powered python DSL for running commands locally, on docker, or over ssh. 4 | 5 | [![CircleCI](https://circleci.com/gh/joshbuddy/pitcrew.svg?style=svg)](https://circleci.com/gh/joshbuddy/pitcrew) 6 | 7 | ## What does Pitcrew do? 8 | 9 | 10 | 11 | 12 | 15 | 16 | 17 | 18 | 21 | 22 | 23 | 24 | 27 | 28 | 29 | 30 | 33 | 34 | 35 | 37 | 40 | 41 | 42 | 43 | 46 | 47 | 48 | 49 | 53 | 54 | 55 | 56 | 59 | 60 |
Pitcrew can run commands 13 | $ crew sh date 14 |
...or over ssh 19 | $ crew sh -p providers.ssh -P '{"hosts": ["192.168.0.1"]}' date 20 |
on hundreds of hosts! 25 | $ crew sh -p providers.ssh -P '{"hosts": ["192.168.0.1-100"]}' date 26 |
Crew can also run tasks 31 | $ crew run install.homebrew 32 |
Tasks are either other shell commands, or other tasks, for example, 36 | this provisions Cloudfront, SSL and S3 and builds and deploys docs to pitcrew.io 38 | $ crew run examples.deploy_pitcrew 39 |
You can list available tasks 44 | $ crew list 45 |
...edit an existing task 50 | $ crew edit examples.deploy_pitcrew 51 | # opens in $EDITOR 52 |
or create a new task! 57 | $ crew new some.new.task 58 |
61 | 62 | ## Installation 63 | 64 | ### From binary 65 | 66 | To install pitcrew in your home directory, run the following: 67 | 68 | ``` 69 | curl -fsSL "https://github.com/joshbuddy/pitcrew/releases/latest/download/crew-$(uname)" > crew 70 | chmod u+x crew 71 | ./crew run crew.install --dest="$HOME/crew" 72 | ``` 73 | 74 | ### From PyPi 75 | 76 | To install from the Python Package Index, run the following: 77 | 78 | ``` 79 | pip install pitcrew 80 | crew run crew.install --dest="$HOME/crew" 81 | ``` 82 | 83 | ### From source 84 | 85 | ``` 86 | git clone https://github.com/joshbuddy/pitcrew 87 | cd pitcrew 88 | python3.6 -m venv env 89 | source env/bin/activate 90 | pip install -e . 91 | ``` 92 | 93 | ## Concepts 94 | 95 | A command or set of commands is called a **task**. A **context** runs tasks either locally, on docker or over ssh. 96 | A **provider** generates contexts. 97 | 98 | ### Tasks 99 | 100 | Tasks are either composed from other tasks or invoking a command on a shell. 101 | 102 | An example of a *task* might be reading a file. `fs.read(path)` reads a file as bytes and returns it: 103 | 104 | ### `pitcrew/tasks/fs/read.py` 105 | 106 | ```python 107 | import base64 108 | from pitcrew import task 109 | 110 | 111 | @task.arg("path", desc="The file to read", type=str) 112 | @task.returns("The bytes of the file") 113 | class FsRead(task.BaseTask): 114 | """Read value of path into bytes""" 115 | 116 | async def run(self) -> bytes: 117 | code, out, err = await self.sh_with_code(f"cat {self.params.esc_path}") 118 | assert code == 0, "exitcode was not zero" 119 | return out 120 | 121 | ``` 122 | 123 | Other tasks might include writing a file, installing xcode or cloning a git repository. All the currently available 124 | tasks are listed at [docs/tasks.md](docs/tasks.md). The api available in a task is available at [docs/api.md#crewtask](docs/api.md#crewtask). 125 | 126 | ### Contexts 127 | 128 | An example of a *context* might be over ssh, or even locally. Learn more about contexts at [docs/api.md#crewcontext](docs/api.md#crewcontext). 129 | 130 | ### Providers 131 | 132 | A *provider* is a task with a specific return type. The return type is an async iterator which returns contexts. 133 | 134 | ## Usage 135 | 136 | For detailed usage, see [docs/cli.md](docs/cli.md) for more details. 137 | 138 | ### Run a command 139 | 140 | Pitcrew allows running a command using `bin/crew sh -- [shell command]`. 141 | 142 | For example `crew sh ls /` will list the "/" directory locally. 143 | 144 | You can run this across three hosts via ssh using `crew sh -p providers.ssh -P '{"hosts": ["192.168.0.1", "192.168.0.2", "192.168.0.3"]}' ls /`. 145 | 146 | ### Run a task 147 | 148 | Pitcrew allows running a task using `crew run [task name] `. This will normally target your local machine unless you use the `-p` flag to select a different provider. 149 | 150 | ### See available tasks 151 | 152 | To see all the available tasks run `crew list`. This will show all available tasks which are stored in `crew/tasks`. 153 | 154 | ### Make a new task 155 | 156 | To see all the available tasks run `crew new [task_name]`. This will create a template of a new task. 157 | 158 | ### Run tests 159 | 160 | To run an ad-hoc command use . For tasks use `crew run [task-name] `. 161 | 162 | ### Get CLI help 163 | 164 | To see the whole list of commands available from the command-line, run `crew help`. 165 | -------------------------------------------------------------------------------- /docs/api.md: -------------------------------------------------------------------------------- 1 | # pitcrew.task 2 | 3 | Tasks are defined by putting them in the crew/tasks directory. Tasks must inherit 4 | from `crew.task.BaseTask`. Inside a task the main ways of interacting with the 5 | system is through running other tasks, using `self.sh` or `self.sh_with_code`. 6 | 7 | ### Running other tasks 8 | 9 | This is accomplished by calling through `self.{task_name}`. For instance, to 10 | run the task `fs.write` located at `crew/tasks/fs/write.py` you'd call 11 | `self.fs.write("/tmp/path", b"some contents")`. 12 | 13 | ### Running commands 14 | 15 | There are two ways of running commands, `self.sh` and `self.sh_with_code`. 16 | 17 | `self.sh` accepts a command an optional environment. It returns the stdout 18 | of the run command as a string encoded with utf-8. If the command exits 19 | with a non-zero status an assertion exception is raised. 20 | 21 | `self.sh_with_code` accepts a command and an optional environment. It returns 22 | a tuple of (code, stdout, stderr) where stdout and stderr are bytes. 23 | 24 | ### Writing tasks 25 | 26 | New tasks are created through `crew new `. Then use `crew edit ` to 27 | open the task file in your `$EDITOR`. 28 | 29 | ### Lifecycle of a task 30 | 31 | Tasks come in two flavors, tasks with verification and tasks without verification. 32 | Tasks with verification are typically _idempotent_. 33 | 34 | If a task has a validate method it performs the following: 35 | 36 | 1. Run `validate()`, return response and stop if no assertion error is raised 37 | 2. Run `run()` method 38 | 3. Run `validate()` method and raise any errors it produces, or return its return value. 39 | 40 | If a task doesn't have a validate method it performs the following: 41 | 42 | 1. Run `run()` method. 43 | 44 | ### Tests 45 | 46 | To add tests to a task, add test classes to your test file. For example: 47 | 48 | ```python 49 | class FsDigestsSha256Test(task.TaskTest): 50 | @task.TaskTest.ubuntu 51 | async def test_ubuntu(self): 52 | content = b"Some delicious bytes" 53 | await self.fs.write("/tmp/some-file", content) 54 | expected_digest = hashlib.sha256(content).hexdigest() 55 | actual_digest = await self.fs.digests.sha256("/tmp/some-file") 56 | assert expected_digest == actual_digest, "digests are not equal" 57 | ``` 58 | 59 | ## BaseTask 60 | ```python 61 | BaseTask(self, /, *args, **kwargs) 62 | ``` 63 | 64 | ### memoize 65 | bool(x) -> bool 66 | 67 | Returns True when the argument x is true, False otherwise. 68 | The builtins True and False are the only two instances of the class bool. 69 | The class bool is a subclass of the class int, and cannot be subclassed. 70 | ### nodoc 71 | bool(x) -> bool 72 | 73 | Returns True when the argument x is true, False otherwise. 74 | The builtins True and False are the only two instances of the class bool. 75 | The class bool is a subclass of the class int, and cannot be subclassed. 76 | ### tests 77 | Built-in mutable sequence. 78 | 79 | If no argument is given, the constructor creates a new empty list. 80 | The argument must be an iterable if specified. 81 | ### use_coersion 82 | bool(x) -> bool 83 | 84 | Returns True when the argument x is true, False otherwise. 85 | The builtins True and False are the only two instances of the class bool. 86 | The class bool is a subclass of the class int, and cannot be subclassed. 87 | ### invoke_sync 88 | ```python 89 | BaseTask.invoke_sync(self, *args, **kwargs) 90 | ``` 91 | Invokes the task synchronously and returns the result. 92 | ### task_file 93 | ```python 94 | BaseTask.task_file(self, path) 95 | ``` 96 | Gets a file relative to the task being executed. 97 | ## arg 98 | ```python 99 | arg(name, type=None, **kwargs) 100 | ``` 101 | Decorator to add a required argument to the task. 102 | ## opt 103 | ```python 104 | opt(name, type=None, **kwargs) 105 | ``` 106 | Decorator to add an optional argument to the task. 107 | ## returns 108 | ```python 109 | returns(desc) 110 | ``` 111 | Decorator to describe the return type. 112 | ## memoize 113 | ```python 114 | memoize() 115 | ``` 116 | Decorator to instruct task to memoize return within the context's cache. 117 | ## nodoc 118 | ```python 119 | nodoc() 120 | ``` 121 | Decorator to instruct task to not generate documentation for test. 122 | # pitcrew.context 123 | Contexts allow execution of tasks. There are currently three types of 124 | contexts: local, ssh and docker. 125 | 126 | Local contexts run commands on the host computer running crew. 127 | 128 | SSH contexts run commands over SSH on the target computer. 129 | 130 | Docker contexts run commands on a running docker container. 131 | 132 | ## ChangeUser 133 | ```python 134 | ChangeUser(self, context, new_user) 135 | ``` 136 | Context manager to allow changing the user within a context 137 | ## ChangeDirectory 138 | ```python 139 | ChangeDirectory(self, context, new_directory) 140 | ``` 141 | Context manager to allow changing the current directory within a context 142 | ## Context 143 | ```python 144 | Context(self, app, loader, user=None, parent_context=None, directory=None) 145 | ``` 146 | Abstract base class for all contexts. 147 | ### password 148 | ```python 149 | Context.password(self, prompt) -> str 150 | ``` 151 | Present the user with a password prompt using the prompt given. If two 152 | identical prompts are supplied, the user is only asked once, and subsequent calls will 153 | provide the password given. 154 | ### sh 155 | ```python 156 | Context.sh(self, command, stdin=None, env=None) -> str 157 | ``` 158 | Runs a shell command within the given context. Raises an AssertionError if it exits with 159 | a non-zero exitcode. Returns STDOUT encoded with utf-8. 160 | ### docker_context 161 | ```python 162 | Context.docker_context(self, *args, **kwargs) -> 'DockerContext' 163 | ``` 164 | Creates a new docker context with the given container id. 165 | ### ssh_context 166 | ```python 167 | Context.ssh_context(self, *args, **kwargs) -> 'SSHContext' 168 | ``` 169 | Creates a new ssh context with the given container id. 170 | ### with_user 171 | ```python 172 | Context.with_user(self, user) 173 | ``` 174 | Returns a context handler for defining the user 175 | ### cd 176 | ```python 177 | Context.cd(self, directory) 178 | ``` 179 | Returns a context handler for changing the directory 180 | ### invoke 181 | ```python 182 | Context.invoke(self, fn, *args, **kwargs) 183 | ``` 184 | Allows invoking of an async function within this context. 185 | ## LocalContext 186 | ```python 187 | LocalContext(self, app, loader, user=None, parent_context=None, directory=None) 188 | ``` 189 | 190 | ### LocalFile 191 | ```python 192 | LocalContext.LocalFile(self, context, path) 193 | ``` 194 | A reference to a file on the local machine executing pitcrew 195 | ## SSHContext 196 | ```python 197 | SSHContext(self, app, loader, host, port=22, user=None, parent_context=None, **connection_kwargs) 198 | ``` 199 | 200 | ### SSHFile 201 | ```python 202 | SSHContext.SSHFile(self, context, path) 203 | ``` 204 | A reference to a file on a remote host accessible via SSH 205 | ## DockerContext 206 | ```python 207 | DockerContext(self, app, loader, container_id, **kwargs) 208 | ``` 209 | 210 | ### DockerFile 211 | ```python 212 | DockerContext.DockerFile(self, context, path) 213 | ``` 214 | A reference to a file on a Docker container 215 | # pitcrew.file 216 | File objects are created through their respective contexts. A file object can be copied into 217 | another context via a file reference for the destination. For example, if operating in an SSH 218 | context, this would copy from the local filesystem to that destination: 219 | 220 | self.local_context("/some/file").copy_to(self.file("/some/other")) 221 | 222 | 223 | For convenience `owner`, `group` and `mode` arguments are available on the `copy_to` method to 224 | allow setting those attributes post-copy. 225 | 226 | ## File 227 | ```python 228 | File(self, context, path) 229 | ``` 230 | Abstract base class for file-based operations 231 | ### copy_to 232 | ```python 233 | File.copy_to(self, dest, archive=False, owner=None, group=None, mode=None) 234 | ``` 235 | Copies a file from the source to the destination. 236 | ## LocalFile 237 | ```python 238 | LocalFile(self, context, path) 239 | ``` 240 | A reference to a file on the local machine executing pitcrew 241 | ## DockerFile 242 | ```python 243 | DockerFile(self, context, path) 244 | ``` 245 | A reference to a file on a Docker container 246 | ## SSHFile 247 | ```python 248 | SSHFile(self, context, path) 249 | ``` 250 | A reference to a file on a remote host accessible via SSH 251 | -------------------------------------------------------------------------------- /docs/cli.md: -------------------------------------------------------------------------------- 1 | # CLI 2 | 3 | ## `crew sh [shell-command]` 4 | 5 | This allows an ad-hoc command to be run across a set of contexts. 6 | 7 | ### Arguments 8 | 9 | `-p` The provider task to use 10 | `-P` The arguments to pass to the provider encoded as json 11 | 12 | ### Examples 13 | 14 | List your root folder locally 15 | 16 | `crew sh ls /` 17 | 18 | Get the time on 100 machines 19 | 20 | `crew sh -p providers.ssh -P '{"user": "root", "hosts": ["192.168.0.1-100"]}' date` 21 | 22 | ## `crew run [task-name] ` 23 | 24 | This runs a command across a set of contexts. 25 | 26 | ### Arguments 27 | 28 | `-p` The provider task to use 29 | `-P` The arguments to pass to the provider encoded as json 30 | 31 | ### Examples 32 | 33 | Create a file at `./foo` with the contents "bar". 34 | 35 | `crew run fs.write foo bar` 36 | 37 | Install an apt package over ssh 38 | 39 | `crew run -p providers.ssh -P '{"user": "root", "hosts": ["192.168.0.1"]}' apt_get.install python3.6` 40 | 41 | ## `crew list` 42 | 43 | This will list all tasks currently available. 44 | 45 | ## `crew info [task-name]` 46 | 47 | Displays information about a single task 48 | 49 | ## `crew new [task-name]` 50 | 51 | This will create a new task file. 52 | 53 | ## `crew test ` 54 | 55 | Run tests for crew tasks. If you specify a prefix, it will only run tests which belong to tasks 56 | matching the prefix 57 | 58 | ### Examples 59 | 60 | This will run all task tests. 61 | 62 | `crew test` 63 | 64 | To only run tests for tasks starting with `fs.` run: 65 | 66 | `crew test fs.` 67 | 68 | ## `crew docs` 69 | 70 | Generates documentation for the currently available tasks in the file `docs/tasks.md`. 71 | 72 | ## `crew help` 73 | 74 | Get help for crew commands 75 | -------------------------------------------------------------------------------- /docs/tasks.md: -------------------------------------------------------------------------------- 1 | # Tasks 2 | 3 | ## apt_get.install 4 | 5 | Install a package using apt-get 6 | 7 | ### Arguments 8 | 9 | 10 | - packages *(str)* : The package to install 11 | 12 | 13 | ### Returns 14 | 15 | *(list)* The version of the installed package 16 | 17 | 18 |
19 | Show source 20 | 21 | ```python 22 | import re 23 | from pitcrew import task 24 | 25 | 26 | @task.varargs("packages", type=str, desc="The package to install") 27 | @task.returns("The version of the installed package") 28 | class AptgetInstall(task.BaseTask): 29 | """Install a package using apt-get""" 30 | 31 | async def verify(self) -> list: 32 | versions = [] 33 | for p in self.params.packages: 34 | versions.append(await self.get_version(p)) 35 | return versions 36 | 37 | async def run(self): 38 | packages = " ".join(map(lambda p: self.esc(p), self.params.packages)) 39 | return await self.sh(f"apt-get install -y {packages}") 40 | 41 | async def available(self) -> bool: 42 | code, _, _ = await self.sh_with_code("which apt-get") 43 | return code == 0 44 | 45 | async def get_version(self, name) -> str: 46 | policy_output = await self.sh(f"apt-cache policy {self.esc(name)}") 47 | m = re.search("Installed: (.*?)\n", policy_output) 48 | assert m, "no version found" 49 | installed_version = m.group(1) 50 | assert installed_version != "(none)", "Installed version is (none)" 51 | return installed_version 52 | 53 | ``` 54 | 55 |
56 | 57 | ------------------------------------------------- 58 | 59 | ## apt_get.update 60 | 61 | Performs `apt-get update` 62 | 63 | 64 | 65 | 66 | 67 |
68 | Show source 69 | 70 | ```python 71 | from pitcrew import task 72 | 73 | 74 | class AptgetUpdate(task.BaseTask): 75 | """Performs `apt-get update`""" 76 | 77 | async def run(self): 78 | await self.sh("apt-get update") 79 | 80 | ``` 81 | 82 |
83 | 84 | ------------------------------------------------- 85 | 86 | ## crew.install 87 | 88 | Installs crew in the path specified 89 | 90 | ### Arguments 91 | 92 | 93 | - dest *(list)* : The directory to install crew in 94 | 95 | 96 | 97 |
98 | Show source 99 | 100 | ```python 101 | from pitcrew import task 102 | 103 | 104 | @task.opt("dest", desc="The directory to install crew in", type=list, default="pitcrew") 105 | class CrewInstall(task.BaseTask): 106 | """Installs crew in the path specified""" 107 | 108 | async def verify(self): 109 | with self.cd(self.params.dest): 110 | await self.sh("./env/bin/crew --help") 111 | 112 | async def run(self): 113 | platform = await self.facts.system.uname() 114 | if platform == "darwin": 115 | await self.install.xcode_cli() 116 | await self.install.homebrew() 117 | await self.install("git") 118 | await self.git.clone( 119 | "https://github.com/joshbuddy/pitcrew.git", self.params.dest 120 | ) 121 | with self.cd(self.params.dest): 122 | await self.homebrew.install("python3") 123 | await self.sh("python3 -m venv --clear env") 124 | await self.sh("env/bin/pip install -e .") 125 | elif platform == "linux": 126 | if await self.sh_ok("which apt-get"): 127 | await self.apt_get.update() 128 | await self.apt_get.install( 129 | "apt-utils", "git", "python3.7", "python3.7-dev", "python3.7-venv" 130 | ) 131 | await self.sh( 132 | "apt-get install -y python3.7-distutils", 133 | env={"DEBIAN_FRONTEND": "noninteractive"}, 134 | ) 135 | else: 136 | raise Exception(f"cannot install on this platform {platform}") 137 | 138 | await self.git.clone( 139 | "https://github.com/joshbuddy/pitcrew.git", self.params.dest 140 | ) 141 | with self.cd(self.params.dest): 142 | await self.sh("python3.7 -m venv env") 143 | await self.sh("env/bin/pip install --upgrade pip wheel") 144 | await self.sh("env/bin/pip install -e .") 145 | 146 | else: 147 | raise Exception(f"cannot install on this platform {platform}") 148 | 149 | 150 | class CrewInstallTest(task.TaskTest): 151 | @task.TaskTest.ubuntu 152 | async def test_ubuntu(self): 153 | with self.cd("/tmp"): 154 | # put this in to test the local copy you've got 155 | await self.local_context.file(".").copy_to(self.file("/tmp/pitcrew")) 156 | await self.sh("rm -rf /tmp/pitcrew/env") 157 | await self.fs.write( 158 | "/tmp/pitcrew/.git/config", 159 | b"""[core] 160 | repositoryformatversion = 0 161 | filemode = true 162 | bare = false 163 | logallrefupdates = true 164 | ignorecase = true 165 | precomposeunicode = true 166 | [remote "origin"] 167 | url = https://github.com/joshbuddy/pitcrew.git 168 | fetch = +refs/heads/*:refs/remotes/origin/* 169 | """, 170 | ) 171 | await self.crew.install() 172 | 173 | ``` 174 | 175 |
176 | 177 | ------------------------------------------------- 178 | 179 | ## crew.release 180 | 181 | This creates a release for crew 182 | 183 | ### Arguments 184 | 185 | 186 | - dryrun *(bool)* : Dry run mode 187 | 188 | 189 | 190 |
191 | Show source 192 | 193 | ```python 194 | import re 195 | import asyncio 196 | import pitcrew 197 | from pitcrew import task 198 | 199 | 200 | @task.opt("dryrun", desc="Dry run mode", type=bool, default=True) 201 | class CrewRelease(task.BaseTask): 202 | """This creates a release for crew""" 203 | 204 | async def run(self): 205 | if not self.params.dryrun: 206 | current_branch = (await self.sh("git rev-parse --abbrev-ref HEAD")).strip() 207 | assert "master" == current_branch, "dryrun=False must be run on master" 208 | 209 | await self.sh("pip install -r requirements-build.txt") 210 | version = pitcrew.__version__ 211 | await self.sh("mkdir -p pkg") 212 | await asyncio.gather( 213 | self.crew.release.darwin(version), self.crew.release.linux(version) 214 | ) 215 | await self.sh( 216 | f"env/bin/githubrelease release joshbuddy/pitcrew create {version} {self.esc('pkg/*')}" 217 | ) 218 | if self.params.dryrun: 219 | await self.sh("env/bin/python setup.py upload_test") 220 | else: 221 | await self.sh("env/bin/python setup.py upload") 222 | print("Don't forget to go to github and hit publish!") 223 | 224 | ``` 225 | 226 |
227 | 228 | ------------------------------------------------- 229 | 230 | ## crew.release.darwin 231 | 232 | This creates a PyInstaller build for crew on Darwin 233 | 234 | ### Arguments 235 | 236 | 237 | - version *(str)* : The version to release 238 | 239 | 240 | 241 |
242 | Show source 243 | 244 | ```python 245 | from pitcrew import task 246 | 247 | 248 | @task.arg("version", desc="The version to release", type=str) 249 | class CrewBuildDarwin(task.BaseTask): 250 | """This creates a PyInstaller build for crew on Darwin""" 251 | 252 | async def run(self): 253 | assert await self.facts.system.uname() == "darwin" 254 | await self.sh("make build") 255 | target = f"pkg/crew-Darwin" 256 | await self.sh(f"cp dist/crew {target}") 257 | 258 | ``` 259 | 260 |
261 | 262 | ------------------------------------------------- 263 | 264 | ## crew.release.linux 265 | 266 | This creates a PyInstaller build for crew on Linux 267 | 268 | ### Arguments 269 | 270 | 271 | - version *(str)* : The version to release 272 | 273 | 274 | 275 |
276 | Show source 277 | 278 | ```python 279 | from pitcrew import task 280 | 281 | 282 | @task.arg("version", desc="The version to release", type=str) 283 | class CrewBuildLinux(task.BaseTask): 284 | """This creates a PyInstaller build for crew on Linux""" 285 | 286 | async def run(self): 287 | container_id = await self.docker.run("ubuntu", detach=True, interactive=True) 288 | docker_ctx = self.docker_context(container_id, user="root") 289 | 290 | async with docker_ctx: 291 | assert ( 292 | await docker_ctx.facts.system.uname() == "linux" 293 | ), "the platform is not linux!" 294 | await self.file(".").copy_to(docker_ctx.file("/tmp/crew")) 295 | await docker_ctx.apt_get.update() 296 | await docker_ctx.apt_get.install("python3.6") 297 | await docker_ctx.apt_get.install("python3.6-dev") 298 | await docker_ctx.apt_get.install("python3-venv") 299 | await docker_ctx.apt_get.install("build-essential") 300 | with docker_ctx.cd("/tmp/crew"): 301 | await docker_ctx.sh("python3.6 -m venv --clear env") 302 | await docker_ctx.sh("env/bin/pip install -r requirements-build.txt") 303 | await docker_ctx.sh("make build") 304 | target = f"pkg/crew-Linux" 305 | await docker_ctx.file("/tmp/crew/dist/crew").copy_to(self.file(target)) 306 | 307 | ``` 308 | 309 |
310 | 311 | ------------------------------------------------- 312 | 313 | ## docker.run 314 | 315 | Runs a specific docker image 316 | 317 | ### Arguments 318 | 319 | 320 | - image *(str)* : The image to run 321 | - detach *(bool)* : Run container in background and print container ID 322 | - tty *(bool)* : Allocate a pseudo-TTY 323 | - interactive *(bool)* : Interactive mode 324 | - publish *(list)* : Publish ports 325 | 326 | 327 | ### Returns 328 | 329 | *(str)* The container id 330 | 331 | 332 |
333 | Show source 334 | 335 | ```python 336 | from pitcrew import task 337 | 338 | 339 | @task.arg("image", desc="The image to run", type=str) 340 | @task.opt( 341 | "detach", 342 | desc="Run container in background and print container ID", 343 | default=False, 344 | type=bool, 345 | ) 346 | @task.opt("tty", desc="Allocate a pseudo-TTY", default=False, type=bool) 347 | @task.opt("interactive", desc="Interactive mode", default=False, type=bool) 348 | @task.opt("publish", desc="Publish ports", type=list) 349 | @task.returns("The container id") 350 | class DockerRun(task.BaseTask): 351 | """Runs a specific docker image""" 352 | 353 | async def run(self) -> str: 354 | flags = [] 355 | if self.params.detach: 356 | flags.append("d") 357 | if self.params.tty: 358 | flags.append("t") 359 | if self.params.interactive: 360 | flags.append("i") 361 | 362 | flag_string = f" -{''.join(flags)}" if flags else "" 363 | 364 | if self.params.publish: 365 | flag_string += f" -p {' '.join(self.params.publish)}" 366 | 367 | out = await self.sh(f"docker run{flag_string} {self.params.esc_image}") 368 | return out.strip() 369 | 370 | ``` 371 | 372 |
373 | 374 | ------------------------------------------------- 375 | 376 | ## docker.stop 377 | 378 | Stops docker container with specified id 379 | 380 | ### Arguments 381 | 382 | 383 | - container_id *(str)* : The container id to stop 384 | - time *(int)* : Seconds to wait for stop before killing it 385 | 386 | 387 | 388 |
389 | Show source 390 | 391 | ```python 392 | from pitcrew import task 393 | 394 | 395 | @task.arg("container_id", desc="The container id to stop", type=str) 396 | @task.opt( 397 | "time", desc="Seconds to wait for stop before killing it", type=int, default=10 398 | ) 399 | class DockerStop(task.BaseTask): 400 | """Stops docker container with specified id""" 401 | 402 | async def run(self): 403 | command = "docker stop" 404 | if self.params.time is not None: 405 | command += f" -t {self.params.time}" 406 | command += f" {self.params.esc_container_id}" 407 | await self.sh(command) 408 | 409 | ``` 410 | 411 |
412 | 413 | ------------------------------------------------- 414 | 415 | ## ensure.aws.route53.has_records 416 | 417 | Ensure route53 has the set of records 418 | 419 | ### Arguments 420 | 421 | 422 | - zone_id *(str)* : The zone id to operate on 423 | - records *(list)* : A list of records to ensure are set 424 | 425 | 426 | 427 |
428 | Show source 429 | 430 | ```python 431 | import json 432 | import asyncio 433 | from pitcrew import task 434 | 435 | 436 | @task.arg("zone_id", desc="The zone id to operate on", type=str) 437 | @task.arg("records", desc="A list of records to ensure are set", type=list) 438 | class HasRecords(task.BaseTask): 439 | """Ensure route53 has the set of records""" 440 | 441 | async def verify(self): 442 | json_out = await self.sh( 443 | f"aws route53 list-resource-record-sets --hosted-zone-id {self.params.esc_zone_id}" 444 | ) 445 | out = json.loads(json_out) 446 | existing_record_sets = out["ResourceRecordSets"] 447 | for record in self.params.records: 448 | assert record in existing_record_sets, "cannot find record" 449 | 450 | async def run(self): 451 | changes = map( 452 | lambda c: {"Action": "UPSERT", "ResourceRecordSet": c}, self.params.records 453 | ) 454 | change_batch = {"Changes": list(changes)} 455 | change_id = json.loads( 456 | await self.sh( 457 | f"aws route53 change-resource-record-sets --hosted-zone-id {self.params.esc_zone_id} --change-batch {self.esc(json.dumps(change_batch))}" 458 | ) 459 | )["ChangeInfo"]["Id"] 460 | while ( 461 | json.loads( 462 | await self.sh(f"aws route53 get-change --id {self.esc(change_id)}") 463 | )["ChangeInfo"]["Status"] 464 | == "PENDING" 465 | ): 466 | await asyncio.sleep(5) 467 | 468 | ``` 469 | 470 |
471 | 472 | ------------------------------------------------- 473 | 474 | ## examples.deploy_pitcrew 475 | 476 | This example builds and deploys pitcrew.io. It uses s3, cloudfront and acm to deploy 477 | this website using ssl. 478 | 479 | 480 | 481 | 482 | 483 |
484 | Show source 485 | 486 | ```python 487 | import json 488 | import asyncio 489 | from pitcrew import task 490 | from uuid import uuid4 491 | 492 | 493 | class DeployPitcrew(task.BaseTask): 494 | """This example builds and deploys pitcrew.io. It uses s3, cloudfront and acm to deploy 495 | this website using ssl. """ 496 | 497 | async def run(self): 498 | # make sure the build requirements are installed 499 | await self.sh("pip install -r requirements-build.txt") 500 | # create the bucket 501 | await self.sh("aws s3api create-bucket --bucket pitcrew-site") 502 | # setup aws & build + upload site 503 | await asyncio.gather(self.setup_aws(), self.build_and_sync()) 504 | 505 | async def setup_aws(self): 506 | # first find the zone 507 | zones = json.loads(await self.sh("aws route53 list-hosted-zones"))[ 508 | "HostedZones" 509 | ] 510 | zone_id = None 511 | for zone in zones: 512 | if zone["Name"] == "pitcrew.io.": 513 | zone_id = zone["Id"] 514 | break 515 | 516 | assert zone_id, "no zone_id found for pitcrew.io" 517 | 518 | # setup the certificate 519 | cert_arn = await self.setup_acm(zone_id) 520 | # setup the CDN 521 | cf_id = await self.setup_cloudfront(zone_id, cert_arn) 522 | dist = json.loads( 523 | await self.sh(f"aws cloudfront get-distribution --id {self.esc(cf_id)}") 524 | )["Distribution"] 525 | domain_name = dist["DomainName"] 526 | 527 | # add the DNS records 528 | await self.ensure.aws.route53.has_records( 529 | zone_id, 530 | [ 531 | { 532 | "Name": "pitcrew.io.", 533 | "Type": "A", 534 | "AliasTarget": { 535 | "HostedZoneId": "Z2FDTNDATAQYW2", 536 | "DNSName": f"{domain_name}.", 537 | "EvaluateTargetHealth": False, 538 | }, 539 | }, 540 | { 541 | "Name": "pitcrew.io.", 542 | "Type": "AAAA", 543 | "AliasTarget": { 544 | "HostedZoneId": "Z2FDTNDATAQYW2", 545 | "DNSName": f"{domain_name}.", 546 | "EvaluateTargetHealth": False, 547 | }, 548 | }, 549 | { 550 | "Name": "www.pitcrew.io.", 551 | "Type": "CNAME", 552 | "TTL": 300, 553 | "ResourceRecords": [{"Value": domain_name}], 554 | }, 555 | ], 556 | ) 557 | 558 | async def setup_acm(self, zone_id) -> str: 559 | # look for the certificate 560 | certs = json.loads( 561 | await self.sh("aws acm list-certificates --certificate-statuses ISSUED") 562 | )["CertificateSummaryList"] 563 | for cert in certs: 564 | if cert["DomainName"] == "pitcrew.io": 565 | return cert["CertificateArn"] 566 | 567 | # if it doesn't exist, create it 568 | arn = json.loads( 569 | await self.sh( 570 | f"aws acm request-certificate --domain-name pitcrew.io --validation-method DNS --subject-alternative-names {self.esc('*.pitcrew.io')}" 571 | ) 572 | )["CertificateArn"] 573 | cert_description = json.loads( 574 | await self.sh( 575 | f"aws acm describe-certificate --certificate-arn {self.esc(arn)}" 576 | ) 577 | ) 578 | 579 | validation = cert_description["Certificate"]["DomainValidationOptions"][0] 580 | await self.ensure.aws.route53.has_records( 581 | zone_id, 582 | [ 583 | { 584 | "Name": validation["ResourceRecord"]["Name"], 585 | "Type": validation["ResourceRecord"]["Type"], 586 | "TTL": 60, 587 | "ResourceRecords": [ 588 | {"Value": validation["ResourceRecord"]["Value"]} 589 | ], 590 | } 591 | ], 592 | ) 593 | 594 | await self.sh( 595 | f"aws acm wait certificate-validated --certificate-arn {self.esc(arn)}" 596 | ) 597 | return arn 598 | 599 | async def setup_cloudfront(self, zone_id, cert_arn) -> str: 600 | s3_origin = "pitcrew-site.s3.amazonaws.com" 601 | 602 | list_distributions = json.loads( 603 | await self.sh(f"aws cloudfront list-distributions") 604 | ) 605 | for dist in list_distributions["DistributionList"]["Items"]: 606 | if dist["Origins"]["Items"][0]["DomainName"] == s3_origin: 607 | return dist["Id"] 608 | 609 | config = { 610 | "DefaultRootObject": "index.html", 611 | "Aliases": {"Quantity": 2, "Items": ["pitcrew.io", "www.pitcrew.io"]}, 612 | "Origins": { 613 | "Quantity": 1, 614 | "Items": [ 615 | { 616 | "Id": "pitcrew-origin", 617 | "DomainName": s3_origin, 618 | "S3OriginConfig": {"OriginAccessIdentity": ""}, 619 | } 620 | ], 621 | }, 622 | "DefaultCacheBehavior": { 623 | "TargetOriginId": "pitcrew-origin", 624 | "ForwardedValues": { 625 | "QueryString": True, 626 | "Cookies": {"Forward": "none"}, 627 | }, 628 | "TrustedSigners": {"Enabled": False, "Quantity": 0}, 629 | "ViewerProtocolPolicy": "redirect-to-https", 630 | "MinTTL": 180, 631 | }, 632 | "CallerReference": str(uuid4()), 633 | "Comment": "Created by crew", 634 | "Enabled": True, 635 | "ViewerCertificate": { 636 | "ACMCertificateArn": cert_arn, 637 | "SSLSupportMethod": "sni-only", 638 | }, 639 | } 640 | create_distribution = json.loads( 641 | await self.sh( 642 | f"aws cloudfront create-distribution --distribution-config {self.esc(json.dumps(config))}" 643 | ) 644 | ) 645 | cf_id = create_distribution["Distribution"]["Id"] 646 | await self.sh(f"aws cloudfront wait distribution-deployed --id {cf_id}") 647 | return cf_id 648 | 649 | async def build_and_sync(self): 650 | await self.examples.deploy_pitcrew.build() 651 | await self.sh("aws s3 sync --acl public-read out/ s3://pitcrew-site/") 652 | 653 | ``` 654 | 655 |
656 | 657 | ------------------------------------------------- 658 | 659 | ## examples.deploy_pitcrew.build 660 | 661 | Builds the website in the `out` directory. 662 | 663 | 664 | 665 | 666 | 667 |
668 | Show source 669 | 670 | ```python 671 | import os 672 | import re 673 | import asyncio 674 | from pitcrew import task 675 | 676 | 677 | class Build(task.BaseTask): 678 | """Builds the website in the `out` directory.""" 679 | 680 | async def run(self): 681 | # create docs for python stuff 682 | await self.sh("make docs") 683 | # create task specific docs 684 | await self.sh("crew docs") 685 | # re-create out directory 686 | await self.sh("rm -rf out") 687 | await self.sh("mkdir out") 688 | # copy our css 689 | await self.task_file("water.css").copy_to(self.file("out/water.css")) 690 | 691 | docs = [] 692 | files = await self.fs.list("docs") 693 | for f in files: 694 | name = f.split("/")[-1] 695 | target = f"out/docs/{os.path.splitext(name)[0]}.html" 696 | docs.append(self.generate_doc(f"docs/{f}", target)) 697 | docs.append(self.generate_doc("README.md", "out/index.html")) 698 | await asyncio.gather(*docs) 699 | 700 | async def generate_doc(self, source, target): 701 | out = await self.sh( 702 | f"env/bin/python -m markdown2 -x fenced-code-blocks -x header-ids {source}" 703 | ) 704 | out = re.sub(r"\.md", ".html", out) 705 | await self.sh(f"mkdir -p {self.esc(os.path.split(target)[0])}") 706 | page = self.template("doc.html.j2").render_as_bytes(body=out) 707 | await self.fs.write(target, page) 708 | 709 | ``` 710 | 711 |
712 | 713 | ------------------------------------------------- 714 | 715 | ## facts.system.uname 716 | 717 | Returns the lowercase name of the platform 718 | 719 | 720 | 721 | 722 | ### Returns 723 | 724 | *(str)* The name of the platform 725 | 726 | 727 |
728 | Show source 729 | 730 | ```python 731 | from pitcrew import task 732 | 733 | 734 | @task.returns("The name of the platform") 735 | @task.memoize() 736 | class Uname(task.BaseTask): 737 | """Returns the lowercase name of the platform""" 738 | 739 | async def run(self) -> str: 740 | return (await self.sh("uname")).strip().lower() 741 | 742 | ``` 743 | 744 |
745 | 746 | ------------------------------------------------- 747 | 748 | ## fs.chmod 749 | 750 | Changes the file mode of the specified path 751 | 752 | ### Arguments 753 | 754 | 755 | - path *(str)* : The path to change the mode of 756 | - mode *(str)* : The mode 757 | 758 | 759 | 760 |
761 | Show source 762 | 763 | ```python 764 | from pitcrew import task 765 | 766 | 767 | @task.arg("path", desc="The path to change the mode of", type=str) 768 | @task.arg("mode", desc="The mode", type=str) 769 | @task.returns("The bytes of the file") 770 | class FsChmod(task.BaseTask): 771 | """Changes the file mode of the specified path""" 772 | 773 | async def run(self): 774 | return await self.sh(f"chmod {self.params.esc_mode} {self.params.esc_path}") 775 | 776 | 777 | class FsChmodTest(task.TaskTest): 778 | @task.TaskTest.ubuntu 779 | async def test_ubuntu(self): 780 | with self.cd("/tmp"): 781 | await self.fs.touch("some-file") 782 | await self.fs.chmod("some-file", "644") 783 | assert (await self.fs.stat("some-file")).mode == "100644" 784 | await self.fs.chmod("some-file", "o+x") 785 | assert (await self.fs.stat("some-file")).mode == "100645" 786 | 787 | ``` 788 | 789 |
790 | 791 | ------------------------------------------------- 792 | 793 | ## fs.chown 794 | 795 | Changes the file mode of the specified path 796 | 797 | ### Arguments 798 | 799 | 800 | - path *(str)* : The path to change the mode of 801 | - owner *(str)* : The owner 802 | - group *(str)* : The owner 803 | 804 | 805 | 806 |
807 | Show source 808 | 809 | ```python 810 | from pitcrew import task 811 | 812 | 813 | @task.arg("path", desc="The path to change the mode of", type=str) 814 | @task.arg("owner", desc="The owner", type=str) 815 | @task.opt("group", desc="The owner", type=str) 816 | @task.returns("The bytes of the file") 817 | class FsChown(task.BaseTask): 818 | """Changes the file mode of the specified path""" 819 | 820 | async def run(self): 821 | owner_str = self.params.owner 822 | if self.params.group: 823 | owner_str += f":{self.params.group}" 824 | return await self.sh(f"chown {self.esc(owner_str)} {self.params.esc_path}") 825 | 826 | ``` 827 | 828 |
829 | 830 | ------------------------------------------------- 831 | 832 | ## fs.digests.md5 833 | 834 | Gets md5 digest of path 835 | 836 | ### Arguments 837 | 838 | 839 | - path *(str)* : The path of the file to digest 840 | 841 | 842 | ### Returns 843 | 844 | *(str)* The md5 digest in hexadecimal 845 | 846 | 847 |
848 | Show source 849 | 850 | ```python 851 | import hashlib 852 | from pitcrew import task 853 | 854 | 855 | @task.arg("path", desc="The path of the file to digest", type=str) 856 | @task.returns("The md5 digest in hexadecimal") 857 | class FsDigestsMd5(task.BaseTask): 858 | """Gets md5 digest of path""" 859 | 860 | async def run(self) -> str: 861 | platform = await self.facts.system.uname() 862 | if platform == "darwin": 863 | out = await self.sh(f"md5 {self.params.esc_path}") 864 | return out.strip().split(" ")[-1] 865 | elif platform == "linux": 866 | out = await self.sh(f"md5sum {self.params.esc_path}") 867 | return out.split(" ")[0] 868 | else: 869 | raise Exception("not supported") 870 | 871 | 872 | class FsDigestsMd5Test(task.TaskTest): 873 | @task.TaskTest.ubuntu 874 | async def test_ubuntu(self): 875 | content = b"Some delicious bytes" 876 | await self.fs.write("/tmp/some-file", content) 877 | expected_digest = hashlib.md5(content).hexdigest() 878 | actual_digest = await self.fs.digests.md5("/tmp/some-file") 879 | assert expected_digest == actual_digest, "digests are not equal" 880 | 881 | ``` 882 | 883 |
884 | 885 | ------------------------------------------------- 886 | 887 | ## fs.digests.sha256 888 | 889 | Gets sha256 digest of path 890 | 891 | ### Arguments 892 | 893 | 894 | - path *(str)* : The path of the file to digest 895 | 896 | 897 | ### Returns 898 | 899 | *(str)* The sha256 digest in hexadecimal 900 | 901 | 902 |
903 | Show source 904 | 905 | ```python 906 | import hashlib 907 | from pitcrew import task 908 | 909 | 910 | @task.arg("path", desc="The path of the file to digest", type=str) 911 | @task.returns("The sha256 digest in hexadecimal") 912 | class FsDigestsSha256(task.BaseTask): 913 | """Gets sha256 digest of path""" 914 | 915 | async def run(self) -> str: 916 | platform = await self.facts.system.uname() 917 | if platform == "darwin": 918 | out = await self.sh(f"shasum -a256 {self.params.esc_path}") 919 | return out.split(" ")[0] 920 | elif platform == "linux": 921 | out = await self.sh(f"sha256sum {self.params.esc_path}") 922 | return out.split(" ")[0] 923 | else: 924 | raise Exception("not supported") 925 | 926 | 927 | class FsDigestsSha256Test(task.TaskTest): 928 | @task.TaskTest.ubuntu 929 | async def test_ubuntu(self): 930 | content = b"Some delicious bytes" 931 | await self.fs.write("/tmp/some-file", content) 932 | expected_digest = hashlib.sha256(content).hexdigest() 933 | actual_digest = await self.fs.digests.sha256("/tmp/some-file") 934 | assert expected_digest == actual_digest, "digests are not equal" 935 | 936 | ``` 937 | 938 |
939 | 940 | ------------------------------------------------- 941 | 942 | ## fs.is_directory 943 | 944 | Checks if the path is a directory 945 | 946 | ### Arguments 947 | 948 | 949 | - path *(str)* : The path to check 950 | 951 | 952 | ### Returns 953 | 954 | *(bool)* Indicates if target path is a directory 955 | 956 | 957 |
958 | Show source 959 | 960 | ```python 961 | from pitcrew import task 962 | 963 | 964 | @task.arg("path", desc="The path to check") 965 | @task.returns("Indicates if target path is a directory") 966 | class FsIsDirectory(task.BaseTask): 967 | """Checks if the path is a directory""" 968 | 969 | async def run(self) -> bool: 970 | code, _, _ = await self.sh_with_code(f"test -d {self.params.esc_path}") 971 | return code == 0 972 | 973 | ``` 974 | 975 |
976 | 977 | ------------------------------------------------- 978 | 979 | ## fs.is_file 980 | 981 | Checks if the path is a file 982 | 983 | ### Arguments 984 | 985 | 986 | - path *(str)* : The path to check 987 | 988 | 989 | ### Returns 990 | 991 | *(bool)* Indicates if target path is a file 992 | 993 | 994 |
995 | Show source 996 | 997 | ```python 998 | from pitcrew import task 999 | 1000 | 1001 | @task.arg("path", desc="The path to check") 1002 | @task.returns("Indicates if target path is a file") 1003 | class FsIsFile(task.BaseTask): 1004 | """Checks if the path is a file""" 1005 | 1006 | async def run(self) -> bool: 1007 | code, _, _ = await self.sh_with_code(f"test -f {self.params.esc_path}") 1008 | return code == 0 1009 | 1010 | ``` 1011 | 1012 |
1013 | 1014 | ------------------------------------------------- 1015 | 1016 | ## fs.list 1017 | 1018 | List the files in a directory. 1019 | 1020 | ### Arguments 1021 | 1022 | 1023 | - path *(str)* : The file to read 1024 | 1025 | 1026 | ### Returns 1027 | 1028 | *(list)* The bytes of the file 1029 | 1030 | 1031 |
1032 | Show source 1033 | 1034 | ```python 1035 | from pitcrew import task 1036 | 1037 | 1038 | @task.arg("path", desc="The file to read", type=str) 1039 | @task.returns("The bytes of the file") 1040 | class FsList(task.BaseTask): 1041 | """List the files in a directory.""" 1042 | 1043 | async def run(self) -> list: 1044 | out = await self.sh(f"ls -1 {self.params.esc_path}") 1045 | return out.strip().split("\n") 1046 | 1047 | ``` 1048 | 1049 |
1050 | 1051 | ------------------------------------------------- 1052 | 1053 | ## fs.read 1054 | 1055 | Read value of path into bytes 1056 | 1057 | ### Arguments 1058 | 1059 | 1060 | - path *(str)* : The file to read 1061 | 1062 | 1063 | ### Returns 1064 | 1065 | *(bytes)* The bytes of the file 1066 | 1067 | 1068 |
1069 | Show source 1070 | 1071 | ```python 1072 | from pitcrew import task 1073 | 1074 | 1075 | @task.arg("path", desc="The file to read", type=str) 1076 | @task.returns("The bytes of the file") 1077 | class FsRead(task.BaseTask): 1078 | """Read value of path into bytes""" 1079 | 1080 | async def run(self) -> bytes: 1081 | code, out, err = await self.sh_with_code(f"cat {self.params.esc_path}") 1082 | assert code == 0, "exitcode was not zero" 1083 | return out 1084 | 1085 | ``` 1086 | 1087 |
1088 | 1089 | ------------------------------------------------- 1090 | 1091 | ## fs.stat 1092 | 1093 | Get stat info for path 1094 | 1095 | ### Arguments 1096 | 1097 | 1098 | - path *(str)* : The path of the file to stat 1099 | 1100 | 1101 | ### Returns 1102 | 1103 | *(Stat)* the stat object for the file 1104 | 1105 | 1106 |
1107 | Show source 1108 | 1109 | ```python 1110 | from pitcrew import task 1111 | 1112 | 1113 | class Stat: 1114 | inode: int 1115 | mode: str 1116 | user_id: int 1117 | group_id: int 1118 | size: int 1119 | access_time: int 1120 | modify_time: int 1121 | create_time: int 1122 | block_size: int 1123 | blocks: int 1124 | 1125 | def __str__(self): 1126 | return f"inode={self.inode} mode={self.mode} user_id={self.user_id} group_id={self.group_id} size={self.size} access_time={self.access_time} modify_time={self.modify_time} create_time={self.create_time} block_size={self.block_size} blocks={self.blocks}" 1127 | 1128 | 1129 | @task.arg("path", desc="The path of the file to stat", type=str) 1130 | @task.returns("the stat object for the file") 1131 | class FsStat(task.BaseTask): 1132 | 1133 | """Get stat info for path""" 1134 | 1135 | async def run(self) -> Stat: 1136 | stat = Stat() 1137 | platform = await self.facts.system.uname() 1138 | if platform == "darwin": 1139 | out = await self.sh( 1140 | f'stat -f "%i %p %u %g %z %a %m %c %k %b" {self.params.esc_path}' 1141 | ) 1142 | parts = out.strip().split(" ", 9) 1143 | elif platform == "linux": 1144 | out = await self.sh( 1145 | f'stat --format "%i %f %u %g %s %X %Y %W %B %b" {self.params.esc_path}' 1146 | ) 1147 | parts = out.strip().split(" ", 9) 1148 | else: 1149 | raise Exception(f"Can't support {platform}") 1150 | stat.inode = int(parts[0]) 1151 | stat.mode = "{0:o}".format(int(parts[1], 16)) 1152 | stat.user_id = int(parts[2]) 1153 | stat.group_id = int(parts[3]) 1154 | stat.size = int(parts[4]) 1155 | stat.access_time = int(parts[5]) 1156 | stat.modify_time = int(parts[6]) 1157 | stat.create_time = int(parts[7]) 1158 | stat.block_size = int(parts[8]) 1159 | stat.blocks = int(parts[9]) 1160 | return stat 1161 | 1162 | 1163 | class FsStatTest(task.TaskTest): 1164 | @task.TaskTest.ubuntu 1165 | async def test_ubuntu(self): 1166 | await self.fs.write("/tmp/some-file", b"Some delicious bytes") 1167 | stat = await self.fs.stat("/tmp/some-file") 1168 | assert stat.size == 20, "size is incorrect" 1169 | 1170 | ``` 1171 | 1172 |
1173 | 1174 | ------------------------------------------------- 1175 | 1176 | ## fs.touch 1177 | 1178 | Touches a file 1179 | 1180 | ### Arguments 1181 | 1182 | 1183 | - path *(str)* : The path to change the mode of 1184 | 1185 | 1186 | 1187 |
1188 | Show source 1189 | 1190 | ```python 1191 | from pitcrew import task 1192 | 1193 | 1194 | @task.arg("path", desc="The path to change the mode of", type=str) 1195 | class FsTouch(task.BaseTask): 1196 | """Touches a file""" 1197 | 1198 | async def run(self): 1199 | return await self.sh(f"touch {self.params.esc_path}") 1200 | 1201 | ``` 1202 | 1203 |
1204 | 1205 | ------------------------------------------------- 1206 | 1207 | ## fs.write 1208 | 1209 | Write bytes to a file 1210 | 1211 | ### Arguments 1212 | 1213 | 1214 | - path *(str)* : The path of the file to write to 1215 | - content *(bytes)* : The contents to write 1216 | 1217 | 1218 | 1219 |
1220 | Show source 1221 | 1222 | ```python 1223 | import hashlib 1224 | from pitcrew import task 1225 | 1226 | 1227 | @task.arg("path", type=str, desc="The path of the file to write to") 1228 | @task.arg("content", type=bytes, desc="The contents to write") 1229 | class FsWrite(task.BaseTask): 1230 | """Write bytes to a file""" 1231 | 1232 | async def verify(self): 1233 | stat = await self.fs.stat(self.params.path) 1234 | assert len(self.params.content) == stat.size 1235 | expected_digest = hashlib.sha256(self.params.content).hexdigest() 1236 | actual_digest = await self.fs.digests.sha256(self.params.path) 1237 | assert actual_digest == expected_digest 1238 | 1239 | async def run(self): 1240 | await self.sh( 1241 | f"tee {self.params.esc_path} > /dev/null", stdin=self.params.content 1242 | ) 1243 | 1244 | 1245 | class FsWriteTest(task.TaskTest): 1246 | @task.TaskTest.ubuntu 1247 | async def test_ubuntu(self): 1248 | with self.cd("/tmp"): 1249 | await self.fs.write("some-file", b"some content") 1250 | out = await self.sh("cat some-file") 1251 | assert out == "some content" 1252 | 1253 | ``` 1254 | 1255 |
1256 | 1257 | ------------------------------------------------- 1258 | 1259 | ## git.clone 1260 | 1261 | Installs a package, optionally allowing the version number to specified. 1262 | 1263 | This task defers exection to package-manager specific installation tasks, such as 1264 | homebrew or apt-get. 1265 | 1266 | 1267 | ### Arguments 1268 | 1269 | 1270 | - url *(str)* : The url to clone 1271 | - destination *(str)* : The destination 1272 | 1273 | 1274 | 1275 |
1276 | Show source 1277 | 1278 | ```python 1279 | import os 1280 | from pitcrew import task 1281 | 1282 | 1283 | @task.arg("url", desc="The url to clone", type=str) 1284 | @task.arg("destination", desc="The destination", type=str) 1285 | class GitClone(task.BaseTask): 1286 | """Installs a package, optionally allowing the version number to specified. 1287 | 1288 | This task defers exection to package-manager specific installation tasks, such as 1289 | homebrew or apt-get. 1290 | """ 1291 | 1292 | async def verify(self): 1293 | git_config = await self.fs.read( 1294 | os.path.join(self.params.destination, ".git", "config") 1295 | ) 1296 | assert ( 1297 | self.params.url in git_config.decode() 1298 | ), f"url {self.params.url} couldn't be found in the .git/config" 1299 | 1300 | async def run(self): 1301 | command = f"git clone {self.params.esc_url} {self.params.esc_destination}" 1302 | await self.sh(command) 1303 | 1304 | ``` 1305 | 1306 |
1307 | 1308 | ------------------------------------------------- 1309 | 1310 | ## homebrew.install 1311 | 1312 | Read value of path into bytes 1313 | 1314 | ### Arguments 1315 | 1316 | 1317 | - name *(str)* : Package to install 1318 | 1319 | 1320 | ### Returns 1321 | 1322 | *(str)* the version installed 1323 | 1324 | 1325 |
1326 | Show source 1327 | 1328 | ```python 1329 | from pitcrew import task 1330 | 1331 | 1332 | @task.arg("name", type=str, desc="Package to install") 1333 | @task.returns("the version installed") 1334 | class HomebrewInstall(task.BaseTask): 1335 | """Read value of path into bytes""" 1336 | 1337 | async def verify(self) -> str: 1338 | code, out, err = await self.sh_with_code( 1339 | f"brew ls --versions {self.params.esc_name}" 1340 | ) 1341 | lines = out.decode().strip().split("\n") 1342 | if lines != [""]: 1343 | for line in lines: 1344 | _, version = line.split(" ", 1) 1345 | return version 1346 | assert False, f"no version found for {self.params.name}" 1347 | 1348 | async def run(self): 1349 | await self.sh(f"brew install {self.params.esc_name}") 1350 | 1351 | async def available(self) -> bool: 1352 | code, _, _ = await self.sh_with_code("which brew") 1353 | return code == 0 1354 | 1355 | ``` 1356 | 1357 |
1358 | 1359 | ------------------------------------------------- 1360 | 1361 | ## install 1362 | 1363 | Installs a package, optionally allowing the version number to specified. 1364 | 1365 | This task defers exection to package-manager specific installation tasks, such as 1366 | homebrew or apt-get. 1367 | 1368 | 1369 | ### Arguments 1370 | 1371 | 1372 | - name *(str)* : The name of the package to install 1373 | 1374 | 1375 | ### Returns 1376 | 1377 | *(str)* The version of the package installed 1378 | 1379 | 1380 |
1381 | Show source 1382 | 1383 | ```python 1384 | from pitcrew import task 1385 | 1386 | 1387 | @task.arg("name", desc="The name of the package to install", type=str) 1388 | @task.returns("The version of the package installed") 1389 | class Install(task.BaseTask): 1390 | """Installs a package, optionally allowing the version number to specified. 1391 | 1392 | This task defers exection to package-manager specific installation tasks, such as 1393 | homebrew or apt-get. 1394 | """ 1395 | 1396 | async def run(self) -> str: 1397 | installer_tasks = [self.homebrew.install, self.apt_get.install] 1398 | for pkg in installer_tasks: 1399 | task = pkg.task() 1400 | if await task.available(): 1401 | return await task.invoke(name=self.params.name) 1402 | raise Exception("cannot find a package manager to defer to") 1403 | 1404 | ``` 1405 | 1406 |
1407 | 1408 | ------------------------------------------------- 1409 | 1410 | ## install.homebrew 1411 | 1412 | Installs the homebrew package manager 1413 | 1414 | 1415 | 1416 | 1417 | 1418 |
1419 | Show source 1420 | 1421 | ```python 1422 | from pitcrew import task 1423 | 1424 | 1425 | class InstallHomebrew(task.BaseTask): 1426 | """Installs the homebrew package manager""" 1427 | 1428 | async def verify(self): 1429 | await self.install.xcode_cli() 1430 | assert await self.sh("which brew") 1431 | 1432 | async def run(self): 1433 | await self.sh( 1434 | '/usr/bin/ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)"' 1435 | ) 1436 | 1437 | ``` 1438 | 1439 |
1440 | 1441 | ------------------------------------------------- 1442 | 1443 | ## install.xcode_cli 1444 | 1445 | Installs xcode cli tools 1446 | 1447 | 1448 | 1449 | 1450 | 1451 |
1452 | Show source 1453 | 1454 | ```python 1455 | from pitcrew import task 1456 | 1457 | 1458 | class InstallXcodeCli(task.BaseTask): 1459 | """Installs xcode cli tools""" 1460 | 1461 | async def verify(self): 1462 | assert await self.fs.is_directory("/Library/Developer/CommandLineTools") 1463 | 1464 | async def run(self): 1465 | await self.sh("xcode-select --install") 1466 | await self.poll(self.verify) 1467 | 1468 | ``` 1469 | 1470 |
1471 | 1472 | ------------------------------------------------- 1473 | 1474 | ## providers.docker 1475 | 1476 | A provider for ssh contexts 1477 | 1478 | ### Arguments 1479 | 1480 | 1481 | - container_ids *(list)* : The container ids to use 1482 | 1483 | 1484 | ### Returns 1485 | 1486 | *(DockerProvider)* An async generator that gives ssh contexts 1487 | 1488 | 1489 |
1490 | Show source 1491 | 1492 | ```python 1493 | from pitcrew import task 1494 | 1495 | 1496 | class DockerProvider: 1497 | def __init__(self, context, container_ids): 1498 | self.context = context 1499 | self.container_ids = container_ids 1500 | self.index = 0 1501 | 1502 | async def __aenter__(self): 1503 | return self 1504 | 1505 | async def __aexit__(self, exc_type, exc_value, traceback): 1506 | if exc_type: 1507 | raise exc_value.with_traceback(traceback) 1508 | 1509 | def __aiter__(self): 1510 | return self 1511 | 1512 | async def __anext__(self): 1513 | if self.index == len(self.container_ids): 1514 | raise StopAsyncIteration 1515 | docker_ctx = self.context.docker_context( 1516 | container_id=self.container_ids[self.index] 1517 | ) 1518 | self.index += 1 1519 | return docker_ctx 1520 | 1521 | def __str__(self): 1522 | return f"DockerProvider(container_ids={self.container_ids})" 1523 | 1524 | 1525 | @task.returns("An async generator that gives ssh contexts") 1526 | @task.arg("container_ids", type=list, desc="The container ids to use") 1527 | class ProvidersDocker(task.BaseTask): 1528 | """A provider for ssh contexts""" 1529 | 1530 | async def run(self) -> DockerProvider: 1531 | return DockerProvider(self.context, self.params.container_ids) 1532 | 1533 | ``` 1534 | 1535 |
1536 | 1537 | ------------------------------------------------- 1538 | 1539 | ## providers.local 1540 | 1541 | A provider for a local context 1542 | 1543 | 1544 | 1545 | 1546 | ### Returns 1547 | 1548 | *(LocalProvider)* An async generator that gives a local context 1549 | 1550 | 1551 |
1552 | Show source 1553 | 1554 | ```python 1555 | from pitcrew import task 1556 | 1557 | 1558 | class LocalProvider: 1559 | def __init__(self, local_context): 1560 | self.returned = False 1561 | self.local_context = local_context 1562 | 1563 | async def __aenter__(self): 1564 | return self 1565 | 1566 | async def __aexit__(self, exc_type, exc_value, traceback): 1567 | if exc_type: 1568 | raise exc_value.with_traceback(traceback) 1569 | 1570 | def __aiter__(self): 1571 | return self 1572 | 1573 | async def __anext__(self): 1574 | if not self.returned: 1575 | self.returned = True 1576 | return self.local_context 1577 | else: 1578 | raise StopAsyncIteration 1579 | 1580 | def __str__(self): 1581 | return "LocalProvider" 1582 | 1583 | 1584 | @task.returns("An async generator that gives a local context") 1585 | class ProvidersLocal(task.BaseTask): 1586 | """A provider for a local context""" 1587 | 1588 | async def run(self) -> LocalProvider: 1589 | return LocalProvider(self.context.local_context) 1590 | 1591 | 1592 | class ProvidersLocalTest(task.TaskTest): 1593 | @task.TaskTest.ubuntu 1594 | async def test_ubuntu(self): 1595 | async for p in await self.providers.local(): 1596 | assert p == self.context.local_context 1597 | 1598 | ``` 1599 | 1600 |
1601 | 1602 | ------------------------------------------------- 1603 | 1604 | ## providers.ssh 1605 | 1606 | A provider for ssh contexts 1607 | 1608 | ### Arguments 1609 | 1610 | 1611 | - hosts *(list)* : The hosts to use for ssh contexts 1612 | - tunnels *(list)* : The set of tunnels to connect through 1613 | - user *(str)* : The user to use for the ssh contexts 1614 | - agent_forwarding *(bool)* : Specify if forwarding is enabled 1615 | - agent_path *(str)* : Specify if forwarding is enabled 1616 | - ask_password *(str)* : The prompt to use for asking for a password 1617 | 1618 | 1619 | ### Returns 1620 | 1621 | *(SSHProvider)* An async generator that gives ssh contexts 1622 | 1623 | 1624 |
1625 | Show source 1626 | 1627 | ```python 1628 | import asyncssh 1629 | from pitcrew import task 1630 | from netaddr.ip.nmap import iter_nmap_range 1631 | 1632 | 1633 | class SSHProvider: 1634 | def __init__(self, context, hosts, user, tunnels=[], **connection_args): 1635 | self.context = context 1636 | self.hosts = hosts 1637 | self.tunnels = tunnels 1638 | self.connection_args = connection_args 1639 | self.flattened_hosts = self.__generate_flattened_hosts() 1640 | self.user = user 1641 | self.index = 0 1642 | self.tunnel_contexts = [] 1643 | 1644 | async def __aenter__(self): 1645 | last_tunnel = None 1646 | for tunnel in self.tunnels: 1647 | context = self.context.ssh_context(tunnel=last_tunnel, **tunnel) 1648 | self.tunnel_contexts.append(context) 1649 | await context.__aenter__() 1650 | last_tunnel = context.connection 1651 | 1652 | async def __aexit__(self, exc_type, exc_value, traceback): 1653 | for context in reversed(self.tunnel_contexts): 1654 | try: 1655 | await context.__aexit__() 1656 | except: 1657 | pass 1658 | if exc_type: 1659 | raise exc_value.with_traceback(traceback) 1660 | 1661 | def __aiter__(self): 1662 | return self 1663 | 1664 | async def __anext__(self): 1665 | if self.index == len(self.flattened_hosts): 1666 | raise StopAsyncIteration 1667 | 1668 | tunnel = self.tunnel_contexts[-1].connection if self.tunnel_contexts else None 1669 | ssh_ctx = self.context.ssh_context( 1670 | host=self.flattened_hosts[self.index], 1671 | user=self.user, 1672 | tunnel=tunnel, 1673 | **self.connection_args, 1674 | ) 1675 | self.index += 1 1676 | return ssh_ctx 1677 | 1678 | def __str__(self): 1679 | return f"SSHProvider(user={self.user} hosts={self.hosts})" 1680 | 1681 | def __generate_flattened_hosts(self): 1682 | hosts = [] 1683 | for host in self.hosts: 1684 | try: 1685 | hosts.append(map(lambda ip: str(ip), list(iter_nmap_range(host)))) 1686 | except: 1687 | hosts.append(host) 1688 | return hosts 1689 | 1690 | 1691 | @task.returns("An async generator that gives ssh contexts") 1692 | @task.arg("hosts", type=list, desc="The hosts to use for ssh contexts") 1693 | @task.arg( 1694 | "tunnels", type=list, desc="The set of tunnels to connect through", default=[] 1695 | ) 1696 | @task.opt("user", type=str, desc="The user to use for the ssh contexts") 1697 | @task.opt( 1698 | "agent_forwarding", 1699 | type=bool, 1700 | default=False, 1701 | desc="Specify if forwarding is enabled", 1702 | ) 1703 | @task.opt("agent_path", type=str, desc="Specify if forwarding is enabled") 1704 | @task.opt("ask_password", type=str, desc="The prompt to use for asking for a password") 1705 | class ProvidersSsh(task.BaseTask): 1706 | """A provider for ssh contexts""" 1707 | 1708 | async def run(self) -> SSHProvider: 1709 | extra_args = {} 1710 | if self.params.agent_path: 1711 | extra_args["agent_path"] = self.params.agent_path 1712 | if self.params.ask_password: 1713 | extra_args["password"] = await self.password(self.params.ask_password) 1714 | return SSHProvider( 1715 | self.context, 1716 | self.params.hosts, 1717 | self.params.user, 1718 | tunnels=self.params.tunnels, 1719 | agent_forwarding=self.params.agent_forwarding, 1720 | **extra_args, 1721 | ) 1722 | 1723 | ``` 1724 | 1725 |
1726 | 1727 | ------------------------------------------------- 1728 | 1729 | -------------------------------------------------------------------------------- /pitcrew.spec: -------------------------------------------------------------------------------- 1 | # -*- mode: python -*- 2 | 3 | from PyInstaller.utils.hooks import collect_data_files 4 | 5 | block_cipher = None 6 | 7 | 8 | a = Analysis(['pitcrew/cli.py'], 9 | pathex=['pitcrew'], 10 | binaries=[], 11 | datas=collect_data_files("pitcrew.tasks", include_py_files=True), 12 | hiddenimports=[], 13 | hookspath=[], 14 | runtime_hooks=[], 15 | excludes=[], 16 | win_no_prefer_redirects=False, 17 | win_private_assemblies=False, 18 | cipher=block_cipher, 19 | noarchive=False) 20 | pyz = PYZ(a.pure, a.zipped_data, 21 | cipher=block_cipher) 22 | exe = EXE(pyz, 23 | a.scripts, 24 | a.binaries, 25 | a.zipfiles, 26 | a.datas, 27 | [], 28 | name='crew', 29 | debug=False, 30 | bootloader_ignore_signals=False, 31 | strip=False, 32 | upx=True, 33 | runtime_tmpdir=None, 34 | console=True ) 35 | -------------------------------------------------------------------------------- /pitcrew/__init__.py: -------------------------------------------------------------------------------- 1 | from .__version__ import __version__ 2 | -------------------------------------------------------------------------------- /pitcrew/__version__.py: -------------------------------------------------------------------------------- 1 | __name__ = "pitcrew" 2 | __description__ = ( 3 | "AsyncIO-powered python DSL for running commands locally, on docker, or over ssh." 4 | ) 5 | __url__ = "http://pitcrew.io" 6 | __author__ = "joshbuddy" 7 | __author_email__ = "joshbuddy@gmail.com" 8 | __license__ = "Apache 2.0" 9 | __version__ = "0.0.3" 10 | -------------------------------------------------------------------------------- /pitcrew/app.py: -------------------------------------------------------------------------------- 1 | import re 2 | import os 3 | import atexit 4 | import shutil 5 | import jinja2 6 | from pitcrew.loader import Loader 7 | from pitcrew.context import LocalContext 8 | from pitcrew.docs import Docs 9 | from pitcrew.test import TestRunner 10 | from pitcrew.executor import Executor 11 | from pitcrew.passwords import Passwords 12 | 13 | 14 | class App: 15 | def __init__(self): 16 | self.template_render_path = os.path.join("/tmp", "crew", "templates") 17 | atexit.register(self.delete_rendered_templates) 18 | os.makedirs(self.template_render_path, exist_ok=True) 19 | self.loader = Loader() 20 | self.passwords = Passwords() 21 | self.local_context = LocalContext(self, self.loader) 22 | 23 | def executor(self, *args, **kwargs): 24 | return Executor(*args, **kwargs) 25 | 26 | def load(self, task_name): 27 | task = self.loader.load(task_name, self.local_context) 28 | return task 29 | 30 | def docs(self): 31 | return Docs(self) 32 | 33 | def test_runner(self): 34 | return TestRunner(self, LocalContext(self, self.loader)) 35 | 36 | def create_task(self, name): 37 | path = os.path.realpath(os.path.join(__file__, "..", "templates")) 38 | templateLoader = jinja2.FileSystemLoader(searchpath=path) 39 | templateEnv = jinja2.Environment(loader=templateLoader) 40 | word_parts = re.split(r"[._]", name) 41 | path_parts = name.split(".") 42 | class_name = "".join(map(lambda p: p.capitalize(), word_parts)) 43 | template = templateEnv.get_template("new_task.py.j2") 44 | rendered_task = template.render(task_class_name=class_name) 45 | base_task_path = os.path.realpath(os.path.join(__file__, "..", "tasks")) 46 | task_path = os.path.join(base_task_path, *path_parts) + ".py" 47 | if os.path.isfile(task_path): 48 | raise Exception(f"there is already something in the way {task_path}") 49 | base_path = os.path.dirname(task_path) 50 | os.makedirs(base_path, exist_ok=True) 51 | for i in range(len(path_parts) - 1): 52 | potential_task_path = ( 53 | os.path.join(base_task_path, *path_parts[0 : i + 1]) + ".py" 54 | ) 55 | if os.path.isfile(potential_task_path): 56 | new_path = os.path.join( 57 | base_task_path, *path_parts[0 : i + 1], "__init__.py" 58 | ) 59 | os.rename(potential_task_path, new_path) 60 | 61 | with open(task_path, "w") as fh: 62 | fh.write(rendered_task) 63 | 64 | async def __aenter__(self): 65 | return self 66 | 67 | async def __aexit__(self, exc_type, exc, tb): 68 | pass 69 | 70 | def delete_rendered_templates(self): 71 | shutil.rmtree(self.template_render_path, ignore_errors=True) 72 | -------------------------------------------------------------------------------- /pitcrew/cli.py: -------------------------------------------------------------------------------- 1 | import os 2 | import json 3 | import sys 4 | import asyncio 5 | import click 6 | import argparse 7 | from subprocess import call 8 | from pitcrew.app import App 9 | from pitcrew.util import ResultsPrinter 10 | 11 | 12 | @click.group(invoke_without_command=False) 13 | @click.version_option() 14 | @click.pass_context 15 | def cli(ctx): 16 | pass 17 | 18 | 19 | @cli.command( 20 | short_help="run a shell command", 21 | context_settings={ 22 | "ignore_unknown_options": True, 23 | "allow_extra_args": True, 24 | "allow_interspersed_args": True, 25 | }, 26 | ) 27 | @click.argument("shell_command", nargs=-1, type=click.UNPROCESSED) 28 | @click.option("--provider", "-p", default="providers.local") 29 | @click.option("--provider-args", "provider_json", "-P", default="{}") 30 | @click.pass_context 31 | def sh(ctx, *, provider, provider_json, shell_command): 32 | """Allows running a shell command.""" 33 | 34 | async def run_task(): 35 | async with App() as app: 36 | provider_args = json.loads(provider_json) 37 | joined_command = " ".join(shell_command) 38 | sys.stderr.write( 39 | f"Invoking \033[1m{joined_command}\033[0m with\n provider \033[1m{provider} {provider_args}\033[0m\n" 40 | ) 41 | 42 | async def fn(self): 43 | return await self.sh(joined_command) 44 | 45 | provider_task = app.load(provider) 46 | provider_instance = await provider_task.invoke(**provider_args) 47 | async with app.executor(provider_instance) as executor: 48 | results = await executor.invoke(fn) 49 | 50 | ResultsPrinter(results).print() 51 | 52 | loop = asyncio.get_event_loop() 53 | loop.run_until_complete(run_task()) 54 | 55 | 56 | @cli.command( 57 | short_help="run a task", 58 | context_settings={ 59 | "ignore_unknown_options": True, 60 | "allow_extra_args": True, 61 | "allow_interspersed_args": True, 62 | }, 63 | ) 64 | @click.argument("task_name") 65 | @click.argument("extra_args", nargs=-1, type=click.UNPROCESSED) 66 | @click.option("--provider", "-p", default="providers.local") 67 | @click.option("--provider-args", "provider_json", "-P", default="{}") 68 | @click.pass_context 69 | def run(ctx, *, provider, provider_json, task_name, extra_args): 70 | """Allows running a task in crew/tasks. Parameters after task name are 71 | interpretted as task arguments.""" 72 | 73 | async def run_task(): 74 | async with App() as app: 75 | task = app.load(task_name) 76 | task.coerce_inputs(True) 77 | parser = argparse.ArgumentParser(description=task.__doc__) 78 | for arg in task.args: 79 | if arg.required: 80 | parser.add_argument(arg.name, help=arg.desc) 81 | else: 82 | parser.add_argument(f"--{arg.name}", help=arg.desc) 83 | 84 | parsed_task_args = parser.parse_args(extra_args) 85 | provider_args = json.loads(provider_json) 86 | dict_args = vars(parsed_task_args) 87 | sys.stderr.write( 88 | f"Invoking \033[1m{task_name}\033[0m \033[1m{dict_args}\033[0m with\n provider \033[1m{provider} {provider_args}\033[0m\n" 89 | ) 90 | 91 | provider_task = app.load(provider) 92 | provider_instance = await provider_task.invoke(**provider_args) 93 | async with app.executor(provider_instance) as executor: 94 | results = await executor.run_task(task, **dict_args) 95 | 96 | ResultsPrinter(results).print() 97 | 98 | loop = asyncio.get_event_loop() 99 | loop.run_until_complete(run_task()) 100 | 101 | 102 | @cli.command(short_help="list all tasks") 103 | @click.pass_context 104 | def list(ctx): 105 | """Lists available crew tasks.""" 106 | 107 | app = App() 108 | for task in app.loader.each_task(): 109 | if task.nodoc: 110 | continue 111 | short_desc = ( 112 | task.__doc__.split("\n")[0] 113 | if task.__doc__ 114 | else "\033[5m(no description)\033[0m" 115 | ) 116 | print(f"\033[1m{task.task_name}\033[0m {short_desc}") 117 | 118 | 119 | @cli.command(short_help="info a command") 120 | @click.argument("task_name") 121 | @click.pass_context 122 | def info(ctx, *, task_name): 123 | """Shows information about a single task.""" 124 | 125 | task = App().load(task_name) 126 | print("\033[1mName\033[0m") 127 | print(task_name) 128 | print("\033[1mPath\033[0m") 129 | print(task.source_path()) 130 | print("\033[1mDescription\033[0m") 131 | print(task.__doc__) 132 | print("\033[1mArguments\033[0m") 133 | for arg in task.args: 134 | print(f"{arg.name} ({arg.type.__name__}): {arg.desc}") 135 | 136 | if task.has_return_type(): 137 | print("\033[1mReturns\033[0m") 138 | print(f"{task.expected_return_type().__name__}: {task.return_desc}") 139 | 140 | 141 | @cli.command(short_help="generate docs") 142 | @click.option("--check/--no-check", "-c", default=False) 143 | @click.pass_context 144 | def docs(ctx, check): 145 | """Regenerates docs.""" 146 | 147 | docs = App().docs() 148 | if check: 149 | docs.check() 150 | html = docs.generate() 151 | with open("docs/tasks.md", "w") as fh: 152 | fh.write(html) 153 | 154 | print(f"Docs generated at docs/tasks.md") 155 | 156 | 157 | @cli.command(short_help="run tests") 158 | @click.argument("task_prefix", nargs=-1) 159 | @click.pass_context 160 | def test(ctx, task_prefix): 161 | """Run task tests.""" 162 | 163 | async def run_tests(): 164 | async with App() as app: 165 | await app.test_runner().run(task_prefix) 166 | 167 | loop = asyncio.get_event_loop() 168 | loop.run_until_complete(run_tests()) 169 | print(f"Tests complete") 170 | 171 | 172 | @cli.command(short_help="create a new task") 173 | @click.argument("task_name") 174 | @click.pass_context 175 | def new(ctx, task_name): 176 | app = App() 177 | app.create_task(task_name) 178 | print(f"Task created!") 179 | 180 | 181 | @cli.command(short_help="edit a task") 182 | @click.argument("task_name") 183 | @click.pass_context 184 | def edit(ctx, task_name): 185 | task = App().load(task_name) 186 | editor = os.environ["EDITOR"] 187 | call([editor, task.source_path()]) 188 | 189 | 190 | @cli.command(short_help="show help") 191 | @click.pass_context 192 | def help(ctx): 193 | click.echo(ctx.parent.get_help()) 194 | 195 | 196 | if __name__ == "__main__": 197 | cli() 198 | -------------------------------------------------------------------------------- /pitcrew/context.py: -------------------------------------------------------------------------------- 1 | """Contexts allow execution of tasks. There are currently three types of 2 | contexts: local, ssh and docker. 3 | 4 | Local contexts run commands on the host computer running crew. 5 | 6 | SSH contexts run commands over SSH on the target computer. 7 | 8 | Docker contexts run commands on a running docker container. 9 | """ 10 | 11 | import os 12 | import shlex 13 | import asyncio 14 | import asyncssh 15 | import getpass 16 | from typing import Tuple 17 | from pitcrew.file import LocalFile, DockerFile, SSHFile 18 | from pitcrew.logger import logger 19 | from abc import ABC, abstractmethod 20 | 21 | 22 | class ChangeUser: 23 | """Context manager to allow changing the user within a context""" 24 | 25 | def __init__(self, context, new_user): 26 | self.context = context 27 | self.old_user = context.user 28 | self.new_user = new_user 29 | 30 | def __enter__(self): 31 | self.context.user = self.new_user 32 | return self.context 33 | 34 | def __exit__(self, exc, value, tb): 35 | self.context.user = self.old_user 36 | 37 | 38 | class ChangeDirectory: 39 | """Context manager to allow changing the current directory within a context""" 40 | 41 | def __init__(self, context, new_directory): 42 | self.context = context 43 | self.old_directory = context.directory 44 | if not self.old_directory: 45 | self.old_directory = "." 46 | if new_directory.startswith("/"): 47 | self.new_directory = new_directory 48 | else: 49 | self.new_directory = os.path.join(self.old_directory, new_directory) 50 | 51 | def __enter__(self): 52 | self.context.directory = self.new_directory 53 | return self.context 54 | 55 | def __exit__(self, exc, value, tb): 56 | self.context.directory = self.old_directory 57 | 58 | 59 | class Context(ABC): 60 | """Abstract base class for all contexts.""" 61 | 62 | def __init__(self, app, loader, user=None, parent_context=None, directory=None): 63 | self.app = app 64 | self.loader = loader 65 | self.user = user or getpass.getuser() 66 | self.directory = directory 67 | self.actual_user = None 68 | self.parent_context = parent_context 69 | self.cache = {} 70 | 71 | @abstractmethod 72 | async def sh_with_code(command, stdin=None, env=None) -> Tuple[int, bytes, bytes]: 73 | pass 74 | 75 | @abstractmethod 76 | async def raw_sh_with_code(command) -> Tuple[int, bytes, bytes]: 77 | pass 78 | 79 | @abstractmethod 80 | def descriptor(self) -> str: 81 | pass 82 | 83 | async def password(self, prompt) -> str: 84 | """Present the user with a password prompt using the prompt given. If two 85 | identical prompts are supplied, the user is only asked once, and subsequent calls will 86 | provide the password given.""" 87 | return await self.app.passwords.get_password(prompt) 88 | 89 | async def sh_ok(self, command, stdin=None, env=None) -> bool: 90 | code, _, _ = await self.sh_with_code(command, stdin=stdin, env=env) 91 | return code == 0 92 | 93 | async def sh(self, command, stdin=None, env=None) -> str: 94 | """Runs a shell command within the given context. Raises an AssertionError if it exits with 95 | a non-zero exitcode. Returns STDOUT encoded with utf-8.""" 96 | 97 | logger.shell_start(self, command) 98 | code, out, err = await self.sh_with_code(command, stdin=stdin, env=env) 99 | logger.shell_stop(self, code, out, err) 100 | assert ( 101 | code == 0 102 | ), f"expected exit code of 0, got {code} when running\n:COMMAND: {command}\nOUT: {out.decode()}\n\nERR {err.decode()}" 103 | return out.decode() 104 | 105 | def docker_context(self, *args, **kwargs) -> "DockerContext": 106 | """Creates a new docker context with the given container id.""" 107 | return DockerContext( 108 | self.app, self.loader, *args, **kwargs, parent_context=self 109 | ) 110 | 111 | def ssh_context(self, *args, **kwargs) -> "SSHContext": 112 | """Creates a new ssh context with the given container id.""" 113 | return SSHContext(self.app, self.loader, *args, **kwargs, parent_context=self) 114 | 115 | async def fill_actual_user(self): 116 | if self.actual_user: 117 | return 118 | code, out, err = await self.raw_sh_with_code("whoami") 119 | assert code == 0, "unable to run whoami to determine the user" 120 | self.actual_user = out.decode().strip() 121 | if self.actual_user != self.user: 122 | print("Escalating user!") 123 | 124 | def with_user(self, user): 125 | """Returns a context handler for defining the user""" 126 | return ChangeUser(self, user) 127 | 128 | def cd(self, directory): 129 | """Returns a context handler for changing the directory""" 130 | return ChangeDirectory(self, directory) 131 | 132 | async def __aenter__(self): 133 | return self 134 | 135 | async def __aexit__(self, exc_type, exc_value, traceback): 136 | if exc_type: 137 | raise exc_value.with_traceback(traceback) 138 | 139 | async def invoke(self, fn, *args, **kwargs): 140 | """Allows invoking of an async function within this context.""" 141 | task = self.create_task(fn) 142 | return await task.run(*args, **kwargs) 143 | 144 | def create_task(self, fn): 145 | from pitcrew.task import BaseTask 146 | 147 | task_cls = type(fn.__name__, (BaseTask,), dict(run=fn)) 148 | task = task_cls() 149 | task.context = self 150 | return task 151 | 152 | def has_package(self, name): 153 | return self.loader.has_package(name) 154 | 155 | def package(self, name): 156 | return self.loader.package(name, self) 157 | 158 | @property 159 | def local_context(self) -> "LocalContext": 160 | return self.app.local_context 161 | 162 | def file(self, path): 163 | return self.file_class(self, path) 164 | 165 | def esc(self, text): 166 | return shlex.quote(text) 167 | 168 | def __getattr__(self, name): 169 | if self.has_package(name): 170 | return self.package(name) 171 | else: 172 | return self.__getattribute__(name) 173 | 174 | async def _prepare_command(self, command): 175 | await self.fill_actual_user() 176 | if self.directory: 177 | command = f"cd {self.esc(self.directory)} && {command}" 178 | if self.actual_user != self.user: 179 | command = f"sudo -u {self.esc(self.user)} -- /bin/sh -c {self.esc(command)}" 180 | return command 181 | 182 | 183 | class LocalContext(Context): 184 | _singleton = None 185 | file_class = LocalFile 186 | 187 | def __new__(cls, *args, **kwargs): 188 | if not cls._singleton: 189 | cls._singleton = object.__new__(LocalContext) 190 | return cls._singleton 191 | 192 | async def sh_with_code(self, command, stdin=None, env=None): 193 | command = await self._prepare_command(command) 194 | new_env = os.environ.copy() 195 | new_env.pop("__PYVENV_LAUNCHER__", None) 196 | if env: 197 | new_env.update(env) 198 | 199 | kwargs = { 200 | "stdout": asyncio.subprocess.PIPE, 201 | "stderr": asyncio.subprocess.PIPE, 202 | "stdin": asyncio.subprocess.PIPE, 203 | "env": new_env, 204 | } 205 | proc = await asyncio.create_subprocess_shell(command, **kwargs) 206 | stdout, stderr = await proc.communicate(input=stdin) 207 | return (proc.returncode, stdout, stderr) 208 | 209 | async def raw_sh_with_code(self, command): 210 | kwargs = {"stdout": asyncio.subprocess.PIPE, "stderr": asyncio.subprocess.PIPE} 211 | proc = await asyncio.create_subprocess_shell(command, **kwargs) 212 | stdout, stderr = await proc.communicate() 213 | return (proc.returncode, stdout, stderr) 214 | 215 | def descriptor(self): 216 | return f"{self.user}@local" 217 | 218 | 219 | class SSHContext(Context): 220 | file_class = SSHFile 221 | 222 | def __init__( 223 | self, 224 | app, 225 | loader, 226 | host, 227 | port=22, 228 | user=None, 229 | parent_context=None, 230 | **connection_kwargs, 231 | ): 232 | self.host = host 233 | self.port = port 234 | self.async_helper = None 235 | self.connection = None 236 | self.connect_timeout = 1 237 | self.connection_kwargs = connection_kwargs 238 | super().__init__(app, loader, user=user, parent_context=parent_context) 239 | 240 | async def sh_with_code(self, command, stdin=None, env=None): 241 | command = await self._prepare_command(command) 242 | proc = await self.connection.run( 243 | command, stdin=stdin, env=env or {}, encoding=None 244 | ) 245 | return (proc.exit_status, proc.stdout, proc.stderr) 246 | 247 | async def raw_sh_with_code(self, command): 248 | proc = await self.connection.run(command, encoding=None) 249 | return (proc.exit_status, proc.stdout, proc.stderr) 250 | 251 | async def __aenter__(self): 252 | gen = asyncio.wait_for( 253 | asyncssh.connect( 254 | self.host, port=self.port, username=self.user, **self.connection_kwargs 255 | ), 256 | timeout=self.connect_timeout, 257 | ) 258 | self.connection = await gen 259 | return self 260 | 261 | async def __aexit__(self, exc_type, exc_value, traceback): 262 | self.connection.close() 263 | await super().__aexit__(exc_type, exc_value, traceback) 264 | 265 | def descriptor(self): 266 | return f"ssh:{self.user}@{self.host}" 267 | 268 | 269 | class DockerContext(Context): 270 | file_class = DockerFile 271 | 272 | def __init__(self, app, loader, container_id, **kwargs): 273 | self.container_id = container_id 274 | super().__init__(app, loader, **kwargs) 275 | 276 | async def sh_with_code(self, command, stdin=None, env=None): 277 | command = await self._prepare_command(command) 278 | env_string = "" 279 | if env: 280 | for k, v in env.items(): 281 | env_string += f"-e {self.esc(k)}={self.esc(v)} " 282 | 283 | cmd = f"docker exec -i {env_string}{self.container_id} /bin/sh -c {self.esc(command)}" 284 | return await self.local_context.sh_with_code(cmd, stdin=stdin) 285 | 286 | async def raw_sh_with_code(self, command): 287 | return await self.local_context.raw_sh_with_code( 288 | f"docker exec -i {self.container_id} /bin/sh -c {self.esc(command)}" 289 | ) 290 | 291 | def descriptor(self): 292 | return f"docker:{self.user}@{self.container_id[0:6]}" 293 | 294 | async def __aexit__(self, exc_type, exc_value, traceback): 295 | await self.local_context.docker.stop(self.container_id, time=0) 296 | await super().__aexit__(exc_type, exc_value, traceback) 297 | -------------------------------------------------------------------------------- /pitcrew/docs.py: -------------------------------------------------------------------------------- 1 | import os 2 | import jinja2 3 | 4 | 5 | class Docs: 6 | def __init__(self, app): 7 | self.app = app 8 | self.undoced_tasks = [] 9 | 10 | def generate(self): 11 | path = os.path.realpath(os.path.join(__file__, "..", "templates")) 12 | templateLoader = jinja2.FileSystemLoader(searchpath=path) 13 | templateEnv = jinja2.Environment(loader=templateLoader) 14 | template = templateEnv.get_template("task.md.j2") 15 | tasks = [] 16 | for task in self.app.loader.each_task(): 17 | if task.nodoc: 18 | continue 19 | desc = task.desc() 20 | if not desc: 21 | self.undoced_tasks.append(task) 22 | tasks.append(template.render(task=task, desc=desc)) 23 | big_template = templateEnv.get_template("tasks.md.j2") 24 | return big_template.render(tasks=tasks) 25 | 26 | def check(self): 27 | self.undoced_tasks = [] 28 | self.generate() 29 | if self.undoced_tasks: 30 | tasks = ", ".join(map(lambda t: t.task_name, self.undoced_tasks)) 31 | raise Exception(f"there are tasks undoced {tasks}") 32 | -------------------------------------------------------------------------------- /pitcrew/executor.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | 3 | 4 | class WorkItem: 5 | def __init__(self, context, fn, args, kwargs): 6 | self.context = context 7 | self.fn = fn 8 | self.args = args 9 | self.kwargs = kwargs 10 | 11 | 12 | class ExecutionResult: 13 | def __init__(self, context, result, exception): 14 | self.context = context 15 | self.result = result 16 | self.exception = exception 17 | 18 | 19 | class ResultsList: 20 | def __init__(self): 21 | self.results = [] 22 | self.passed = [] 23 | self.failed = [] 24 | self.errored = [] 25 | 26 | def append(self, result): 27 | self.results.append(result) 28 | 29 | if result.exception and isinstance(result.exception, AssertionError): 30 | self.failed.append(result) 31 | elif result.exception: 32 | self.errored.append(result) 33 | else: 34 | self.passed.append(result) 35 | 36 | 37 | class Executor: 38 | def __init__(self, provider, concurrency=100): 39 | self.provider = provider 40 | self.concurrency = concurrency 41 | self.queue = asyncio.Queue(maxsize=self.concurrency) 42 | self.results = ResultsList() 43 | self.workers = [] 44 | 45 | async def run_task(self, task, *args, **kwargs) -> ResultsList: 46 | async def execute_task(ctx, *args, **kwargs): 47 | return await task.invoke_with_context(ctx, *args, **kwargs) 48 | 49 | return await self.invoke(execute_task, *args, **kwargs) 50 | 51 | async def invoke(self, fn, *args, **kwargs) -> ResultsList: 52 | enqueuer_future = asyncio.ensure_future( 53 | self._start_provider_enquerer(fn, *args, **kwargs) 54 | ) 55 | await asyncio.wait_for(enqueuer_future, None) 56 | await self.queue.join() 57 | return self.results 58 | 59 | async def __aenter__(self): 60 | return self 61 | 62 | async def __aexit__(self, exc_type, exc_value, traceback): 63 | for w in self.workers: 64 | w.cancel() 65 | 66 | for w in asyncio.as_completed(self.workers): 67 | await w 68 | 69 | def _start_worker(self): 70 | self.workers.append(asyncio.ensure_future(self._worker_loop())) 71 | 72 | async def _worker_loop(self): 73 | try: 74 | while True: 75 | item = await self.queue.get() 76 | try: 77 | async with item.context: 78 | try: 79 | result = await item.context.invoke( 80 | item.fn, *item.args, **item.kwargs 81 | ) 82 | self.results.append( 83 | ExecutionResult(item.context, result, None) 84 | ) 85 | except Exception as e: 86 | self.results.append(ExecutionResult(item.context, None, e)) 87 | except Exception as e: 88 | self.results.append(ExecutionResult(item.context, None, e)) 89 | finally: 90 | self.queue.task_done() 91 | except asyncio.CancelledError: 92 | pass 93 | 94 | async def _start_provider_enquerer(self, fn, *args, **kwargs): 95 | async with self.provider: 96 | async for context in self.provider: 97 | await self.queue.put(WorkItem(context, fn, args, kwargs)) 98 | self._start_worker() 99 | -------------------------------------------------------------------------------- /pitcrew/file.py: -------------------------------------------------------------------------------- 1 | """File objects are created through their respective contexts. A file object can be copied into 2 | another context via a file reference for the destination. For example, if operating in an SSH 3 | context, this would copy from the local filesystem to that destination: 4 | 5 | self.local_context("/some/file").copy_to(self.file("/some/other")) 6 | 7 | 8 | For convenience `owner`, `group` and `mode` arguments are available on the `copy_to` method to 9 | allow setting those attributes post-copy. 10 | """ 11 | 12 | import os 13 | import asyncssh 14 | from pitcrew.logger import logger 15 | from abc import ABC 16 | 17 | 18 | class File(ABC): 19 | """Abstract base class for file-based operations""" 20 | 21 | def __init__(self, context, path): 22 | self.context = context 23 | self.path = path 24 | 25 | def __str__(self): 26 | return f"{self.context.descriptor()}:{self.path}" 27 | 28 | async def copy_to(self, dest, archive=False, owner=None, group=None, mode=None): 29 | """Copies a file from the source to the destination.""" 30 | with logger.with_copy(self, dest): 31 | pair = (self.__class__, dest.__class__) 32 | if pair in copiers: 33 | await copiers[pair](self, dest, archive=archive) 34 | if owner: 35 | await dest.context.fs.chown(dest.path, owner, group=group) 36 | if mode: 37 | await dest.context.fs.chmod(dest.path, mode) 38 | else: 39 | raise Exception(f"cannot find a copier for {pair}") 40 | 41 | 42 | class LocalFile(File): 43 | """A reference to a file on the local machine executing pitcrew""" 44 | 45 | def __init__(self, context, path): 46 | self.context = context 47 | self.path = os.path.expanduser(path) 48 | 49 | 50 | class DockerFile(File): 51 | """A reference to a file on a Docker container""" 52 | 53 | pass 54 | 55 | 56 | class SSHFile(File): 57 | """A reference to a file on a remote host accessible via SSH""" 58 | 59 | pass 60 | 61 | 62 | async def local_to_local_copier(src, dest, archive=False): 63 | ctx = src.context 64 | command = "cp" 65 | if archive: 66 | command += " -a" 67 | command += f" {ctx.esc(src.path)} {ctx.esc(dest.path)}" 68 | await ctx.sh(command) 69 | 70 | 71 | async def ssh_to_local_copier(src, dest, archive=False): 72 | ctx = src.context 73 | await asyncssh.scp( 74 | (ctx.connection, src.path), dest.path, recursive=archive, preserve=archive 75 | ) 76 | 77 | 78 | async def local_to_ssh_copier(src, dest, archive=False): 79 | ctx = dest.context 80 | await asyncssh.scp( 81 | src.path, (ctx.connection, dest.path), recursive=archive, preserve=archive 82 | ) 83 | 84 | 85 | async def docker_to_local_copier(src, dest, archive=False): 86 | ctx = dest.context 87 | command = "docker cp" 88 | if archive: 89 | command += " -a" 90 | command += ( 91 | f" {ctx.esc(src.context.container_id)}:{ctx.esc(src.path)} {ctx.esc(dest.path)}" 92 | ) 93 | await ctx.sh(command) 94 | 95 | 96 | async def local_to_docker_copier(src, dest, archive=False): 97 | ctx = src.context 98 | command = "docker cp" 99 | if archive: 100 | command += " -a" 101 | command += f" {ctx.esc(src.path)} {ctx.esc(dest.context.container_id)}:{ctx.esc(dest.path)}" 102 | await ctx.sh(command) 103 | 104 | 105 | copiers = { 106 | (LocalFile, LocalFile): local_to_local_copier, 107 | (SSHFile, LocalFile): ssh_to_local_copier, 108 | (LocalFile, SSHFile): local_to_ssh_copier, 109 | (DockerFile, LocalFile): docker_to_local_copier, 110 | (LocalFile, DockerFile): local_to_docker_copier, 111 | } 112 | -------------------------------------------------------------------------------- /pitcrew/loader.py: -------------------------------------------------------------------------------- 1 | import inspect 2 | import os 3 | import importlib 4 | import pitcrew.tasks 5 | from pitcrew import task 6 | from pitcrew.logger import logger 7 | 8 | 9 | class NoTaskException(Exception): 10 | pass 11 | 12 | 13 | class Package: 14 | def __init__(self, loader, context, name, base=None): 15 | self.loader = loader 16 | self.context = context 17 | self.name = name 18 | self.base = base 19 | 20 | def __getattr__(self, name): 21 | return Package(self.loader, self.context, name, base=self) 22 | 23 | async def __call__(self, *args, **kwargs): 24 | return await self.task().invoke(*args, **kwargs) 25 | 26 | def _prefix(self): 27 | if self.base is None: 28 | return [self.name] 29 | else: 30 | return self.base._prefix() + [self.name] 31 | 32 | def task(self): 33 | task_name = ".".join(self._prefix()) 34 | task = self.loader.load(task_name, self.context) 35 | return task 36 | 37 | 38 | class Loader: 39 | def __init__(self): 40 | self.tasks = {} 41 | self.task_dir = os.path.abspath(os.path.join(__file__, "..", "tasks")) 42 | 43 | def load(self, name, context): 44 | return self.create_task(name, context) 45 | 46 | def package(self, name, context): 47 | return Package(self, context, name) 48 | 49 | def has_package(self, name): 50 | if os.path.isfile(os.path.join(self.task_dir, name) + ".py"): 51 | return True 52 | elif os.path.isdir(os.path.join(self.task_dir, name)): 53 | return True 54 | else: 55 | return False 56 | 57 | def create_task(self, task_name, context): 58 | self.populate_task(task_name) 59 | task = self.tasks[task_name]() 60 | task.context = context 61 | task.name = task_name 62 | return task 63 | 64 | def each_task(self, prefix=[]): 65 | path = os.path.join(self.task_dir, *prefix) 66 | if os.path.isfile(path) and path.endswith(".py"): 67 | if prefix[-1] == "__init__.py": 68 | prefix.pop() 69 | else: 70 | prefix[-1] = os.path.splitext(prefix[-1])[0] 71 | if prefix: 72 | yield self.populate_task(".".join(prefix)) 73 | elif os.path.isdir(path): 74 | paths = os.listdir(path) 75 | paths.sort() 76 | for p in paths: 77 | if p != "__init__.py" and p.startswith("__"): 78 | continue 79 | new_prefix = prefix + [p] 80 | for t in self.each_task(new_prefix): 81 | yield t 82 | 83 | def populate_task(self, task_name): 84 | try: 85 | if task_name not in self.tasks: 86 | pkg = importlib.import_module(f".{task_name}", "pitcrew.tasks") 87 | c = None 88 | tests = [] 89 | for name, val in pkg.__dict__.items(): 90 | if inspect.isclass(val): 91 | if issubclass(val, task.BaseTask): 92 | task_cls = getattr(pkg, name) 93 | task_cls._args() 94 | c = task_cls 95 | c.task_name = task_name 96 | elif issubclass(val, task.TaskTest): 97 | tests.append(getattr(pkg, name)) 98 | c.tests = tests 99 | self.tasks[task_name] = c 100 | except Exception: 101 | logger.info(f"error loading {task_name}") 102 | raise 103 | return self.tasks[task_name] 104 | -------------------------------------------------------------------------------- /pitcrew/logger.py: -------------------------------------------------------------------------------- 1 | import time 2 | import sys 3 | 4 | 5 | def truncated_value(val): 6 | if not isinstance(val, str) and not isinstance(val, bytes): 7 | return val 8 | elif len(val) < 100: 9 | return val 10 | else: 11 | return f"{val[0:100]}..." 12 | 13 | 14 | class TaskLogger: 15 | def __init__(self, logger, task): 16 | self.logger = logger 17 | self.task = task 18 | self.colors = ["\033[1;36m", "\033[1;34m", "\033[1;35m"] 19 | 20 | def __enter__(self): 21 | self.start_time = time.time() 22 | line = " " * self.logger.depth() 23 | line += self.colors[self.logger.depth() % len(self.colors)] 24 | line += f"> {self.task.name}" 25 | line += "\033[0m" 26 | for key, val in self.task.params.__dict__().items(): 27 | line += f" {key}=\033[1m{truncated_value(val)}\033[0m" 28 | logger.info(line) 29 | 30 | self.logger.task_stack.append(self) 31 | 32 | def __exit__(self, exc, value, tb): 33 | self.logger.task_stack.pop() 34 | 35 | duration = time.time() - self.start_time 36 | line = " " * self.logger.depth() 37 | line += self.colors[self.logger.depth() % len(self.colors)] 38 | line += f"< {self.task.name}" 39 | line += " \033[0m\033[3m ({0:.2f}s)\033[0m".format(duration) 40 | if exc: 41 | line += f" \033[31m✗\033[0m {exc.__name__} {value}" 42 | else: 43 | line += f" \033[32m✓\033[0m" 44 | if self.task.return_value: 45 | line += f" << {truncated_value(self.task.return_value)}" 46 | logger.info(line) 47 | 48 | 49 | class CopyLogger: 50 | def __init__(self, logger, src, dest): 51 | self.logger = logger 52 | self.src = src 53 | self.dest = dest 54 | 55 | def __enter__(self): 56 | self.start_time = time.time() 57 | line = " " * self.logger.depth() 58 | line += f"💾 Copying \033[1m{self.src}\033[0m ==> \033[1m{self.dest}\033[0m" 59 | self.logger.info(line) 60 | self.logger.task_stack.append(self) 61 | 62 | def __exit__(self, exc, tb, something): 63 | self.logger.task_stack.pop() 64 | duration = time.time() - self.start_time 65 | line = " " * self.logger.depth() 66 | if exc: 67 | line += "\033[31m✗\033[0m copying failed ({0:.2f}s)".format(duration) 68 | else: 69 | line += "\033[32m✓\033[0m done copying ({0:.2f}s)".format(duration) 70 | self.logger.info(line) 71 | 72 | 73 | class TestLogger: 74 | def __init__(self, logger, task, name): 75 | self.logger = logger 76 | self.task = task 77 | self.name = name 78 | 79 | def __enter__(self): 80 | self.start_time = time.time() 81 | line = " " * self.logger.depth() 82 | line += f"🏃 Running {self.task.task_name} > \033[1m{self.name}\033[0m" 83 | self.logger.info(line) 84 | self.logger.task_stack.append(self) 85 | 86 | def __exit__(self, exc, tb, something): 87 | self.logger.task_stack.pop() 88 | duration = time.time() - self.start_time 89 | line = " " * self.logger.depth() 90 | if exc: 91 | line += "\033[31m✗\033[0m test failed ({0:.2f}s)".format(duration) 92 | else: 93 | line += "\033[32m✓\033[0m test succeeded ({0:.2f}s)".format(duration) 94 | self.logger.info(line) 95 | 96 | 97 | class Logger: 98 | def __init__(self, writer): 99 | self.writer = writer 100 | self.task_stack = [] 101 | 102 | def with_task(self, task): 103 | return TaskLogger(self, task) 104 | 105 | def with_test(self, task, name): 106 | return TestLogger(self, task, name) 107 | 108 | def shell_start(self, context, command): 109 | line = " " * len(self.task_stack) 110 | line += f"\033[33m${context.descriptor()}\033[0m {truncated_value(command)}" 111 | self.writer.write(line) 112 | self.writer.flush() 113 | 114 | def shell_stop(self, context, code, out, err): 115 | self.info( 116 | f" # code={code} out={truncated_value(out)} err={truncated_value(err)}" 117 | ) 118 | 119 | def with_copy(self, src, dest): 120 | return CopyLogger(self, src, dest) 121 | 122 | def info(self, line): 123 | self.writer.write(line) 124 | self.writer.write("\n") 125 | self.writer.flush() 126 | 127 | def depth(self): 128 | return len(self.task_stack) 129 | 130 | 131 | logger = Logger(sys.stderr) 132 | -------------------------------------------------------------------------------- /pitcrew/passwords.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | import getpass 3 | 4 | 5 | class Passwords: 6 | def __init__(self): 7 | self.passwords = {} 8 | 9 | async def get_password(self, prompt): 10 | if prompt in self.passwords: 11 | return await self.passwords[prompt] 12 | 13 | loop = asyncio.get_running_loop() 14 | fut = loop.create_future() 15 | self.passwords[prompt] = fut 16 | await loop.run_in_executor(None, self.__get_password, fut, prompt) 17 | return await fut 18 | 19 | def __get_password(self, fut, prompt): 20 | password = getpass.getpass(prompt=prompt) 21 | fut.set_result(password) 22 | -------------------------------------------------------------------------------- /pitcrew/task.py: -------------------------------------------------------------------------------- 1 | """ 2 | Tasks are defined by putting them in the crew/tasks directory. Tasks must inherit 3 | from `crew.task.BaseTask`. Inside a task the main ways of interacting with the 4 | system is through running other tasks, using `self.sh` or `self.sh_with_code`. 5 | 6 | ### Running other tasks 7 | 8 | This is accomplished by calling through `self.{task_name}`. For instance, to 9 | run the task `fs.write` located at `crew/tasks/fs/write.py` you'd call 10 | `self.fs.write("/tmp/path", b"some contents")`. 11 | 12 | ### Running commands 13 | 14 | There are two ways of running commands, `self.sh` and `self.sh_with_code`. 15 | 16 | `self.sh` accepts a command an optional environment. It returns the stdout 17 | of the run command as a string encoded with utf-8. If the command exits 18 | with a non-zero status an assertion exception is raised. 19 | 20 | `self.sh_with_code` accepts a command and an optional environment. It returns 21 | a tuple of (code, stdout, stderr) where stdout and stderr are bytes. 22 | 23 | ### Writing tasks 24 | 25 | New tasks are created through `crew new `. Then use `crew edit ` to 26 | open the task file in your `$EDITOR`. 27 | 28 | ### Lifecycle of a task 29 | 30 | Tasks come in two flavors, tasks with verification and tasks without verification. 31 | Tasks with verification are typically _idempotent_. 32 | 33 | If a task has a validate method it performs the following: 34 | 35 | 1. Run `validate()`, return response and stop if no assertion error is raised 36 | 2. Run `run()` method 37 | 3. Run `validate()` method and raise any errors it produces, or return its return value. 38 | 39 | If a task doesn't have a validate method it performs the following: 40 | 41 | 1. Run `run()` method. 42 | 43 | ### Tests 44 | 45 | To add tests to a task, add test classes to your test file. For example: 46 | 47 | ```python 48 | class FsDigestsSha256Test(task.TaskTest): 49 | @task.TaskTest.ubuntu 50 | async def test_ubuntu(self): 51 | content = b"Some delicious bytes" 52 | await self.fs.write("/tmp/some-file", content) 53 | expected_digest = hashlib.sha256(content).hexdigest() 54 | actual_digest = await self.fs.digests.sha256("/tmp/some-file") 55 | assert expected_digest == actual_digest, "digests are not equal" 56 | ``` 57 | """ 58 | 59 | import os 60 | import shlex 61 | import inspect 62 | import asyncio 63 | from pitcrew.logger import logger 64 | from pitcrew.template import Template 65 | from pitcrew.test.util import ubuntu_decorator 66 | from abc import ABC, abstractmethod 67 | 68 | 69 | class Parameters: 70 | def __init__(self): 71 | self.__dict = {} 72 | 73 | def _set_attr(self, name, val): 74 | setattr(self, name, val) 75 | self.__dict[name] = val 76 | 77 | def __dict__(self): 78 | return self.__dict 79 | 80 | 81 | class TaskFailureError(Exception): 82 | pass 83 | 84 | 85 | class BaseTask(ABC): 86 | tests = [] 87 | context = None 88 | use_coersion = False 89 | name = None 90 | return_value = None 91 | return_desc = None 92 | memoize = False 93 | nodoc = False 94 | 95 | @classmethod 96 | def expected_return_type(cls): 97 | if hasattr(cls, "verify"): 98 | sig = inspect.signature(cls.verify) 99 | else: 100 | sig = inspect.signature(cls.run) 101 | return sig.return_annotation 102 | 103 | @classmethod 104 | def has_return_type(cls): 105 | return cls.expected_return_type() != inspect.Signature.empty 106 | 107 | @classmethod 108 | def arg_struct(cls): 109 | s = Parameters() 110 | for arg in cls._args(): 111 | s._set_attr(arg.name, arg.default) 112 | if arg.type == str: 113 | setattr(s, f"esc_{arg.name}", None) 114 | return s 115 | 116 | @classmethod 117 | def _args(cls): 118 | if not hasattr(cls, "args"): 119 | cls.args = [] 120 | return cls.args 121 | 122 | @classmethod 123 | def esc(cls, text): 124 | return shlex.quote(text) 125 | 126 | @classmethod 127 | def name(cls): 128 | return cls.__name__ 129 | 130 | @classmethod 131 | def desc(cls): 132 | return cls.__doc__ 133 | 134 | @classmethod 135 | def source(cls): 136 | with open(cls.source_path()) as fh: 137 | return fh.read() 138 | 139 | @classmethod 140 | def source_path(cls): 141 | return inspect.getfile(cls) 142 | 143 | def __getattr__(self, name): 144 | return getattr(self.context, name) 145 | 146 | def _process_args(self, *incoming_args, **incoming_kwargs): 147 | incoming_args = list(incoming_args) 148 | params = self.__class__.arg_struct() 149 | for arg in self.args: 150 | value = arg.default 151 | if arg.required and len(incoming_args) != 0: 152 | if arg.remaining: 153 | value = [] 154 | while len(incoming_args) != 0: 155 | value.append( 156 | arg.process(incoming_args.pop(0), self.use_coersion) 157 | ) 158 | else: 159 | value = arg.process(incoming_args.pop(0), self.use_coersion) 160 | elif arg.name in incoming_kwargs: 161 | value = arg.process( 162 | incoming_kwargs.pop(arg.name, None), self.use_coersion 163 | ) 164 | elif arg.required: 165 | raise Exception(f"missing argument '{arg.name}'") 166 | params._set_attr(arg.name, value) 167 | if value and arg.type == str and not arg.remaining: 168 | esc_value = self.esc(value) 169 | setattr(params, f"esc_{arg.name}", esc_value) 170 | 171 | if incoming_args: 172 | raise TypeError(f"got unexpected positional arguments {incoming_args}") 173 | 174 | if incoming_kwargs: 175 | raise TypeError(f"got unexpected keyword arguments {incoming_kwargs}") 176 | 177 | self.params = params 178 | 179 | def _enforce_return_type(self, value): 180 | expected_return_type = self.__class__.expected_return_type() 181 | if expected_return_type == inspect.Signature.empty and not value: 182 | self.return_value = value 183 | else: 184 | if not isinstance(value, expected_return_type): 185 | raise TypeError( 186 | f"return value {value} does not conform to expected type {expected_return_type}" 187 | ) 188 | self.return_value = value 189 | if self.__class__.memoize: 190 | self.context.cache[self.__class__] = value 191 | return value 192 | 193 | def coerce_inputs(self, use_coersion=True): 194 | self.use_coersion = use_coersion 195 | 196 | def invoke_sync(self, *args, **kwargs): 197 | """Invokes the task synchronously and returns the result.""" 198 | if inspect.iscoroutinefunction(self.invoke): 199 | loop = asyncio.get_event_loop() 200 | ret_value = loop.run_until_complete(self.invoke(*args, **kwargs)) 201 | loop.close() 202 | return ret_value 203 | else: 204 | return self.invoke(*args, **kwargs) 205 | 206 | def task_file(self, path): 207 | """Gets a file relative to the task being executed.""" 208 | file_path = os.path.abspath( 209 | os.path.join(inspect.getfile(self.__class__), "..", path) 210 | ) 211 | 212 | return self.context.local_context.file(file_path) 213 | 214 | @abstractmethod 215 | async def run(self): 216 | pass 217 | 218 | async def invoke_with_context(self, context, *args, **kwargs): 219 | self.context = context 220 | return await self.invoke(*args, **kwargs) 221 | 222 | async def invoke(self, *args, **kwargs): 223 | self._process_args(*args, **kwargs) 224 | 225 | if self.__class__.memoize and self.__class__ in self.context.cache: 226 | return self.context.cache[self.__class__] 227 | 228 | with logger.with_task(self): 229 | if hasattr(self, "verify"): 230 | return await self._invoke_with_verify() 231 | else: 232 | return await self._invoke_without_verify() 233 | 234 | async def _invoke_with_verify(self): 235 | try: 236 | return self._enforce_return_type(await self.verify()) 237 | except AssertionError: 238 | await self.run() 239 | try: 240 | return self._enforce_return_type(await self.verify()) 241 | except AssertionError: 242 | raise TaskFailureError("this task failed to run") 243 | 244 | async def _invoke_without_verify(self): 245 | return self._enforce_return_type(await self.run()) 246 | 247 | def template(self, name): 248 | template_path = os.path.abspath( 249 | os.path.join(inspect.getfile(self.__class__), "..", name) 250 | ) 251 | return Template(self, template_path) 252 | 253 | async def poll(self, fn): 254 | while True: 255 | try: 256 | await fn() 257 | break 258 | except AssertionError: 259 | await asyncio.sleep(1) 260 | 261 | 262 | class Argument: 263 | def __init__( 264 | self, 265 | task_class, 266 | name, 267 | type=None, 268 | default=None, 269 | required=True, 270 | desc=None, 271 | remaining=False, 272 | ): 273 | if name == "env": 274 | raise ValueError("`env' is a reserved argument name") 275 | self.task_class = task_class 276 | self.name = name 277 | self.type = type or str 278 | self.default = default 279 | self.required = required 280 | self.desc = desc 281 | self.remaining = remaining 282 | 283 | def process(self, value, use_coersion=False): 284 | if use_coersion: 285 | value = self.coerce_from_string(value) 286 | 287 | if value is None: 288 | if self.required: 289 | raise Exception(f"value is required for {self.name}") 290 | return self.default 291 | elif self.type != any and not isinstance(value, self.type): 292 | raise Exception( 293 | "this doesn't match %s %s with value %s" % (self.name, self.type, value) 294 | ) 295 | return value 296 | 297 | def coerce_from_string(self, value): 298 | if self.type == bytes: 299 | return value.encode() 300 | elif self.type == int: 301 | return int(value) 302 | elif self.type == bool: 303 | if value.lower() in ["true", "yes", "1"]: 304 | return True 305 | elif value.lower() in ["false", "no", "0"]: 306 | return False 307 | else: 308 | raise Exception(f"value {value} cannot be converted to bool") 309 | else: 310 | return value 311 | 312 | def __str__(self): 313 | return f"arg {self.name} {self.type}" 314 | 315 | 316 | class TaskTest: 317 | def __init__(self, context): 318 | self.context = context 319 | 320 | def __getattr__(self, name): 321 | return getattr(self.context, name) 322 | 323 | ubuntu = ubuntu_decorator 324 | 325 | 326 | def arg(name, type=None, **kwargs): 327 | """Decorator to add a required argument to the task.""" 328 | 329 | def decorator(cls): 330 | cls._args().insert(0, Argument(cls, name, type, **kwargs)) 331 | return cls 332 | 333 | return decorator 334 | 335 | 336 | def varargs(name, type=None, **kwargs): 337 | def decorator(cls): 338 | cls._args().insert(0, Argument(cls, name, type, remaining=True, **kwargs)) 339 | return cls 340 | 341 | return decorator 342 | 343 | 344 | def opt(name, type=None, **kwargs): 345 | """Decorator to add an optional argument to the task.""" 346 | 347 | def decorator(cls): 348 | cls._args().insert(0, Argument(cls, name, type, required=False, **kwargs)) 349 | return cls 350 | 351 | return decorator 352 | 353 | 354 | def returns(desc): 355 | """Decorator to describe the return type.""" 356 | 357 | def decorator(cls): 358 | cls.return_desc = desc 359 | return cls 360 | 361 | return decorator 362 | 363 | 364 | def memoize(): 365 | """Decorator to instruct task to memoize return within the context's cache.""" 366 | 367 | def decorator(cls): 368 | cls.memoize = True 369 | return cls 370 | 371 | return decorator 372 | 373 | 374 | def nodoc(): 375 | """Decorator to instruct task to not generate documentation for test.""" 376 | 377 | def decorator(cls): 378 | cls.nodoc = True 379 | return cls 380 | 381 | return decorator 382 | -------------------------------------------------------------------------------- /pitcrew/tasks/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/joshbuddy/pitcrew/ce9231cd4a4fb0a732be0856678ba956b6b4fedb/pitcrew/tasks/__init__.py -------------------------------------------------------------------------------- /pitcrew/tasks/apt_get/install.py: -------------------------------------------------------------------------------- 1 | import re 2 | from pitcrew import task 3 | 4 | 5 | @task.varargs("packages", type=str, desc="The package to install") 6 | @task.returns("The version of the installed package") 7 | class AptgetInstall(task.BaseTask): 8 | """Install a package using apt-get""" 9 | 10 | async def verify(self) -> list: 11 | versions = [] 12 | for p in self.params.packages: 13 | versions.append(await self.get_version(p)) 14 | return versions 15 | 16 | async def run(self): 17 | packages = " ".join(map(lambda p: self.esc(p), self.params.packages)) 18 | return await self.sh(f"apt-get install -y {packages}") 19 | 20 | async def available(self) -> bool: 21 | code, _, _ = await self.sh_with_code("which apt-get") 22 | return code == 0 23 | 24 | async def get_version(self, name) -> str: 25 | policy_output = await self.sh(f"apt-cache policy {self.esc(name)}") 26 | m = re.search("Installed: (.*?)\n", policy_output) 27 | assert m, "no version found" 28 | installed_version = m.group(1) 29 | assert installed_version != "(none)", "Installed version is (none)" 30 | return installed_version 31 | -------------------------------------------------------------------------------- /pitcrew/tasks/apt_get/update.py: -------------------------------------------------------------------------------- 1 | from pitcrew import task 2 | 3 | 4 | class AptgetUpdate(task.BaseTask): 5 | """Performs `apt-get update`""" 6 | 7 | async def run(self): 8 | await self.sh("apt-get update") 9 | -------------------------------------------------------------------------------- /pitcrew/tasks/crew/install.py: -------------------------------------------------------------------------------- 1 | from pitcrew import task 2 | 3 | 4 | @task.opt("dest", desc="The directory to install crew in", type=list, default="pitcrew") 5 | class CrewInstall(task.BaseTask): 6 | """Installs crew in the path specified""" 7 | 8 | async def verify(self): 9 | with self.cd(self.params.dest): 10 | await self.sh("./env/bin/crew --help") 11 | 12 | async def run(self): 13 | platform = await self.facts.system.uname() 14 | if platform == "darwin": 15 | await self.install.xcode_cli() 16 | await self.install.homebrew() 17 | await self.install("git") 18 | await self.git.clone( 19 | "https://github.com/joshbuddy/pitcrew.git", self.params.dest 20 | ) 21 | with self.cd(self.params.dest): 22 | await self.homebrew.install("python3") 23 | await self.sh("python3 -m venv --clear env") 24 | await self.sh("env/bin/pip install -e .") 25 | elif platform == "linux": 26 | if await self.sh_ok("which apt-get"): 27 | await self.apt_get.update() 28 | await self.apt_get.install( 29 | "apt-utils", "git", "python3.7", "python3.7-dev", "python3.7-venv" 30 | ) 31 | await self.sh( 32 | "apt-get install -y python3.7-distutils", 33 | env={"DEBIAN_FRONTEND": "noninteractive"}, 34 | ) 35 | else: 36 | raise Exception(f"cannot install on this platform {platform}") 37 | 38 | await self.git.clone( 39 | "https://github.com/joshbuddy/pitcrew.git", self.params.dest 40 | ) 41 | with self.cd(self.params.dest): 42 | await self.sh("python3.7 -m venv env") 43 | await self.sh("env/bin/pip install --upgrade pip wheel") 44 | await self.sh("env/bin/pip install -e .") 45 | 46 | else: 47 | raise Exception(f"cannot install on this platform {platform}") 48 | 49 | 50 | class CrewInstallTest(task.TaskTest): 51 | @task.TaskTest.ubuntu 52 | async def test_ubuntu(self): 53 | with self.cd("/tmp"): 54 | # put this in to test the local copy you've got 55 | await self.local_context.file(".").copy_to(self.file("/tmp/pitcrew")) 56 | await self.sh("rm -rf /tmp/pitcrew/env") 57 | await self.fs.write( 58 | "/tmp/pitcrew/.git/config", 59 | b"""[core] 60 | repositoryformatversion = 0 61 | filemode = true 62 | bare = false 63 | logallrefupdates = true 64 | ignorecase = true 65 | precomposeunicode = true 66 | [remote "origin"] 67 | url = https://github.com/joshbuddy/pitcrew.git 68 | fetch = +refs/heads/*:refs/remotes/origin/* 69 | """, 70 | ) 71 | await self.crew.install() 72 | -------------------------------------------------------------------------------- /pitcrew/tasks/crew/release/__init__.py: -------------------------------------------------------------------------------- 1 | import re 2 | import asyncio 3 | import pitcrew 4 | from pitcrew import task 5 | 6 | 7 | @task.opt("dryrun", desc="Dry run mode", type=bool, default=True) 8 | class CrewRelease(task.BaseTask): 9 | """This creates a release for crew""" 10 | 11 | async def run(self): 12 | if not self.params.dryrun: 13 | current_branch = (await self.sh("git rev-parse --abbrev-ref HEAD")).strip() 14 | assert "master" == current_branch, "dryrun=False must be run on master" 15 | 16 | await self.sh("pip install -r requirements-build.txt") 17 | version = pitcrew.__version__ 18 | await self.sh("mkdir -p pkg") 19 | await asyncio.gather( 20 | self.crew.release.darwin(version), self.crew.release.linux(version) 21 | ) 22 | await self.sh( 23 | f"env/bin/githubrelease release joshbuddy/pitcrew create {version} {self.esc('pkg/*')}" 24 | ) 25 | if self.params.dryrun: 26 | await self.sh("env/bin/python setup.py upload_test") 27 | else: 28 | await self.sh("env/bin/python setup.py upload") 29 | print("Don't forget to go to github and hit publish!") 30 | -------------------------------------------------------------------------------- /pitcrew/tasks/crew/release/darwin.py: -------------------------------------------------------------------------------- 1 | from pitcrew import task 2 | 3 | 4 | @task.arg("version", desc="The version to release", type=str) 5 | class CrewBuildDarwin(task.BaseTask): 6 | """This creates a PyInstaller build for crew on Darwin""" 7 | 8 | async def run(self): 9 | assert await self.facts.system.uname() == "darwin" 10 | await self.sh("make build") 11 | target = f"pkg/crew-Darwin" 12 | await self.sh(f"cp dist/crew {target}") 13 | -------------------------------------------------------------------------------- /pitcrew/tasks/crew/release/linux.py: -------------------------------------------------------------------------------- 1 | from pitcrew import task 2 | 3 | 4 | @task.arg("version", desc="The version to release", type=str) 5 | class CrewBuildLinux(task.BaseTask): 6 | """This creates a PyInstaller build for crew on Linux""" 7 | 8 | async def run(self): 9 | container_id = await self.docker.run("ubuntu", detach=True, interactive=True) 10 | docker_ctx = self.docker_context(container_id, user="root") 11 | 12 | async with docker_ctx: 13 | assert ( 14 | await docker_ctx.facts.system.uname() == "linux" 15 | ), "the platform is not linux!" 16 | await self.file(".").copy_to(docker_ctx.file("/tmp/crew")) 17 | await docker_ctx.apt_get.update() 18 | await docker_ctx.apt_get.install("python3.6") 19 | await docker_ctx.apt_get.install("python3.6-dev") 20 | await docker_ctx.apt_get.install("python3-venv") 21 | await docker_ctx.apt_get.install("build-essential") 22 | with docker_ctx.cd("/tmp/crew"): 23 | await docker_ctx.sh("python3.6 -m venv --clear env") 24 | await docker_ctx.sh("env/bin/pip install -r requirements-build.txt") 25 | await docker_ctx.sh("make build") 26 | target = f"pkg/crew-Linux" 27 | await docker_ctx.file("/tmp/crew/dist/crew").copy_to(self.file(target)) 28 | -------------------------------------------------------------------------------- /pitcrew/tasks/docker/run.py: -------------------------------------------------------------------------------- 1 | from pitcrew import task 2 | 3 | 4 | @task.arg("image", desc="The image to run", type=str) 5 | @task.opt( 6 | "detach", 7 | desc="Run container in background and print container ID", 8 | default=False, 9 | type=bool, 10 | ) 11 | @task.opt("tty", desc="Allocate a pseudo-TTY", default=False, type=bool) 12 | @task.opt("interactive", desc="Interactive mode", default=False, type=bool) 13 | @task.opt("publish", desc="Publish ports", type=list) 14 | @task.returns("The container id") 15 | class DockerRun(task.BaseTask): 16 | """Runs a specific docker image""" 17 | 18 | async def run(self) -> str: 19 | flags = [] 20 | if self.params.detach: 21 | flags.append("d") 22 | if self.params.tty: 23 | flags.append("t") 24 | if self.params.interactive: 25 | flags.append("i") 26 | 27 | flag_string = f" -{''.join(flags)}" if flags else "" 28 | 29 | if self.params.publish: 30 | flag_string += f" -p {' '.join(self.params.publish)}" 31 | 32 | out = await self.sh(f"docker run{flag_string} {self.params.esc_image}") 33 | return out.strip() 34 | -------------------------------------------------------------------------------- /pitcrew/tasks/docker/stop.py: -------------------------------------------------------------------------------- 1 | from pitcrew import task 2 | 3 | 4 | @task.arg("container_id", desc="The container id to stop", type=str) 5 | @task.opt( 6 | "time", desc="Seconds to wait for stop before killing it", type=int, default=10 7 | ) 8 | class DockerStop(task.BaseTask): 9 | """Stops docker container with specified id""" 10 | 11 | async def run(self): 12 | command = "docker stop" 13 | if self.params.time is not None: 14 | command += f" -t {self.params.time}" 15 | command += f" {self.params.esc_container_id}" 16 | await self.sh(command) 17 | -------------------------------------------------------------------------------- /pitcrew/tasks/ensure/aws/route53/has_records.py: -------------------------------------------------------------------------------- 1 | import json 2 | import asyncio 3 | from pitcrew import task 4 | 5 | 6 | @task.arg("zone_id", desc="The zone id to operate on", type=str) 7 | @task.arg("records", desc="A list of records to ensure are set", type=list) 8 | class HasRecords(task.BaseTask): 9 | """Ensure route53 has the set of records""" 10 | 11 | async def verify(self): 12 | json_out = await self.sh( 13 | f"aws route53 list-resource-record-sets --hosted-zone-id {self.params.esc_zone_id}" 14 | ) 15 | out = json.loads(json_out) 16 | existing_record_sets = out["ResourceRecordSets"] 17 | for record in self.params.records: 18 | assert record in existing_record_sets, "cannot find record" 19 | 20 | async def run(self): 21 | changes = map( 22 | lambda c: {"Action": "UPSERT", "ResourceRecordSet": c}, self.params.records 23 | ) 24 | change_batch = {"Changes": list(changes)} 25 | change_id = json.loads( 26 | await self.sh( 27 | f"aws route53 change-resource-record-sets --hosted-zone-id {self.params.esc_zone_id} --change-batch {self.esc(json.dumps(change_batch))}" 28 | ) 29 | )["ChangeInfo"]["Id"] 30 | while ( 31 | json.loads( 32 | await self.sh(f"aws route53 get-change --id {self.esc(change_id)}") 33 | )["ChangeInfo"]["Status"] 34 | == "PENDING" 35 | ): 36 | await asyncio.sleep(5) 37 | -------------------------------------------------------------------------------- /pitcrew/tasks/examples/deploy_pitcrew/__init__.py: -------------------------------------------------------------------------------- 1 | import json 2 | import asyncio 3 | from pitcrew import task 4 | from uuid import uuid4 5 | 6 | 7 | class DeployPitcrew(task.BaseTask): 8 | """This example builds and deploys pitcrew.io. It uses s3, cloudfront and acm to deploy 9 | this website using ssl. """ 10 | 11 | async def run(self): 12 | # make sure the build requirements are installed 13 | await self.sh("pip install -r requirements-build.txt") 14 | # create the bucket 15 | await self.sh("aws s3api create-bucket --bucket pitcrew-site") 16 | # setup aws & build + upload site 17 | await asyncio.gather(self.setup_aws(), self.build_and_sync()) 18 | 19 | async def setup_aws(self): 20 | # first find the zone 21 | zones = json.loads(await self.sh("aws route53 list-hosted-zones"))[ 22 | "HostedZones" 23 | ] 24 | zone_id = None 25 | for zone in zones: 26 | if zone["Name"] == "pitcrew.io.": 27 | zone_id = zone["Id"] 28 | break 29 | 30 | assert zone_id, "no zone_id found for pitcrew.io" 31 | 32 | # setup the certificate 33 | cert_arn = await self.setup_acm(zone_id) 34 | # setup the CDN 35 | cf_id = await self.setup_cloudfront(zone_id, cert_arn) 36 | dist = json.loads( 37 | await self.sh(f"aws cloudfront get-distribution --id {self.esc(cf_id)}") 38 | )["Distribution"] 39 | domain_name = dist["DomainName"] 40 | 41 | # add the DNS records 42 | await self.ensure.aws.route53.has_records( 43 | zone_id, 44 | [ 45 | { 46 | "Name": "pitcrew.io.", 47 | "Type": "A", 48 | "AliasTarget": { 49 | "HostedZoneId": "Z2FDTNDATAQYW2", 50 | "DNSName": f"{domain_name}.", 51 | "EvaluateTargetHealth": False, 52 | }, 53 | }, 54 | { 55 | "Name": "pitcrew.io.", 56 | "Type": "AAAA", 57 | "AliasTarget": { 58 | "HostedZoneId": "Z2FDTNDATAQYW2", 59 | "DNSName": f"{domain_name}.", 60 | "EvaluateTargetHealth": False, 61 | }, 62 | }, 63 | { 64 | "Name": "www.pitcrew.io.", 65 | "Type": "CNAME", 66 | "TTL": 300, 67 | "ResourceRecords": [{"Value": domain_name}], 68 | }, 69 | ], 70 | ) 71 | 72 | async def setup_acm(self, zone_id) -> str: 73 | # look for the certificate 74 | certs = json.loads( 75 | await self.sh("aws acm list-certificates --certificate-statuses ISSUED") 76 | )["CertificateSummaryList"] 77 | for cert in certs: 78 | if cert["DomainName"] == "pitcrew.io": 79 | return cert["CertificateArn"] 80 | 81 | # if it doesn't exist, create it 82 | arn = json.loads( 83 | await self.sh( 84 | f"aws acm request-certificate --domain-name pitcrew.io --validation-method DNS --subject-alternative-names {self.esc('*.pitcrew.io')}" 85 | ) 86 | )["CertificateArn"] 87 | cert_description = json.loads( 88 | await self.sh( 89 | f"aws acm describe-certificate --certificate-arn {self.esc(arn)}" 90 | ) 91 | ) 92 | 93 | validation = cert_description["Certificate"]["DomainValidationOptions"][0] 94 | await self.ensure.aws.route53.has_records( 95 | zone_id, 96 | [ 97 | { 98 | "Name": validation["ResourceRecord"]["Name"], 99 | "Type": validation["ResourceRecord"]["Type"], 100 | "TTL": 60, 101 | "ResourceRecords": [ 102 | {"Value": validation["ResourceRecord"]["Value"]} 103 | ], 104 | } 105 | ], 106 | ) 107 | 108 | await self.sh( 109 | f"aws acm wait certificate-validated --certificate-arn {self.esc(arn)}" 110 | ) 111 | return arn 112 | 113 | async def setup_cloudfront(self, zone_id, cert_arn) -> str: 114 | s3_origin = "pitcrew-site.s3.amazonaws.com" 115 | 116 | list_distributions = json.loads( 117 | await self.sh(f"aws cloudfront list-distributions") 118 | ) 119 | for dist in list_distributions["DistributionList"]["Items"]: 120 | if dist["Origins"]["Items"][0]["DomainName"] == s3_origin: 121 | return dist["Id"] 122 | 123 | config = { 124 | "DefaultRootObject": "index.html", 125 | "Aliases": {"Quantity": 2, "Items": ["pitcrew.io", "www.pitcrew.io"]}, 126 | "Origins": { 127 | "Quantity": 1, 128 | "Items": [ 129 | { 130 | "Id": "pitcrew-origin", 131 | "DomainName": s3_origin, 132 | "S3OriginConfig": {"OriginAccessIdentity": ""}, 133 | } 134 | ], 135 | }, 136 | "DefaultCacheBehavior": { 137 | "TargetOriginId": "pitcrew-origin", 138 | "ForwardedValues": { 139 | "QueryString": True, 140 | "Cookies": {"Forward": "none"}, 141 | }, 142 | "TrustedSigners": {"Enabled": False, "Quantity": 0}, 143 | "ViewerProtocolPolicy": "redirect-to-https", 144 | "MinTTL": 180, 145 | }, 146 | "CallerReference": str(uuid4()), 147 | "Comment": "Created by crew", 148 | "Enabled": True, 149 | "ViewerCertificate": { 150 | "ACMCertificateArn": cert_arn, 151 | "SSLSupportMethod": "sni-only", 152 | }, 153 | } 154 | create_distribution = json.loads( 155 | await self.sh( 156 | f"aws cloudfront create-distribution --distribution-config {self.esc(json.dumps(config))}" 157 | ) 158 | ) 159 | cf_id = create_distribution["Distribution"]["Id"] 160 | await self.sh(f"aws cloudfront wait distribution-deployed --id {cf_id}") 161 | return cf_id 162 | 163 | async def build_and_sync(self): 164 | await self.examples.deploy_pitcrew.build() 165 | await self.sh("aws s3 sync --acl public-read out/ s3://pitcrew-site/") 166 | -------------------------------------------------------------------------------- /pitcrew/tasks/examples/deploy_pitcrew/build.py: -------------------------------------------------------------------------------- 1 | import os 2 | import re 3 | import asyncio 4 | from pitcrew import task 5 | 6 | 7 | class Build(task.BaseTask): 8 | """Builds the website in the `out` directory.""" 9 | 10 | async def run(self): 11 | # create docs for python stuff 12 | await self.sh("make docs") 13 | # create task specific docs 14 | await self.sh("crew docs") 15 | # re-create out directory 16 | await self.sh("rm -rf out") 17 | await self.sh("mkdir out") 18 | # copy our css 19 | await self.task_file("water.css").copy_to(self.file("out/water.css")) 20 | 21 | docs = [] 22 | files = await self.fs.list("docs") 23 | for f in files: 24 | name = f.split("/")[-1] 25 | target = f"out/docs/{os.path.splitext(name)[0]}.html" 26 | docs.append(self.generate_doc(f"docs/{f}", target)) 27 | docs.append(self.generate_doc("README.md", "out/index.html")) 28 | await asyncio.gather(*docs) 29 | 30 | async def generate_doc(self, source, target): 31 | out = await self.sh( 32 | f"env/bin/python -m markdown2 -x fenced-code-blocks -x header-ids {source}" 33 | ) 34 | out = re.sub(r"\.md", ".html", out) 35 | await self.sh(f"mkdir -p {self.esc(os.path.split(target)[0])}") 36 | page = self.template("doc.html.j2").render_as_bytes(body=out) 37 | await self.fs.write(target, page) 38 | -------------------------------------------------------------------------------- /pitcrew/tasks/examples/deploy_pitcrew/doc.html.j2: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 78 | 79 | 80 | {{ body|safe }} 81 | 82 | -------------------------------------------------------------------------------- /pitcrew/tasks/examples/deploy_pitcrew/water.css: -------------------------------------------------------------------------------- 1 | body{font-family:system-ui,-apple-system,BlinkMacSystemFont,Segoe UI,Roboto,Oxygen,Ubuntu,Cantarell,Fira Sans,Droid Sans,Helvetica Neue,sans-serif;line-height:1.4;max-width:800px;margin:20px auto;padding:0 10px;color:#dbdbdb;background:#202b38;text-rendering:optimizeLegibility}button,input,textarea{transition:background-color .1s linear,border-color .1s linear,color .1s linear,box-shadow .1s linear,transform .1s ease}h1{font-size:2.2em;margin-top:0}h1,h2,h3,h4,h5,h6{margin-bottom:12px}h1,h2,h3,h4,h5,h6,strong{color:#fff}b,h1,h2,h3,h4,h5,h6,strong,th{font-weight:600}button,input[type=button],input[type=checkbox],input[type=submit]{cursor:pointer}input:not([type=checkbox]),select{display:block}button,input,select,textarea{color:#fff;background-color:#161f27;font-family:inherit;font-size:inherit;margin-right:6px;margin-bottom:6px;padding:10px;border:none;border-radius:6px;outline:none}button,input:not([type=checkbox]),select,textarea{-webkit-appearance:none}textarea{margin-right:0;width:100%;box-sizing:border-box;resize:vertical}button,input[type=button],input[type=submit]{padding-right:30px;padding-left:30px}button:hover,input[type=button]:hover,input[type=submit]:hover{background:#324759}button:focus,input:focus,select:focus,textarea:focus{box-shadow:0 0 0 2px rgba(0,150,191,.67)}button:active,input[type=button]:active,input[type=checkbox]:active,input[type=submit]:active{transform:translateY(2px)}input:disabled{cursor:not-allowed;opacity:.5}::-webkit-input-placeholder{color:#a9a9a9}:-ms-input-placeholder{color:#a9a9a9}::-ms-input-placeholder{color:#a9a9a9}::placeholder{color:#a9a9a9}a{text-decoration:none;color:#41adff}a:hover{text-decoration:underline}code,kbd{background:#161f27;color:#ffbe85;padding:5px;border-radius:6px}pre>code{padding:10px;display:block;overflow-x:auto}img{max-width:100%}hr{border:none;border-top:1px solid #dbdbdb}table{border-collapse:collapse;margin-bottom:10px;width:100%}td,th{padding:6px;text-align:left}th{border-bottom:1px solid #dbdbdb}tbody tr:nth-child(2n){background-color:#161f27}::-webkit-scrollbar{height:10px;width:10px}::-webkit-scrollbar-track{background:#161f27;border-radius:6px}::-webkit-scrollbar-thumb{background:#324759;border-radius:6px}::-webkit-scrollbar-thumb:hover{background:#415c73} 2 | /*# sourceMappingURL=dark.css.map */ 3 | -------------------------------------------------------------------------------- /pitcrew/tasks/facts/system/uname.py: -------------------------------------------------------------------------------- 1 | from pitcrew import task 2 | 3 | 4 | @task.returns("The name of the platform") 5 | @task.memoize() 6 | class Uname(task.BaseTask): 7 | """Returns the lowercase name of the platform""" 8 | 9 | async def run(self) -> str: 10 | return (await self.sh("uname")).strip().lower() 11 | -------------------------------------------------------------------------------- /pitcrew/tasks/fs/chmod.py: -------------------------------------------------------------------------------- 1 | from pitcrew import task 2 | 3 | 4 | @task.arg("path", desc="The path to change the mode of", type=str) 5 | @task.arg("mode", desc="The mode", type=str) 6 | @task.returns("The bytes of the file") 7 | class FsChmod(task.BaseTask): 8 | """Changes the file mode of the specified path""" 9 | 10 | async def run(self): 11 | return await self.sh(f"chmod {self.params.esc_mode} {self.params.esc_path}") 12 | 13 | 14 | class FsChmodTest(task.TaskTest): 15 | @task.TaskTest.ubuntu 16 | async def test_ubuntu(self): 17 | with self.cd("/tmp"): 18 | await self.fs.touch("some-file") 19 | await self.fs.chmod("some-file", "644") 20 | assert (await self.fs.stat("some-file")).mode == "100644" 21 | await self.fs.chmod("some-file", "o+x") 22 | assert (await self.fs.stat("some-file")).mode == "100645" 23 | -------------------------------------------------------------------------------- /pitcrew/tasks/fs/chown.py: -------------------------------------------------------------------------------- 1 | from pitcrew import task 2 | 3 | 4 | @task.arg("path", desc="The path to change the mode of", type=str) 5 | @task.arg("owner", desc="The owner", type=str) 6 | @task.opt("group", desc="The owner", type=str) 7 | @task.returns("The bytes of the file") 8 | class FsChown(task.BaseTask): 9 | """Changes the file mode of the specified path""" 10 | 11 | async def run(self): 12 | owner_str = self.params.owner 13 | if self.params.group: 14 | owner_str += f":{self.params.group}" 15 | return await self.sh(f"chown {self.esc(owner_str)} {self.params.esc_path}") 16 | -------------------------------------------------------------------------------- /pitcrew/tasks/fs/digests/md5.py: -------------------------------------------------------------------------------- 1 | import hashlib 2 | from pitcrew import task 3 | 4 | 5 | @task.arg("path", desc="The path of the file to digest", type=str) 6 | @task.returns("The md5 digest in hexadecimal") 7 | class FsDigestsMd5(task.BaseTask): 8 | """Gets md5 digest of path""" 9 | 10 | async def run(self) -> str: 11 | platform = await self.facts.system.uname() 12 | if platform == "darwin": 13 | out = await self.sh(f"md5 {self.params.esc_path}") 14 | return out.strip().split(" ")[-1] 15 | elif platform == "linux": 16 | out = await self.sh(f"md5sum {self.params.esc_path}") 17 | return out.split(" ")[0] 18 | else: 19 | raise Exception("not supported") 20 | 21 | 22 | class FsDigestsMd5Test(task.TaskTest): 23 | @task.TaskTest.ubuntu 24 | async def test_ubuntu(self): 25 | content = b"Some delicious bytes" 26 | await self.fs.write("/tmp/some-file", content) 27 | expected_digest = hashlib.md5(content).hexdigest() 28 | actual_digest = await self.fs.digests.md5("/tmp/some-file") 29 | assert expected_digest == actual_digest, "digests are not equal" 30 | -------------------------------------------------------------------------------- /pitcrew/tasks/fs/digests/sha256.py: -------------------------------------------------------------------------------- 1 | import hashlib 2 | from pitcrew import task 3 | 4 | 5 | @task.arg("path", desc="The path of the file to digest", type=str) 6 | @task.returns("The sha256 digest in hexadecimal") 7 | class FsDigestsSha256(task.BaseTask): 8 | """Gets sha256 digest of path""" 9 | 10 | async def run(self) -> str: 11 | platform = await self.facts.system.uname() 12 | if platform == "darwin": 13 | out = await self.sh(f"shasum -a256 {self.params.esc_path}") 14 | return out.split(" ")[0] 15 | elif platform == "linux": 16 | out = await self.sh(f"sha256sum {self.params.esc_path}") 17 | return out.split(" ")[0] 18 | else: 19 | raise Exception("not supported") 20 | 21 | 22 | class FsDigestsSha256Test(task.TaskTest): 23 | @task.TaskTest.ubuntu 24 | async def test_ubuntu(self): 25 | content = b"Some delicious bytes" 26 | await self.fs.write("/tmp/some-file", content) 27 | expected_digest = hashlib.sha256(content).hexdigest() 28 | actual_digest = await self.fs.digests.sha256("/tmp/some-file") 29 | assert expected_digest == actual_digest, "digests are not equal" 30 | -------------------------------------------------------------------------------- /pitcrew/tasks/fs/is_directory.py: -------------------------------------------------------------------------------- 1 | from pitcrew import task 2 | 3 | 4 | @task.arg("path", desc="The path to check") 5 | @task.returns("Indicates if target path is a directory") 6 | class FsIsDirectory(task.BaseTask): 7 | """Checks if the path is a directory""" 8 | 9 | async def run(self) -> bool: 10 | code, _, _ = await self.sh_with_code(f"test -d {self.params.esc_path}") 11 | return code == 0 12 | -------------------------------------------------------------------------------- /pitcrew/tasks/fs/is_file.py: -------------------------------------------------------------------------------- 1 | from pitcrew import task 2 | 3 | 4 | @task.arg("path", desc="The path to check") 5 | @task.returns("Indicates if target path is a file") 6 | class FsIsFile(task.BaseTask): 7 | """Checks if the path is a file""" 8 | 9 | async def run(self) -> bool: 10 | code, _, _ = await self.sh_with_code(f"test -f {self.params.esc_path}") 11 | return code == 0 12 | -------------------------------------------------------------------------------- /pitcrew/tasks/fs/list.py: -------------------------------------------------------------------------------- 1 | from pitcrew import task 2 | 3 | 4 | @task.arg("path", desc="The file to read", type=str) 5 | @task.returns("The bytes of the file") 6 | class FsList(task.BaseTask): 7 | """List the files in a directory.""" 8 | 9 | async def run(self) -> list: 10 | out = await self.sh(f"ls -1 {self.params.esc_path}") 11 | return out.strip().split("\n") 12 | -------------------------------------------------------------------------------- /pitcrew/tasks/fs/read.py: -------------------------------------------------------------------------------- 1 | from pitcrew import task 2 | 3 | 4 | @task.arg("path", desc="The file to read", type=str) 5 | @task.returns("The bytes of the file") 6 | class FsRead(task.BaseTask): 7 | """Read value of path into bytes""" 8 | 9 | async def run(self) -> bytes: 10 | code, out, err = await self.sh_with_code(f"cat {self.params.esc_path}") 11 | assert code == 0, "exitcode was not zero" 12 | return out 13 | -------------------------------------------------------------------------------- /pitcrew/tasks/fs/stat.py: -------------------------------------------------------------------------------- 1 | from pitcrew import task 2 | 3 | 4 | class Stat: 5 | inode: int 6 | mode: str 7 | user_id: int 8 | group_id: int 9 | size: int 10 | access_time: int 11 | modify_time: int 12 | create_time: int 13 | block_size: int 14 | blocks: int 15 | 16 | def __str__(self): 17 | return f"inode={self.inode} mode={self.mode} user_id={self.user_id} group_id={self.group_id} size={self.size} access_time={self.access_time} modify_time={self.modify_time} create_time={self.create_time} block_size={self.block_size} blocks={self.blocks}" 18 | 19 | 20 | @task.arg("path", desc="The path of the file to stat", type=str) 21 | @task.returns("the stat object for the file") 22 | class FsStat(task.BaseTask): 23 | 24 | """Get stat info for path""" 25 | 26 | async def run(self) -> Stat: 27 | stat = Stat() 28 | platform = await self.facts.system.uname() 29 | if platform == "darwin": 30 | out = await self.sh( 31 | f'stat -f "%i %p %u %g %z %a %m %c %k %b" {self.params.esc_path}' 32 | ) 33 | parts = out.strip().split(" ", 9) 34 | elif platform == "linux": 35 | out = await self.sh( 36 | f'stat --format "%i %f %u %g %s %X %Y %W %B %b" {self.params.esc_path}' 37 | ) 38 | parts = out.strip().split(" ", 9) 39 | else: 40 | raise Exception(f"Can't support {platform}") 41 | stat.inode = int(parts[0]) 42 | stat.mode = "{0:o}".format(int(parts[1], 16)) 43 | stat.user_id = int(parts[2]) 44 | stat.group_id = int(parts[3]) 45 | stat.size = int(parts[4]) 46 | stat.access_time = int(parts[5]) 47 | stat.modify_time = int(parts[6]) 48 | stat.create_time = int(parts[7]) 49 | stat.block_size = int(parts[8]) 50 | stat.blocks = int(parts[9]) 51 | return stat 52 | 53 | 54 | class FsStatTest(task.TaskTest): 55 | @task.TaskTest.ubuntu 56 | async def test_ubuntu(self): 57 | await self.fs.write("/tmp/some-file", b"Some delicious bytes") 58 | stat = await self.fs.stat("/tmp/some-file") 59 | assert stat.size == 20, "size is incorrect" 60 | -------------------------------------------------------------------------------- /pitcrew/tasks/fs/touch.py: -------------------------------------------------------------------------------- 1 | from pitcrew import task 2 | 3 | 4 | @task.arg("path", desc="The path to change the mode of", type=str) 5 | class FsTouch(task.BaseTask): 6 | """Touches a file""" 7 | 8 | async def run(self): 9 | return await self.sh(f"touch {self.params.esc_path}") 10 | -------------------------------------------------------------------------------- /pitcrew/tasks/fs/write.py: -------------------------------------------------------------------------------- 1 | import hashlib 2 | from pitcrew import task 3 | 4 | 5 | @task.arg("path", type=str, desc="The path of the file to write to") 6 | @task.arg("content", type=bytes, desc="The contents to write") 7 | class FsWrite(task.BaseTask): 8 | """Write bytes to a file""" 9 | 10 | async def verify(self): 11 | stat = await self.fs.stat(self.params.path) 12 | assert len(self.params.content) == stat.size 13 | expected_digest = hashlib.sha256(self.params.content).hexdigest() 14 | actual_digest = await self.fs.digests.sha256(self.params.path) 15 | assert actual_digest == expected_digest 16 | 17 | async def run(self): 18 | await self.sh( 19 | f"tee {self.params.esc_path} > /dev/null", stdin=self.params.content 20 | ) 21 | 22 | 23 | class FsWriteTest(task.TaskTest): 24 | @task.TaskTest.ubuntu 25 | async def test_ubuntu(self): 26 | with self.cd("/tmp"): 27 | await self.fs.write("some-file", b"some content") 28 | out = await self.sh("cat some-file") 29 | assert out == "some content" 30 | -------------------------------------------------------------------------------- /pitcrew/tasks/git/clone.py: -------------------------------------------------------------------------------- 1 | import os 2 | from pitcrew import task 3 | 4 | 5 | @task.arg("url", desc="The url to clone", type=str) 6 | @task.arg("destination", desc="The destination", type=str) 7 | class GitClone(task.BaseTask): 8 | """Installs a package, optionally allowing the version number to specified. 9 | 10 | This task defers exection to package-manager specific installation tasks, such as 11 | homebrew or apt-get. 12 | """ 13 | 14 | async def verify(self): 15 | git_config = await self.fs.read( 16 | os.path.join(self.params.destination, ".git", "config") 17 | ) 18 | assert ( 19 | self.params.url in git_config.decode() 20 | ), f"url {self.params.url} couldn't be found in the .git/config" 21 | 22 | async def run(self): 23 | command = f"git clone {self.params.esc_url} {self.params.esc_destination}" 24 | await self.sh(command) 25 | -------------------------------------------------------------------------------- /pitcrew/tasks/homebrew/install.py: -------------------------------------------------------------------------------- 1 | from pitcrew import task 2 | 3 | 4 | @task.arg("name", type=str, desc="Package to install") 5 | @task.returns("the version installed") 6 | class HomebrewInstall(task.BaseTask): 7 | """Read value of path into bytes""" 8 | 9 | async def verify(self) -> str: 10 | code, out, err = await self.sh_with_code( 11 | f"brew ls --versions {self.params.esc_name}" 12 | ) 13 | lines = out.decode().strip().split("\n") 14 | if lines != [""]: 15 | for line in lines: 16 | _, version = line.split(" ", 1) 17 | return version 18 | assert False, f"no version found for {self.params.name}" 19 | 20 | async def run(self): 21 | await self.sh(f"brew install {self.params.esc_name}") 22 | 23 | async def available(self) -> bool: 24 | code, _, _ = await self.sh_with_code("which brew") 25 | return code == 0 26 | -------------------------------------------------------------------------------- /pitcrew/tasks/install/__init__.py: -------------------------------------------------------------------------------- 1 | from pitcrew import task 2 | 3 | 4 | @task.arg("name", desc="The name of the package to install", type=str) 5 | @task.returns("The version of the package installed") 6 | class Install(task.BaseTask): 7 | """Installs a package, optionally allowing the version number to specified. 8 | 9 | This task defers exection to package-manager specific installation tasks, such as 10 | homebrew or apt-get. 11 | """ 12 | 13 | async def run(self) -> str: 14 | installer_tasks = [self.homebrew.install, self.apt_get.install] 15 | for pkg in installer_tasks: 16 | task = pkg.task() 17 | if await task.available(): 18 | return await task.invoke(name=self.params.name) 19 | raise Exception("cannot find a package manager to defer to") 20 | -------------------------------------------------------------------------------- /pitcrew/tasks/install/homebrew.py: -------------------------------------------------------------------------------- 1 | from pitcrew import task 2 | 3 | 4 | class InstallHomebrew(task.BaseTask): 5 | """Installs the homebrew package manager""" 6 | 7 | async def verify(self): 8 | await self.install.xcode_cli() 9 | assert await self.sh("which brew") 10 | 11 | async def run(self): 12 | await self.sh( 13 | '/usr/bin/ruby -e "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install)"' 14 | ) 15 | -------------------------------------------------------------------------------- /pitcrew/tasks/install/xcode_cli.py: -------------------------------------------------------------------------------- 1 | from pitcrew import task 2 | 3 | 4 | class InstallXcodeCli(task.BaseTask): 5 | """Installs xcode cli tools""" 6 | 7 | async def verify(self): 8 | assert await self.fs.is_directory("/Library/Developer/CommandLineTools") 9 | 10 | async def run(self): 11 | await self.sh("xcode-select --install") 12 | await self.poll(self.verify) 13 | -------------------------------------------------------------------------------- /pitcrew/tasks/providers/docker.py: -------------------------------------------------------------------------------- 1 | from pitcrew import task 2 | 3 | 4 | class DockerProvider: 5 | def __init__(self, context, container_ids): 6 | self.context = context 7 | self.container_ids = container_ids 8 | self.index = 0 9 | 10 | async def __aenter__(self): 11 | return self 12 | 13 | async def __aexit__(self, exc_type, exc_value, traceback): 14 | if exc_type: 15 | raise exc_value.with_traceback(traceback) 16 | 17 | def __aiter__(self): 18 | return self 19 | 20 | async def __anext__(self): 21 | if self.index == len(self.container_ids): 22 | raise StopAsyncIteration 23 | docker_ctx = self.context.docker_context( 24 | container_id=self.container_ids[self.index] 25 | ) 26 | self.index += 1 27 | return docker_ctx 28 | 29 | def __str__(self): 30 | return f"DockerProvider(container_ids={self.container_ids})" 31 | 32 | 33 | @task.returns("An async generator that gives ssh contexts") 34 | @task.arg("container_ids", type=list, desc="The container ids to use") 35 | class ProvidersDocker(task.BaseTask): 36 | """A provider for ssh contexts""" 37 | 38 | async def run(self) -> DockerProvider: 39 | return DockerProvider(self.context, self.params.container_ids) 40 | -------------------------------------------------------------------------------- /pitcrew/tasks/providers/local.py: -------------------------------------------------------------------------------- 1 | from pitcrew import task 2 | 3 | 4 | class LocalProvider: 5 | def __init__(self, local_context): 6 | self.returned = False 7 | self.local_context = local_context 8 | 9 | async def __aenter__(self): 10 | return self 11 | 12 | async def __aexit__(self, exc_type, exc_value, traceback): 13 | if exc_type: 14 | raise exc_value.with_traceback(traceback) 15 | 16 | def __aiter__(self): 17 | return self 18 | 19 | async def __anext__(self): 20 | if not self.returned: 21 | self.returned = True 22 | return self.local_context 23 | else: 24 | raise StopAsyncIteration 25 | 26 | def __str__(self): 27 | return "LocalProvider" 28 | 29 | 30 | @task.returns("An async generator that gives a local context") 31 | class ProvidersLocal(task.BaseTask): 32 | """A provider for a local context""" 33 | 34 | async def run(self) -> LocalProvider: 35 | return LocalProvider(self.context.local_context) 36 | 37 | 38 | class ProvidersLocalTest(task.TaskTest): 39 | @task.TaskTest.ubuntu 40 | async def test_ubuntu(self): 41 | async for p in await self.providers.local(): 42 | assert p == self.context.local_context 43 | -------------------------------------------------------------------------------- /pitcrew/tasks/providers/ssh.py: -------------------------------------------------------------------------------- 1 | import asyncssh 2 | from pitcrew import task 3 | from netaddr.ip.nmap import iter_nmap_range 4 | 5 | 6 | class SSHProvider: 7 | def __init__(self, context, hosts, user, tunnels=[], **connection_args): 8 | self.context = context 9 | self.hosts = hosts 10 | self.tunnels = tunnels 11 | self.connection_args = connection_args 12 | self.flattened_hosts = self.__generate_flattened_hosts() 13 | self.user = user 14 | self.index = 0 15 | self.tunnel_contexts = [] 16 | 17 | async def __aenter__(self): 18 | last_tunnel = None 19 | for tunnel in self.tunnels: 20 | context = self.context.ssh_context(tunnel=last_tunnel, **tunnel) 21 | self.tunnel_contexts.append(context) 22 | await context.__aenter__() 23 | last_tunnel = context.connection 24 | 25 | async def __aexit__(self, exc_type, exc_value, traceback): 26 | for context in reversed(self.tunnel_contexts): 27 | try: 28 | await context.__aexit__() 29 | except: 30 | pass 31 | if exc_type: 32 | raise exc_value.with_traceback(traceback) 33 | 34 | def __aiter__(self): 35 | return self 36 | 37 | async def __anext__(self): 38 | if self.index == len(self.flattened_hosts): 39 | raise StopAsyncIteration 40 | 41 | tunnel = self.tunnel_contexts[-1].connection if self.tunnel_contexts else None 42 | ssh_ctx = self.context.ssh_context( 43 | host=self.flattened_hosts[self.index], 44 | user=self.user, 45 | tunnel=tunnel, 46 | **self.connection_args, 47 | ) 48 | self.index += 1 49 | return ssh_ctx 50 | 51 | def __str__(self): 52 | return f"SSHProvider(user={self.user} hosts={self.hosts})" 53 | 54 | def __generate_flattened_hosts(self): 55 | hosts = [] 56 | for host in self.hosts: 57 | try: 58 | hosts.append(map(lambda ip: str(ip), list(iter_nmap_range(host)))) 59 | except: 60 | hosts.append(host) 61 | return hosts 62 | 63 | 64 | @task.returns("An async generator that gives ssh contexts") 65 | @task.arg("hosts", type=list, desc="The hosts to use for ssh contexts") 66 | @task.arg( 67 | "tunnels", type=list, desc="The set of tunnels to connect through", default=[] 68 | ) 69 | @task.opt("user", type=str, desc="The user to use for the ssh contexts") 70 | @task.opt( 71 | "agent_forwarding", 72 | type=bool, 73 | default=False, 74 | desc="Specify if forwarding is enabled", 75 | ) 76 | @task.opt("agent_path", type=str, desc="Specify if forwarding is enabled") 77 | @task.opt("ask_password", type=str, desc="The prompt to use for asking for a password") 78 | class ProvidersSsh(task.BaseTask): 79 | """A provider for ssh contexts""" 80 | 81 | async def run(self) -> SSHProvider: 82 | extra_args = {} 83 | if self.params.agent_path: 84 | extra_args["agent_path"] = self.params.agent_path 85 | if self.params.ask_password: 86 | extra_args["password"] = await self.password(self.params.ask_password) 87 | return SSHProvider( 88 | self.context, 89 | self.params.hosts, 90 | self.params.user, 91 | tunnels=self.params.tunnels, 92 | agent_forwarding=self.params.agent_forwarding, 93 | **extra_args, 94 | ) 95 | -------------------------------------------------------------------------------- /pitcrew/template.py: -------------------------------------------------------------------------------- 1 | import os 2 | import uuid 3 | from jinja2 import Template as JinjaTemplate 4 | from pitcrew.file import File 5 | 6 | 7 | class Template: 8 | def __init__(self, task, path): 9 | self.task = task 10 | self.path = path 11 | 12 | self.rendered_path = os.path.join( 13 | task.context.app.template_render_path, 14 | f"{uuid.uuid4()}-{os.path.basename(self.path)}", 15 | ) 16 | 17 | def render(self, **kwargs) -> File: 18 | with open(self.rendered_path, "wb") as out: 19 | out.write(self.render_as_bytes(**kwargs)) 20 | return self.task.context.app.local_context.file(self.rendered_path) 21 | 22 | def render_as_bytes(self, **kwargs) -> bytes: 23 | with open(self.path) as fh: 24 | template = JinjaTemplate(fh.read()) 25 | return template.render(**kwargs).encode() 26 | -------------------------------------------------------------------------------- /pitcrew/templates/new_task.py.j2: -------------------------------------------------------------------------------- 1 | from pitcrew import task 2 | 3 | 4 | # @task.arg("an_argument", desc="Describe the argument", type=str) 5 | # @task.opt("an_optional_argument", desc="Describe the optional argument", type=str) 6 | # @task.returns("Describe the return value") 7 | class {{ task_class_name }}(task.BaseTask): 8 | """The description of the task""" 9 | 10 | # use this if you want to verify the task 11 | # async def verify(self): 12 | # pass 13 | 14 | async def run(self): 15 | pass 16 | 17 | class {{ task_class_name }}Test(task.TaskTest): 18 | @task.TaskTest.ubuntu 19 | async def test_ubuntu(self): 20 | pass 21 | -------------------------------------------------------------------------------- /pitcrew/templates/task.md.j2: -------------------------------------------------------------------------------- 1 | ## {{ task.task_name }} 2 | 3 | {% if desc -%} 4 | {{ desc }} 5 | {%- else -%} 6 | ⚠️ *No task description set* 7 | {%- endif %} 8 | 9 | {% if task.args -%} 10 | ### Arguments 11 | 12 | {% for arg in task.args %} 13 | - {{ arg.name }} *({{ arg.type.__name__}})* {% if arg.desc -%} 14 | : {{ arg.desc }} 15 | {%- else -%} 16 | : ⚠️ *no description* 17 | {%- endif -%} 18 | {%- endfor %} 19 | {%- endif %} 20 | 21 | {% if task.has_return_type() %} 22 | ### Returns 23 | 24 | *({{ task.expected_return_type().__name__ }})* {% if task.return_desc -%} 25 | {{ task.return_desc }} 26 | {%- else -%} 27 | ⚠️ *No task return value description set* 28 | {%- endif %} 29 | {% endif %} 30 | 31 |
32 | Show source 33 | 34 | ```python 35 | {{ task.source() }} 36 | ``` 37 | 38 |
39 | -------------------------------------------------------------------------------- /pitcrew/templates/tasks.md.j2: -------------------------------------------------------------------------------- 1 | # Tasks 2 | 3 | {% for task in tasks %} 4 | {{- task }} 5 | 6 | ------------------------------------------------- 7 | 8 | {% endfor %} 9 | -------------------------------------------------------------------------------- /pitcrew/test/__init__.py: -------------------------------------------------------------------------------- 1 | from pitcrew.logger import logger 2 | 3 | 4 | class TestRunner: 5 | def __init__(self, app, context): 6 | self.app = app 7 | self.context = context 8 | 9 | async def run(self, prefix=None): 10 | for task in self.app.loader.each_task(): 11 | if prefix and not task.task_name.startswith(prefix): 12 | continue 13 | for test_cls in task.tests: 14 | test = test_cls(self.context) 15 | for name, val in test_cls.__dict__.items(): 16 | if name.startswith("test_"): 17 | test_method = getattr(test, name) 18 | with logger.with_test(task, name): 19 | await test_method() 20 | -------------------------------------------------------------------------------- /pitcrew/test/util.py: -------------------------------------------------------------------------------- 1 | def ubuntu_decorator(f): 2 | async def wrapper(self, *args, **kwargs): 3 | container_id = await self.docker.run("ubuntu", detach=True, interactive=True) 4 | docker_ctx = self.docker_context(container_id, user="root") 5 | previous_context = self.context 6 | try: 7 | async with docker_ctx: 8 | self.context = docker_ctx 9 | return await f(self, *args, **kwargs) 10 | finally: 11 | self.context = previous_context 12 | 13 | return wrapper 14 | -------------------------------------------------------------------------------- /pitcrew/util.py: -------------------------------------------------------------------------------- 1 | from pitcrew.executor import ExecutionResult 2 | import base64 3 | import sys 4 | import json 5 | 6 | 7 | class OutputEncoder(json.JSONEncoder): 8 | def default(self, obj): 9 | if isinstance(obj, bytes): 10 | try: 11 | return obj.decode("utf-8", "strict") 12 | except UnicodeDecodeError: 13 | return base64.b64encode(obj) 14 | elif isinstance(obj, Exception): 15 | return str(obj) 16 | elif isinstance(obj, ExecutionResult): 17 | return { 18 | "context": obj.context.descriptor(), 19 | "result": obj.result, 20 | "exception": obj.exception, 21 | } 22 | else: 23 | return json.JSONEncoder.default(self, obj) 24 | 25 | 26 | class ResultsPrinter: 27 | def __init__(self, results): 28 | self.results = results 29 | 30 | def print(self): 31 | results = self.results 32 | if results.passed: 33 | sys.stderr.write("Passed:\n") 34 | sys.stdout.write(json.dumps(results.passed, cls=OutputEncoder)) 35 | sys.stdout.flush() 36 | sys.stderr.write("\n") 37 | if results.failed: 38 | sys.stderr.write("Failed:\n") 39 | sys.stderr.write(json.dumps(results.failed, cls=OutputEncoder)) 40 | sys.stderr.write("\n") 41 | sys.stderr.flush() 42 | if results.errored: 43 | sys.stderr.write("Errored:\n") 44 | sys.stderr.write(json.dumps(results.errored, cls=OutputEncoder)) 45 | sys.stderr.write("\n") 46 | sys.stderr.flush() 47 | 48 | sys.stderr.write(f"\n 🔧🔧🔧 Finished 🔧🔧🔧\n") 49 | 50 | sys.stderr.write("\nSummary") 51 | if len(results.passed) != 0: 52 | sys.stderr.write(f" \033[32mpassed={len(results.passed)}\033[0m") 53 | else: 54 | sys.stderr.write(f" passed={len(results.passed)}") 55 | 56 | if len(results.failed) != 0: 57 | sys.stderr.write(f" \033[31mfailed={len(results.failed)}\033[0m") 58 | else: 59 | sys.stderr.write(f" failed={len(results.failed)}") 60 | 61 | if len(results.errored) != 0: 62 | sys.stderr.write(f" \033[31;1merrored={len(results.errored)}\033[0m") 63 | else: 64 | sys.stderr.write(f" errored={len(results.errored)}") 65 | 66 | sys.stderr.write("\n") 67 | sys.stderr.flush() 68 | -------------------------------------------------------------------------------- /requirements-build.txt: -------------------------------------------------------------------------------- 1 | -r requirements.txt 2 | altgraph==0.16.1 3 | bleach==3.1.0 4 | certifi==2019.3.9 5 | chardet==3.0.4 6 | docutils==0.14 7 | future==0.17.1 8 | githubrelease==1.5.8 9 | idna==2.8 10 | LinkHeader==0.4.3 11 | livereload==2.6.1 12 | macholib==1.11 13 | Markdown==3.1 14 | markdown2==2.3.7 15 | mkdocs==1.0.4 16 | pefile==2019.4.18 17 | pkginfo==1.5.0.1 18 | pydoc-markdown==2.0.5 19 | Pygments==2.4.0 20 | PyInstaller==3.4 21 | PyYAML==5.1 22 | readme-renderer==24.0 23 | requests==2.22.0 24 | requests-toolbelt==0.9.1 25 | tornado==6.0.2 26 | tqdm==4.32.1 27 | twine==1.13.0 28 | urllib3==1.25.2 29 | webencodings==0.5.1 30 | wheel==0.33.4 31 | -------------------------------------------------------------------------------- /requirements-development.txt: -------------------------------------------------------------------------------- 1 | -r requirements.txt 2 | aiounittest==1.1.0 3 | appdirs==1.4.3 4 | attrs==19.1.0 5 | black==19.3b0 6 | entrypoints==0.3 7 | flake8==3.7.7 8 | livereload==2.6.0 9 | Markdown==3.1 10 | mccabe==0.6.1 11 | mkdocs==1.0.4 12 | pycodestyle==2.5.0 13 | pydoc-markdown==2.0.5 14 | pyflakes==2.1.1 15 | PyYAML==5.1 16 | toml==0.10.0 17 | tornado==6.0.2 18 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | Click==7.0 2 | Jinja2==2.10.1 3 | MarkupSafe==1.1.1 4 | asn1crypto==0.24.0 5 | asyncssh==1.16.1 6 | cffi==1.12.3 7 | cryptography==2.6.1 8 | netaddr==0.7.19 9 | pycparser==2.19 10 | six==1.12.0 11 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | 4 | # Note: To use the 'upload' functionality of this file, you must: 5 | # $ pip install twine 6 | 7 | import io 8 | import os 9 | import sys 10 | import subprocess 11 | from shutil import rmtree 12 | 13 | from setuptools import find_packages, setup, Command 14 | 15 | here = os.path.abspath(os.path.dirname(__file__)) 16 | 17 | about = {} 18 | with open(os.path.join(here, 'pitcrew', '__version__.py')) as f: 19 | exec(f.read(), about) 20 | 21 | # Package meta-data. 22 | NAME = about['__name__'] 23 | DESCRIPTION = about['__description__'] 24 | URL = about['__url__'] 25 | AUTHOR = about['__author__'] 26 | EMAIL = about['__author_email__'] 27 | VERSION = about['__version__'] 28 | REQUIRES_PYTHON = '>=3.6.0' 29 | EXTRAS = { } 30 | with open(os.path.join(here, "requirements.txt")) as f: 31 | REQUIRED = list(map(lambda l: l.strip(), f.readlines())) 32 | 33 | # The rest you shouldn't have to touch too much :) 34 | # ------------------------------------------------ 35 | # Except, perhaps the License and Trove Classifiers! 36 | # If you do change the License, remember to change the Trove Classifier for that! 37 | # Import the README and use it as the long-description. 38 | # Note: this will only work if 'README.md' is present in your MANIFEST.in file! 39 | try: 40 | with io.open(os.path.join(here, 'README.md'), encoding='utf-8') as f: 41 | long_description = '\n' + f.read() 42 | except FileNotFoundError: 43 | long_description = DESCRIPTION 44 | 45 | 46 | class UploadCommand(Command): 47 | """Support setup.py upload.""" 48 | 49 | description = 'Build and publish the package.' 50 | user_options = [] 51 | 52 | @staticmethod 53 | def status(s): 54 | """Prints things in bold.""" 55 | print('\033[1m{0}\033[0m'.format(s)) 56 | 57 | def initialize_options(self): 58 | pass 59 | 60 | def finalize_options(self): 61 | pass 62 | 63 | def run(self): 64 | try: 65 | self.status('Removing previous builds…') 66 | rmtree(os.path.join(here, 'dist')) 67 | except OSError: 68 | pass 69 | 70 | self.status('Building Source and Wheel (universal) distribution…') 71 | subprocess.check_call([sys.executable, "setup.py", "sdist", "bdist_wheel", "--universal"]) 72 | 73 | self.status('Uploading the package to PyPI via Twine…') 74 | subprocess.check_call(self.twine_command()) 75 | 76 | sys.exit() 77 | 78 | def twine_command(self): 79 | return ['twine', 'upload', 'dist/*'] 80 | 81 | class UploadTestCommand(UploadCommand): 82 | def twine_command(self): 83 | return ['twine', 'upload', '-r', 'pypitest', 'dist/*'] 84 | 85 | 86 | # Where the magic happens: 87 | setup( 88 | name=NAME, 89 | version=about['__version__'], 90 | description=DESCRIPTION, 91 | long_description=long_description, 92 | long_description_content_type='text/markdown', 93 | author=AUTHOR, 94 | author_email=EMAIL, 95 | python_requires=REQUIRES_PYTHON, 96 | url=URL, 97 | packages=find_packages(exclude=('tests',)), 98 | entry_points={ 99 | 'console_scripts': ['crew=pitcrew.cli:cli'], 100 | }, 101 | install_requires=REQUIRED, 102 | extras_require=EXTRAS, 103 | include_package_data=True, 104 | license='MIT', 105 | classifiers=[ 106 | # Trove classifiers 107 | # Full list: https://pypi.python.org/pypi?%3Aaction=list_classifiers 108 | 'License :: OSI Approved :: MIT License', 109 | 'Programming Language :: Python', 110 | 'Programming Language :: Python :: 3', 111 | 'Programming Language :: Python :: 3.6', 112 | 'Programming Language :: Python :: Implementation :: CPython', 113 | 'Programming Language :: Python :: Implementation :: PyPy' 114 | ], 115 | # $ setup.py publish support. 116 | cmdclass={ 117 | 'upload_test': UploadTestCommand, 118 | 'upload': UploadCommand, 119 | }, 120 | ) 121 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/joshbuddy/pitcrew/ce9231cd4a4fb0a732be0856678ba956b6b4fedb/tests/__init__.py -------------------------------------------------------------------------------- /tests/test_cli.py: -------------------------------------------------------------------------------- 1 | import base64 2 | import os 3 | import getpass 4 | import json 5 | import shutil 6 | from click.testing import CliRunner 7 | import unittest 8 | from pitcrew.cli import cli 9 | 10 | 11 | class TestCli(unittest.TestCase): 12 | def test_help(self): 13 | runner = CliRunner() 14 | result = runner.invoke(cli, ["help"]) 15 | self.assertEqual(result.exit_code, 0) 16 | self.assertTrue("Commands" in result.output) 17 | self.assertTrue("Usage" in result.output) 18 | self.assertTrue("Options" in result.output) 19 | 20 | def test_docs(self): 21 | with open("docs/tasks.md") as fh: 22 | expected_docs = fh.read() 23 | os.remove("docs/tasks.md") 24 | runner = CliRunner() 25 | result = runner.invoke(cli, ["docs"]) 26 | self.assertEqual(result.exit_code, 0) 27 | with open("docs/tasks.md") as fh: 28 | actual_docs = fh.read() 29 | self.assertEqual(actual_docs, expected_docs) 30 | 31 | def test_info(self): 32 | runner = CliRunner() 33 | result = runner.invoke(cli, ["info", "fs.write"]) 34 | self.assertEqual(result.exit_code, 0) 35 | self.assertTrue("fs.write\n" in result.output) 36 | 37 | def test_list(self): 38 | runner = CliRunner() 39 | result = runner.invoke(cli, ["list"]) 40 | self.assertEqual(result.exit_code, 0) 41 | self.assertTrue(len(result.output.split("\n")) > 10) 42 | 43 | def test_new(self): 44 | task_path = os.path.abspath( 45 | os.path.join( 46 | __file__, 47 | "..", 48 | "..", 49 | "pitcrew", 50 | "tasks", 51 | "some", 52 | "kind", 53 | "of", 54 | "task.py", 55 | ) 56 | ) 57 | try: 58 | runner = CliRunner() 59 | result = runner.invoke(cli, ["new", "some.kind.of.task"]) 60 | self.assertEqual(result.exit_code, 0) 61 | self.assertTrue(os.path.isfile(task_path)) 62 | finally: 63 | os.remove(task_path) 64 | 65 | def test_new_rename(self): 66 | base_path = os.path.abspath( 67 | os.path.join(__file__, "..", "..", "pitcrew", "tasks") 68 | ) 69 | try: 70 | runner = CliRunner() 71 | result = runner.invoke(cli, ["new", "some.kind.of.task"]) 72 | self.assertEqual(result.exit_code, 0) 73 | self.assertTrue(os.path.isfile(base_path + "/some/kind/of/task.py")) 74 | result = runner.invoke(cli, ["new", "some.kind.of.task.lower"]) 75 | self.assertEqual(result.exit_code, 0) 76 | self.assertFalse(os.path.isfile(base_path + "/some/kind/of/task.py")) 77 | self.assertTrue( 78 | os.path.isfile(base_path + "/some/kind/of/task/__init__.py") 79 | ) 80 | self.assertTrue(os.path.isfile(base_path + "/some/kind/of/task/lower.py")) 81 | finally: 82 | shutil.rmtree(base_path + "/some/kind") 83 | 84 | def test_run(self): 85 | runner = CliRunner(mix_stderr=False) 86 | result = runner.invoke(cli, ["run", "fs.read", "requirements.txt"]) 87 | self.assertEqual(result.exit_code, 0) 88 | 89 | with open("requirements.txt", "r") as fh: 90 | expected_output = json.dumps( 91 | [ 92 | { 93 | "context": f"{getpass.getuser()}@local", 94 | "result": fh.read(), 95 | "exception": None, 96 | } 97 | ] 98 | ) 99 | self.assertEqual(result.stdout_bytes.decode(), expected_output) 100 | 101 | def test_run_with_binary(self): 102 | base64_data = "CUGhip285YEjnHE4Cel0/lA5OLPV5gEsuEGMEfR7" 103 | with open("test_data", "wb") as fh: 104 | fh.write(base64.b64decode(base64_data)) 105 | runner = CliRunner(mix_stderr=False) 106 | result = runner.invoke(cli, ["run", "fs.read", "test_data"]) 107 | self.assertEqual(result.exit_code, 0) 108 | expected_output = json.dumps( 109 | [ 110 | { 111 | "context": f"{getpass.getuser()}@local", 112 | "result": base64_data, 113 | "exception": None, 114 | } 115 | ] 116 | ) 117 | self.assertEqual(result.stdout_bytes.decode(), expected_output) 118 | 119 | def test_test(self): 120 | runner = CliRunner() 121 | result = runner.invoke(cli, ["test", "fs.digests.md5"]) 122 | self.assertEqual(result.exit_code, 0) 123 | -------------------------------------------------------------------------------- /tests/test_task.py: -------------------------------------------------------------------------------- 1 | from pitcrew import task 2 | import aiounittest 3 | 4 | 5 | class TestTask(aiounittest.AsyncTestCase): 6 | async def test_normal_arg(self): 7 | @task.arg("normal") 8 | class Task(task.BaseTask): 9 | async def run(self): 10 | pass 11 | 12 | task_instance = Task() 13 | await task_instance.invoke(normal="arg") 14 | 15 | async def test_normal_args(self): 16 | @task.arg("one") 17 | @task.arg("two") 18 | @task.arg("three") 19 | class Task(task.BaseTask): 20 | async def run(self): 21 | assert self.params.one == "one" 22 | assert self.params.two == "two" 23 | assert self.params.three == "three" 24 | 25 | task_instance = Task() 26 | await task_instance.invoke("one", "two", "three") 27 | 28 | async def test_normal_args_via_keyword(self): 29 | @task.arg("one") 30 | @task.arg("two") 31 | @task.arg("three") 32 | class Task(task.BaseTask): 33 | async def run(self): 34 | assert self.params.one == "one" 35 | assert self.params.two == "two" 36 | assert self.params.three == "three" 37 | 38 | task_instance = Task() 39 | await task_instance.invoke(one="one", two="two", three="three") 40 | 41 | async def test_normal_kwargs(self): 42 | @task.opt("optional") 43 | class Task(task.BaseTask): 44 | async def run(self): 45 | assert self.params.optional is None 46 | 47 | task_instance = Task() 48 | await task_instance.invoke() 49 | 50 | async def test_extra_kwargs(self): 51 | class Task(task.BaseTask): 52 | async def run(self): 53 | pass 54 | 55 | task_instance = Task() 56 | with self.assertRaises(TypeError): 57 | await task_instance.invoke(extra="arg") 58 | 59 | async def test_extra_args(self): 60 | @task.arg("one") 61 | class Task(task.BaseTask): 62 | async def run(self): 63 | pass 64 | 65 | task_instance = Task() 66 | with self.assertRaises(TypeError): 67 | await task_instance.invoke("asd", "qwe") 68 | 69 | async def test_varargs(self): 70 | @task.varargs("one", type=str) 71 | class Task(task.BaseTask): 72 | async def run(self): 73 | assert self.params.one == ["one", "two", "three"] 74 | 75 | task_instance = Task() 76 | await task_instance.invoke("one", "two", "three") 77 | --------------------------------------------------------------------------------