├── .github └── workflows │ └── push.yml ├── .gitignore ├── CONTRIBUTING.md ├── LICENSE ├── MANIFEST.in ├── Makefile ├── README.md ├── docs ├── architecture.md ├── make_examples │ ├── Makefile │ ├── README.md │ └── peru.yaml ├── peru.gif └── release.md ├── fastentrypoints.py ├── packaging ├── arch │ ├── PKGBUILD │ └── makelocal.sh ├── build_ubuntu_source_package.sh └── debian │ ├── changelog │ ├── compat │ ├── control │ ├── copyright │ ├── docs │ ├── rules │ └── source │ └── format ├── peru.py ├── peru ├── VERSION ├── __init__.py ├── __main__.py ├── async_exit_stack.py ├── async_helpers.py ├── cache.py ├── compat.py ├── display.py ├── docopt │ ├── LICENSE-MIT │ ├── __init__.py │ ├── _version.py │ └── py.typed ├── edit_yaml.py ├── error.py ├── glob.py ├── imports.py ├── keyval.py ├── main.py ├── merge.py ├── module.py ├── parser.py ├── plugin.py ├── resources │ └── plugins │ │ ├── bat │ │ ├── bat_plugin.bat │ │ └── plugin.yaml │ │ ├── cp │ │ ├── cp_plugin.py │ │ └── plugin.yaml │ │ ├── curl │ │ ├── curl_plugin.py │ │ └── plugin.yaml │ │ ├── empty │ │ ├── empty_plugin.py │ │ └── plugin.yaml │ │ ├── git │ │ ├── git_plugin.py │ │ └── plugin.yaml │ │ ├── hg │ │ ├── hg_plugin.py │ │ └── plugin.yaml │ │ ├── noop_cache │ │ ├── noop_cache_plugin.py │ │ └── plugin.yaml │ │ ├── print │ │ ├── plugin.yaml │ │ └── print_plugin.py │ │ ├── rsync │ │ ├── plugin.yaml │ │ └── rsync_plugin.sh │ │ └── svn │ │ ├── plugin.yaml │ │ └── svn_plugin.py ├── rule.py ├── runtime.py └── scope.py ├── requirements-dev.txt ├── setup.py ├── test.py └── tests ├── .coveragerc ├── __init__.py ├── resources ├── absolute_path.tar ├── absolute_path.zip ├── from_windows.zip ├── illegal_symlink_absolute.tar ├── illegal_symlink_dots.tar ├── leading_dots.tar ├── leading_dots.zip ├── legal_symlink_dots.tar ├── with_exe.tar └── with_exe.zip ├── shared.py ├── test_async.py ├── test_cache.py ├── test_compat.py ├── test_curl_plugin.py ├── test_display.py ├── test_edit_yaml.py ├── test_git_plugin.py ├── test_glob.py ├── test_keyval.py ├── test_merge.py ├── test_parallelism.py ├── test_parser.py ├── test_paths.py ├── test_plugins.py ├── test_reup.py ├── test_rule.py ├── test_runtime.py ├── test_scope.py ├── test_sync.py └── test_test_shared.py /.github/workflows/push.yml: -------------------------------------------------------------------------------- 1 | name: tests 2 | 3 | on: 4 | push: 5 | branches: 6 | - "*" 7 | pull_request: 8 | 9 | jobs: 10 | test: 11 | name: Python ${{ matrix.python-version }} on ${{ matrix.os }} 12 | runs-on: ${{ matrix.os }} 13 | strategy: 14 | fail-fast: false 15 | matrix: 16 | python-version: ["3.8", "3.9", "3.10", "3.11", "3.12", "3.13"] 17 | os: [ubuntu-latest, windows-latest, macOS-latest] 18 | 19 | steps: 20 | - uses: actions/checkout@v4 21 | - name: Set up Python ${{ matrix.python-version }} 22 | uses: actions/setup-python@v5 23 | with: 24 | python-version: ${{ matrix.python-version }} 25 | # Confirm that the `python` maps to the version we just installed. 26 | # PowerShell on Windows does this differently than the Unix shells. 27 | - run: which python ; python --version ; which pip ; pip --version 28 | if: matrix.os != 'windows-latest' 29 | - run: get-command python ; python --version ; get-command pip ; pip --version 30 | if: matrix.os == 'windows-latest' 31 | - run: git --version 32 | # macOS doesn't currently have hg or svnadmin installed by default. 33 | - run: brew install subversion 34 | if: matrix.os == 'macOS-latest' 35 | - run: pip install wheel && pip install mercurial 36 | if: matrix.os == 'macOS-latest' 37 | # Install test dependencies, like flake8. 38 | - run: pip install -r ./requirements-dev.txt 39 | # Run tests. 40 | - run: python test.py -v 41 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | __pycache__ 2 | *.pyc 3 | .coverage 4 | peru.egg-info 5 | build/ 6 | dist/ 7 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing 2 | 3 | We always like contributions here in `peru`! 4 | 5 | First of all, if you're looking for something to work or you have some idea for a new feature or you found a bug, then check out our [issue tracker](https://github.com/buildinspace/peru/issues) for this. 6 | 7 | In the issue, discuss your idea and implementation. 8 | 9 | Then if you want to make a contribution to `peru` then please raise a 10 | [Pull Request](https://github.com/buildinspace/peru/pulls) on GitHub. 11 | 12 | To help speed up the review process please ensure the following: 13 | 14 | - The PR addresses an open issue. 15 | - The project passes linting with `make check` or `flake8 peru tests`. 16 | - All tests are passing locally with (includes linting): `make test` or `python test.py`. 17 | - If adding a new feature you also add documentation. 18 | 19 | ## Developing 20 | 21 | The minimal Python version supported is 3.5. If you are developing in newer versions, be aware of functions not backwards compatible. The Github Workflow will make this check in the pull request. 22 | 23 | To check out a local copy of the project you can [fork the project on GitHub](https://github.com/buildinspace/peru/fork) 24 | and then clone it locally. If you are using https, then you should adapt to it. 25 | 26 | ```bash 27 | git clone git@github.com:yourusername/peru.git 28 | cd peru 29 | ``` 30 | 31 | This project uses `flake8` for linting. To configure your local environment, please install these development dependencies. 32 | You may want to do this in a virtualenv; use `make venv` to create it in 33 | `.venv` in the current directory. 34 | 35 | ```bash 36 | make deps-dev 37 | # OR 38 | pip install -r requirements-dev.txt 39 | ``` 40 | 41 | then you can run `flake8` with 42 | 43 | ```bash 44 | make check 45 | # OR 46 | flake8 peru tests 47 | ``` 48 | 49 | ## Testing 50 | 51 | You can check that things are working correctly by calling the tests. 52 | 53 | ```bash 54 | make test 55 | # OR 56 | python test.py -v 57 | ``` 58 | 59 | ``` 60 | $ python test.py -v 61 | test_safe_communicate (test_async.AsyncTest) ... ok 62 | test_basic_export (test_cache.CacheTest) ... ok 63 | . 64 | . 65 | . 66 | test_assert_contents (test_test_shared.SharedTestCodeTest) ... ok 67 | test_create_dir (test_test_shared.SharedTestCodeTest) ... ok 68 | test_read_dir (test_test_shared.SharedTestCodeTest) ... ok 69 | ---------------------------------------------------------------------- 70 | Ran 152 tests in 45.11s 71 | 72 | OK (skipped=1) 73 | ``` 74 | 75 | These checks will be run automatically when you make a pull request. 76 | 77 | You should always have a skipped test, because this is a platform specific tests. 78 | 79 | If you are working on a new feature please add tests to ensure the feature works as expected. If you are working on a bug fix then please add a test to ensure there is no regression. 80 | 81 | Tests are stored in `peru/tests` and verify the current implementation to see how your test will fit in. 82 | 83 | ## Making a Pull Request 84 | 85 | Once you have made your changes and are ready to make a Pull Request please ensure tests and linting pass locally before pushing to GitHub. 86 | 87 | When making your Pull Request please include a short description of the changes, but more importantly why they are important. 88 | 89 | Perhaps by writing a before and after paragraph with user examples. 90 | 91 | ``` 92 | # New feature short description here 93 | 94 | Closes #56 95 | 96 | **Changes** 97 | 98 | This PR includes a new feature that ... 99 | 100 | **Before** 101 | 102 | If a user tried to pull a repository ... 103 | 104 | ```python 105 | > code example 106 | > 107 | ``` 108 | **After** 109 | 110 | If a user tries to pull a repository now ... 111 | 112 | ```python 113 | > code example 114 | > 115 | ``` 116 | ``` 117 | 118 | After that you should wait the review and perform possible changes in the submitted code. 119 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy 4 | of this software and associated documentation files (the "Software"), to deal 5 | in the Software without restriction, including without limitation the rights 6 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 7 | copies of the Software, and to permit persons to whom the Software is 8 | furnished to do so, subject to the following conditions: 9 | 10 | The above copyright notice and this permission notice shall be included in 11 | all copies or substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 19 | THE SOFTWARE. 20 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | # Without this line, the PyPI source distribution will be missing fastentrypoints. 2 | include fastentrypoints.py 3 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | PYTHON ?= python3 2 | FLAKE8 ?= flake8 3 | PIP ?= pip3 4 | 5 | VENV ?= .venv 6 | 7 | ##@ Code Quality 8 | 9 | .PHONY: all 10 | all: check test ## Run all checks and tests 11 | 12 | .PHONY: test 13 | test: ## Run all tests 14 | $(PYTHON) test.py -v 15 | 16 | .PHONY: check 17 | check: ## Run all checks 18 | $(FLAKE8) peru tests 19 | 20 | ##@ Dev Env Setup 21 | 22 | .PHONY: venv 23 | venv: ## Create a venv 24 | $(PYTHON) -m venv --clear $(VENV) 25 | @echo "Activate the venv with 'source $(VENV)/bin/activate'" 26 | 27 | .PHONY: deps-dev 28 | deps-dev: ## Install development dependencies 29 | $(PIP) install -r requirements-dev.txt 30 | 31 | ##@ Utility 32 | 33 | .PHONY: help 34 | help: ## Display this help 35 | @awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n make \033[36m\033[0m\n"} /^[\#a-zA-Z0-9_-]+:.*?##/ { printf " \033[36m%-15s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST) 36 | -------------------------------------------------------------------------------- /docs/architecture.md: -------------------------------------------------------------------------------- 1 | # Architecture 2 | 3 | When you run `peru sync`, here's what happens: 4 | 5 | 1. Peru checks the main cache to see whether any modules need to be 6 | fetched. 7 | 2. For modules that aren't already cached, peru executes the plugin 8 | corresponding to the module's type (git, hg, etc.). 9 | 3. These plugin scripts do all the actual work to download files. Each 10 | job gets a temporary directory where it puts the files it fetches. It 11 | also gets a persistent, plugin-specific caching directory where it 12 | can keep clones or whatever else to speed up future fetches. 13 | 4. When a plugin job is done, the files that it fetched are read into 14 | the main cache. 15 | 5. When everything is in cache, your `imports` tree is merged together 16 | and checked out to your project. 17 | 18 | And when you run `peru reup`, here's what happens: 19 | 20 | 1. For each module, peru executes the corresponding plugin script. This 21 | is a lot like `peru sync` above, but instead of telling the plugins 22 | to fetch, peru tells them to reup. (Not all plugins support this, but 23 | the important ones do.) 24 | 2. Each job finds the most up-to-date information for its module. The 25 | git plugin, for example, runs `git fetch` and reads the latest rev of 26 | the appropriate branch. 27 | 3. Each job then writes updated module fields formatted as YAML to a 28 | temporary file. The git plugin would write something like 29 | 30 | ```yaml 31 | rev: 381f7c737f5d53cf7915163b583b537f2fd5fc0d 32 | ``` 33 | 34 | to reflect the new value of the `rev` field. 35 | 4. Peru reads these new fields when each job finishes and writes them to 36 | the `peru.yaml` file in your project. 37 | 38 | ## Plugins 39 | 40 | The goal of the plugin interface is that plugins should do as little as 41 | possible. They only download files and new fields, and they don't know 42 | anything about what happens to things after they're fetched. 43 | 44 | Most of the builtin plugins are written in Python for portability, but 45 | they don't actually run in the peru process; instead, we run them as 46 | subprocesses. That's partly to enforce a clean separation, and partly to 47 | allow you to write plugins in whatever language you want. 48 | 49 | A plugin definition is a directory that contains a `plugin.yaml` file 50 | and at least one executable. The `plugin.yaml` file defines the module 51 | type's fields, and how the executable(s) should be invoked. Here's the 52 | `plugin.yaml` from `peru/resources/plugins/git`: 53 | 54 | ```yaml 55 | sync exe: git_plugin.py 56 | reup exe: git_plugin.py 57 | required fields: 58 | - url 59 | optional fields: 60 | - rev 61 | - reup 62 | cache fields: 63 | - url 64 | ``` 65 | 66 | - The name of the plugin directory determines the name of the module 67 | type. 68 | - `sync exe` is required, and it tells peru what to execute when it 69 | wants the plugin to sync. 70 | - `reup exe` is optional; it declares that the plugin supports reup and 71 | how to execute it. This can be the same script as `sync exe`, as it 72 | is here, in which case the script should decide what to do based on 73 | the `PERU_PLUGIN_COMMAND` environment variable described below. 74 | - `required fields` is required, and it tells peru which fields are 75 | mandatory for a module of this type. 76 | - `optional fields` is optional, and it lists any fields that are 77 | allowed but not required for a module of this type. 78 | - `cache fields` specifies that the plugin would like a cache directory, 79 | in addition to its output directory, where it can keep long-lived 80 | clones and things like that. The list that follows is the set of 81 | fields that this cache dir should be keyed off of. In this case, if 82 | two git modules share the same url, they will share the same cache 83 | dir. (Because there's no reason to clone a repo twice just to get two 84 | different revs.) Peru guarantees that no two jobs that share the same 85 | cache dir will ever run at the same time, so plugins don't need to 86 | worry about locking. 87 | 88 | The other part of a plugin definition is the executable script(s). These 89 | are invoked with no arguments, and several environment variables are 90 | defined to tell the plugin what to do: 91 | 92 | - `PERU_PLUGIN_COMMAND` is either `sync` or `reup`, depending on what 93 | peru needs the plugin to do. 94 | - `PERU_PLUGIN_CACHE` points to the plugin's cache directory. If 95 | `plugin.yaml` doesn't include `cache fields`, this path will be 96 | `/dev/null` (or `nul` on Windows). 97 | - `PERU_PLUGIN_TMP` points to a temp directory that will be deleted 98 | after the job is finished. 99 | - `PERU_MODULE_*`: Each module field is provided as a variable of this 100 | form. For example, the git plugin gets its `url` field as 101 | `PERU_MODULE_URL`. The variables for optional fields that aren't 102 | present in the module are defined but empty. 103 | - `PERU_SYNC_DEST` points to the temporary directory where the plugin 104 | should put the files it downloads. This is only defined for sync 105 | jobs. 106 | - `PERU_REUP_OUTPUT` points to the temporary file where the plugin 107 | should write updated field values, formatted as YAML. This is only 108 | defined for reup jobs. 109 | 110 | Plugins are always invoked with your project root (where your 111 | `peru.yaml` file lives) as the working directory. That means that you 112 | can use relative paths like `url: ../foo` in your `peru.yaml`. But 113 | plugin scripts that want to refer to other files in the plugin folder 114 | need to use paths based on `argv[0]`; simple relative paths won't work 115 | for that. 116 | 117 | You can install your own plugins by putting them in one of the directories that 118 | peru searches. On Posix systems, those are: 119 | 120 | 1. `$XDG_CONFIG_HOME/peru/plugins/` (default `~/.config/peru/plugins/`) 121 | 2. `/usr/local/lib/peru/plugins/` 122 | 3. `/usr/lib/peru/plugins/` 123 | 124 | On Windows, the plugin paths are: 125 | 126 | 1. `%LOCALAPPDATA%\peru\plugins\` 127 | 2. `%PROGRAMFILES%\peru\plugins\` 128 | 129 | ## Caching 130 | 131 | There are two types of caching in peru: plugin caching and the main tree 132 | cache. Both of these live inside the `.peru` directory that peru creates 133 | at the root of your project. We described plugin caching in the section 134 | above; it's the directories that hold plugin-specific things like cloned 135 | repos. Peru itself is totally agnostic about what goes in those 136 | directories. 137 | 138 | The tree cache (see `peru/cache.py`) is what peru itself really cares 139 | about. When a plugin is finished fetching files for a module, all those 140 | files are read into the tree cache. And when peru is ready to write out 141 | all your imports, it's the tree cache that's responsible for merging all 142 | those file trees and checking them out to disk. 143 | 144 | Under the covers, the tree cache is actually another git repo, which 145 | lives at `.peru/cache/trees`. (Note: This is *totally separate* from the 146 | git *plugin*, which fetches code from other people's repos just like the 147 | rest of the plugins.) If you're familiar with how git does things, it's 148 | a bare repo with only blob and tree objects, no commit objects. Using 149 | git to power our cache helps us get a lot of speed without actually 150 | doing any hard work! When you run `peru sync`, what happens first is 151 | basically a `git status` comparing the last tree peru checked out to 152 | what you have on disk. That's pretty fast even for large trees, and if 153 | the status is clean, peru doesn't have to do anything. Likewise, writing 154 | your imports to disk is basically a `git checkout`. 155 | 156 | The cache's export method is where we enforce most of peru's on-disk 157 | behavior. Like git, peru will refuse to overwrite pre-existing or 158 | modified files. Unlike git, peru will restore deleted files without 159 | complaining. Peru keeps track of the last tree it checked out 160 | (`.peru/lastimports`), so it can clean up old files when your imports 161 | change, though it will notice modified files and throw an error. 162 | 163 | Modules in the tree cache are keyed off of a hash of all their fields. 164 | So if you happen to have two identical modules that just have different 165 | names, you'll notice that only one of them appears to get fetched; the 166 | second will always be a cache hit. This also helps to understand the 167 | behavior of modules that fetch from `master` or similar, rather than a 168 | fixed rev: Once the module is in cache, it won't be fetched again unless 169 | you either change one of its fields or blow away the whole cache. In the 170 | future we could provide a mechanism to selectively clear the cache for 171 | one module, but for now you can just delete `.peru/cache`. The 172 | association between cache keys and git trees is maintained with a simple 173 | directory of key-value pairs (`.peru/cache/keyval`). 174 | 175 | Currently we just call regular command line git to accomplish all of 176 | this. In the future we could switch to libgit2, which might be 177 | substantially faster and cleaner. 178 | -------------------------------------------------------------------------------- /docs/make_examples/Makefile: -------------------------------------------------------------------------------- 1 | # Declaring phony targets explicitly in your Makefiles is a good habit. If for 2 | # some strange reason we created a file called "run" or "phony", make could get 3 | # tricked into thinking that those rules didn't need to run. This directive 4 | # means that make will never look for those files. 5 | .PHONY: run phony 6 | 7 | run: hello 8 | ./hello 9 | 10 | # Depending on the lastimports file means that this rule will only run when the 11 | # imports actually change, even though the sync runs every time. Peru will not 12 | # touch this file if nothing has changed since the last sync. 13 | hello: .peru/lastimports 14 | gcc -o hello c.c 15 | 16 | # Depending on a phony target causes this rule to run every time it's 17 | # referenced. This is what we want; it ensures that peru will check local 18 | # overrides for any changes, even though make doesn't know about them. 19 | .peru/lastimports: phony 20 | peru sync 21 | 22 | phony: 23 | -------------------------------------------------------------------------------- /docs/make_examples/README.md: -------------------------------------------------------------------------------- 1 | # Using peru with make 2 | 3 | *[Editor's node: The dude who wrote this doesn't actually know very much 4 | about make. If he screwed anything up, please file a bug!]* 5 | 6 | Getting peru and make to play nicely together can be a little tricky. 7 | Here are a few examples, going roughly from the simplest to the most 8 | correct. 9 | 10 | For these examples, we'll pretend that we want to fetch a Hello World C 11 | program, compile it, and run it. The 12 | [leachim6/hello-world](https://github.com/leachim6/hello-world) project 13 | offers a Hello World example in every language, so we'll get our code 14 | from there. Here's the `peru.yaml` file to fetch just their C example 15 | (`c.c`) into the root of our project 16 | 17 | ```yaml 18 | imports: 19 | helloworld: ./ 20 | 21 | git module helloworld: 22 | url: https://github.com/leachim6/hello-world 23 | pick: c/c.c 24 | export: c/ 25 | ``` 26 | 27 | And here's a simple `Makefile` to get us started: 28 | 29 | ```make 30 | run: hello 31 | ./hello 32 | 33 | hello: peru 34 | gcc -o hello c.c 35 | 36 | peru: 37 | peru sync 38 | ``` 39 | 40 | Because the `peru` target doesn't produce a file called `peru`, make 41 | will run it every time. That's both good and bad. It's good because if 42 | you ever use the `peru override` command, make isn't going to have any 43 | idea when your overrides have changed, so running `peru sync` on every 44 | build is the only way to get overrides right. But it's bad because now 45 | make is going to run the `hello` target every time too, even if the C 46 | file hasn't changed. It's not such a big deal for just one C file, but 47 | if we were building a project that took a long time, it would be 48 | annoying. Here's one way to fix that problem: 49 | 50 | ```make 51 | run: hello 52 | ./hello 53 | 54 | hello: .peru/lastimports 55 | gcc -o hello c.c 56 | 57 | .peru/lastimports: peru.yaml 58 | peru sync 59 | ``` 60 | 61 | Here `gcc` will only run when the imports have changed. We make this 62 | work by referring to the `lastimports` file that peru generates. That 63 | file contains the git hash of all the files peru has put on disk, and 64 | importantly, peru promises not to touch that file when the hash hasn't 65 | changed. 66 | 67 | That's what we want for `gcc`. What about for `peru sync`? Because we 68 | threw in the explicit dependency on `peru.yaml`, make will kindly rerun 69 | the sync for us if we change the YAML file. If you don't use overrides, 70 | that might be all you need. But if you do use overrides, you'll need an 71 | extra hack: 72 | 73 | ```make 74 | run: hello 75 | ./hello 76 | 77 | hello: .peru/lastimports 78 | gcc -o hello c.c 79 | 80 | .peru/lastimports: phony 81 | peru sync 82 | 83 | phony: 84 | ``` 85 | 86 | This is similar to the last example, in that `gcc` only runs when the 87 | imported files actually change. But we've added a phony target, which 88 | produces no files. Depending on that forces make to run the `peru sync` 89 | rule every time, so overrides will work properly. A `peru sync` with 90 | everything in cache amounts to a single `git status`, so you shouldn't 91 | notice a slowdown unless your dependencies are extremely large. 92 | 93 | The `Makefile` in this directory reproduces the last example, with some 94 | comments and a real `.PHONY` declaration (which keeps make from getting 95 | confused if we ever do create a file called "phony" for some reason). 96 | -------------------------------------------------------------------------------- /docs/make_examples/peru.yaml: -------------------------------------------------------------------------------- 1 | imports: 2 | helloworld: ./ 3 | 4 | git module helloworld: 5 | url: https://github.com/leachim6/hello-world 6 | pick: c/c.c 7 | export: c/ 8 | -------------------------------------------------------------------------------- /docs/peru.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/buildinspace/peru/7007451b7f6e2ff23f9dfc23cc28dae7280205d2/docs/peru.gif -------------------------------------------------------------------------------- /docs/release.md: -------------------------------------------------------------------------------- 1 | Notes to self about making a release: 2 | 3 | 1. Make a commit bumping peru/VERSION. Note changes in the commit message. 4 | 2. Make a tag pointing to that commit named after the new version. 5 | 3. `git push && git push --tags` 6 | 4. Copy the commit message to https://groups.google.com/forum/#!forum/peru-tool. 7 | 5. `python3 setup.py sdist` 8 | 6. `twine upload dist/*` 9 | - Full instructions here: https://packaging.python.org/tutorials/packaging-projects 10 | 7. Bump the AUR package. 11 | - `git clone ssh+git://aur@aur.archlinux.org/peru` 12 | - Update the pkgver and pkgrel. 13 | - Update the package hash with the help of `makepkg -g`. 14 | - `makepkg -d && makepkg --printsrcinfo > .SRCINFO` 15 | - Commit and push. 16 | - `git clone ssh+git://aur@aur.archlinux.org/peru-git` 17 | - Same procedure, but leave this one alone if it's just a version bump. 18 | 8. Poke Sean to update the Ubuntu PPA :) 19 | -------------------------------------------------------------------------------- /fastentrypoints.py: -------------------------------------------------------------------------------- 1 | # Copyright (c) 2016, Aaron Christianson 2 | # All rights reserved. 3 | # 4 | # Redistribution and use in source and binary forms, with or without 5 | # modification, are permitted provided that the following conditions are 6 | # met: 7 | # 8 | # 1. Redistributions of source code must retain the above copyright 9 | # notice, this list of conditions and the following disclaimer. 10 | # 11 | # 2. Redistributions in binary form must reproduce the above copyright 12 | # notice, this list of conditions and the following disclaimer in the 13 | # documentation and/or other materials provided with the distribution. 14 | # 15 | # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS 16 | # IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 17 | # TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A 18 | # PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 19 | # HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 20 | # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED 21 | # TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 22 | # PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF 23 | # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING 24 | # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 25 | # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26 | ''' 27 | Monkey patch setuptools to write faster console_scripts with this format: 28 | 29 | import sys 30 | from mymodule import entry_function 31 | sys.exit(entry_function()) 32 | 33 | This is better. 34 | 35 | (c) 2016, Aaron Christianson 36 | http://github.com/ninjaaron/fast-entry_points 37 | ''' 38 | from setuptools.command import easy_install 39 | import re 40 | TEMPLATE = '''\ 41 | # -*- coding: utf-8 -*- 42 | # EASY-INSTALL-ENTRY-SCRIPT: '{3}','{4}','{5}' 43 | __requires__ = '{3}' 44 | import re 45 | import sys 46 | 47 | from {0} import {1} 48 | 49 | if __name__ == '__main__': 50 | sys.argv[0] = re.sub(r'(-script\.pyw?|\.exe)?$', '', sys.argv[0]) 51 | sys.exit({2}())''' 52 | 53 | 54 | @classmethod 55 | def get_args(cls, dist, header=None): 56 | """ 57 | Yield write_script() argument tuples for a distribution's 58 | console_scripts and gui_scripts entry points. 59 | """ 60 | if header is None: 61 | header = cls.get_header() 62 | spec = str(dist.as_requirement()) 63 | for type_ in 'console', 'gui': 64 | group = type_ + '_scripts' 65 | for name, ep in dist.get_entry_map(group).items(): 66 | # ensure_safe_name 67 | if re.search(r'[\\/]', name): 68 | raise ValueError("Path separators not allowed in script names") 69 | script_text = TEMPLATE.format(ep.module_name, ep.attrs[0], 70 | '.'.join(ep.attrs), spec, group, 71 | name) 72 | args = cls._get_script_args(type_, name, header, script_text) 73 | for res in args: 74 | yield res 75 | 76 | 77 | easy_install.ScriptWriter.get_args = get_args 78 | 79 | 80 | def main(): 81 | import os 82 | import re 83 | import shutil 84 | import sys 85 | dests = sys.argv[1:] or ['.'] 86 | filename = re.sub('\.pyc$', '.py', __file__) 87 | 88 | for dst in dests: 89 | shutil.copy(filename, dst) 90 | manifest_path = os.path.join(dst, 'MANIFEST.in') 91 | setup_path = os.path.join(dst, 'setup.py') 92 | 93 | # Insert the include statement to MANIFEST.in if not present 94 | with open(manifest_path, 'a+') as manifest: 95 | manifest.seek(0) 96 | manifest_content = manifest.read() 97 | if not 'include fastentrypoints.py' in manifest_content: 98 | manifest.write(('\n' if manifest_content else '') + 99 | 'include fastentrypoints.py') 100 | 101 | # Insert the import statement to setup.py if not present 102 | with open(setup_path, 'a+') as setup: 103 | setup.seek(0) 104 | setup_content = setup.read() 105 | if not 'import fastentrypoints' in setup_content: 106 | setup.seek(0) 107 | setup.truncate() 108 | setup.write('import fastentrypoints\n' + setup_content) 109 | 110 | 111 | print(__name__) 112 | -------------------------------------------------------------------------------- /packaging/arch/PKGBUILD: -------------------------------------------------------------------------------- 1 | # Maintainer: Jack O'Connor 2 | 3 | pkgname=peru-git 4 | pkgdesc='A tool for fetching code' 5 | url='https://github.com/buildinspace/peru' 6 | license=('MIT') 7 | pkgver=0 8 | pkgver() { 9 | cd "$srcdir/peru" 10 | echo $(git rev-list --count master).$(git rev-parse --short master) 11 | } 12 | pkgrel=1 13 | arch=('any') 14 | # Asyncio and pathlib are standard in Python 3.4, so they're not in the 15 | # dependencies list. 16 | depends=(python python-yaml git) 17 | makedepends=(python-setuptools) 18 | optdepends=( 19 | 'mercurial: fetching from hg repos' 20 | 'subversion: fetching from svn repos' 21 | ) 22 | source=('git://github.com/buildinspace/peru') 23 | md5sums=('SKIP') 24 | 25 | package() { 26 | cd "$srcdir/peru" 27 | python3 setup.py install --root="$pkgdir" 28 | } 29 | -------------------------------------------------------------------------------- /packaging/arch/makelocal.sh: -------------------------------------------------------------------------------- 1 | #! /bin/bash 2 | 3 | # By default, makepkg clones the source repo from github before building. To 4 | # package your local copy of the sources (including any changes you've made), 5 | # use this script. 6 | # 7 | # See `man makepkg` for relevant options, like -d to ignore missing 8 | # dependencies. (The only dependency needed for packaging is python.) 9 | 10 | set -e 11 | 12 | cd $(dirname "$BASH_SOURCE") 13 | rm -rf src 14 | mkdir src 15 | ln -s ../../.. src/peru 16 | 17 | makepkg -e "$@" 18 | -------------------------------------------------------------------------------- /packaging/build_ubuntu_source_package.sh: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env bash 2 | 3 | # $ build_ubuntu_source_package.sh [series [package-version]] 4 | # Generates a Debian source package suitable for upload to a Launchpad PPA. This 5 | # script only builds the package artifacts, it does NOT upload them. Use dput to 6 | # upload to a PPA after building. 7 | # See https://launchpad.net/~buildinspace/+archive/ubuntu/peru 8 | 9 | set -e 10 | 11 | # Change to the repo root directory. 12 | cd $(dirname "$BASH_SOURCE")/.. 13 | repo_root=`pwd` 14 | 15 | # Bail if the repo is dirty. 16 | if [ -n "$(git status --porcelain)" ] ; then 17 | git status 18 | echo "The source repository is dirty. Aborting." 19 | exit 1 20 | fi 21 | 22 | # Get the series and package version from the command line, otherwise assume 23 | # "utopic" and version "1". Get the current version from the repo and append 24 | # package versioning information. Launchpad will reject changes to uploads for 25 | # the same version as derived from the source tarball. Bump the package version 26 | # to upload changes for the same version of peru. 27 | series="${1:-utopic}" 28 | version="$( Mon, 09 Mar 2015 15:04:39 -0700 6 | 7 | peru (0.1.3ubuntu~utopic1) utopic; urgency=low 8 | 9 | * https://phabricator.buildinspace.com/D168 10 | 11 | -- Sean Olson Fri, 09 Jan 2015 20:43:01 -0800 12 | 13 | peru (0.1.2ubuntu~utopic1-1) utopic; urgency=low 14 | 15 | * https://phabricator.buildinspace.com/D147 16 | 17 | -- Sean Olson Sun, 04 Jan 2015 15:17:52 -0800 18 | 19 | peru (0.1-1) trusty; urgency=low 20 | 21 | * initial release 22 | 23 | -- Sean Olson Mon, 24 Nov 2014 13:42:21 -0800 24 | -------------------------------------------------------------------------------- /packaging/debian/compat: -------------------------------------------------------------------------------- 1 | 9 2 | -------------------------------------------------------------------------------- /packaging/debian/control: -------------------------------------------------------------------------------- 1 | Source: peru 2 | Section: utils 3 | Priority: optional 4 | Maintainer: Sean Olson 5 | Build-Depends: debhelper (>= 8.0.0), dh-python, python3, python3-setuptools (>= 3.3) 6 | Standards-Version: 3.9.4 7 | Homepage: https://github.com/buildinspace/peru 8 | Vcs-Git: https://github.com/buildinspace/peru 9 | Vcs-Browser: https://github.com/buildinspace/peru 10 | 11 | Package: peru 12 | Architecture: all 13 | Depends: ${misc:Depends}, ${python3:Depends}, git 14 | Suggests: mercurial, subversion 15 | Description: A tool for fetching code 16 | -------------------------------------------------------------------------------- /packaging/debian/copyright: -------------------------------------------------------------------------------- 1 | Format: http://www.debian.org/doc/packaging-manuals/copyright-format/1.0/ 2 | Upstream-Name: peru 3 | Source: 4 | 5 | Files: * 6 | Copyright: 2014 Jack O'Connor 7 | 2014 Sean Olson 8 | License: MIT 9 | 10 | License: MIT 11 | Permission is hereby granted, free of charge, to any person obtaining a 12 | copy of this software and associated documentation files (the "Software"), 13 | to deal in the Software without restriction, including without limitation 14 | the rights to use, copy, modify, merge, publish, distribute, sublicense, 15 | and/or sell copies of the Software, and to permit persons to whom the 16 | Software is furnished to do so, subject to the following conditions: 17 | . 18 | The above copyright notice and this permission notice shall be included 19 | in all copies or substantial portions of the Software. 20 | . 21 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS 22 | OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 23 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. 24 | IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY 25 | CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, 26 | TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE 27 | SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 28 | -------------------------------------------------------------------------------- /packaging/debian/docs: -------------------------------------------------------------------------------- 1 | README.md 2 | -------------------------------------------------------------------------------- /packaging/debian/rules: -------------------------------------------------------------------------------- 1 | #!/usr/bin/make -f 2 | # -*- makefile -*- 3 | 4 | #export DH_VERBOSE=1 5 | export PYBUILD_NAME=peru 6 | 7 | %: 8 | dh $@ --with python3 --buildsystem=pybuild 9 | -------------------------------------------------------------------------------- /packaging/debian/source/format: -------------------------------------------------------------------------------- 1 | 3.0 (quilt) 2 | -------------------------------------------------------------------------------- /peru.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python3 2 | 3 | # This script is for running peru directly from the repo, mainly for 4 | # development. This isn't what gets installed when you install peru. That would 5 | # be a script generated by setup.py, which calls peru.main.main(). 6 | 7 | import os 8 | import sys 9 | 10 | repo_root = os.path.dirname(os.path.realpath(__file__)) 11 | 12 | sys.path.insert(0, repo_root) 13 | 14 | import peru.main # noqa: E402 15 | 16 | sys.exit(peru.main.main()) 17 | -------------------------------------------------------------------------------- /peru/VERSION: -------------------------------------------------------------------------------- 1 | 1.3.3 2 | -------------------------------------------------------------------------------- /peru/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/buildinspace/peru/7007451b7f6e2ff23f9dfc23cc28dae7280205d2/peru/__init__.py -------------------------------------------------------------------------------- /peru/__main__.py: -------------------------------------------------------------------------------- 1 | import sys 2 | 3 | from .main import main 4 | 5 | sys.exit(main()) 6 | -------------------------------------------------------------------------------- /peru/async_exit_stack.py: -------------------------------------------------------------------------------- 1 | # This definition of AsyncExitStack is copied entirely from 2 | # https://github.com/sorcio/async_exit_stack, in order to support Python 3.5 3 | # and 3.6. 4 | # 5 | # flake8: noqa 6 | 7 | from collections import deque 8 | import sys 9 | from types import MethodType 10 | 11 | 12 | class AsyncExitStack: 13 | """Async context manager for dynamic management of a stack of exit 14 | callbacks. 15 | For example: 16 | async with AsyncExitStack() as stack: 17 | connections = [await stack.enter_async_context(get_connection()) 18 | for i in range(5)] 19 | # All opened connections will automatically be released at the 20 | # end of the async with statement, even if attempts to open a 21 | # connection later in the list raise an exception. 22 | """ 23 | 24 | ### _BaseExitStack staticmethods 25 | 26 | @staticmethod 27 | def _create_exit_wrapper(cm, cm_exit): 28 | return MethodType(cm_exit, cm) 29 | 30 | @staticmethod 31 | def _create_cb_wrapper(callback, *args, **kwds): 32 | def _exit_wrapper(exc_type, exc, tb): 33 | callback(*args, **kwds) 34 | 35 | return _exit_wrapper 36 | 37 | ### AsyncExitStack staticmethods 38 | 39 | @staticmethod 40 | def _create_async_exit_wrapper(cm, cm_exit): 41 | return MethodType(cm_exit, cm) 42 | 43 | @staticmethod 44 | def _create_async_cb_wrapper(callback, *args, **kwds): 45 | async def _exit_wrapper(exc_type, exc, tb): 46 | await callback(*args, **kwds) 47 | 48 | return _exit_wrapper 49 | 50 | ### _BaseExitStack methods 51 | 52 | def __init__(self): 53 | self._exit_callbacks = deque() 54 | 55 | def pop_all(self): 56 | """Preserve the context stack by transferring it to a new instance.""" 57 | new_stack = type(self)() 58 | new_stack._exit_callbacks = self._exit_callbacks 59 | self._exit_callbacks = deque() 60 | return new_stack 61 | 62 | def push(self, exit): 63 | """Registers a callback with the standard __exit__ method signature. 64 | Can suppress exceptions the same way __exit__ method can. 65 | Also accepts any object with an __exit__ method (registering a call 66 | to the method instead of the object itself). 67 | """ 68 | # We use an unbound method rather than a bound method to follow 69 | # the standard lookup behaviour for special methods. 70 | _cb_type = type(exit) 71 | 72 | try: 73 | exit_method = _cb_type.__exit__ 74 | except AttributeError: 75 | # Not a context manager, so assume it's a callable. 76 | self._push_exit_callback(exit) 77 | else: 78 | self._push_cm_exit(exit, exit_method) 79 | return exit # Allow use as a decorator. 80 | 81 | def enter_context(self, cm): 82 | """Enters the supplied context manager. 83 | If successful, also pushes its __exit__ method as a callback and 84 | returns the result of the __enter__ method. 85 | """ 86 | # We look up the special methods on the type to match the with 87 | # statement. 88 | _cm_type = type(cm) 89 | _exit = _cm_type.__exit__ 90 | result = _cm_type.__enter__(cm) 91 | self._push_cm_exit(cm, _exit) 92 | return result 93 | 94 | def callback(self, callback, *args, **kwds): 95 | """Registers an arbitrary callback and arguments. 96 | Cannot suppress exceptions. 97 | """ 98 | _exit_wrapper = self._create_cb_wrapper(callback, *args, **kwds) 99 | 100 | # We changed the signature, so using @wraps is not appropriate, but 101 | # setting __wrapped__ may still help with introspection. 102 | _exit_wrapper.__wrapped__ = callback 103 | self._push_exit_callback(_exit_wrapper) 104 | return callback # Allow use as a decorator 105 | 106 | def _push_cm_exit(self, cm, cm_exit): 107 | """Helper to correctly register callbacks to __exit__ methods.""" 108 | _exit_wrapper = self._create_exit_wrapper(cm, cm_exit) 109 | self._push_exit_callback(_exit_wrapper, True) 110 | 111 | def _push_exit_callback(self, callback, is_sync=True): 112 | self._exit_callbacks.append((is_sync, callback)) 113 | 114 | ### AsyncExitStack methods 115 | 116 | async def enter_async_context(self, cm): 117 | """Enters the supplied async context manager. 118 | If successful, also pushes its __aexit__ method as a callback and 119 | returns the result of the __aenter__ method. 120 | """ 121 | _cm_type = type(cm) 122 | _exit = _cm_type.__aexit__ 123 | result = await _cm_type.__aenter__(cm) 124 | self._push_async_cm_exit(cm, _exit) 125 | return result 126 | 127 | def push_async_exit(self, exit): 128 | """Registers a coroutine function with the standard __aexit__ method 129 | signature. 130 | Can suppress exceptions the same way __aexit__ method can. 131 | Also accepts any object with an __aexit__ method (registering a call 132 | to the method instead of the object itself). 133 | """ 134 | _cb_type = type(exit) 135 | try: 136 | exit_method = _cb_type.__aexit__ 137 | except AttributeError: 138 | # Not an async context manager, so assume it's a coroutine function 139 | self._push_exit_callback(exit, False) 140 | else: 141 | self._push_async_cm_exit(exit, exit_method) 142 | return exit # Allow use as a decorator 143 | 144 | def push_async_callback(self, callback, *args, **kwds): 145 | """Registers an arbitrary coroutine function and arguments. 146 | Cannot suppress exceptions. 147 | """ 148 | _exit_wrapper = self._create_async_cb_wrapper(callback, *args, **kwds) 149 | 150 | # We changed the signature, so using @wraps is not appropriate, but 151 | # setting __wrapped__ may still help with introspection. 152 | _exit_wrapper.__wrapped__ = callback 153 | self._push_exit_callback(_exit_wrapper, False) 154 | return callback # Allow use as a decorator 155 | 156 | async def aclose(self): 157 | """Immediately unwind the context stack.""" 158 | await self.__aexit__(None, None, None) 159 | 160 | def _push_async_cm_exit(self, cm, cm_exit): 161 | """Helper to correctly register coroutine function to __aexit__ 162 | method.""" 163 | _exit_wrapper = self._create_async_exit_wrapper(cm, cm_exit) 164 | self._push_exit_callback(_exit_wrapper, False) 165 | 166 | async def __aenter__(self): 167 | return self 168 | 169 | async def __aexit__(self, *exc_details): 170 | received_exc = exc_details[0] is not None 171 | 172 | # We manipulate the exception state so it behaves as though 173 | # we were actually nesting multiple with statements 174 | frame_exc = sys.exc_info()[1] 175 | 176 | def _fix_exception_context(new_exc, old_exc): 177 | # Context may not be correct, so find the end of the chain 178 | while 1: 179 | exc_context = new_exc.__context__ 180 | if exc_context is old_exc: 181 | # Context is already set correctly (see issue 20317) 182 | return 183 | if exc_context is None or exc_context is frame_exc: 184 | break 185 | new_exc = exc_context 186 | # Change the end of the chain to point to the exception 187 | # we expect it to reference 188 | new_exc.__context__ = old_exc 189 | 190 | # Callbacks are invoked in LIFO order to match the behaviour of 191 | # nested context managers 192 | suppressed_exc = False 193 | pending_raise = False 194 | while self._exit_callbacks: 195 | is_sync, cb = self._exit_callbacks.pop() 196 | try: 197 | if is_sync: 198 | cb_suppress = cb(*exc_details) 199 | else: 200 | cb_suppress = await cb(*exc_details) 201 | 202 | if cb_suppress: 203 | suppressed_exc = True 204 | pending_raise = False 205 | exc_details = (None, None, None) 206 | except: 207 | new_exc_details = sys.exc_info() 208 | # simulate the stack of exceptions by setting the context 209 | _fix_exception_context(new_exc_details[1], exc_details[1]) 210 | pending_raise = True 211 | exc_details = new_exc_details 212 | if pending_raise: 213 | try: 214 | # bare "raise exc_details[1]" replaces our carefully 215 | # set-up context 216 | fixed_ctx = exc_details[1].__context__ 217 | raise exc_details[1] 218 | except BaseException: 219 | exc_details[1].__context__ = fixed_ctx 220 | raise 221 | return received_exc and suppressed_exc 222 | -------------------------------------------------------------------------------- /peru/async_helpers.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | import atexit 3 | import codecs 4 | import contextlib 5 | import io 6 | import os 7 | import subprocess 8 | import sys 9 | import traceback 10 | 11 | from .error import PrintableError 12 | 13 | 14 | # Prior to Python 3.8 (which switched to the ProactorEventLoop by default on 15 | # Windows), the default event loop on Windows doesn't support subprocesses, so 16 | # we need to use the proactor loop. See: 17 | # https://docs.python.org/3/library/asyncio-eventloops.html#available-event-loops 18 | # Because the event loop is essentially a global variable, we have to set this 19 | # at import time. Otherwise asyncio objects that get instantiated early 20 | # (particularly Locks and Semaphores) could grab a reference to the wrong loop. 21 | # TODO: Importing for side effects isn't very clean. Find a better way. 22 | if os.name == 'nt': 23 | EVENT_LOOP = asyncio.ProactorEventLoop() 24 | else: 25 | EVENT_LOOP = asyncio.new_event_loop() 26 | asyncio.set_event_loop(EVENT_LOOP) 27 | 28 | # We also need to make sure the event loop is explicitly closed, to avoid a bug 29 | # in _UnixSelectorEventLoop.__del__. See http://bugs.python.org/issue23548. 30 | atexit.register(EVENT_LOOP.close) 31 | 32 | 33 | def run_task(coro): 34 | return EVENT_LOOP.run_until_complete(coro) 35 | 36 | 37 | class GatheredExceptions(PrintableError): 38 | def __init__(self, exceptions, reprs): 39 | assert len(exceptions) > 0 40 | self.exceptions = [] 41 | self.reprs = [] 42 | for e, st in zip(exceptions, reprs): 43 | # Flatten in the exceptions list of any other GatheredExceptions we 44 | # see. (This happens, for example, if something throws inside a 45 | # recursive module.) 46 | if isinstance(e, GatheredExceptions): 47 | self.exceptions.extend(e.exceptions) 48 | else: 49 | self.exceptions.append(e) 50 | 51 | # Don't flatten the reprs. This would make us lose PrintableError 52 | # context. TODO: Represent context in a more structured way? 53 | self.reprs.append(st) 54 | 55 | self.message = "\n\n".join(self.reprs) 56 | 57 | 58 | async def gather_coalescing_exceptions(coros, display, *, verbose): 59 | '''The tricky thing about running multiple coroutines in parallel is what 60 | we're supposed to do when one of them raises an exception. The approach 61 | we're using here is to catch exceptions and keep waiting for other tasks to 62 | finish. At the end, we reraise a GatheredExceptions error, if any 63 | exceptions were caught. 64 | 65 | Another minor detail: We also want to make sure to start coroutines in the 66 | order given, so that they end up appearing to the user alphabetically in 67 | the fancy display. Note that asyncio.gather() puts coroutines in a set 68 | internally, so we schedule coroutines *before* we give them to gather(). 69 | ''' 70 | 71 | exceptions = [] 72 | reprs = [] 73 | 74 | async def catching_wrapper(coro): 75 | try: 76 | return (await coro) 77 | except Exception as e: 78 | exceptions.append(e) 79 | if isinstance(e, PrintableError) and not verbose: 80 | reprs.append(e.message) 81 | else: 82 | reprs.append(traceback.format_exc()) 83 | return None 84 | 85 | # Suppress a deprecation warning in Python 3.5, while continuing to support 86 | # 3.3 and early 3.4 releases. 87 | if hasattr(asyncio, 'ensure_future'): 88 | schedule = getattr(asyncio, 'ensure_future') 89 | else: 90 | schedule = getattr(asyncio, 'async') 91 | 92 | futures = [schedule(catching_wrapper(coro)) for coro in coros] 93 | 94 | results = await asyncio.gather(*futures) 95 | 96 | if exceptions: 97 | raise GatheredExceptions(exceptions, reprs) 98 | else: 99 | return results 100 | 101 | 102 | async def create_subprocess_with_handle(command, 103 | display_handle, 104 | *, 105 | shell=False, 106 | cwd, 107 | **kwargs): 108 | '''Writes subprocess output to a display handle as it comes in, and also 109 | returns a copy of it as a string. Throws if the subprocess returns an 110 | error. Note that cwd is a required keyword-only argument, on theory that 111 | peru should never start child processes "wherever I happen to be running 112 | right now."''' 113 | 114 | # We're going to get chunks of bytes from the subprocess, and it's possible 115 | # that one of those chunks ends in the middle of a unicode character. An 116 | # incremental decoder keeps those dangling bytes around until the next 117 | # chunk arrives, so that split characters get decoded properly. Use 118 | # stdout's encoding, but provide a default for the case where stdout has 119 | # been redirected to a StringIO. (This happens in tests.) 120 | encoding = sys.stdout.encoding or 'utf8' 121 | decoder_factory = codecs.getincrementaldecoder(encoding) 122 | decoder = decoder_factory(errors='replace') 123 | 124 | output_copy = io.StringIO() 125 | 126 | # Display handles are context managers. Entering and exiting the display 127 | # handle lets the display know when the job starts and stops. 128 | with display_handle: 129 | stdin = asyncio.subprocess.DEVNULL 130 | stdout = asyncio.subprocess.PIPE 131 | stderr = asyncio.subprocess.STDOUT 132 | if shell: 133 | proc = await asyncio.create_subprocess_shell( 134 | command, 135 | stdin=stdin, 136 | stdout=stdout, 137 | stderr=stderr, 138 | cwd=cwd, 139 | **kwargs) 140 | else: 141 | proc = await asyncio.create_subprocess_exec( 142 | *command, 143 | stdin=stdin, 144 | stdout=stdout, 145 | stderr=stderr, 146 | cwd=cwd, 147 | **kwargs) 148 | 149 | # Read all the output from the subprocess as its comes in. 150 | while True: 151 | outputbytes = await proc.stdout.read(4096) 152 | if not outputbytes: 153 | break 154 | outputstr = decoder.decode(outputbytes) 155 | outputstr_unified = _unify_newlines(outputstr) 156 | display_handle.write(outputstr_unified) 157 | output_copy.write(outputstr_unified) 158 | 159 | returncode = await proc.wait() 160 | 161 | if returncode != 0: 162 | raise subprocess.CalledProcessError(returncode, command, 163 | output_copy.getvalue()) 164 | 165 | if hasattr(decoder, 'buffer'): 166 | # The utf8 decoder has this attribute, but some others don't. 167 | assert not decoder.buffer, 'decoder nonempty: ' + repr(decoder.buffer) 168 | 169 | return output_copy.getvalue() 170 | 171 | 172 | def _unify_newlines(s): 173 | r'''Because all asyncio subprocess output is read in binary mode, we don't 174 | get universal newlines for free. But it's the right thing to do, because we 175 | do all our printing with strings in text mode, which translates "\n" back 176 | into the platform-appropriate line separator. So for example, "\r\n" in a 177 | string on Windows will become "\r\r\n" when it gets printed. This function 178 | ensures that all newlines are represented as "\n" internally, which solves 179 | that problem and also helps our tests work on Windows. Right now we only 180 | handle Windows, but we can expand this if there's ever another newline 181 | style we have to support.''' 182 | 183 | return s.replace('\r\n', '\n') 184 | 185 | 186 | async def safe_communicate(process, input=None): 187 | '''Asyncio's communicate method has a bug where `communicate(input=b"")` is 188 | treated the same as `communicate(). That means that child processes can 189 | hang waiting for input, when their stdin should be closed. See 190 | https://bugs.python.org/issue26848. The issue is fixed upstream in 191 | https://github.com/python/asyncio/commit/915b6eaa30e1e3744e6f8223f996e197c1c9b91d, 192 | but we will probably always need this workaround for old versions.''' 193 | if input is not None and len(input) == 0: 194 | process.stdin.close() 195 | return (await process.communicate()) 196 | else: 197 | return (await process.communicate(input)) 198 | 199 | 200 | class RaisesGatheredContainer: 201 | def __init__(self): 202 | self.exception = None 203 | 204 | 205 | @contextlib.contextmanager 206 | def raises_gathered(error_type): 207 | '''For use in tests. Many tests expect a single error to be thrown, and 208 | want it to be of a specific type. This is a helper method for when that 209 | type is inside a gathered exception.''' 210 | container = RaisesGatheredContainer() 211 | try: 212 | yield container 213 | except GatheredExceptions as e: 214 | # Make sure there is exactly one exception. 215 | if len(e.exceptions) != 1: 216 | raise 217 | inner = e.exceptions[0] 218 | # Make sure the exception is the right type. 219 | if not isinstance(inner, error_type): 220 | raise 221 | # Success. 222 | container.exception = inner 223 | -------------------------------------------------------------------------------- /peru/compat.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | 4 | # In Python versions prior to 3.4, __file__ returns a relative path. This path 5 | # is fixed at load time, so if the program later cd's (as we do in tests, at 6 | # least) __file__ is no longer valid. As a workaround, compute the absolute 7 | # path at load time. 8 | MODULE_ROOT = os.path.abspath(os.path.dirname(__file__)) 9 | 10 | 11 | def makedirs(path): 12 | '''os.makedirs() has an exist_ok param, but it still throws errors when the 13 | path exists with non-default permissions. This isn't fixed until 3.4. 14 | Pathlib won't be getting an exist_ok param until 3.5.''' 15 | path = str(path) # compatibility with pathlib 16 | # Use isdir to avoid silently returning if the path exists but isn't a dir. 17 | if not os.path.isdir(path): 18 | os.makedirs(path) 19 | 20 | 21 | def is_fancy_terminal(): 22 | '''The Windows terminal does not support most of the fancy things we want 23 | to do with colors and formatting. This is a quick and dirty way to make 24 | sure we default to simple output on Windows.''' 25 | return sys.stdout.isatty() and os.name != 'nt' 26 | -------------------------------------------------------------------------------- /peru/docopt/LICENSE-MIT: -------------------------------------------------------------------------------- 1 | Copyright (c) 2012 Vladimir Keleshev, 2 | 3 | Permission is hereby granted, free of charge, to any person 4 | obtaining a copy of this software and associated 5 | documentation files (the "Software"), to deal in the Software 6 | without restriction, including without limitation the rights 7 | to use, copy, modify, merge, publish, distribute, sublicense, 8 | and/or sell copies of the Software, and to permit persons to 9 | whom the Software is furnished to do so, subject to the 10 | following conditions: 11 | 12 | The above copyright notice and this permission notice shall 13 | be included in all copies or substantial portions of the 14 | Software. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY 17 | KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE 18 | WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR 19 | PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 20 | COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 21 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR 22 | OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE 23 | SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 24 | -------------------------------------------------------------------------------- /peru/docopt/_version.py: -------------------------------------------------------------------------------- 1 | __version__ = "0.9.0" 2 | -------------------------------------------------------------------------------- /peru/docopt/py.typed: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/buildinspace/peru/7007451b7f6e2ff23f9dfc23cc28dae7280205d2/peru/docopt/py.typed -------------------------------------------------------------------------------- /peru/edit_yaml.py: -------------------------------------------------------------------------------- 1 | import yaml 2 | 3 | 4 | def set_module_field_in_file(yaml_file_path, module_name, field_name, new_val): 5 | with open(yaml_file_path) as f: 6 | yaml_text = f.read() 7 | new_yaml_text = set_module_field(yaml_text, module_name, field_name, 8 | new_val) 9 | with open(yaml_file_path, "w") as f: 10 | f.write(new_yaml_text) 11 | 12 | 13 | def set_module_field(yaml_text, module_name, field_name, new_val): 14 | yaml_dict = _parse_yaml_text(yaml_text) 15 | bounds = _get_module_field_bounds(yaml_dict, module_name, field_name) 16 | quoted_val = _maybe_quote(new_val) 17 | if bounds: 18 | # field exists, modify it 19 | return yaml_text[:bounds[0]] + quoted_val + yaml_text[bounds[1]:] 20 | else: 21 | # field is new, hack it in 22 | return _append_module_field(yaml_text, yaml_dict, module_name, 23 | field_name, quoted_val) 24 | 25 | 26 | def _maybe_quote(val): 27 | '''All of our values should be strings. Usually those can be passed in as 28 | bare words, but if they're parseable as an int or float we need to quote 29 | them.''' 30 | assert isinstance(val, str), 'We should never set non-string values.' 31 | needs_quoting = False 32 | try: 33 | int(val) 34 | needs_quoting = True 35 | except Exception: 36 | pass 37 | try: 38 | float(val) 39 | needs_quoting = True 40 | except Exception: 41 | pass 42 | if needs_quoting: 43 | return '"{}"'.format(val) 44 | else: 45 | return val 46 | 47 | 48 | def _append_module_field(yaml_text, yaml_dict, module_name, field_name, 49 | new_val): 50 | module_fields = yaml_dict[module_name] 51 | # use the last field to determine position and indentation 52 | assert len(module_fields) > 0, "There aren't any fields here!" 53 | last_key = module_fields.keys[-1] 54 | last_val = module_fields.vals[-1] 55 | indentation = " " * last_key.start_mark.column 56 | yaml_lines = yaml_text.split("\n") 57 | 58 | # We want to append the new field at the end of the module. Unfortunately, 59 | # the end_mark of a multi-line field is actually the first line of the next 60 | # toplevel dict. Check for this. 61 | if last_val.end_mark.column > 0: 62 | new_line_number = last_val.end_mark.line + 1 63 | else: 64 | new_line_number = last_val.end_mark.line 65 | # If the module ended with a line of whitespace, insert before that. 66 | prev_line = yaml_lines[new_line_number - 1] 67 | if prev_line == "" or prev_line.isspace(): 68 | new_line_number -= 1 69 | 70 | new_line = "{}{}: {}".format(indentation, field_name, new_val) 71 | new_yaml_lines = (yaml_lines[:new_line_number] + [new_line] + 72 | yaml_lines[new_line_number:]) 73 | return "\n".join(new_yaml_lines) 74 | 75 | 76 | def _get_module_field_bounds(yaml_dict, module_name, field_name): 77 | module_fields = yaml_dict[module_name] 78 | if field_name not in module_fields: 79 | return None 80 | field_val = module_fields[field_name] 81 | return (field_val.start_mark.index, field_val.end_mark.index) 82 | 83 | 84 | def _parse_yaml_text(yaml_text): 85 | events_list = list(yaml.parse(yaml_text)) 86 | return _parse_events_list(events_list) 87 | 88 | 89 | def _parse_events_list(events_list): 90 | event = events_list.pop(0) 91 | if (isinstance(event, yaml.StreamStartEvent) 92 | or isinstance(event, yaml.DocumentStartEvent)): 93 | ret = _parse_events_list(events_list) 94 | events_list.pop(-1) 95 | return ret 96 | elif (isinstance(event, yaml.ScalarEvent) 97 | or isinstance(event, yaml.AliasEvent) 98 | or isinstance(event, yaml.SequenceEndEvent) 99 | or isinstance(event, yaml.MappingEndEvent)): 100 | return event 101 | elif isinstance(event, yaml.SequenceStartEvent): 102 | contents = [] 103 | while True: 104 | item = _parse_events_list(events_list) 105 | if isinstance(item, yaml.SequenceEndEvent): 106 | end_event = item 107 | return YamlList(event, end_event, contents) 108 | contents.append(item) 109 | elif isinstance(event, yaml.MappingStartEvent): 110 | keys = [] 111 | vals = [] 112 | while True: 113 | key = _parse_events_list(events_list) 114 | if isinstance(key, yaml.MappingEndEvent): 115 | end_event = key 116 | return YamlDict(event, end_event, keys, vals) 117 | keys.append(key) 118 | val = _parse_events_list(events_list) 119 | vals.append(val) 120 | else: 121 | raise RuntimeError("Unknown parse event type", event) 122 | 123 | 124 | class YamlDict: 125 | def __init__(self, start_event, end_event, keys, vals): 126 | assert all(isinstance(key, yaml.ScalarEvent) for key in keys) 127 | assert len(keys) == len(vals) 128 | self.keys = keys 129 | self.key_map = {key.value: key for key in keys} 130 | self.vals = vals 131 | self.val_map = {key.value: val for key, val in zip(keys, vals)} 132 | self.start_event = start_event 133 | self.end_event = end_event 134 | self.start_mark = start_event.start_mark 135 | self.end_mark = end_event.end_mark 136 | 137 | def __contains__(self, key): 138 | return key in self.key_map 139 | 140 | def __getitem__(self, key): 141 | return self.val_map[key] 142 | 143 | def __iter__(self): 144 | return iter(self.key_map) 145 | 146 | def __len__(self): 147 | return len(self.keys) 148 | 149 | 150 | class YamlList: 151 | def __init__(self, start_event, end_event, contents): 152 | self._contents = contents 153 | self.start_event = start_event 154 | self.end_event = end_event 155 | self.start_mark = start_event.start_mark 156 | self.end_mark = end_event.end_mark 157 | 158 | def __contains__(self, item): 159 | return item in self._contents 160 | 161 | def __getitem__(self, index): 162 | return self._contents[index] 163 | 164 | def __iter__(self): 165 | return iter(self._contents) 166 | 167 | def __len__(self): 168 | return len(self._contents) 169 | -------------------------------------------------------------------------------- /peru/error.py: -------------------------------------------------------------------------------- 1 | from contextlib import contextmanager 2 | from textwrap import indent 3 | 4 | 5 | class PrintableError(Exception): 6 | def __init__(self, message, *args, **kwargs): 7 | self.message = message.format(*args, **kwargs) 8 | 9 | def __str__(self): 10 | return self.message 11 | 12 | def add_context(self, context): 13 | # TODO: Something more structured? 14 | self.message = 'In {}:\n{}'.format(context, indent(self.message, ' ')) 15 | 16 | 17 | @contextmanager 18 | def error_context(context): 19 | try: 20 | yield 21 | except PrintableError as e: 22 | e.add_context(context) 23 | raise 24 | -------------------------------------------------------------------------------- /peru/glob.py: -------------------------------------------------------------------------------- 1 | from pathlib import PurePosixPath 2 | import re 3 | 4 | from .error import PrintableError 5 | 6 | UNESCAPED_STAR_EXPR = ( 7 | r'(? "%PERU_SYNC_DEST%\%PERU_MODULE_FILENAME%" 7 | -------------------------------------------------------------------------------- /peru/resources/plugins/bat/plugin.yaml: -------------------------------------------------------------------------------- 1 | sync exe: bat_plugin.bat 2 | required fields: 3 | - filename 4 | - message 5 | -------------------------------------------------------------------------------- /peru/resources/plugins/cp/cp_plugin.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python3 2 | 3 | import os 4 | import shutil 5 | 6 | shutil.copytree( 7 | os.environ['PERU_MODULE_PATH'], 8 | os.environ['PERU_SYNC_DEST'], 9 | symlinks=True, 10 | dirs_exist_ok=True, 11 | ) 12 | -------------------------------------------------------------------------------- /peru/resources/plugins/cp/plugin.yaml: -------------------------------------------------------------------------------- 1 | sync exe: cp_plugin.py 2 | required fields: 3 | - path 4 | -------------------------------------------------------------------------------- /peru/resources/plugins/curl/curl_plugin.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python3 2 | 3 | import hashlib 4 | import os 5 | import pathlib 6 | import re 7 | import stat 8 | import sys 9 | import tarfile 10 | from urllib.error import HTTPError, URLError 11 | from urllib.parse import urlsplit 12 | from urllib.request import Request 13 | import peru.main 14 | import urllib.request 15 | import zipfile 16 | 17 | 18 | def add_user_agent_to_request(request): 19 | components = [ 20 | "peru/%s" % peru.main.get_version(), 21 | urllib.request.URLopener.version 22 | ] 23 | request.add_header("User-agent", " ".join(components)) 24 | return request 25 | 26 | 27 | def build_request(url): 28 | request = Request(url) 29 | return add_user_agent_to_request(request) 30 | 31 | 32 | def get_request_filename(request): 33 | '''Figure out the filename for an HTTP download.''' 34 | # Check to see if a filename is specified in the HTTP headers. 35 | if 'Content-Disposition' in request.info(): 36 | disposition = request.info()['Content-Disposition'] 37 | pieces = re.split(r'\s*;\s*', disposition) 38 | for piece in pieces: 39 | if piece.startswith('filename='): 40 | filename = piece[len('filename='):] 41 | # Strip exactly one " from each end. 42 | if filename.startswith('"'): 43 | filename = filename[1:] 44 | if filename.endswith('"'): 45 | filename = filename[:-1] 46 | # Interpret backslashed quotes. 47 | filename = filename.replace('\\"', '"') 48 | return filename 49 | # If no filename was specified, pick a reasonable default. 50 | return os.path.basename(urlsplit(request.url).path) or 'index.html' 51 | 52 | 53 | def format_bytes(num_bytes): 54 | for threshold, unit in ((10**9, 'GB'), (10**6, 'MB'), (10**3, 'KB')): 55 | if num_bytes >= threshold: 56 | # Truncate floats instead of rounding. 57 | float_str = str(num_bytes / threshold) 58 | decimal_index = float_str.index('.') 59 | truncated_float = float_str[:decimal_index + 2] 60 | return truncated_float + unit 61 | return '{}B'.format(num_bytes) 62 | 63 | 64 | def download_file(request, output_file, stdout=sys.stdout): 65 | digest = hashlib.sha1() 66 | file_size_str = request.info().get('Content-Length') 67 | file_size = int(file_size_str) if file_size_str is not None else None 68 | bytes_read = 0 69 | while True: 70 | buf = request.read(4096) 71 | if not buf: 72 | break 73 | digest.update(buf) 74 | if output_file: 75 | output_file.write(buf) 76 | bytes_read += len(buf) 77 | percentage = '' 78 | kb_downloaded = format_bytes(bytes_read) 79 | total_kb = '' 80 | if file_size: 81 | percentage = ' {}%'.format(round(100 * bytes_read / file_size)) 82 | total_kb = '/' + format_bytes(file_size) 83 | print( 84 | 'downloaded{} {}{}'.format(percentage, kb_downloaded, total_kb), 85 | file=stdout) 86 | return digest.hexdigest() 87 | 88 | 89 | def plugin_sync(url, sha1): 90 | unpack = os.environ['PERU_MODULE_UNPACK'] 91 | dest = os.environ['PERU_SYNC_DEST'] 92 | if unpack: 93 | # Download to the tmp dir for later unpacking. 94 | download_dir = os.environ['PERU_PLUGIN_TMP'] 95 | else: 96 | # Download directly to the destination dir. 97 | download_dir = dest 98 | 99 | with urllib.request.urlopen(build_request(url)) as request: 100 | filename = os.environ['PERU_MODULE_FILENAME'] 101 | if not filename: 102 | filename = get_request_filename(request) 103 | full_filepath = os.path.join(download_dir, filename) 104 | with open(full_filepath, 'wb') as output_file: 105 | digest = download_file(request, output_file) 106 | 107 | if sha1 and digest != sha1: 108 | print( 109 | 'Bad checksum!\n url: {}\nexpected: {}\n actual: {}'.format( 110 | url, sha1, digest), 111 | file=sys.stderr) 112 | sys.exit(1) 113 | 114 | try: 115 | if unpack == 'tar': 116 | extract_tar(full_filepath, dest) 117 | elif unpack == 'zip': 118 | extract_zip(full_filepath, dest) 119 | elif unpack: 120 | print('Unknown value for "unpack":', unpack, file=sys.stderr) 121 | sys.exit(1) 122 | except EvilArchiveError as e: 123 | print(e.message, file=sys.stderr) 124 | sys.exit(1) 125 | 126 | 127 | def extract_tar(archive_path, dest): 128 | with tarfile.open(archive_path) as t: 129 | for info in t.getmembers(): 130 | validate_filename(info.path) 131 | if info.issym(): 132 | validate_symlink(info.path, info.linkname) 133 | # Python 3.12 added the `filter` kwarg, which should make our 134 | # validation redundant. (It was also added to patch releases of earlier 135 | # Python versions.) Python 3.13 made it a warning to omit this 136 | # argument, because Python 3.14 will change the default to "data". 137 | # That's the behavior we want, and specifying it here lets us get it on 138 | # Python 3.12/3.13 and silences the warning. 139 | kwargs = {} 140 | if sys.version_info >= (3, 12): 141 | kwargs["filter"] = "data" 142 | t.extractall(dest, **kwargs) 143 | 144 | 145 | def extract_zip(archive_path, dest): 146 | with zipfile.ZipFile(archive_path) as z: 147 | for name in z.namelist(): 148 | validate_filename(name) 149 | z.extractall(dest) 150 | # Set file permissions. Tar does this by default, but with zip we need 151 | # to do it ourselves. 152 | for info in z.filelist: 153 | if not info.filename.endswith('/'): 154 | # This is how to get file permissions out of a zip archive, 155 | # according to http://stackoverflow.com/q/434641/823869 and 156 | # http://bugs.python.org/file34873/issue15795_cleaned.patch. 157 | mode = (info.external_attr >> 16) & 0o777 158 | # Don't copy the whole mode, just set the executable bit. Two 159 | # reasons for this. 1) This is all going to end up in a git 160 | # tree, which only records the executable bit anyway. 2) Zip's 161 | # support for Unix file modes is nonstandard, so the mode field 162 | # is often zero and could be garbage. Mistakenly setting a file 163 | # executable isn't a big deal, but e.g. removing read 164 | # permissions would cause an error. 165 | if mode & stat.S_IXUSR: 166 | os.chmod(os.path.join(dest, info.filename), 0o755) 167 | 168 | 169 | def validate_filename(name): 170 | path = pathlib.PurePosixPath(name) 171 | if path.is_absolute() or ".." in path.parts: 172 | raise EvilArchiveError("Illegal path in archive: " + name) 173 | 174 | 175 | def validate_symlink(name, target): 176 | # We might do this twice but that's fine. 177 | validate_filename(name) 178 | 179 | allowed_parent_parts = len(pathlib.PurePosixPath(name).parts) - 1 180 | 181 | target_path = pathlib.PurePosixPath(target) 182 | if target_path.is_absolute(): 183 | raise EvilArchiveError("Illegal symlink target in archive: " + target) 184 | leading_parent_parts = 0 185 | for part in target_path.parts: 186 | if part != "..": 187 | break 188 | leading_parent_parts += 1 189 | if leading_parent_parts > allowed_parent_parts: 190 | raise EvilArchiveError("Illegal symlink target in archive: " + target) 191 | if ".." in target_path.parts[leading_parent_parts:]: 192 | raise EvilArchiveError("Illegal symlink target in archive: " + target) 193 | 194 | 195 | class EvilArchiveError(RuntimeError): 196 | def __init__(self, message): 197 | self.message = message 198 | 199 | 200 | def plugin_reup(url, sha1): 201 | reup_output = os.environ['PERU_REUP_OUTPUT'] 202 | with urllib.request.urlopen(build_request(url)) as request: 203 | digest = download_file(request, None) 204 | with open(reup_output, 'w') as output_file: 205 | print('sha1:', digest, file=output_file) 206 | 207 | 208 | def main(): 209 | url = os.environ['PERU_MODULE_URL'] 210 | sha1 = os.environ['PERU_MODULE_SHA1'] 211 | command = os.environ['PERU_PLUGIN_COMMAND'] 212 | try: 213 | if command == 'sync': 214 | plugin_sync(url, sha1) 215 | elif command == 'reup': 216 | plugin_reup(url, sha1) 217 | else: 218 | raise RuntimeError('unknown command: ' + repr(command)) 219 | except (HTTPError, URLError) as e: 220 | print("Error fetching", url) 221 | print(e) 222 | return 1 223 | 224 | 225 | if __name__ == '__main__': 226 | sys.exit(main()) 227 | -------------------------------------------------------------------------------- /peru/resources/plugins/curl/plugin.yaml: -------------------------------------------------------------------------------- 1 | sync exe: curl_plugin.py 2 | reup exe: curl_plugin.py 3 | required fields: 4 | - url 5 | optional fields: 6 | - sha1 7 | - filename 8 | - unpack 9 | -------------------------------------------------------------------------------- /peru/resources/plugins/empty/empty_plugin.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python3 2 | 3 | # This plugin does nothing. 4 | # This allows for a module spec. without actually fetching anything. 5 | -------------------------------------------------------------------------------- /peru/resources/plugins/empty/plugin.yaml: -------------------------------------------------------------------------------- 1 | sync exe: empty_plugin.py 2 | required fields: [] 3 | -------------------------------------------------------------------------------- /peru/resources/plugins/git/git_plugin.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python3 2 | 3 | from collections import namedtuple 4 | import configparser 5 | import hashlib 6 | import os 7 | import subprocess 8 | import sys 9 | 10 | Result = namedtuple("Result", ["returncode", "output"]) 11 | 12 | 13 | def git(*args, git_dir=None, capture_output=False, checked=True): 14 | # Avoid forgetting this arg. 15 | assert git_dir is None or os.path.isdir(git_dir) 16 | 17 | command = ['git'] 18 | if git_dir: 19 | command.append('--git-dir={0}'.format(git_dir)) 20 | command.extend(args) 21 | 22 | stdout = subprocess.PIPE if capture_output else None 23 | # Always let stderr print to the caller. 24 | process = subprocess.Popen( 25 | command, 26 | stdin=subprocess.DEVNULL, 27 | stdout=stdout, 28 | universal_newlines=True) 29 | output, _ = process.communicate() 30 | if checked and process.returncode != 0: 31 | sys.exit(1) 32 | 33 | return Result(process.returncode, output) 34 | 35 | 36 | def has_clone(url): 37 | return os.path.exists(repo_cache_path(url)) 38 | 39 | 40 | def clone_if_needed(url): 41 | repo_path = repo_cache_path(url) 42 | if not has_clone(url): 43 | # We look for this print in test, to count the number of clones we did. 44 | print('git clone ' + url) 45 | git('clone', '--mirror', '--progress', url, repo_path) 46 | return repo_path 47 | 48 | 49 | def repo_cache_path(url): 50 | # Because peru gives each plugin a unique cache dir based on its cacheable 51 | # fields (in this case, url) we could clone directly into cache_root. 52 | # However, because the git plugin needs to handle git submodules as well, 53 | # it still has to separate things out by repo url. 54 | CACHE_ROOT = os.environ['PERU_PLUGIN_CACHE'] 55 | 56 | # If we just concatenate the escaped repo URL into the path, we start to 57 | # run up against the 260-character path limit on Windows. 58 | url_hash = hashlib.sha1(url.encode()).hexdigest() 59 | 60 | return os.path.join(CACHE_ROOT, url_hash) 61 | 62 | 63 | def git_fetch(url, repo_path): 64 | print('git fetch ' + url) 65 | git('fetch', '--prune', git_dir=repo_path) 66 | 67 | 68 | def already_has_rev(repo, rev): 69 | # Make sure the rev exists. 70 | cat_result = git('cat-file', '-e', rev, git_dir=repo, checked=False) 71 | if cat_result.returncode != 0: 72 | return False 73 | # Get the hash for the rev. 74 | parse_result = git( 75 | 'rev-parse', rev, git_dir=repo, checked=False, capture_output=True) 76 | if parse_result.returncode != 0: 77 | return False 78 | # Only return True for revs that are absolute hashes. 79 | # We could consider treating tags the way, but... 80 | # 1) Tags actually can change. 81 | # 2) It's not clear at a glance if something is a branch or a tag. 82 | # Keep it simple. 83 | return parse_result.output.strip() == rev 84 | 85 | 86 | def checkout_tree(url, rev, dest): 87 | repo_path = clone_if_needed(url) 88 | if not already_has_rev(repo_path, rev): 89 | git_fetch(url, repo_path) 90 | # If we just use `git checkout rev -- .` here, we get an error when rev is 91 | # an empty commit. 92 | git('--work-tree=' + dest, 'read-tree', rev, git_dir=repo_path) 93 | git('--work-tree=' + dest, 'checkout-index', '--all', git_dir=repo_path) 94 | checkout_submodules(url, repo_path, rev, dest) 95 | 96 | 97 | def checkout_submodules(parent_url, repo_path, rev, work_tree): 98 | if os.environ['PERU_MODULE_SUBMODULES'] == 'false': 99 | return 100 | 101 | gitmodules = os.path.join(work_tree, '.gitmodules') 102 | if not os.path.exists(gitmodules): 103 | return 104 | 105 | parser = configparser.ConfigParser() 106 | parser.read(gitmodules) 107 | for section in parser.sections(): 108 | sub_relative_path = parser[section]['path'] 109 | sub_full_path = os.path.join(work_tree, sub_relative_path) 110 | raw_sub_url = parser[section]['url'] 111 | # Submodules can begin with ./ or ../, in which case they're relative 112 | # to the parent's URL. Handle this case. 113 | sub_url = expand_relative_submodule_url(raw_sub_url, parent_url) 114 | ls_tree = git( 115 | 'ls-tree', 116 | rev, 117 | sub_relative_path, 118 | git_dir=repo_path, 119 | capture_output=True).output 120 | # Normally when you run `git submodule add ...`, git puts two things in 121 | # your repo: an entry in .gitmodules, and a commit object at the 122 | # appropriate path inside your repo. However, it's possible for those 123 | # two to get out of sync, especially if you use mv/rm on a directory 124 | # followed by `git add`, instead of the smarter `git mv`/`git rm`. If 125 | # we run into one of these missing submodules, just skip it. 126 | if len(ls_tree.strip()) == 0: 127 | print('WARNING: submodule ' + sub_relative_path + 128 | ' is configured in .gitmodules, but missing in the repo') 129 | continue 130 | sub_rev = ls_tree.split()[2] 131 | checkout_tree(sub_url, sub_rev, sub_full_path) 132 | 133 | 134 | # According to comments in its own source code, git's implementation of 135 | # relative submodule URLs is full of unintended corner cases. See: 136 | # https://github.com/git/git/blob/v2.20.1/builtin/submodule--helper.c#L135 137 | # 138 | # We absolutely give up on trying to replicate their logic -- which probably 139 | # isn't stable in any case -- and instead we just leave the dots in and let the 140 | # host make sense of it. A quick sanity check on GitHub confirmed that that 141 | # seems to work for now. 142 | def expand_relative_submodule_url(raw_sub_url, parent_url): 143 | if not raw_sub_url.startswith("./") and not raw_sub_url.startswith("../"): 144 | return raw_sub_url 145 | new_path = parent_url 146 | if not new_path.endswith("/"): 147 | new_path += "/" 148 | new_path += raw_sub_url 149 | return new_path 150 | 151 | 152 | def plugin_sync(url, rev): 153 | checkout_tree(url, rev, os.environ['PERU_SYNC_DEST']) 154 | 155 | 156 | def plugin_reup(url, reup): 157 | reup_output = os.environ['PERU_REUP_OUTPUT'] 158 | repo_path = clone_if_needed(url) 159 | git_fetch(url, repo_path) 160 | output = git( 161 | 'rev-parse', reup, git_dir=repo_path, capture_output=True).output 162 | with open(reup_output, 'w') as out_file: 163 | print('rev:', output.strip(), file=out_file) 164 | 165 | 166 | def git_default_branch(url) -> str: 167 | """ 168 | This function checks if the default branch is master. 169 | If it is not found, then it assumes it is main. 170 | For other default branches, user should use the 'rev' option. 171 | 172 | Args: 173 | url (str): url from the target repository to be checked. 174 | Returns: 175 | str: returns a possible match for the git default branch. 176 | """ 177 | repo_path = clone_if_needed(url) 178 | output = git('show-ref', '--verify', '--quiet', 'refs/heads/master', 179 | git_dir=repo_path, checked=False, capture_output=True) 180 | if output.returncode == 0: 181 | return 'master' 182 | else: 183 | return 'main' 184 | 185 | 186 | def main(): 187 | URL = os.environ['PERU_MODULE_URL'] 188 | default_branch = git_default_branch(URL) 189 | REV = os.environ['PERU_MODULE_REV'] or default_branch 190 | REUP = os.environ['PERU_MODULE_REUP'] or default_branch 191 | 192 | command = os.environ['PERU_PLUGIN_COMMAND'] 193 | if command == 'sync': 194 | plugin_sync(URL, REV) 195 | elif command == 'reup': 196 | plugin_reup(URL, REUP) 197 | else: 198 | raise RuntimeError('Unknown command: ' + repr(command)) 199 | 200 | 201 | if __name__ == "__main__": 202 | main() 203 | -------------------------------------------------------------------------------- /peru/resources/plugins/git/plugin.yaml: -------------------------------------------------------------------------------- 1 | sync exe: git_plugin.py 2 | reup exe: git_plugin.py 3 | required fields: 4 | - url 5 | optional fields: 6 | - rev 7 | - reup 8 | - submodules 9 | cache fields: 10 | - url 11 | -------------------------------------------------------------------------------- /peru/resources/plugins/hg/hg_plugin.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python3 2 | 3 | from collections import namedtuple 4 | import os 5 | import subprocess 6 | import sys 7 | import textwrap 8 | 9 | CACHE_PATH = os.environ['PERU_PLUGIN_CACHE'] 10 | URL = os.environ['PERU_MODULE_URL'] 11 | REV = os.environ['PERU_MODULE_REV'] or 'default' 12 | REUP = os.environ['PERU_MODULE_REUP'] or 'default' 13 | 14 | Result = namedtuple("Result", ["returncode", "output"]) 15 | 16 | 17 | def hg(*args, hg_dir=None, capture_output=False, checked=True): 18 | # Avoid forgetting this arg. 19 | assert hg_dir is None or os.path.isdir(hg_dir) 20 | 21 | command = ['hg'] 22 | if hg_dir: 23 | command.append('--repository') 24 | command.append(hg_dir) 25 | command.extend(args) 26 | 27 | stdout = subprocess.PIPE if capture_output else None 28 | # Always let stderr print to the caller. 29 | process = subprocess.Popen( 30 | command, 31 | stdin=subprocess.DEVNULL, 32 | stdout=stdout, 33 | universal_newlines=True) 34 | output, _ = process.communicate() 35 | if checked and process.returncode != 0: 36 | sys.exit(1) 37 | 38 | return Result(process.returncode, output) 39 | 40 | 41 | def clone_if_needed(url, verbose=False): 42 | if not os.path.exists(os.path.join(CACHE_PATH, '.hg')): 43 | if verbose: 44 | print('hg clone', url) 45 | hg('clone', '--noupdate', url, CACHE_PATH) 46 | configure(CACHE_PATH) 47 | 48 | 49 | def configure(repo_path): 50 | # Set configs needed for cached repos. 51 | hgrc_path = os.path.join(repo_path, '.hg', 'hgrc') 52 | with open(hgrc_path, 'a') as f: 53 | f.write( 54 | textwrap.dedent('''\ 55 | [ui] 56 | # prevent 'hg archive' from creating '.hg_archival.txt' files. 57 | archivemeta = false 58 | ''')) 59 | 60 | 61 | def hg_pull(url, repo_path): 62 | print('hg pull', url) 63 | hg('pull', hg_dir=repo_path) 64 | 65 | 66 | def already_has_rev(repo, rev): 67 | res = hg( 68 | 'identify', 69 | '--debug', 70 | '--rev', 71 | rev, 72 | hg_dir=repo, 73 | capture_output=True, 74 | checked=False) 75 | if res.returncode != 0: 76 | return False 77 | 78 | # Only return True for revs that are absolute hashes. 79 | # We could consider treating tags the way, but... 80 | # 1) Tags actually can change. 81 | # 2) It's not clear at a glance whether something is a branch or a tag. 82 | # Keep it simple. 83 | return res.output.split()[0] == rev 84 | 85 | 86 | def plugin_sync(): 87 | dest = os.environ['PERU_SYNC_DEST'] 88 | clone_if_needed(URL, verbose=True) 89 | if not already_has_rev(CACHE_PATH, REV): 90 | hg_pull(URL, CACHE_PATH) 91 | # TODO: Should this handle subrepos? 92 | hg('archive', '--type', 'files', '--rev', REV, dest, hg_dir=CACHE_PATH) 93 | 94 | 95 | def plugin_reup(): 96 | reup_output = os.environ['PERU_REUP_OUTPUT'] 97 | 98 | clone_if_needed(URL, CACHE_PATH) 99 | hg_pull(URL, CACHE_PATH) 100 | output = hg( 101 | 'identify', 102 | '--debug', 103 | '--rev', 104 | REUP, 105 | hg_dir=CACHE_PATH, 106 | capture_output=True).output 107 | 108 | with open(reup_output, 'w') as output_file: 109 | print('rev:', output.split()[0], file=output_file) 110 | 111 | 112 | command = os.environ['PERU_PLUGIN_COMMAND'] 113 | if command == 'sync': 114 | plugin_sync() 115 | elif command == 'reup': 116 | plugin_reup() 117 | else: 118 | raise RuntimeError('Unknown command: ' + repr(command)) 119 | -------------------------------------------------------------------------------- /peru/resources/plugins/hg/plugin.yaml: -------------------------------------------------------------------------------- 1 | sync exe: hg_plugin.py 2 | reup exe: hg_plugin.py 3 | required fields: 4 | - url 5 | optional fields: 6 | - rev 7 | - reup 8 | cache fields: 9 | - url 10 | -------------------------------------------------------------------------------- /peru/resources/plugins/noop_cache/noop_cache_plugin.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python3 2 | 3 | # no-op 4 | -------------------------------------------------------------------------------- /peru/resources/plugins/noop_cache/plugin.yaml: -------------------------------------------------------------------------------- 1 | # This plugin is for testing purposes. The cache dir is not used. The nonce is 2 | # there to let us change module fields without changing plugin cache fields. 3 | sync exe: noop_cache_plugin.py 4 | required fields: 5 | - path 6 | optional fields: 7 | - nonce 8 | cache fields: 9 | - path 10 | -------------------------------------------------------------------------------- /peru/resources/plugins/print/plugin.yaml: -------------------------------------------------------------------------------- 1 | sync exe: print_plugin.py 2 | required fields: 3 | - nonce 4 | -------------------------------------------------------------------------------- /peru/resources/plugins/print/print_plugin.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python3 2 | 3 | # This plugin is for testing the fancy display. It prints some output without 4 | # making any network requests. I needed it on an airplane :) 5 | 6 | import random 7 | import sys 8 | import time 9 | 10 | for i in range(1, 6): 11 | print(i) 12 | sys.stdout.flush() 13 | time.sleep(0.5 + random.random()) 14 | -------------------------------------------------------------------------------- /peru/resources/plugins/rsync/plugin.yaml: -------------------------------------------------------------------------------- 1 | sync exe: rsync_plugin.sh 2 | required fields: 3 | - path 4 | -------------------------------------------------------------------------------- /peru/resources/plugins/rsync/rsync_plugin.sh: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env bash 2 | 3 | # The cp plugin is implemented in Python. This is a similar plugin implemented 4 | # in bash, as an example of how to implement plugins in other languages, and to 5 | # force us to keep our plugin interface simple. Not intended for serious use. 6 | 7 | set -e -u -o pipefail 8 | 9 | # Don't perform the copy without a source. Generally, plugins should not need 10 | # to worry about this, and peru should ensure that required fields are set, but 11 | # the validation may break, and that results in a destructive rsync command 12 | # that will copy root to the destination. 13 | if [ -z "$PERU_MODULE_PATH" ]; then 14 | echo >&2 "No source path has been set for rsync. Aborting." 15 | exit 1 16 | fi 17 | 18 | # Do the copy. Always append a trailing slash to the path, so that the 19 | # contents are copied rather than the directory itself. 20 | rsync -r "$PERU_MODULE_PATH/" "$PERU_SYNC_DEST" 21 | -------------------------------------------------------------------------------- /peru/resources/plugins/svn/plugin.yaml: -------------------------------------------------------------------------------- 1 | sync exe: svn_plugin.py 2 | reup exe: svn_plugin.py 3 | required fields: 4 | - url 5 | optional fields: 6 | - rev 7 | - reup 8 | -------------------------------------------------------------------------------- /peru/resources/plugins/svn/svn_plugin.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python3 2 | 3 | import os 4 | import subprocess 5 | import sys 6 | 7 | 8 | def svn(*args, svn_dir=None, capture_output=False): 9 | # Avoid forgetting this arg. 10 | assert svn_dir is None or os.path.isdir(svn_dir) 11 | 12 | command = ['svn', '--non-interactive'] 13 | command.extend(args) 14 | 15 | stdout = subprocess.PIPE if capture_output else None 16 | # Always let stderr print to the caller. 17 | process = subprocess.Popen( 18 | command, 19 | stdin=subprocess.DEVNULL, 20 | stdout=stdout, 21 | cwd=svn_dir, 22 | universal_newlines=True) 23 | output, _ = process.communicate() 24 | if process.returncode != 0: 25 | sys.exit(1) 26 | 27 | return output 28 | 29 | 30 | def remote_head_rev(url): 31 | print('svn info', url) 32 | info = svn('info', url, capture_output=True).split('\n') 33 | for item in info: 34 | if item.startswith('Revision: '): 35 | return item.split()[1] 36 | 37 | print('svn revision info not found', file=sys.stderr) 38 | sys.exit(1) 39 | 40 | 41 | def plugin_sync(): 42 | # Just fetch the target revision and strip the metadata. 43 | # Plugin-level caching for Subversion is futile. 44 | svn('export', '--force', '--revision', os.environ['PERU_MODULE_REV'] 45 | or 'HEAD', os.environ['PERU_MODULE_URL'], os.environ['PERU_SYNC_DEST']) 46 | 47 | 48 | def plugin_reup(): 49 | url = os.environ['PERU_MODULE_URL'] 50 | rev = remote_head_rev(url) 51 | output_file = os.environ['PERU_REUP_OUTPUT'] 52 | with open(output_file, 'w') as f: 53 | # Quote Subversion revisions to prevent integer intepretation. 54 | print('rev:', '"{}"'.format(rev), file=f) 55 | 56 | 57 | command = os.environ['PERU_PLUGIN_COMMAND'] 58 | if command == 'sync': 59 | plugin_sync() 60 | elif command == 'reup': 61 | plugin_reup() 62 | else: 63 | raise RuntimeError('Unknown command: ' + repr(command)) 64 | -------------------------------------------------------------------------------- /peru/rule.py: -------------------------------------------------------------------------------- 1 | from pathlib import PurePosixPath 2 | import re 3 | 4 | from . import cache 5 | from .error import PrintableError 6 | from . import glob 7 | 8 | 9 | class Rule: 10 | def __init__(self, name, copy, move, executable, drop, pick, export): 11 | self.name = name 12 | self.copy = copy 13 | self.move = move 14 | self.executable = executable 15 | self.drop = drop 16 | self.pick = pick 17 | self.export = export 18 | 19 | def _cache_key(self, input_tree): 20 | return cache.compute_key({ 21 | 'input_tree': input_tree, 22 | 'copy': self.copy, 23 | 'move': self.move, 24 | 'executable': self.executable, 25 | 'drop': self.drop, 26 | 'pick': self.pick, 27 | 'export': self.export, 28 | }) 29 | 30 | async def get_tree(self, runtime, input_tree): 31 | key = self._cache_key(input_tree) 32 | 33 | # As with Module, take a lock on the cache key to avoid running the 34 | # same rule (or identical rules) twice with the same input. 35 | cache_lock = runtime.cache_key_locks[key] 36 | async with cache_lock: 37 | if key in runtime.cache.keyval: 38 | return runtime.cache.keyval[key] 39 | 40 | tree = input_tree 41 | if self.copy: 42 | tree = await copy_files(runtime.cache, tree, self.copy) 43 | if self.move: 44 | tree = await move_files(runtime.cache, tree, self.move) 45 | if self.drop: 46 | tree = await drop_files(runtime.cache, tree, self.drop) 47 | if self.pick: 48 | tree = await pick_files(runtime.cache, tree, self.pick) 49 | if self.executable: 50 | tree = await make_files_executable(runtime.cache, tree, 51 | self.executable) 52 | if self.export: 53 | tree = await get_export_tree(runtime.cache, tree, self.export) 54 | 55 | runtime.cache.keyval[key] = tree 56 | 57 | return tree 58 | 59 | 60 | async def _copy_files_modifications(_cache, tree, paths_multimap): 61 | modifications = {} 62 | for source in paths_multimap: 63 | source_info_dict = await _cache.ls_tree(tree, source) 64 | if not source_info_dict: 65 | raise NoMatchingFilesError( 66 | 'Path "{}" does not exist.'.format(source)) 67 | source_info = list(source_info_dict.items())[0][1] 68 | for dest in paths_multimap[source]: 69 | # If dest is a directory, put the source inside dest instead of 70 | # overwriting dest entirely. 71 | dest_is_dir = False 72 | dest_info_dict = await _cache.ls_tree(tree, dest) 73 | if dest_info_dict: 74 | dest_info = list(dest_info_dict.items())[0][1] 75 | dest_is_dir = (dest_info.type == cache.TREE_TYPE) 76 | adjusted_dest = dest 77 | if dest_is_dir: 78 | adjusted_dest = str( 79 | PurePosixPath(dest) / PurePosixPath(source).name) 80 | modifications[adjusted_dest] = source_info 81 | return modifications 82 | 83 | 84 | async def copy_files(_cache, tree, paths_multimap): 85 | modifications = await _copy_files_modifications(_cache, tree, 86 | paths_multimap) 87 | tree = await _cache.modify_tree(tree, modifications) 88 | return tree 89 | 90 | 91 | async def move_files(_cache, tree, paths_multimap): 92 | # First obtain the copies from the original tree. Moves are not ordered but 93 | # happen all at once, so if you move a->b and b->c, the contents of c will 94 | # always end up being b rather than a. 95 | modifications = await _copy_files_modifications(_cache, tree, 96 | paths_multimap) 97 | # Now add in deletions, but be careful not to delete a file that just got 98 | # moved. Note that if "a" gets moved into "dir", it will end up at "dir/a", 99 | # even if "dir" is deleted (because modify_tree always modifies parents 100 | # before decending into children, and deleting a dir is a modification of 101 | # that dir's parent). 102 | for source in paths_multimap: 103 | if source not in modifications: 104 | modifications[source] = None 105 | tree = await _cache.modify_tree(tree, modifications) 106 | return tree 107 | 108 | 109 | async def _get_glob_entries(_cache, tree, globs_list): 110 | matches = {} 111 | for glob_str in globs_list: 112 | # Do an in-memory match of all the paths in the tree against the 113 | # glob expression. As an optimization, if the glob is something 114 | # like 'a/b/**/foo', only list the paths under 'a/b'. 115 | regex = glob.glob_to_path_regex(glob_str) 116 | prefix = glob.unglobbed_prefix(glob_str) 117 | entries = await _cache.ls_tree(tree, prefix, recursive=True) 118 | found = False 119 | for path, entry in entries.items(): 120 | if re.match(regex, path): 121 | matches[path] = entry 122 | found = True 123 | if not found: 124 | raise NoMatchingFilesError( 125 | '"{}" didn\'t match any files.'.format(glob_str)) 126 | return matches 127 | 128 | 129 | async def pick_files(_cache, tree, globs_list): 130 | picks = await _get_glob_entries(_cache, tree, globs_list) 131 | tree = await _cache.modify_tree(None, picks) 132 | return tree 133 | 134 | 135 | async def drop_files(_cache, tree, globs_list): 136 | drops = await _get_glob_entries(_cache, tree, globs_list) 137 | for path in drops: 138 | drops[path] = None 139 | tree = await _cache.modify_tree(tree, drops) 140 | return tree 141 | 142 | 143 | async def make_files_executable(_cache, tree, globs_list): 144 | entries = await _get_glob_entries(_cache, tree, globs_list) 145 | exes = {} 146 | for path, entry in entries.items(): 147 | # Ignore directories. 148 | if entry.type == cache.BLOB_TYPE: 149 | exes[path] = entry._replace(mode=cache.EXECUTABLE_FILE_MODE) 150 | tree = await _cache.modify_tree(tree, exes) 151 | return tree 152 | 153 | 154 | async def get_export_tree(_cache, tree, export_path): 155 | entries = await _cache.ls_tree(tree, export_path) 156 | if not entries: 157 | raise NoMatchingFilesError( 158 | 'Export path "{}" doesn\'t exist.'.format(export_path)) 159 | entry = list(entries.values())[0] 160 | if entry.type != cache.TREE_TYPE: 161 | raise NoMatchingFilesError( 162 | 'Export path "{}" is not a directory.'.format(export_path)) 163 | return entry.hash 164 | 165 | 166 | class NoMatchingFilesError(PrintableError): 167 | pass 168 | -------------------------------------------------------------------------------- /peru/runtime.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | import collections 3 | import os 4 | from pathlib import Path 5 | import tempfile 6 | 7 | from . import cache 8 | from . import compat 9 | from .error import PrintableError 10 | from . import display 11 | from .keyval import KeyVal 12 | from . import parser 13 | from . import plugin 14 | 15 | 16 | async def Runtime(args, env): 17 | 'This is the async constructor for the _Runtime class.' 18 | r = _Runtime(args, env) 19 | await r._init_cache() 20 | return r 21 | 22 | 23 | class _Runtime: 24 | def __init__(self, args, env): 25 | "Don't instantiate this class directly. Use the Runtime() constructor." 26 | self._set_paths(args, env) 27 | 28 | compat.makedirs(self.state_dir) 29 | 30 | self._tmp_root = os.path.join(self.state_dir, 'tmp') 31 | compat.makedirs(self._tmp_root) 32 | 33 | self.overrides = KeyVal( 34 | os.path.join(self.state_dir, 'overrides'), self._tmp_root) 35 | self._used_overrides = set() 36 | 37 | self.force = args.get('--force', False) 38 | if args['--quiet'] and args['--verbose']: 39 | raise PrintableError( 40 | "Peru can't be quiet and verbose at the same time.") 41 | self.quiet = args['--quiet'] 42 | self.verbose = args['--verbose'] 43 | self.no_overrides = args.get('--no-overrides', False) 44 | self.no_cache = args.get('--no-cache', False) 45 | 46 | # Use a semaphore (a lock that allows N holders at once) to limit the 47 | # number of fetches that can run in parallel. 48 | num_fetches = _get_parallel_fetch_limit(args) 49 | self.fetch_semaphore = asyncio.BoundedSemaphore(num_fetches) 50 | 51 | # Use locks to make sure the same cache keys don't get double fetched. 52 | self.cache_key_locks = collections.defaultdict(asyncio.Lock) 53 | 54 | # Use a different set of locks to make sure that plugin cache dirs are 55 | # only used by one job at a time. 56 | self.plugin_cache_locks = collections.defaultdict(asyncio.Lock) 57 | 58 | self.display = get_display(args) 59 | 60 | async def _init_cache(self): 61 | self.cache = await cache.Cache(self.cache_dir) 62 | 63 | def _set_paths(self, args, env): 64 | explicit_peru_file = args['--file'] 65 | explicit_sync_dir = args['--sync-dir'] 66 | explicit_basename = args['--file-basename'] 67 | if explicit_peru_file and explicit_basename: 68 | raise CommandLineError( 69 | 'Cannot use both --file and --file-basename at the same time.') 70 | if explicit_peru_file and explicit_sync_dir: 71 | self.peru_file = explicit_peru_file 72 | self.sync_dir = explicit_sync_dir 73 | elif explicit_peru_file or explicit_sync_dir: 74 | raise CommandLineError('If the --file or --sync-dir is set, ' 75 | 'the other must also be set.') 76 | else: 77 | basename = explicit_basename or parser.DEFAULT_PERU_FILE_NAME 78 | self.peru_file = find_project_file(os.getcwd(), basename) 79 | self.sync_dir = os.path.dirname(self.peru_file) 80 | self.state_dir = (args['--state-dir'] 81 | or os.path.join(self.sync_dir, '.peru')) 82 | self.cache_dir = (args['--cache-dir'] or env.get('PERU_CACHE_DIR') 83 | or os.path.join(self.state_dir, 'cache')) 84 | 85 | def tmp_dir(self): 86 | dir = tempfile.TemporaryDirectory(dir=self._tmp_root) 87 | return dir 88 | 89 | def get_plugin_context(self): 90 | return plugin.PluginContext( 91 | # Plugin cwd is always the directory containing peru.yaml, even if 92 | # the sync_dir has been explicitly set elsewhere. That's because 93 | # relative paths in peru.yaml should respect the location of that 94 | # file. 95 | cwd=str(Path(self.peru_file).parent), 96 | plugin_cache_root=self.cache.plugins_root, 97 | parallelism_semaphore=self.fetch_semaphore, 98 | plugin_cache_locks=self.plugin_cache_locks, 99 | tmp_root=self._tmp_root) 100 | 101 | def set_override(self, name, path): 102 | if not os.path.isabs(path): 103 | # We can't store relative paths as given, because peru could be 104 | # running from a different working dir next time. But we don't want 105 | # to absolutify everything, because the user might want the paths 106 | # to be relative (for example, so a whole workspace can be moved as 107 | # a group while preserving all the overrides). So reinterpret all 108 | # relative paths from the project root. 109 | path = os.path.relpath(path, start=self.sync_dir) 110 | self.overrides[name] = path 111 | 112 | def get_override(self, name): 113 | if self.no_overrides or name not in self.overrides: 114 | return None 115 | path = self.overrides[name] 116 | if not os.path.isabs(path): 117 | # Relative paths are stored relative to the project root. 118 | # Reinterpret them relative to the cwd. See the above comment in 119 | # set_override. 120 | path = os.path.relpath(os.path.join(self.sync_dir, path)) 121 | return path 122 | 123 | def mark_override_used(self, name): 124 | '''Marking overrides as used lets us print a warning when an override 125 | is unused.''' 126 | self._used_overrides.add(name) 127 | 128 | def print_overrides(self): 129 | if self.quiet or self.no_overrides: 130 | return 131 | names = sorted(self.overrides) 132 | if not names: 133 | return 134 | self.display.print('syncing with overrides:') 135 | for name in names: 136 | self.display.print(' {}: {}'.format(name, 137 | self.get_override(name))) 138 | 139 | def warn_unused_overrides(self): 140 | if self.quiet or self.no_overrides: 141 | return 142 | unused_names = set(self.overrides) - self._used_overrides 143 | if not unused_names: 144 | return 145 | self.display.print('WARNING unused overrides:') 146 | for name in sorted(unused_names): 147 | self.display.print(' ' + name) 148 | 149 | 150 | def find_project_file(start_dir, basename): 151 | '''Walk up the directory tree until we find a file of the given name.''' 152 | prefix = os.path.abspath(start_dir) 153 | while True: 154 | candidate = os.path.join(prefix, basename) 155 | if os.path.isfile(candidate): 156 | return candidate 157 | if os.path.exists(candidate): 158 | raise PrintableError( 159 | "Found {}, but it's not a file.".format(candidate)) 160 | if os.path.dirname(prefix) == prefix: 161 | # We've walked all the way to the top. Bail. 162 | raise PrintableError("Can't find " + basename) 163 | # Not found at this level. We must go...shallower. 164 | prefix = os.path.dirname(prefix) 165 | 166 | 167 | def _get_parallel_fetch_limit(args): 168 | jobs = args.get('--jobs') 169 | if jobs is None: 170 | return plugin.DEFAULT_PARALLEL_FETCH_LIMIT 171 | try: 172 | parallel = int(jobs) 173 | if parallel <= 0: 174 | raise PrintableError('Argument to --jobs must be 1 or more.') 175 | return parallel 176 | except Exception: 177 | raise PrintableError('Argument to --jobs must be a number.') 178 | 179 | 180 | def get_display(args): 181 | if args['--quiet']: 182 | return display.QuietDisplay() 183 | elif args['--verbose']: 184 | return display.VerboseDisplay() 185 | elif compat.is_fancy_terminal(): 186 | return display.FancyDisplay() 187 | else: 188 | return display.QuietDisplay() 189 | 190 | 191 | class CommandLineError(PrintableError): 192 | pass 193 | -------------------------------------------------------------------------------- /peru/scope.py: -------------------------------------------------------------------------------- 1 | from .error import PrintableError 2 | 3 | SCOPE_SEPARATOR = '.' 4 | RULE_SEPARATOR = '|' 5 | 6 | 7 | class Scope: 8 | '''A Scope holds the elements that are parsed out of a single peru.yaml 9 | file. This is kept separate from a Runtime, because recursive modules need 10 | to work with a Scope that makes sense to them, rather than a single global 11 | scope.''' 12 | 13 | def __init__(self, modules, rules): 14 | self.modules = modules 15 | self.rules = rules 16 | 17 | async def parse_target(self, runtime, target_str): 18 | '''A target is a pipeline of a module into zero or more rules, and each 19 | module and rule can itself be scoped with zero or more module names.''' 20 | pipeline_parts = target_str.split(RULE_SEPARATOR) 21 | module = await self.resolve_module(runtime, pipeline_parts[0], 22 | target_str) 23 | rules = [] 24 | for part in pipeline_parts[1:]: 25 | rule = await self.resolve_rule(runtime, part) 26 | rules.append(rule) 27 | return module, tuple(rules) 28 | 29 | async def resolve_module(self, 30 | runtime, 31 | module_str, 32 | logging_target_name=None): 33 | logging_target_name = logging_target_name or module_str 34 | module_names = module_str.split(SCOPE_SEPARATOR) 35 | return (await self._resolve_module_from_names(runtime, module_names, 36 | logging_target_name)) 37 | 38 | async def _resolve_module_from_names(self, runtime, module_names, 39 | logging_target_name): 40 | next_module = self._get_module_checked(module_names[0]) 41 | for name in module_names[1:]: 42 | next_scope = await _get_scope_or_fail(runtime, logging_target_name, 43 | next_module) 44 | if name not in next_scope.modules: 45 | _error(logging_target_name, 'module {} not found in {}', name, 46 | next_module.name) 47 | next_module = next_scope._get_module_checked(name) 48 | return next_module 49 | 50 | async def resolve_rule(self, runtime, rule_str, logging_target_name=None): 51 | logging_target_name = logging_target_name or rule_str 52 | *module_names, rule_name = rule_str.split(SCOPE_SEPARATOR) 53 | scope = self 54 | location_str = '' 55 | if module_names: 56 | module = await self._resolve_module_from_names( 57 | runtime, module_names, logging_target_name) 58 | scope = await _get_scope_or_fail(runtime, logging_target_name, 59 | module) 60 | location_str = ' in module ' + module.name 61 | if rule_name not in scope.rules: 62 | _error(logging_target_name, 'rule {} not found{}', rule_name, 63 | location_str) 64 | return scope._get_rule_checked(rule_name) 65 | 66 | def get_modules_for_reup(self, names): 67 | for name in names: 68 | if SCOPE_SEPARATOR in name: 69 | raise PrintableError( 70 | 'Can\'t reup module "{}"; it belongs to another project.'. 71 | format(name)) 72 | return [self._get_module_checked(name) for name in names] 73 | 74 | def _get_module_checked(self, name): 75 | if name not in self.modules: 76 | raise PrintableError('Module "{}" doesn\'t exist.', name) 77 | return self.modules[name] 78 | 79 | def _get_rule_checked(self, name): 80 | if name not in self.rules: 81 | raise PrintableError('Rule "{}" doesn\'t exist.', name) 82 | return self.rules[name] 83 | 84 | 85 | async def _get_scope_or_fail(runtime, logging_target_name, module): 86 | scope, imports = await module.parse_peru_file(runtime) 87 | if not scope: 88 | _error(logging_target_name, 'module {} is not a peru project', 89 | module.name) 90 | return scope 91 | 92 | 93 | def _error(logging_target_name, text, *text_format_args): 94 | text = text.format(*text_format_args) 95 | raise PrintableError('Error in target {}: {}'.format( 96 | logging_target_name, text)) 97 | -------------------------------------------------------------------------------- /requirements-dev.txt: -------------------------------------------------------------------------------- 1 | coverage 2 | flake8 3 | 4 | -e . 5 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | import os 2 | import setuptools 3 | import sys 4 | 5 | # Importing fastentrypoints monkey-patches setuptools to avoid generating slow 6 | # executables from the entry_points directive. See 7 | # https://github.com/ninjaaron/fast-entry_points. 8 | import fastentrypoints 9 | 10 | # Written according to the docs at 11 | # https://packaging.python.org/en/latest/distributing.html 12 | 13 | project_root = os.path.dirname(__file__) 14 | readme_file = os.path.join(project_root, 'README.md') 15 | module_root = os.path.join(project_root, 'peru') 16 | version_file = os.path.join(module_root, 'VERSION') 17 | 18 | 19 | def get_version(): 20 | with open(version_file) as f: 21 | return f.read().strip() 22 | 23 | 24 | def get_all_resources_filepaths(): 25 | resources_paths = ['VERSION'] 26 | resources_dir = os.path.join(module_root, 'resources') 27 | for dirpath, dirnames, filenames in os.walk(resources_dir): 28 | relpaths = [ 29 | os.path.relpath(os.path.join(dirpath, f), start=module_root) 30 | for f in filenames 31 | ] 32 | resources_paths.extend(relpaths) 33 | return resources_paths 34 | 35 | 36 | def get_install_requires(): 37 | dependencies = ['PyYAML'] 38 | if sys.version_info < (3, 5): 39 | raise RuntimeError('The minimum supported Python version is 3.5.') 40 | return dependencies 41 | 42 | 43 | def readme_text(): 44 | with open(readme_file) as f: 45 | return f.read().strip() 46 | 47 | 48 | setuptools.setup( 49 | name='peru', 50 | description='A tool for fetching code', 51 | version=get_version(), 52 | url='https://github.com/buildinspace/peru', 53 | author="Jack O'Connor , " 54 | "Sean Olson ", 55 | license='MIT', 56 | packages=['peru', 'peru.docopt'], 57 | package_data={'peru': get_all_resources_filepaths()}, 58 | entry_points={'console_scripts': [ 59 | 'peru=peru.main:main', 60 | ]}, 61 | install_requires=get_install_requires(), 62 | long_description=readme_text(), 63 | long_description_content_type='text/markdown', 64 | ) 65 | -------------------------------------------------------------------------------- /test.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python3 2 | 3 | import os 4 | import sys 5 | import subprocess 6 | 7 | REPO_ROOT = os.path.dirname(os.path.realpath(__file__)) 8 | TESTS_DIR = os.path.join(REPO_ROOT, 'tests') 9 | 10 | 11 | def get_untracked_files(): 12 | output = subprocess.check_output([ 13 | 'git', 'ls-files', '--other', '--directory', '--exclude-standard', '-z' 14 | ], 15 | cwd=REPO_ROOT) 16 | return set(f for f in output.split(b'\0') if f) 17 | 18 | 19 | def main(): 20 | # Unset any PERU environment variables to make sure test runs don't get 21 | # thrown off by anything in your bashrc. 22 | for var in list(os.environ.keys()): 23 | if var.startswith('PERU_'): 24 | del os.environ[var] 25 | 26 | # Turn debugging features on for the asyncio library, if it's unset. 27 | if 'PYTHONASYNCIODEBUG' not in os.environ: 28 | os.environ['PYTHONASYNCIODEBUG'] = '1' 29 | 30 | # Make sure the tests don't create any garbage files in the repo. That 31 | # tends to happen when we accidentally run something in the current dir 32 | # that should be in a temp dir, and it's hard to track down when it does. 33 | old_untracked = get_untracked_files() 34 | 35 | # Run the actual tests. 36 | env = os.environ.copy() 37 | env['PYTHONPATH'] = REPO_ROOT 38 | args = sys.argv[1:] 39 | # We no longer run --with-coverage in CI, so this branch might bitrot over 40 | # time. 41 | if len(args) > 0 and '--with-coverage' in args: 42 | args.remove('--with-coverage') 43 | command_start = ['coverage', 'run'] 44 | else: 45 | command_start = [sys.executable, '-W', 'all'] 46 | command = command_start + ['-m', 'unittest'] + args 47 | try: 48 | subprocess.check_call(command, env=env, cwd=TESTS_DIR) 49 | except subprocess.CalledProcessError: 50 | print('ERROR: unit tests returned an error code', file=sys.stderr) 51 | sys.exit(1) 52 | 53 | new_untracked = get_untracked_files() 54 | if old_untracked != new_untracked: 55 | print( 56 | 'Tests created untracked files:\n' + '\n'.join( 57 | f.decode() for f in new_untracked - old_untracked), 58 | file=sys.stderr) 59 | sys.exit(1) 60 | 61 | # Run the linter. 62 | try: 63 | subprocess.check_call(['flake8', 'peru', 'tests', '--exclude=peru/docopt'], cwd=REPO_ROOT) 64 | except FileNotFoundError: 65 | print('ERROR: flake8 not found', file=sys.stderr) 66 | sys.exit(1) 67 | except subprocess.CalledProcessError: 68 | print('ERROR: flake8 returned an error code', file=sys.stderr) 69 | sys.exit(1) 70 | 71 | 72 | if __name__ == '__main__': 73 | main() 74 | -------------------------------------------------------------------------------- /tests/.coveragerc: -------------------------------------------------------------------------------- 1 | [run] 2 | source = ../peru 3 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/buildinspace/peru/7007451b7f6e2ff23f9dfc23cc28dae7280205d2/tests/__init__.py -------------------------------------------------------------------------------- /tests/resources/absolute_path.tar: -------------------------------------------------------------------------------- 1 | /tmp/absolute.txt0000644000175000017500000000001712475407015013257 0ustar jackojackoAbsolute path. 2 | -------------------------------------------------------------------------------- /tests/resources/absolute_path.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/buildinspace/peru/7007451b7f6e2ff23f9dfc23cc28dae7280205d2/tests/resources/absolute_path.zip -------------------------------------------------------------------------------- /tests/resources/from_windows.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/buildinspace/peru/7007451b7f6e2ff23f9dfc23cc28dae7280205d2/tests/resources/from_windows.zip -------------------------------------------------------------------------------- /tests/resources/leading_dots.tar: -------------------------------------------------------------------------------- 1 | ../with_dots.txt0000644000175000017500000000001312475406607014243 0ustar jackojacko00000000000000With dots. 2 | -------------------------------------------------------------------------------- /tests/resources/leading_dots.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/buildinspace/peru/7007451b7f6e2ff23f9dfc23cc28dae7280205d2/tests/resources/leading_dots.zip -------------------------------------------------------------------------------- /tests/resources/legal_symlink_dots.tar: -------------------------------------------------------------------------------- 1 | a/0000755000175000017500000000000014371540534010244 5ustar jackojackoa/b/0000755000175000017500000000000014371540534010465 5ustar jackojackoa/b/c/0000755000175000017500000000000014371540547010713 5ustar jackojackoa/b/c/foo.txt0000777000175000017500000000000014371540547014421 2../../../foo.txtustar jackojacko -------------------------------------------------------------------------------- /tests/resources/with_exe.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/buildinspace/peru/7007451b7f6e2ff23f9dfc23cc28dae7280205d2/tests/resources/with_exe.zip -------------------------------------------------------------------------------- /tests/test_async.py: -------------------------------------------------------------------------------- 1 | import asyncio 2 | from asyncio.subprocess import PIPE 3 | import sys 4 | 5 | from peru.async_helpers import safe_communicate 6 | from shared import PeruTest, make_synchronous 7 | 8 | 9 | class AsyncTest(PeruTest): 10 | @make_synchronous 11 | async def test_safe_communicate(self): 12 | # Test safe_communicate with both empty and non-empty input. 13 | cat_command = [ 14 | sys.executable, "-c", 15 | "import sys; sys.stdout.write(sys.stdin.read())" 16 | ] 17 | 18 | proc_empty = await asyncio.create_subprocess_exec( 19 | *cat_command, stdin=PIPE, stdout=PIPE) 20 | stdout, _ = await safe_communicate(proc_empty, b"") 21 | self.assertEqual(stdout, b"") 22 | 23 | proc_nonempty = await asyncio.create_subprocess_exec( 24 | *cat_command, stdin=PIPE, stdout=PIPE) 25 | stdout, _ = await safe_communicate(proc_nonempty, b"foo bar baz") 26 | self.assertEqual(stdout, b"foo bar baz") 27 | 28 | # And test a case with None input as well. 29 | true_command = [sys.executable, "-c", ""] 30 | proc_true = await asyncio.create_subprocess_exec( 31 | *true_command, stdin=PIPE, stdout=PIPE) 32 | stdout, _ = await safe_communicate(proc_true) 33 | self.assertEqual(stdout, b"") 34 | -------------------------------------------------------------------------------- /tests/test_compat.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | import peru.compat as compat 4 | import shared 5 | 6 | 7 | class CompatTest(shared.PeruTest): 8 | def test_makedirs(self): 9 | tmp_dir = shared.tmp_dir() 10 | foo_dir = os.path.join(tmp_dir, "foo") 11 | compat.makedirs(foo_dir) 12 | os.chmod(foo_dir, 0o700) 13 | # Creating the dir again should be a no-op even though the permissions 14 | # have changed. 15 | compat.makedirs(foo_dir) 16 | -------------------------------------------------------------------------------- /tests/test_curl_plugin.py: -------------------------------------------------------------------------------- 1 | import hashlib 2 | import importlib.util 3 | import io 4 | from os.path import abspath, join, dirname 5 | import urllib 6 | 7 | import peru 8 | import shared 9 | 10 | # https://docs.python.org/3/library/importlib.html#importing-a-source-file-directly 11 | curl_plugin_path = abspath( 12 | join( 13 | dirname(peru.__file__), 'resources', 'plugins', 'curl', 14 | 'curl_plugin.py')) 15 | spec = importlib.util.spec_from_file_location("curl_plugin", curl_plugin_path) 16 | curl_plugin = importlib.util.module_from_spec(spec) 17 | spec.loader.exec_module(curl_plugin) 18 | 19 | 20 | class MockRequest: 21 | def __init__(self, url, info, response): 22 | self.url = url 23 | self._info = info 24 | self._response_buffer = io.BytesIO(response) 25 | 26 | def info(self): 27 | return self._info 28 | 29 | def read(self, *args): 30 | return self._response_buffer.read(*args) 31 | 32 | 33 | class CurlPluginTest(shared.PeruTest): 34 | def test_format_bytes(self): 35 | self.assertEqual('0B', curl_plugin.format_bytes(0)) 36 | self.assertEqual('999B', curl_plugin.format_bytes(999)) 37 | self.assertEqual('1.0KB', curl_plugin.format_bytes(1000)) 38 | self.assertEqual('999.9KB', curl_plugin.format_bytes(999999)) 39 | self.assertEqual('1.0MB', curl_plugin.format_bytes(10**6)) 40 | self.assertEqual('1.0GB', curl_plugin.format_bytes(10**9)) 41 | self.assertEqual('1000.0GB', curl_plugin.format_bytes(10**12)) 42 | 43 | def test_get_request_filename(self): 44 | request = MockRequest('http://www.example.com/', {}, b'junk') 45 | self.assertEqual('index.html', 46 | curl_plugin.get_request_filename(request)) 47 | request.url = 'http://www.example.com/foo' 48 | self.assertEqual('foo', curl_plugin.get_request_filename(request)) 49 | request._info = {'Content-Disposition': 'attachment; filename=bar'} 50 | self.assertEqual('bar', curl_plugin.get_request_filename(request)) 51 | # Check quoted filenames. 52 | request._info = {'Content-Disposition': 'attachment; filename="bar"'} 53 | self.assertEqual('bar', curl_plugin.get_request_filename(request)) 54 | # Check backslashed quotes in filenames. 55 | request._info = { 56 | 'Content-Disposition': 'attachment; filename="bar\\""' 57 | } 58 | self.assertEqual('bar"', curl_plugin.get_request_filename(request)) 59 | 60 | def test_download_file_with_length(self): 61 | content = b'xy' * 4096 62 | request = MockRequest('some url', {'Content-Length': len(content)}, 63 | content) 64 | stdout = io.StringIO() 65 | output_file = io.BytesIO() 66 | sha1 = curl_plugin.download_file(request, output_file, stdout) 67 | self.assertEqual( 68 | 'downloaded 50% 4.0KB/8.1KB\ndownloaded 100% 8.1KB/8.1KB\n', 69 | stdout.getvalue()) 70 | self.assertEqual(content, output_file.getvalue()) 71 | self.assertEqual(hashlib.sha1(content).hexdigest(), sha1) 72 | 73 | def test_download_file_without_length(self): 74 | content = b'foo' 75 | request = MockRequest('some url', {}, content) 76 | stdout = io.StringIO() 77 | output_file = io.BytesIO() 78 | sha1 = curl_plugin.download_file(request, output_file, stdout) 79 | self.assertEqual('downloaded 3B\n', stdout.getvalue()) 80 | self.assertEqual(content, output_file.getvalue()) 81 | self.assertEqual(hashlib.sha1(content).hexdigest(), sha1) 82 | 83 | def test_unpack_windows_zip(self): 84 | '''This zip was packed on Windows, so it doesn't include any file 85 | permissions. This checks that our executable-flag-restoring code 86 | doesn't barf when the flag isn't there.''' 87 | test_dir = shared.create_dir() 88 | archive = shared.test_resources / 'from_windows.zip' 89 | curl_plugin.extract_zip(str(archive), test_dir) 90 | shared.assert_contents(test_dir, {'windows_test/test.txt': 'Notepad!'}) 91 | txt_file = join(test_dir, 'windows_test/test.txt') 92 | shared.assert_not_executable(txt_file) 93 | 94 | def test_evil_archives(self): 95 | '''Even though most zip and tar utilities try to prevent absolute paths 96 | and paths starting with '..', it's entirely possible to construct an 97 | archive with either. These should always be an error.''' 98 | dest = shared.create_dir() 99 | for case in 'absolute_path', 'leading_dots': 100 | zip_archive = shared.test_resources / (case + '.zip') 101 | with self.assertRaises(curl_plugin.EvilArchiveError): 102 | curl_plugin.extract_zip(str(zip_archive), dest) 103 | tar_archive = shared.test_resources / (case + '.tar') 104 | with self.assertRaises(curl_plugin.EvilArchiveError): 105 | curl_plugin.extract_tar(str(tar_archive), dest) 106 | 107 | def test_evil_symlink_archives(self): 108 | """Even worse than archives containing bad paths, an archive could 109 | contain a *symlink* pointing to a bad path. Then a subsequent entry in 110 | the *same* archive could write through the symlink.""" 111 | dest = shared.create_dir() 112 | for case in ["illegal_symlink_dots", "illegal_symlink_absolute"]: 113 | tar_archive = shared.test_resources / (case + ".tar") 114 | with self.assertRaises(curl_plugin.EvilArchiveError): 115 | curl_plugin.extract_tar(str(tar_archive), dest) 116 | # But leading dots should be allowed in symlinks, as long as they don't 117 | # escape the root of the archive. 118 | for case in ["legal_symlink_dots"]: 119 | tar_archive = shared.test_resources / (case + ".tar") 120 | curl_plugin.extract_tar(str(tar_archive), dest) 121 | 122 | def test_request_has_user_agent_header(self): 123 | actual = curl_plugin.build_request("http://example.test") 124 | self.assertTrue(actual.has_header("User-agent")) 125 | ua_header = actual.get_header("User-agent") 126 | peru_component, urllib_component = ua_header.split(' ') 127 | _, peru_version = peru_component.split('/') 128 | _, urllib_version = urllib_component.split('/') 129 | self.assertEqual(peru.main.get_version(), peru_version) 130 | self.assertEqual(urllib.request.__version__, urllib_version) 131 | -------------------------------------------------------------------------------- /tests/test_display.py: -------------------------------------------------------------------------------- 1 | import io 2 | import re 3 | import textwrap 4 | 5 | from peru import display 6 | import shared 7 | 8 | 9 | class DisplayTest(shared.PeruTest): 10 | def test_quiet_display(self): 11 | output = io.StringIO() 12 | disp = display.QuietDisplay(output) 13 | with disp.get_handle('title') as handle: 14 | handle.write('some stuff!') 15 | disp.print('other stuff?') 16 | self.assertEqual('other stuff?\n', output.getvalue()) 17 | 18 | def test_verbose_display(self): 19 | output = io.StringIO() 20 | disp = display.VerboseDisplay(output) 21 | with disp.get_handle('title') as handle: 22 | handle.write('in job 1\n') 23 | disp.print('print stuff') 24 | handle.write('in job 2\n') 25 | expected = textwrap.dedent('''\ 26 | === started title === 27 | print stuff 28 | === finished title === 29 | in job 1 30 | in job 2 31 | === 32 | ''') 33 | self.assertEqual(expected, output.getvalue()) 34 | 35 | def test_fancy_display(self): 36 | output = FakeTerminal() 37 | disp = display.FancyDisplay(output) 38 | 39 | handle1 = disp.get_handle('title1') 40 | handle1.__enter__() 41 | handle1.write('something1') 42 | disp._draw() 43 | # We need to test trailing spaces, and the '# noqa: W291' tag stops the 44 | # linter from complaining about these. 45 | expected1 = textwrap.dedent('''\ 46 | ╶ title1: something1 47 | ''') # noqa: W291 48 | self.assertEqual(expected1, output.getlines()) 49 | 50 | handle2 = disp.get_handle('title2') 51 | handle2.__enter__() 52 | handle2.write('something2') 53 | disp._draw() 54 | expected2 = textwrap.dedent('''\ 55 | ┌ title1: something1 56 | └ title2: something2 57 | ''') # noqa: W291 58 | self.assertEqual(expected2, output.getlines()) 59 | 60 | handle3 = disp.get_handle('title3') 61 | handle3.__enter__() 62 | handle3.write('something3') 63 | disp._draw() 64 | expected3 = textwrap.dedent('''\ 65 | ┌ title1: something1 66 | ├ title2: something2 67 | └ title3: something3 68 | ''') # noqa: W291 69 | self.assertEqual(expected3, output.getlines()) 70 | 71 | disp.print('stuff above') 72 | # Calling _draw() should not be necessary after print(). This ensures 73 | # that we won't lose output if the program exits before _draw_later() 74 | # gets another chance to fire. 75 | expected4 = textwrap.dedent('''\ 76 | stuff above 77 | ┌ title1: something1 78 | ├ title2: something2 79 | └ title3: something3 80 | ''') # noqa: W291 81 | self.assertEqual(expected4, output.getlines()) 82 | 83 | handle2.__exit__(None, None, None) 84 | disp._draw() 85 | expected5 = textwrap.dedent('''\ 86 | stuff above 87 | ┌ title1: something1 88 | └ title3: something3 89 | ''') # noqa: W291 90 | self.assertEqual(expected5, output.getlines()) 91 | 92 | handle1.__exit__(None, None, None) 93 | disp._draw() 94 | expected6 = textwrap.dedent('''\ 95 | stuff above 96 | ╶ title3: something3 97 | ''') # noqa: W291 98 | self.assertEqual(expected6, output.getlines()) 99 | 100 | handle3.__exit__(None, None, None) 101 | # _draw() should not be necessary after the last job exits. 102 | expected7 = textwrap.dedent('''\ 103 | stuff above 104 | ''') 105 | self.assertEqual(expected7, output.getlines()) 106 | self.assertEqual(None, disp._draw_later_handle) 107 | 108 | 109 | class FakeTerminal: 110 | '''Emulates a terminal by keeping track of a list of lines. Knows how to 111 | interpret the ANSI escape sequences that are used by FancyDisplay.''' 112 | 113 | def __init__(self): 114 | self.lines = [io.StringIO()] 115 | self.cursor_line = 0 116 | # Flush doesn't actually do anything in fake terminal, but we want to 117 | # make sure it gets called before any lines are read. 118 | self.flushed = False 119 | 120 | def write(self, string): 121 | tokens = [ 122 | display.ANSI_DISABLE_LINE_WRAP, display.ANSI_ENABLE_LINE_WRAP, 123 | display.ANSI_CLEAR_LINE, display.ANSI_CURSOR_UP_ONE_LINE, '\n' 124 | ] 125 | # The parens make this a capturing expression, so the tokens will be 126 | # included in re.split()'s return list. 127 | token_expr = '(' + '|'.join(re.escape(token) for token in tokens) + ')' 128 | pieces = re.split(token_expr, string) 129 | 130 | for piece in pieces: 131 | if piece in (display.ANSI_DISABLE_LINE_WRAP, 132 | display.ANSI_ENABLE_LINE_WRAP): 133 | # Ignore the line wrap codes. TODO: Test for these? 134 | continue 135 | elif piece == display.ANSI_CLEAR_LINE: 136 | buffer = self.lines[self.cursor_line] 137 | buffer.seek(0) 138 | buffer.truncate() 139 | elif piece == display.ANSI_CURSOR_UP_ONE_LINE: 140 | col = self.lines[self.cursor_line].tell() 141 | self.cursor_line -= 1 142 | assert self.cursor_line >= 0 143 | new_buffer = self.lines[self.cursor_line] 144 | new_buffer.seek(col) 145 | elif piece == '\n': 146 | self.cursor_line += 1 147 | if self.cursor_line == len(self.lines): 148 | self.lines.append(io.StringIO()) 149 | self.lines[self.cursor_line].seek(0) 150 | else: 151 | self.lines[self.cursor_line].write(piece) 152 | 153 | def flush(self): 154 | self.flushed = True 155 | 156 | def getlines(self): 157 | # Make sure flush() was called after the last write. 158 | assert self.flushed 159 | self.flushed = False 160 | # Make sure none of the lines at or beyond the cursor have any text. 161 | for i in range(self.cursor_line, len(self.lines)): 162 | assert self.lines[i].getvalue() == '' 163 | # Concatenate all the lines before the cursor, and append trailing 164 | # newlines. 165 | lines = io.StringIO() 166 | for i in range(self.cursor_line): 167 | lines.write(self.lines[i].getvalue()) 168 | lines.write('\n') 169 | return lines.getvalue() 170 | -------------------------------------------------------------------------------- /tests/test_edit_yaml.py: -------------------------------------------------------------------------------- 1 | from textwrap import dedent 2 | 3 | import yaml 4 | 5 | from peru import edit_yaml 6 | import shared 7 | 8 | yaml_template = dedent("""\ 9 | a: 10 | b: [1, 2, 3] 11 | c: {} 12 | d: blarg 13 | """) 14 | 15 | 16 | class EditYamlTest(shared.PeruTest): 17 | def test_replace(self): 18 | start_yaml = yaml_template.format("foo") 19 | new_yaml = edit_yaml.set_module_field(start_yaml, "a", "c", "bar") 20 | self.assertEqual(yaml_template.format("bar"), new_yaml) 21 | 22 | def test_insert(self): 23 | start_yaml = dedent("""\ 24 | a: 25 | b: foo 26 | """) 27 | new_yaml = edit_yaml.set_module_field(start_yaml, "a", "c", "bar") 28 | self.assertEqual(start_yaml + " c: bar\n", new_yaml) 29 | 30 | def test_insert_number_looking_fields(self): 31 | # These all need to be quoted, or else YAML will interpret them as 32 | # literal ints and floats. 33 | start_yaml = dedent('''\ 34 | a: 35 | b: foo 36 | ''') 37 | intermediate = edit_yaml.set_module_field(start_yaml, 'a', 'c', '5') 38 | new_yaml = edit_yaml.set_module_field(intermediate, 'a', 'd', '.0') 39 | expected_yaml = start_yaml + ' c: "5"\n d: ".0"\n' 40 | self.assertEqual(expected_yaml, new_yaml) 41 | self.assertDictEqual( 42 | yaml.safe_load(new_yaml), 43 | {'a': { 44 | 'b': 'foo', 45 | 'c': '5', 46 | 'd': '.0', 47 | }}) 48 | 49 | def test_insert_with_last_field_as_dict(self): 50 | start_yaml = dedent("""\ 51 | a: 52 | b: 53 | foo: bar 54 | baz: bing 55 | x: y 56 | """) 57 | end_yaml = dedent("""\ 58 | a: 59 | b: 60 | foo: bar 61 | baz: bing 62 | c: stuff 63 | x: y 64 | """) 65 | edited_yaml = edit_yaml.set_module_field(start_yaml, "a", "c", "stuff") 66 | self.assertEqual(end_yaml, edited_yaml) 67 | 68 | def test_with_file(self): 69 | tmp_name = shared.tmp_file() 70 | start_yaml = yaml_template.format("foo") 71 | with open(tmp_name, "w") as f: 72 | f.write(start_yaml) 73 | edit_yaml.set_module_field_in_file(tmp_name, "a", "c", "bar") 74 | with open(tmp_name) as f: 75 | new_yaml = f.read() 76 | self.assertEqual(yaml_template.format("bar"), new_yaml) 77 | -------------------------------------------------------------------------------- /tests/test_git_plugin.py: -------------------------------------------------------------------------------- 1 | import importlib.machinery 2 | from os.path import abspath, join, dirname 3 | 4 | import peru 5 | import shared 6 | 7 | # https://docs.python.org/3/library/importlib.html#importing-a-source-file-directly 8 | git_plugin_path = abspath( 9 | join( 10 | dirname(peru.__file__), 'resources', 'plugins', 'git', 11 | 'git_plugin.py')) 12 | spec = importlib.util.spec_from_file_location("git_plugin", git_plugin_path) 13 | git_plugin = importlib.util.module_from_spec(spec) 14 | spec.loader.exec_module(git_plugin) 15 | 16 | 17 | # NOTE: The sync/reup functionality for the git plugin is tested in 18 | # test_plugins.py along with the other plugin types. 19 | class GitPluginTest(shared.PeruTest): 20 | def test_expand_relative_submodule_url(self): 21 | cases = [ 22 | ("http://foo.com/a/b", "c", "c"), 23 | ("http://foo.com/a/b", "./c", "http://foo.com/a/b/./c"), 24 | ("http://foo.com/a/b", "../c", "http://foo.com/a/b/../c"), 25 | ("http://foo.com/a/b", "../../c", "http://foo.com/a/b/../../c"), 26 | ("http://foo.com/a/b", ".//../c", "http://foo.com/a/b/.//../c"), 27 | ] 28 | for (parent, submodule, expected) in cases: 29 | result = git_plugin.expand_relative_submodule_url( 30 | submodule, parent) 31 | assert expected == result, "{} != {}".format(expected, result) 32 | -------------------------------------------------------------------------------- /tests/test_glob.py: -------------------------------------------------------------------------------- 1 | import collections 2 | import re 3 | 4 | import peru.glob as glob 5 | import shared 6 | 7 | 8 | class GlobTest(shared.PeruTest): 9 | def test_split_on_stars_interpreting_backslashes(self): 10 | cases = [ 11 | ('', ['']), 12 | ('*', ['', '']), 13 | ('abc', ['abc']), 14 | ('abc\\', ['abc\\']), 15 | ('abc\\n', ['abc\\n']), 16 | ('abc\\\\', ['abc\\']), 17 | ('ab*c', ['ab', 'c']), 18 | ('*abc*', ['', 'abc', '']), 19 | (r'a\*bc', ['a*bc']), 20 | (r'a\\*bc', ['a\\', 'bc']), 21 | (r'a\\\*bc', ['a\\*bc']), 22 | (r'a\\\\*bc', ['a\\\\', 'bc']), 23 | ] 24 | for input, output in cases: 25 | self.assertEqual( 26 | output, glob.split_on_stars_interpreting_backslashes(input), 27 | 'Failed split for input {}'.format(input)) 28 | 29 | def test_glob_to_path_regex(self): 30 | Case = collections.namedtuple('Case', ['glob', 'matches', 'excludes']) 31 | cases = [ 32 | Case( 33 | glob='a/b/c', 34 | matches=['a/b/c'], 35 | excludes=['a/b', 'a/b/c/', '/a/b/c', 'a/b/c/d']), 36 | # * should be able to match nothing. 37 | Case( 38 | glob='a/*b/c', 39 | matches=['a/b/c', 'a/xb/c'], 40 | excludes=['a/x/c', 'a/c', 'a//c']), 41 | # But * by itself should never match an empty path component. 42 | Case( 43 | glob='a/*/c', 44 | matches=['a/b/c', 'a/boooo/c', 'a/*/c'], 45 | excludes=['a/c', 'a/b/d/c', 'a//c']), 46 | # Similarly, ** does not match empty path components. It's tempting 47 | # to allow this, but we never want '**/c' to match '/c'. 48 | Case( 49 | glob='a/**/c', 50 | matches=['a/b/c', 'a/d/e/f/g/c', 'a/c'], 51 | excludes=['a/b/c/d', 'x/a/c', 'a//c']), 52 | Case( 53 | glob='a/**/**/c', 54 | matches=['a/b/c', 'a/d/e/f/g/c', 'a/c'], 55 | excludes=['a/b/c/d', 'x/a/c', 'a//c']), 56 | Case(glob='**/c', matches=['a/b/c', 'c'], excludes=['/c', 'c/d']), 57 | Case( 58 | glob='**/*/c', matches=['a/b/c', 'a/c'], excludes=['c', '/c']), 59 | # Leading slashes should be preserved if present. 60 | Case(glob='/a', matches=['/a'], excludes=['a']), 61 | Case( 62 | glob='/**/c', 63 | matches=['/a/b/c', '/c'], 64 | excludes=['c', 'a/b/c']), 65 | # Make sure special characters are escaped properly. 66 | Case(glob='a|b', matches=['a|b'], excludes=['a', 'b']), 67 | # Test escaped * characters. 68 | Case(glob='a\\*', matches=['a*'], excludes=['a', 'aa']), 69 | ] 70 | for case in cases: 71 | regex = glob.glob_to_path_regex(case.glob) 72 | for m in case.matches: 73 | assert re.match(regex, m), \ 74 | 'Glob {} (regex: {} ) should match path {}'.format( 75 | case.glob, regex, m) 76 | for e in case.excludes: 77 | assert not re.match(regex, e), \ 78 | 'Glob {} (regex: {} ) should not match path {}'.format( 79 | case.glob, regex, e) 80 | 81 | def test_bad_globs(self): 82 | bad_globs = [ 83 | '**', 84 | 'a/b/**', 85 | 'a/b/**/', 86 | 'a/b/**c/d', 87 | ] 88 | for bad_glob in bad_globs: 89 | with self.assertRaises(glob.GlobError): 90 | glob.glob_to_path_regex(bad_glob) 91 | 92 | def test_unglobbed_prefix(self): 93 | assert glob.unglobbed_prefix('a/b/c*/d') == 'a/b' 94 | assert glob.unglobbed_prefix('a/b/**/d') == 'a/b' 95 | assert glob.unglobbed_prefix('/a/b/*/d') == '/a/b' 96 | assert glob.unglobbed_prefix('*/a/b') == '' 97 | -------------------------------------------------------------------------------- /tests/test_keyval.py: -------------------------------------------------------------------------------- 1 | import shared 2 | 3 | from peru.keyval import KeyVal 4 | 5 | 6 | class KeyValTest(shared.PeruTest): 7 | def test_keyval(self): 8 | root = shared.create_dir() 9 | tmp_dir = shared.create_dir() 10 | keyval = KeyVal(root, tmp_dir) 11 | key = "mykey" 12 | # keyval should be empty 13 | self.assertFalse(key in keyval) 14 | self.assertSetEqual(set(keyval), set()) 15 | # set a key 16 | keyval[key] = "myval" 17 | self.assertEqual(keyval[key], "myval") 18 | self.assertTrue(key in keyval) 19 | self.assertSetEqual(set(keyval), {key}) 20 | # overwrite the value 21 | keyval[key] = "anotherval" 22 | self.assertEqual(keyval[key], "anotherval") 23 | # instantiate a second keyval on the same dir, should have same content 24 | another_keyval = KeyVal(root, tmp_dir) 25 | self.assertTrue(key in another_keyval) 26 | self.assertEqual(another_keyval[key], "anotherval") 27 | self.assertSetEqual(set(another_keyval), {key}) 28 | # test deletions 29 | del keyval[key] 30 | self.assertFalse(key in keyval) 31 | self.assertFalse(key in another_keyval) 32 | -------------------------------------------------------------------------------- /tests/test_merge.py: -------------------------------------------------------------------------------- 1 | from peru.cache import Cache 2 | from peru.merge import merge_imports_tree 3 | 4 | from shared import create_dir, assert_contents, PeruTest, make_synchronous 5 | 6 | 7 | class MergeTest(PeruTest): 8 | @make_synchronous 9 | async def setUp(self): 10 | self.cache_dir = create_dir() 11 | self.cache = await Cache(self.cache_dir) 12 | 13 | # These tests use this simple one-file tree as module contents. 14 | content = {'a': 'a'} 15 | content_dir = create_dir(content) 16 | self.content_tree = await self.cache.import_tree(content_dir) 17 | 18 | @make_synchronous 19 | async def test_merge_from_map(self): 20 | imports = {'foo': ('path1', ), 'bar': ('path2', )} 21 | target_trees = {'foo': self.content_tree, 'bar': self.content_tree} 22 | 23 | merged_tree = await merge_imports_tree(self.cache, imports, 24 | target_trees) 25 | 26 | merged_dir = create_dir() 27 | await self.cache.export_tree(merged_tree, merged_dir) 28 | expected_content = {'path1/a': 'a', 'path2/a': 'a'} 29 | assert_contents(merged_dir, expected_content) 30 | 31 | @make_synchronous 32 | async def test_merge_from_multimap(self): 33 | # This represents a list of key-value pairs in YAML, for example: 34 | # imports: 35 | # foo: 36 | # - path1 37 | # - path2 38 | imports = {'foo': ('path1', 'path2')} 39 | target_trees = {'foo': self.content_tree} 40 | 41 | merged_tree = await merge_imports_tree(self.cache, imports, 42 | target_trees) 43 | 44 | merged_dir = create_dir() 45 | await self.cache.export_tree(merged_tree, merged_dir) 46 | expected_content = {'path1/a': 'a', 'path2/a': 'a'} 47 | assert_contents(merged_dir, expected_content) 48 | -------------------------------------------------------------------------------- /tests/test_parallelism.py: -------------------------------------------------------------------------------- 1 | from textwrap import dedent 2 | 3 | from peru import plugin 4 | 5 | import shared 6 | 7 | 8 | def assert_parallel(n): 9 | # The plugin module keep a global counter of all the jobs that run in 10 | # parallel, so that we can write these tests. 11 | if plugin.DEBUG_PARALLEL_MAX != n: 12 | raise AssertionError('Expected {} parallel {}. Counted {}.'.format( 13 | n, 'job' if n == 1 else 'jobs', plugin.DEBUG_PARALLEL_MAX)) 14 | 15 | 16 | class ParallelismTest(shared.PeruTest): 17 | def setUp(self): 18 | # Make sure nothing is fishy with the jobs counter, and reset the max. 19 | plugin.debug_assert_clean_parallel_count() 20 | plugin.DEBUG_PARALLEL_MAX = 0 21 | 22 | def tearDown(self): 23 | # Make sure nothing is fishy with the jobs counter. No sense in 24 | # resetting the max here, because the rest of our tests don't know to 25 | # reset it anyway. 26 | plugin.debug_assert_clean_parallel_count() 27 | 28 | def test_two_jobs_in_parallel(self): 29 | # This just checks that two different modules can actually be fetched 30 | # in parallel. 31 | foo = shared.create_dir() 32 | bar = shared.create_dir() 33 | peru_yaml = dedent('''\ 34 | imports: 35 | foo: ./ 36 | bar: ./ 37 | 38 | cp module foo: 39 | path: {} 40 | 41 | cp module bar: 42 | path: {} 43 | '''.format(foo, bar)) 44 | test_dir = shared.create_dir({'peru.yaml': peru_yaml}) 45 | shared.run_peru_command(['sync'], test_dir) 46 | assert_parallel(2) 47 | 48 | def test_jobs_flag(self): 49 | # This checks that the --jobs flag is respected, even when two modules 50 | # could have been fetched in parallel. 51 | foo = shared.create_dir() 52 | bar = shared.create_dir() 53 | peru_yaml = dedent('''\ 54 | imports: 55 | foo: ./ 56 | bar: ./ 57 | 58 | cp module foo: 59 | path: {} 60 | 61 | cp module bar: 62 | path: {} 63 | '''.format(foo, bar)) 64 | test_dir = shared.create_dir({'peru.yaml': peru_yaml}) 65 | shared.run_peru_command(['sync', '-j1'], test_dir) 66 | assert_parallel(1) 67 | 68 | def test_identical_fields(self): 69 | # This checks that modules with identical fields are not fetched in 70 | # parallel. This is the same logic that protects us from fetching a 71 | # given module twice, like when it's imported with two different named 72 | # rules. 73 | foo = shared.create_dir() 74 | peru_yaml = dedent('''\ 75 | imports: 76 | foo1: ./ 77 | foo2: ./ 78 | 79 | cp module foo1: 80 | path: {} 81 | 82 | cp module foo2: 83 | path: {} 84 | '''.format(foo, foo)) 85 | test_dir = shared.create_dir({'peru.yaml': peru_yaml}) 86 | shared.run_peru_command(['sync'], test_dir) 87 | assert_parallel(1) 88 | 89 | def test_identical_plugin_cache_fields(self): 90 | # Plugins that use caching also need to avoid running in parallel, if 91 | # their cache directories are the same. The noop_cache plugin (created 92 | # for this test) uses the path field (but not the nonce field) in its 93 | # plugin cache key. Check that these two modules are not fetched in 94 | # parallel, even though their module fields aren't exactly the same. 95 | foo = shared.create_dir() 96 | peru_yaml = dedent('''\ 97 | imports: 98 | foo1: ./ 99 | foo2: ./ 100 | 101 | noop_cache module foo1: 102 | path: {} 103 | # nonce is ignored, but it makes foo1 different from foo2 as 104 | # far as the module cache is concerned 105 | nonce: '1' 106 | 107 | noop_cache module foo2: 108 | path: {} 109 | nonce: '2' 110 | '''.format(foo, foo)) 111 | test_dir = shared.create_dir({'peru.yaml': peru_yaml}) 112 | shared.run_peru_command(['sync'], test_dir) 113 | assert_parallel(1) 114 | -------------------------------------------------------------------------------- /tests/test_parser.py: -------------------------------------------------------------------------------- 1 | from textwrap import dedent 2 | 3 | from peru import parser 4 | from peru.parser import parse_string, ParserError 5 | from peru.module import Module 6 | from peru.rule import Rule 7 | import shared 8 | 9 | 10 | class ParserTest(shared.PeruTest): 11 | def test_parse_empty_file(self): 12 | scope, imports = parse_string('') 13 | self.assertDictEqual(scope.modules, {}) 14 | self.assertDictEqual(scope.rules, {}) 15 | self.assertEqual(imports, {}) 16 | 17 | def test_parse_rule(self): 18 | input = dedent("""\ 19 | rule foo: 20 | export: out/ 21 | """) 22 | scope, imports = parse_string(input) 23 | self.assertIn("foo", scope.rules) 24 | rule = scope.rules["foo"] 25 | self.assertIsInstance(rule, Rule) 26 | self.assertEqual(rule.name, "foo") 27 | self.assertEqual(rule.export, "out/") 28 | 29 | def test_parse_module(self): 30 | input = dedent("""\ 31 | sometype module foo: 32 | url: http://www.example.com/ 33 | rev: abcdefg 34 | """) 35 | scope, imports = parse_string(input) 36 | self.assertIn("foo", scope.modules) 37 | module = scope.modules["foo"] 38 | self.assertIsInstance(module, Module) 39 | self.assertEqual(module.name, "foo") 40 | self.assertEqual(module.type, "sometype") 41 | self.assertDictEqual(module.plugin_fields, { 42 | "url": "http://www.example.com/", 43 | "rev": "abcdefg" 44 | }) 45 | 46 | def test_parse_module_default_rule(self): 47 | input = dedent("""\ 48 | git module bar: 49 | export: bar 50 | """) 51 | scope, imports = parse_string(input) 52 | self.assertIn("bar", scope.modules) 53 | module = scope.modules["bar"] 54 | self.assertIsInstance(module, Module) 55 | self.assertIsInstance(module.default_rule, Rule) 56 | self.assertEqual(module.default_rule.export, "bar") 57 | 58 | def test_parse_toplevel_imports(self): 59 | input = dedent("""\ 60 | imports: 61 | foo: bar/ 62 | """) 63 | scope, imports = parse_string(input) 64 | self.assertDictEqual(scope.modules, {}) 65 | self.assertDictEqual(scope.rules, {}) 66 | self.assertEqual(imports, {'foo': ('bar/', )}) 67 | 68 | def test_parse_multimap_imports(self): 69 | input = dedent('''\ 70 | imports: 71 | foo: 72 | - bar/ 73 | ''') 74 | scope, imports = parse_string(input) 75 | self.assertDictEqual(scope.modules, {}) 76 | self.assertDictEqual(scope.rules, {}) 77 | self.assertEqual(imports, {'foo': ('bar/', )}) 78 | 79 | def test_parse_empty_imports(self): 80 | input = dedent('''\ 81 | imports: 82 | ''') 83 | scope, imports = parse_string(input) 84 | self.assertDictEqual(scope.modules, {}) 85 | self.assertDictEqual(scope.rules, {}) 86 | self.assertEqual(imports, {}) 87 | 88 | def test_parse_wrong_type_imports_throw(self): 89 | with self.assertRaises(ParserError): 90 | parse_string('imports: 5') 91 | 92 | def test_bad_toplevel_field_throw(self): 93 | with self.assertRaises(ParserError): 94 | parse_string("foo: bar") 95 | 96 | def test_bad_rule_field_throw(self): 97 | with self.assertRaises(ParserError): 98 | parse_string( 99 | dedent("""\ 100 | rule foo: 101 | bad_field: junk 102 | """)) 103 | 104 | def test_bad_rule_name_throw(self): 105 | with self.assertRaises(ParserError): 106 | parse_string("rule foo bar:") 107 | 108 | def test_bad_module_name_throw(self): 109 | with self.assertRaises(ParserError): 110 | parse_string("git module abc def:") 111 | with self.assertRaises(ParserError): 112 | parse_string("git module:") 113 | 114 | def test_duplicate_names_throw(self): 115 | # Modules and rules should not conflict. 116 | ok_input = dedent(''' 117 | rule foo: 118 | git module foo: 119 | ''') 120 | parse_string(ok_input) 121 | # But duplicate modules should fail. (Duplicate rules are a not 122 | # currently possible, because their YAML keys would be exact 123 | # duplicates.) 124 | bad_input = dedent(''' 125 | git module foo: 126 | hg module foo: 127 | ''') 128 | with self.assertRaises(ParserError): 129 | parse_string(bad_input) 130 | 131 | def test_non_string_module_field_name(self): 132 | input = dedent('''\ 133 | git module foo: 134 | 12345: bar 135 | ''') 136 | try: 137 | parse_string(input) 138 | except ParserError as e: 139 | assert '12345' in e.message 140 | else: 141 | assert False, 'expected ParserError' 142 | 143 | def test_non_string_module_field_value(self): 144 | input = dedent('''\ 145 | git module foo: 146 | bar: 123 147 | # These booleans should turn into "true" and "false". 148 | baz: yes 149 | bing: no 150 | ''') 151 | scope, imports = parse_string(input) 152 | foo = scope.modules['foo'] 153 | self.assertDictEqual(foo.plugin_fields, { 154 | "bar": "123", 155 | "baz": "true", 156 | "bing": "false", 157 | }) 158 | 159 | def test_build_field_deprecated_message(self): 160 | input = dedent('''\ 161 | rule foo: 162 | build: shell command 163 | ''') 164 | try: 165 | parse_string(input) 166 | except ParserError as e: 167 | assert 'The "build" field is no longer supported.' in e.message 168 | else: 169 | assert False, 'expected ParserError' 170 | 171 | def test_name_prefix(self): 172 | input = dedent('''\ 173 | git module foo: 174 | url: fun stuff 175 | 176 | rule bar: 177 | export: more stuff 178 | ''') 179 | scope, imports = parse_string(input, name_prefix='x') 180 | # Lookup keys should be unaffected, but the names that modules and 181 | # rules give for themselves should have the prefix. 182 | assert scope.modules['foo'].name == 'xfoo' 183 | assert scope.rules['bar'].name == 'xbar' 184 | 185 | def test_forgotten_colon(self): 186 | # There are many different permutations of this error, and this only 187 | # tests the one mentioned in 188 | # https://github.com/keybase/client/issues/242. 189 | # TODO: A more general data validation library might help the parser do 190 | # a better job of checking these things. See 191 | # https://github.com/buildinspace/peru/issues/40. 192 | input = dedent('''\ 193 | rule test: 194 | pick bla 195 | ''') 196 | with self.assertRaises(ParserError): 197 | parse_string(input) 198 | 199 | def test_duplicate_key_heuristic(self): 200 | yaml = dedent('''\ 201 | a: 202 | a: 1 203 | b: 1 204 | b: 205 | a: 1 206 | b: 1 207 | a: 1 208 | a : whitespace before colon 209 | a: stuff 210 | ''') 211 | duplicates = parser._get_duplicate_keys_approximate(yaml) 212 | self.assertEqual([ 213 | ('a', 5, 7), 214 | ('a', 1, 8), 215 | ('a', 8, 9), 216 | ], duplicates) 217 | -------------------------------------------------------------------------------- /tests/test_paths.py: -------------------------------------------------------------------------------- 1 | import os 2 | import shutil 3 | import textwrap 4 | 5 | from peru.compat import makedirs 6 | from peru.runtime import CommandLineError 7 | 8 | import shared 9 | 10 | 11 | class PathsTest(shared.PeruTest): 12 | def setUp(self): 13 | self.test_root = shared.create_dir() 14 | 15 | self.project_dir = os.path.join(self.test_root, 'project') 16 | self.state_dir = os.path.join(self.project_dir, '.peru') 17 | self.cache_dir = os.path.join(self.state_dir, 'cache') 18 | 19 | self.yaml = textwrap.dedent('''\ 20 | imports: 21 | foo: ./ 22 | cp module foo: 23 | # relative paths should always be interpreted from the dir 24 | # containing peru.yaml, even if that's not the sync dir. 25 | path: ../foo 26 | ''') 27 | shared.write_files(self.project_dir, {'peru.yaml': self.yaml}) 28 | self.peru_file = os.path.join(self.project_dir, 'peru.yaml') 29 | 30 | self.foo_dir = os.path.join(self.test_root, 'foo') 31 | shared.write_files(self.foo_dir, {'bar': 'baz'}) 32 | 33 | # We'll run tests from this inner subdirectory, so that we're more 34 | # likely to catch places where we're using the cwd incorrectly. 35 | self.cwd = os.path.join(self.project_dir, 'cwd', 'in', 'here') 36 | makedirs(self.cwd) 37 | 38 | def assert_success(self, sync_dir, state_dir, cache_dir, more_excludes=[]): 39 | shared.assert_contents( 40 | sync_dir, {'bar': 'baz'}, 41 | excludes=['.peru', 'peru.yaml'] + more_excludes) 42 | assert os.path.isfile(os.path.join(state_dir, 'lastimports')) 43 | assert os.path.isdir(os.path.join(cache_dir, 'trees')) 44 | 45 | def test_unmodified_sync(self): 46 | shared.run_peru_command(['sync'], self.cwd) 47 | self.assert_success(self.project_dir, self.state_dir, self.cache_dir) 48 | 49 | def test_peru_file_and_sync_dir_must_be_set_together(self): 50 | for command in [['--sync-dir=junk', 'sync'], ['--file=junk', 'sync']]: 51 | with self.assertRaises(CommandLineError): 52 | shared.run_peru_command(command, cwd=self.cwd) 53 | 54 | def test_file_and_file_basename_incompatible(self): 55 | with self.assertRaises(CommandLineError): 56 | shared.run_peru_command([ 57 | '--file=foo', '--sync-dir=bar', '--file-basename=baz', 'sync' 58 | ], 59 | cwd=self.cwd) 60 | 61 | def test_setting_all_flags(self): 62 | cwd = shared.create_dir() 63 | sync_dir = shared.create_dir() 64 | state_dir = shared.create_dir() 65 | cache_dir = shared.create_dir() 66 | shared.run_peru_command([ 67 | '--file', self.peru_file, '--sync-dir', sync_dir, '--state-dir', 68 | state_dir, '--cache-dir', cache_dir, 'sync' 69 | ], cwd) 70 | self.assert_success(sync_dir, state_dir, cache_dir) 71 | 72 | def test_setting_all_env_vars(self): 73 | cache_dir = shared.create_dir() 74 | shared.run_peru_command(['sync'], 75 | self.cwd, 76 | env={ 77 | 'PERU_CACHE_DIR': cache_dir, 78 | }) 79 | self.assert_success(self.project_dir, self.state_dir, cache_dir) 80 | 81 | def test_flags_override_vars(self): 82 | flag_cache_dir = shared.create_dir() 83 | env_cache_dir = shared.create_dir() 84 | shared.run_peru_command(['--cache-dir', flag_cache_dir, 'sync'], 85 | self.cwd, 86 | env={'PERU_CACHE_DIR': env_cache_dir}) 87 | self.assert_success(self.project_dir, self.state_dir, flag_cache_dir) 88 | 89 | def test_relative_paths(self): 90 | '''We ran into a bug where calling os.path.dirname(peru_file) was 91 | returning "", which got passed as the cwd of a plugin job and blew up. 92 | This test repros that case. We've switched to pathlib.Path.parent to 93 | fix the issue.''' 94 | shared.run_peru_command( 95 | ['--file', 'peru.yaml', '--sync-dir', '.', 'sync'], 96 | cwd=self.project_dir) 97 | self.assert_success(self.project_dir, self.state_dir, self.cache_dir) 98 | 99 | def test_default_file_name(self): 100 | shutil.move(self.peru_file, os.path.join(self.project_dir, 'xxx')) 101 | shared.run_peru_command(['--file-basename', 'xxx', 'sync'], 102 | cwd=self.cwd) 103 | self.assert_success( 104 | self.project_dir, 105 | self.state_dir, 106 | self.cache_dir, 107 | more_excludes=['xxx']) 108 | -------------------------------------------------------------------------------- /tests/test_reup.py: -------------------------------------------------------------------------------- 1 | import os 2 | from textwrap import dedent 3 | 4 | import shared 5 | from shared import run_peru_command, assert_contents 6 | 7 | 8 | class ReupIntegrationTest(shared.PeruTest): 9 | def setUp(self): 10 | self.foo_dir = shared.create_dir({'a': 'b'}) 11 | self.foo_repo = shared.GitRepo(self.foo_dir) 12 | self.foo_master = self.foo_repo.run('git', 'rev-parse', 'master') 13 | self.bar_dir = shared.create_dir() 14 | self.bar_repo = shared.GitRepo(self.bar_dir) 15 | self.bar_repo.run('git', 'checkout', '-q', '-b', 'otherbranch') 16 | with open(os.path.join(self.bar_dir, 'barfile'), 'w') as f: 17 | f.write('new') 18 | self.bar_repo.run('git', 'add', '-A') 19 | self.bar_repo.run('git', 'commit', '-m', 'creating barfile') 20 | self.bar_otherbranch = self.bar_repo.run('git', 'rev-parse', 21 | 'otherbranch') 22 | 23 | def test_single_reup(self): 24 | yaml_without_imports = dedent('''\ 25 | git module foo: 26 | url: {} 27 | rev: master 28 | 29 | git module bar: 30 | url: {} 31 | reup: otherbranch 32 | ''').format(self.foo_dir, self.bar_dir) 33 | test_dir = shared.create_dir({'peru.yaml': yaml_without_imports}) 34 | expected = dedent('''\ 35 | git module foo: 36 | url: {} 37 | rev: {} 38 | 39 | git module bar: 40 | url: {} 41 | reup: otherbranch 42 | ''').format(self.foo_dir, self.foo_master, self.bar_dir) 43 | run_peru_command(['reup', 'foo'], test_dir) 44 | assert_contents(test_dir, {'peru.yaml': expected}, excludes=['.peru']) 45 | 46 | def test_reup_sync(self): 47 | yaml_with_imports = dedent('''\ 48 | imports: 49 | foo: ./ 50 | bar: ./ 51 | 52 | git module foo: 53 | url: {} 54 | rev: {} 55 | 56 | git module bar: 57 | url: {} 58 | reup: otherbranch 59 | ''').format(self.foo_dir, self.foo_master, self.bar_dir) 60 | test_dir = shared.create_dir({'peru.yaml': yaml_with_imports}) 61 | # First reup without the sync. 62 | run_peru_command(['reup', 'foo', '--no-sync'], test_dir) 63 | assert_contents(test_dir, {}, excludes=['.peru', 'peru.yaml']) 64 | # Now do it with the sync. Note that barfile wasn't pulled in, because 65 | # we didn't reup bar. 66 | run_peru_command(['reup', 'foo', '--quiet'], test_dir) 67 | assert_contents(test_dir, {'a': 'b'}, excludes=['.peru', 'peru.yaml']) 68 | 69 | def test_reup_all(self): 70 | yaml_with_imports = dedent('''\ 71 | imports: 72 | foo: ./ 73 | bar: ./ 74 | 75 | git module foo: 76 | url: {} 77 | rev: {} 78 | 79 | git module bar: 80 | url: {} 81 | reup: otherbranch 82 | ''').format(self.foo_dir, self.foo_master, self.bar_dir) 83 | test_dir = shared.create_dir({'peru.yaml': yaml_with_imports}) 84 | expected = dedent('''\ 85 | imports: 86 | foo: ./ 87 | bar: ./ 88 | 89 | git module foo: 90 | url: {} 91 | rev: {} 92 | 93 | git module bar: 94 | url: {} 95 | reup: otherbranch 96 | rev: {} 97 | ''').format(self.foo_dir, self.foo_master, self.bar_dir, 98 | self.bar_otherbranch) 99 | run_peru_command(['reup'], test_dir) 100 | # This time we finally pull in barfile. 101 | assert_contents( 102 | test_dir, { 103 | 'peru.yaml': expected, 104 | 'a': 'b', 105 | 'barfile': 'new' 106 | }, 107 | excludes=['.peru']) 108 | -------------------------------------------------------------------------------- /tests/test_rule.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | from peru import cache 4 | from peru import rule 5 | 6 | import shared 7 | from shared import COLON 8 | 9 | 10 | class RuleTest(shared.PeruTest): 11 | @shared.make_synchronous 12 | async def setUp(self): 13 | self.cache_dir = shared.create_dir() 14 | self.cache = await cache.Cache(self.cache_dir) 15 | # Include a leading colon to test that we prepend ./ to pathspecs. 16 | self.content = {'a': 'foo', 'b/c': 'bar', COLON + 'd': 'baz'} 17 | self.content_dir = shared.create_dir(self.content) 18 | self.content_tree = await self.cache.import_tree(self.content_dir) 19 | self.entries = await self.cache.ls_tree( 20 | self.content_tree, recursive=True) 21 | 22 | @shared.make_synchronous 23 | async def test_copy(self): 24 | # A file copied into a directory should be placed into that directory. 25 | # A directory or file copied into a file should overwrite that file. 26 | copies = {'a': ['x', 'b', 'b/c'], 'b': ['a', 'y']} 27 | tree = await rule.copy_files(self.cache, self.content_tree, copies) 28 | await shared.assert_tree_contents( 29 | self.cache, tree, { 30 | 'a/c': 'bar', 31 | 'b/a': 'foo', 32 | 'b/c': 'foo', 33 | 'x': 'foo', 34 | 'y/c': 'bar', 35 | COLON + 'd': 'baz', 36 | }) 37 | 38 | @shared.make_synchronous 39 | async def test_move(self): 40 | # Same semantics as copy above. Also, make sure that move deletes move 41 | # sources, but does not delete sources that were overwritten by the 42 | # target of another move. 43 | moves = {'a': 'b', 'b': 'a'} 44 | tree = await rule.move_files(self.cache, self.content_tree, moves) 45 | await shared.assert_tree_contents(self.cache, tree, { 46 | 'a/c': 'bar', 47 | 'b/a': 'foo', 48 | COLON + 'd': 'baz', 49 | }) 50 | 51 | @shared.make_synchronous 52 | async def test_drop(self): 53 | drop_dir = await rule.drop_files(self.cache, self.content_tree, ['b']) 54 | await shared.assert_tree_contents(self.cache, drop_dir, { 55 | 'a': 'foo', 56 | COLON + 'd': 'baz' 57 | }) 58 | 59 | drop_file = await rule.drop_files(self.cache, self.content_tree, ['a']) 60 | await shared.assert_tree_contents(self.cache, drop_file, { 61 | 'b/c': 'bar', 62 | COLON + 'd': 'baz' 63 | }) 64 | 65 | drop_colon = await rule.drop_files(self.cache, self.content_tree, 66 | [COLON + 'd']) 67 | await shared.assert_tree_contents(self.cache, drop_colon, { 68 | 'a': 'foo', 69 | 'b/c': 'bar' 70 | }) 71 | 72 | globs = await rule.drop_files(self.cache, self.content_tree, 73 | ['**/c', '**/a']) 74 | await shared.assert_tree_contents(self.cache, globs, 75 | {COLON + 'd': 'baz'}) 76 | 77 | @shared.make_synchronous 78 | async def test_pick(self): 79 | pick_dir = await rule.pick_files(self.cache, self.content_tree, ['b']) 80 | await shared.assert_tree_contents(self.cache, pick_dir, {'b/c': 'bar'}) 81 | 82 | pick_file = await rule.pick_files(self.cache, self.content_tree, ['a']) 83 | await shared.assert_tree_contents(self.cache, pick_file, {'a': 'foo'}) 84 | 85 | pick_colon = await rule.pick_files(self.cache, self.content_tree, 86 | [COLON + 'd']) 87 | await shared.assert_tree_contents(self.cache, pick_colon, 88 | {COLON + 'd': 'baz'}) 89 | 90 | globs = await rule.pick_files(self.cache, self.content_tree, 91 | ['**/c', '**/a']) 92 | await shared.assert_tree_contents(self.cache, globs, { 93 | 'a': 'foo', 94 | 'b/c': 'bar' 95 | }) 96 | 97 | @shared.make_synchronous 98 | async def test_executable(self): 99 | exe = await rule.make_files_executable(self.cache, self.content_tree, 100 | ['b/*']) 101 | new_content_dir = shared.create_dir() 102 | await self.cache.export_tree(exe, new_content_dir) 103 | shared.assert_contents(new_content_dir, self.content) 104 | shared.assert_not_executable(os.path.join(new_content_dir, 'a')) 105 | shared.assert_executable(os.path.join(new_content_dir, 'b/c')) 106 | 107 | @shared.make_synchronous 108 | async def test_export(self): 109 | b = await rule.get_export_tree(self.cache, self.content_tree, 'b') 110 | await shared.assert_tree_contents(self.cache, b, {'c': 'bar'}) 111 | -------------------------------------------------------------------------------- /tests/test_runtime.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | import peru.runtime as runtime 4 | 5 | import shared 6 | 7 | 8 | class RuntimeTest(shared.PeruTest): 9 | def test_find_peru_file(self): 10 | test_dir = shared.create_dir({ 11 | 'a/find_me': 'junk', 12 | 'a/b/c/junk': 'junk', 13 | }) 14 | result = runtime.find_project_file( 15 | os.path.join(test_dir, 'a', 'b', 'c'), 'find_me') 16 | expected = os.path.join(test_dir, 'a', 'find_me') 17 | self.assertEqual(expected, result) 18 | -------------------------------------------------------------------------------- /tests/test_scope.py: -------------------------------------------------------------------------------- 1 | from peru.async_helpers import run_task 2 | import peru.scope 3 | import shared 4 | 5 | 6 | class ScopeTest(shared.PeruTest): 7 | def test_parse_target(self): 8 | scope = scope_tree_to_scope({ 9 | 'modules': { 10 | 'a': { 11 | 'modules': { 12 | 'b': { 13 | 'modules': { 14 | 'c': {} 15 | }, 16 | 'rules': ['r'], 17 | } 18 | } 19 | } 20 | } 21 | }) 22 | c, (r, ) = run_task(scope.parse_target(DummyRuntime(), 'a.b.c|a.b.r')) 23 | assert type(c) is DummyModule and c.name == 'a.b.c' 24 | assert type(r) is DummyRule and r.name == 'a.b.r' 25 | 26 | 27 | def scope_tree_to_scope(tree, prefix=""): 28 | '''This function is for generating dummy scope/module/rule hierarchies for 29 | testing. A scope tree contains a modules dictionary and a rules list, both 30 | optional. The values of the modules dictionary are themselves scope trees. 31 | So if module A contains module B and rule R, that's represented as: 32 | 33 | { 34 | 'modules': { 35 | 'A': { 36 | 'modules': { 37 | 'B': {}, 38 | }, 39 | 'rules': ['R'], 40 | } 41 | } 42 | } 43 | ''' 44 | modules = {} 45 | if 'modules' in tree: 46 | for module_name, sub_tree in tree['modules'].items(): 47 | full_name = prefix + module_name 48 | new_prefix = full_name + peru.scope.SCOPE_SEPARATOR 49 | module_scope = scope_tree_to_scope(sub_tree, new_prefix) 50 | modules[module_name] = DummyModule(full_name, module_scope) 51 | rules = {} 52 | if 'rules' in tree: 53 | for rule_name in tree['rules']: 54 | full_name = prefix + rule_name 55 | rules[rule_name] = DummyRule(full_name) 56 | return peru.scope.Scope(modules, rules) 57 | 58 | 59 | class DummyModule: 60 | def __init__(self, name, scope): 61 | self.name = name 62 | self.scope = scope 63 | 64 | async def parse_peru_file(self, dummy_runtime): 65 | return self.scope, None 66 | 67 | 68 | class DummyRule: 69 | def __init__(self, name): 70 | self.name = name 71 | 72 | 73 | class DummyRuntime: 74 | pass 75 | -------------------------------------------------------------------------------- /tests/test_test_shared.py: -------------------------------------------------------------------------------- 1 | import os 2 | from pathlib import Path 3 | 4 | import shared 5 | 6 | 7 | class SharedTestCodeTest(shared.PeruTest): 8 | def test_create_dir(self): 9 | empty_dir = shared.create_dir() 10 | self.assertListEqual([], os.listdir(empty_dir)) 11 | content = {Path('foo'): 'a', Path('bar/baz'): 'b'} 12 | content_dir = shared.create_dir(content) 13 | # Don't use read_dir, because the read_dir test relies on create_dir. 14 | actual_content = {} 15 | for p in Path(content_dir).glob('**/*'): 16 | if p.is_dir(): 17 | continue 18 | with p.open() as f: 19 | actual_content[p.relative_to(content_dir)] = f.read() 20 | self.assertDictEqual(content, actual_content) 21 | 22 | def test_read_dir(self): 23 | content = {Path('foo'): 'a', Path('bar/baz'): 'b'} 24 | test_dir = shared.create_dir(content) 25 | read_content = shared.read_dir(test_dir) 26 | self.assertDictEqual(content, read_content) 27 | self.assertDictEqual({ 28 | Path('foo'): 'a' 29 | }, shared.read_dir(test_dir, excludes=['bar'])) 30 | self.assertDictEqual({ 31 | Path('foo'): 'a' 32 | }, shared.read_dir(test_dir, excludes=['bar/baz'])) 33 | 34 | def test_assert_contents(self): 35 | content = {'foo': 'a', 'bar/baz': 'b'} 36 | test_dir = shared.create_dir(content) 37 | shared.assert_contents(test_dir, content) 38 | shared.write_files(test_dir, {'bing': 'c'}) 39 | with self.assertRaises(AssertionError): 40 | shared.assert_contents(test_dir, content) 41 | shared.assert_contents(test_dir, content, excludes=['bing']) 42 | try: 43 | shared.assert_contents(test_dir, content, excludes=['foo']) 44 | except AssertionError as e: 45 | assert e.args[0].startswith('EXPECTED FILES WERE EXCLUDED') 46 | --------------------------------------------------------------------------------