├── .flake8 ├── .gitignore ├── .pylintrc ├── LICENSE.txt ├── README.md ├── mypy.ini ├── outrun.gif ├── outrun ├── __init__.py ├── __main__.py ├── args.py ├── config.py ├── constants.py ├── filesystem │ ├── __init__.py │ ├── caching │ │ ├── __init__.py │ │ ├── cache.py │ │ ├── common.py │ │ ├── filesystem.py │ │ ├── prefetching.py │ │ └── service.py │ ├── common.py │ ├── filesystem.py │ ├── fuse │ │ ├── __init__.py │ │ └── fuse.py │ └── service.py ├── logger.py ├── operations │ ├── __init__.py │ ├── common.py │ ├── environment.py │ ├── events.py │ ├── local.py │ └── remote.py ├── rpc │ └── __init__.py └── tests │ ├── __init__.py │ ├── conftest.py │ ├── test_args.py │ ├── test_config.py │ ├── test_filesystem │ ├── __init__.py │ ├── test_caching │ │ ├── __init__.py │ │ ├── test_cache.py │ │ ├── test_common.py │ │ ├── test_filesystem.py │ │ ├── test_prefetching.py │ │ └── test_service.py │ ├── test_common.py │ ├── test_filesystem.py │ ├── test_fuse │ │ ├── __init__.py │ │ └── test_operations.py │ └── test_service.py │ ├── test_logger.py │ ├── test_main.py │ ├── test_operations │ ├── __init__.py │ ├── test_environment.py │ ├── test_events.py │ ├── test_local_ops.py │ └── test_remote_ops.py │ ├── test_rpc.py │ └── vagrant │ ├── .gitignore │ ├── Vagrantfile │ ├── __init__.py │ └── test_vagrant.py └── setup.py /.flake8: -------------------------------------------------------------------------------- 1 | [flake8] 2 | # Black line length. 3 | max-line-length = 88 4 | 5 | # flake8-docstring conflicts with black in case method starts with a nested function. 6 | extend-ignore = D202 7 | 8 | application-import-names = outrun 9 | import-order-style = google 10 | 11 | per-file-ignores = 12 | # No point in documenting thin wrappers around os.* functions or RPC calls 13 | outrun/filesystem/service.py:D102 14 | outrun/filesystem/filesystem.py:D102 15 | 16 | # Ignore lack of comments in unit tests 17 | exclude = outrun/tests/* 18 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | pip-wheel-metadata/ 24 | share/python-wheels/ 25 | *.egg-info/ 26 | .installed.cfg 27 | *.egg 28 | MANIFEST 29 | 30 | # PyInstaller 31 | # Usually these files are written by a python script from a template 32 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 33 | *.manifest 34 | *.spec 35 | 36 | # Installer logs 37 | pip-log.txt 38 | pip-delete-this-directory.txt 39 | 40 | # Unit test / coverage reports 41 | htmlcov/ 42 | .tox/ 43 | .nox/ 44 | .coverage 45 | .coverage.* 46 | .cache 47 | nosetests.xml 48 | coverage.xml 49 | *.cover 50 | *.py,cover 51 | .hypothesis/ 52 | .pytest_cache/ 53 | cover/ 54 | 55 | # Translations 56 | *.mo 57 | *.pot 58 | 59 | # Django stuff: 60 | *.log 61 | local_settings.py 62 | db.sqlite3 63 | db.sqlite3-journal 64 | 65 | # Flask stuff: 66 | instance/ 67 | .webassets-cache 68 | 69 | # Scrapy stuff: 70 | .scrapy 71 | 72 | # Sphinx documentation 73 | docs/_build/ 74 | 75 | # PyBuilder 76 | .pybuilder/ 77 | target/ 78 | 79 | # Jupyter Notebook 80 | .ipynb_checkpoints 81 | 82 | # IPython 83 | profile_default/ 84 | ipython_config.py 85 | 86 | # pyenv 87 | # For a library or package, you might want to ignore these files since the code is 88 | # intended to run in multiple environments; otherwise, check them in: 89 | # .python-version 90 | 91 | # pipenv 92 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 93 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 94 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 95 | # install all needed dependencies. 96 | #Pipfile.lock 97 | 98 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow 99 | __pypackages__/ 100 | 101 | # Celery stuff 102 | celerybeat-schedule 103 | celerybeat.pid 104 | 105 | # SageMath parsed files 106 | *.sage.py 107 | 108 | # Environments 109 | .env 110 | .venv 111 | env/ 112 | venv/ 113 | ENV/ 114 | env.bak/ 115 | venv.bak/ 116 | 117 | # Spyder project settings 118 | .spyderproject 119 | .spyproject 120 | 121 | # Rope project settings 122 | .ropeproject 123 | 124 | # mkdocs documentation 125 | /site 126 | 127 | # mypy 128 | .mypy_cache/ 129 | .dmypy.json 130 | dmypy.json 131 | 132 | # Pyre type checker 133 | .pyre/ 134 | 135 | # pytype static type analyzer 136 | .pytype/ 137 | 138 | # Cython debug symbols 139 | cython_debug/ 140 | 141 | # static files generated from Django application using `collectstatic` 142 | media 143 | static 144 | 145 | # Local development 146 | Pipfile 147 | Pipfile.lock 148 | -------------------------------------------------------------------------------- /.pylintrc: -------------------------------------------------------------------------------- 1 | [MESSAGES CONTROL] 2 | 3 | disable=unsubscriptable-object,too-few-public-methods,too-many-public-methods,too-many-instance-attributes,too-many-arguments,logging-fstring-interpolation,bad-continuation,invalid-name,no-else-return,no-else-raise,no-else-break,no-else-continue,missing-function-docstring,subprocess-popen-preexec-fn,broad-except,duplicate-code 4 | 5 | generated-members=zmq.*,signal.* 6 | 7 | ignore=tests 8 | -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | Copyright 2020 Alexander Overvoorde 179 | 180 | Licensed under the Apache License, Version 2.0 (the "License"); 181 | you may not use this file except in compliance with the License. 182 | You may obtain a copy of the License at 183 | 184 | http://www.apache.org/licenses/LICENSE-2.0 185 | 186 | Unless required by applicable law or agreed to in writing, software 187 | distributed under the License is distributed on an "AS IS" BASIS, 188 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 189 | See the License for the specific language governing permissions and 190 | limitations under the License. 191 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Outrun 2 | 3 | [![](https://img.shields.io/pypi/v/outrun.svg?style=flat-square&label=latest%20stable%20version)](https://pypi.org/project/outrun/) 4 | 5 | Outrun lets you execute a local command using the processing power of another Linux machine. 6 | 7 | ![](https://github.com/Overv/outrun/raw/master/outrun.gif) 8 | 9 | * No need to first install the command on the other machine. 10 | * Reference local files and paths like you would normally. 11 | * Works across wildly different Linux distributions, like Ubuntu and Alpine. 12 | 13 | Contents 14 | 15 | * [Installation](#installation) 16 | * [Usage](#usage) 17 | * [How it works](#how-it-works) 18 | * [Limitations](#limitations) 19 | * [FAQ](#faq) 20 | * [Development](#development) 21 | * [License](#license) 22 | 23 | ## Installation 24 | 25 | Outrun requires Python 3.7 and can simply be installed using [pip](https://pip.pypa.io/en/stable/installing/): 26 | 27 | pip3 install outrun 28 | 29 | It must be installed on your own machine and any machines that you'll be using to run commands. On those other machines you must make sure to install it globally, such that it can be started with a command like `ssh user@host outrun`. 30 | 31 | In addition to outrun itself, you also have to install the [FUSE 3.x](https://github.com/libfuse/libfuse) library. Most Linux distributions include it in a package that is simply named `fuse3`. 32 | 33 | ## Usage 34 | 35 | ### Prerequisites 36 | 37 | You must have root access to the other machine, either by being able to directly SSH into it as `root` or by having `sudo` access. This is necessary because outrun uses [chroot](https://en.wikipedia.org/wiki/Chroot). 38 | 39 | Since the remote machine will have full access to your local file system as your current user, make sure to only use machines that you trust. 40 | 41 | ### Example 42 | 43 | If you would normally run: 44 | 45 | ffmpeg -i input.mp4 -vcodec libx265 -crf 28 output.mp4 46 | 47 | Then you can let outrun execute that same command using the processing power of `user@host` like this: 48 | 49 | outrun user@host ffmpeg -i input.mp4 -vcodec libx265 -crf 28 output.mp4 50 | 51 | FFMPEG does not need to be installed on the other machine and `input.mp4`/`output.mp4` will be read from and written to your local hard drive as you would expect. 52 | 53 | The first time you run a new command, it may take some time to start because its dependencies must first be copied to the other machine. This may happen even if another machine has already run FFMPEG on it before, because it likely has a slightly different version. The time this takes is highly dependent on your bandwidth and latency. 54 | 55 | ### Configuration 56 | 57 | See the output of `outrun --help` for an overview of available options to customize behaviour. The persistent cache on each remote machine can be configured by creating a `~/.outrun/config` file like this: 58 | 59 | ```ini 60 | [cache] 61 | path = ~/.outrun/cache 62 | max_entries = 1024 63 | max_size = 21474836480 # bytes -> 20 GB 64 | ``` 65 | 66 | ## How it works 67 | 68 | ### Making a command work on a different machine 69 | 70 | Let’s consider the `ffmpeg` command from the example and evaluate what’s necessary for it to run on the other machine as if it were running locally. 71 | 72 | First we need some way of running commands and observing their output on another machine in the first place, so it all starts with a normal SSH session. Try running `ssh -tt user@host htop` and you'll see that it's easy to run an interactive remote program with SSH that provides a very similar experience to a locally running program. 73 | 74 | Of course, we're not interested in running software that's installed on the *other* machine, but rather the `ffmpeg` program on our *own* machine. A straightforward way to get started would be to [scp](https://linux.die.net/man/1/scp) the `/usr/bin/ffmpeg` executable to the other machine and try to run it there. Unfortunately you'll likely find that the following will happen when you try to execute it: 75 | 76 | $ ./ffmpeg 77 | ./ffmpeg: not found 78 | 79 | That’s because the executable is [dynamically linked](https://en.wikipedia.org/wiki/Dynamic_linker) and it tries to load its library dependencies from the file system, which don’t exist on the other machine. You can use [ldd](https://linux.die.net/man/1/ldd) to see which shared libraries an ELF executable like this depends on: 80 | 81 | $ ldd `which ffmpeg` 82 | linux-vdso.so.1 (0x00007fffb7796000) 83 | libavdevice.so.58 => /usr/lib/libavdevice.so.58 (0x00007f2407f2a000) 84 | libavfilter.so.7 => /usr/lib/libavfilter.so.7 (0x00007f2407be0000) 85 | libavformat.so.58 => /usr/lib/libavformat.so.58 (0x00007f2407977000) 86 | libavcodec.so.58 => /usr/lib/libavcodec.so.58 (0x00007f2406434000) 87 | libpostproc.so.55 => /usr/lib/libpostproc.so.55 (0x00007f2406414000) 88 | libswresample.so.3 => /usr/lib/libswresample.so.3 (0x00007f24063f4000) 89 | ... 90 | 91 | We could painstakingly copy all of these libraries as well, but that wouldn’t necessarily be the end of it. Software like [Blender](https://www.blender.org/), for example, additionally loads a lot of other dependencies like Python files after it has started running. Even if we were to clutter the remote file system with all of these dependencies, we would still not be able to run the original command since `input.mp4` doesn't exist on the other machine. 92 | 93 | To handle the inevitable reality that programs may need access to any file in the local file system, outrun simply mirrors the entire file system of the local machine by mounting it over the network. This is done by mounting a FUSE file system on the remote machine that forwards all of its operations, like listing files in a directory, to an RPC service on the local machine. This RPC service simply exposes functions like `readdir()` and `stat()` to interact with the local file system. 94 | 95 | You may wonder why we’re not using an existing network file system solution like [NFS](https://en.wikipedia.org/wiki/Network_File_System) or [SSHFS](https://en.wikipedia.org/wiki/SSHFS). The trouble with these is that it’s difficult to automate quick setup for ad-hoc sessions. NFS needs to be set up with configuration files and SSHFS requires the remote machine to be able to SSH back to the local machine. In contrast, the included RPC file system is a lightweight TCP server with a per-session token that can be securely tunneled over the SSH session that we're already using. Having a custom solution also opens up opportunities for a lot of optimizations as we'll see later on. 96 | 97 | Let's say we mount the local machine's file system at `/tmp/local_fs`. We can now easily access FFMPEG at `/tmp/local_fs/usr/bin/ffmpeg` and all of its library dependencies at `/tmp/local_fs/usr/lib`. Unfortunately it doesn't seem like we've made much progress yet: 98 | 99 | $ /tmp/local_fs/usr/bin/ffmpeg 100 | /tmp/local_fs/usr/bin/ffmpeg: not found 101 | 102 | The problem is that we're still looking for its dependencies in `/usr/lib` on the remote machine. We could work around this problem by playing with environment variables like `$LD_LIBRARY_PATH` to make FFMPEG look for libraries in `/tmp/local_fs/usr/lib`, but then we're just back to solving one small problem at a time again. We wouldn't have to worry about this if we could pretend that `/tmp/local_fs` *is* the root file system so that `/usr/lib` automatically redirects to `/tmp/local_fs/usr/lib` instead. We can do exactly that using [chroot](https://en.wikipedia.org/wiki/Chroot). 103 | 104 | $ chroot /tmp/local_fs /bin/bash 105 | # ffmpeg -i input.mp4 -vcodec libx265 -crf 28 output.mp4 106 | ffmpeg version n4.2.3 Copyright (c) 2000-2020 the FFmpeg developers 107 | ... 108 | input.mp4: No such file or directory 109 | 110 | It works! Well, almost. We're still missing some of the context in which the original command was to be executed. To be able to find `input.mp4` and to store `output.mp4` in the right place, we also need to switch to the same original working directory. 111 | 112 | # cd /home/user/videos 113 | # ffmpeg -i input.mp4 -vcodec libx265 -crf 28 output.mp4 114 | ... 115 | # ls 116 | input.mp4 output.mp4 117 | 118 | While FFMPEG already works as expected, we should also bring in the right environment variables. For example, `$HOME` may affect where configuration files are loaded from and `$LANG` may affect the display language of certain programs. 119 | 120 | If you would now go back to your own machine and look at the original `/home/user/videos`, you'll see that `output.mp4` is right there as if we didn't just run FFMPEG on a completely different machine! 121 | 122 | ### Optimization 123 | 124 | While this approach works, the performance leaves a lot to be desired. If you're connecting to a server over the internet, even one in a region close to you, your latency to it will likely be at least 20 ms. This is also how long every file system operation will take and that quickly adds up. For example, on my machine FFMPEG has 110 shared libraries that need to be loaded, which means that just looking up their attributes will already take 2.2 seconds! If we had to naively do every single file system operation over the network like this then outrun wouldn't really be feasible. That's why outrun's network file system comes with two types of optimizations: caching and prefetching. 125 | 126 | Outrun persistently caches all files that are read from system directories like `/usr/bin` and `/lib`, that are known to contain applications and their dependencies. These directories are assumed to remain unchanged for the duration of an outrun session. Each new session will check if any of the cached files have changed and update them as necessary. This simple strategy is sufficient to make the same program start much faster the second time around, and often also helps with other programs since many dependencies like [glibc](https://www.gnu.org/software/libc/) are shared. An [LRU cache policy](https://en.wikipedia.org/wiki/Cache_replacement_policies#Least_recently_used_(LRU)) is used to throw out files that haven’t been used in a while. 127 | 128 | For other directories, outrun lets the kernel perform basic optimizations like reading files in large chunks and buffering small writes into one large write, much like NFS. 129 | 130 | Where outrun's file system really deviates from generic network file systems is its aggressive approach to prefetching. For example, when an executable like `/usr/bin/ffmpeg` is read, it is very likely that it will be executed and its shared libraries will be loaded next. Therefore, when the `open()` call for `/usr/bin/ffmpeg` comes in, it will not just transfer that executable in its entirety ahead of time, but also transfer all of its 110 shared libraries and associated file system metadata in a single operation. If this assumption is correct, which it generally will be, we've just reduced hundreds of `stat()`/`readlink()`/`open()`/`read()`/`close()` calls to a single RPC call with a one-time 20 ms overhead. 131 | 132 | Another example of such a prefetch is the assumption that if a `.py` file is accessed, it is likely being interpreted and Python will soon look for its compiled `.pyc` companion. Therefore that file is immediately sent along with it. If the `__pycache__` directory doesn't exist, then we simply prefetch the `ENOENT` error that would result from trying to access it instead. 133 | 134 | Since compiled files tend to compress well, all file contents that are read in their entirety are also transferred with [LZ4](https://github.com/lz4/lz4) compression to save on bandwidth. 135 | 136 | #### Documentation 137 | 138 | If you would like to read more details about outrun and its design decisions, then have a look at the source code. Each module (file system, RPC, operation orchestration) contains more specific documentation in its docstrings. 139 | 140 | ## Limitations 141 | 142 | There are a couple of things to keep in mind while using outrun: 143 | 144 | * File system performance remains a bottleneck, so the most suitable workloads are computationally constrained tasks like ray tracing and video encoding. Using outrun for something like `git status` works, but is not recommended. 145 | * Since the software to be executed is copied from your own machine to the remote machine, it must be binary compatible. It’s not possible to set up a session from an x86 machine to an ARM machine, for example. 146 | * The command will use the network and date/time of the remote machine. 147 | * If you want to access local endpoints, then they must be explicitly forwarded by using the SSH flags parameter to set up [remote forwarding](https://www.ssh.com/ssh/tunneling/example#remote-forwarding). 148 | 149 | ## FAQ 150 | 151 | * Does outrun support multiple sessions at the same time? 152 | * Yes, you can run software on many different machines simultaneously. Each remote machine can also support many different machines connecting to it at the same time with wildly different file systems. 153 | * Why was outrun written in Python? 154 | * The original prototype for outrun was written in C++, but it turned out that Python and its standard library make it much easier to ~~glue~~ orchestrate processes and operating system interactions. With regards to file system performance, it doesn’t make much of a difference since network latency is by far the greatest bottleneck. 155 | 156 | 157 | ## Development 158 | 159 | ### Static analysis and code style 160 | 161 | Outrun was written to heavily rely on [type hints](https://docs.python.org/3/library/typing.html) for documentation purposes and to allow for static analysis using [mypy](http://mypy-lang.org/). In addition to that, [flake8](https://flake8.pycqa.org/en/latest/) is used to enforce code style and [pylint](https://www.pylint.org/) is used to catch additional problems. 162 | 163 | mypy outrun 164 | flake8 165 | pylint outrun 166 | 167 | ### Testing 168 | 169 | Outrun comes with a test suite for all individual modules based on [pytest](https://docs.pytest.org/en/latest/). 170 | 171 | pytest --fuse --cov=outrun outrun/tests 172 | 173 | Since a lot of the functionality in outrun depends on OS interaction, its test suite also includes full integration tests that simulate usage with a VM connecting to another VM. These are set up using [Vagrant](https://www.vagrantup.com/) and can be run by including the `--vagrant` flag: 174 | 175 | pytest --vagrant --cov=outrun outrun/tests 176 | 177 | ## License 178 | 179 | Outrun is licensed under [version 2.0 of the Apache License](http://www.apache.org/licenses/LICENSE-2.0). 180 | -------------------------------------------------------------------------------- /mypy.ini: -------------------------------------------------------------------------------- 1 | [mypy] 2 | 3 | [mypy-semver.*] 4 | ignore_missing_imports = True 5 | 6 | [mypy-msgpack.*] 7 | ignore_missing_imports = True 8 | 9 | [mypy-zmq.*] 10 | ignore_missing_imports = True 11 | 12 | [mypy-fasteners.*] 13 | ignore_missing_imports = True 14 | 15 | [mypy-lz4.*] 16 | ignore_missing_imports = True 17 | 18 | [mypy-pytest.*] 19 | ignore_missing_imports = True 20 | 21 | [mypy-pytest_cov.*] 22 | ignore_missing_imports = True 23 | 24 | [mypy-setuptools.*] 25 | ignore_missing_imports = True 26 | 27 | [mypy-vagrant.*] 28 | ignore_missing_imports = True 29 | 30 | [mypy-outrun.filesystem.fuse.*] 31 | ignore_errors = True 32 | -------------------------------------------------------------------------------- /outrun.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Overv/outrun/20af1136060ecb0a53f464b73e5cdac913a097c3/outrun.gif -------------------------------------------------------------------------------- /outrun/__init__.py: -------------------------------------------------------------------------------- 1 | """outrun - Delegate execution of a local command to a remote machine.""" 2 | -------------------------------------------------------------------------------- /outrun/__main__.py: -------------------------------------------------------------------------------- 1 | """ 2 | Module implementing the command-line interface and invoking the main logic of outrun. 3 | 4 | outrun is started on the local machine and launches another copy of itself on the remote 5 | machine through an SSH session. The role of the local instance is to expose the local 6 | file system, context (environment variables and working directory), and command to 7 | execute. The remote instance will use this to set up a chroot environment that resembles 8 | the local environment as closely as possible and then executes the command within it. 9 | """ 10 | 11 | import logging 12 | import platform 13 | import signal 14 | import sys 15 | from typing import List, NoReturn, Optional 16 | 17 | from semver import VersionInfo 18 | 19 | import outrun.constants as constants 20 | from outrun.logger import log 21 | import outrun.operations as operations 22 | from .args import Arguments 23 | 24 | 25 | def main(arguments: Optional[List[str]] = None) -> NoReturn: 26 | """ 27 | Run either the local or remote side of outrun with the given arguments. 28 | 29 | Defaults to parsing command-line arguments from sys.argv if none are specified. 30 | """ 31 | # Parse command-line arguments. 32 | args = Arguments.parse(arguments) 33 | 34 | # Check if the local and remote instance use compatible protocols. 35 | if args.protocol.major != VersionInfo.parse(constants.PROTOCOL_VERSION).major: 36 | log.error( 37 | f"incompatible protocol ({args.protocol} != {constants.PROTOCOL_VERSION})" 38 | ) 39 | sys.exit(constants.OUTRUN_ERROR_CODE) 40 | 41 | # Check if the local and remote machines have matching platforms. 42 | if args.platform != platform.machine(): 43 | log.error(f"incompatible platform ({args.platform} != {platform.machine()}") 44 | sys.exit(constants.OUTRUN_ERROR_CODE) 45 | 46 | # Configure debug logging. 47 | if args.debug: 48 | log.setLevel(logging.DEBUG) 49 | else: 50 | log.setLevel(logging.ERROR) 51 | 52 | # Run operations of the local or remote side. 53 | ops: operations.Operations 54 | 55 | if args.remote: 56 | ops = operations.RemoteOperations(args) 57 | else: 58 | ops = operations.LocalOperations(args) 59 | 60 | try: 61 | exit_code = ops.run() 62 | except KeyboardInterrupt: 63 | exit_code = 128 + signal.SIGINT 64 | except Exception as e: 65 | log.error(f"failed to run command: {e}") 66 | exit_code = constants.OUTRUN_ERROR_CODE 67 | 68 | # Exit with either the exit code of the original command, 255 for SSH errors, or 69 | # OUTRUN_ERROR_CODE for outrun failures. 70 | sys.exit(exit_code) 71 | -------------------------------------------------------------------------------- /outrun/args.py: -------------------------------------------------------------------------------- 1 | """Module defining the command-line arguments and providing a parser for them.""" 2 | 3 | from __future__ import annotations 4 | 5 | import argparse 6 | import platform 7 | import shlex 8 | from typing import List, Optional 9 | 10 | import semver 11 | 12 | from outrun.constants import PROTOCOL_VERSION, VERSION 13 | 14 | 15 | class Arguments(argparse.Namespace): 16 | """Parsed command-line arguments.""" 17 | 18 | destination: str 19 | command: str 20 | args: List[str] 21 | 22 | extra_ssh_args: List[str] 23 | 24 | remote: bool 25 | unshare: bool 26 | 27 | protocol: semver.VersionInfo 28 | 29 | config: str 30 | 31 | environment_port: Optional[int] 32 | filesystem_port: Optional[int] 33 | cache_port: Optional[int] 34 | 35 | debug: bool 36 | cache: bool 37 | prefetch: bool 38 | writeback_cache: bool 39 | timeout: int 40 | workers: int 41 | 42 | @classmethod 43 | def parse(cls, args: Optional[List[str]] = None) -> Arguments: 44 | """ 45 | Parse command-line arguments from the given list of strings. 46 | 47 | Defaults to sys.argv if none are specified. 48 | """ 49 | return cls._get_parser().parse_args(args, namespace=cls()) 50 | 51 | @classmethod 52 | def _get_parser(cls) -> argparse.ArgumentParser: 53 | parser = argparse.ArgumentParser( 54 | description="Delegate execution of a local command to a remote machine.", 55 | usage="outrun [option...] destination command [arg...]", 56 | ) 57 | 58 | parser.add_argument( 59 | "--version", 60 | action="version", 61 | version=f"%(prog)s {VERSION} (protocol {PROTOCOL_VERSION})", 62 | help="show the program version and protocol version", 63 | ) 64 | 65 | # Primary arguments 66 | parser.add_argument("destination", type=str, help="remote host to execute on") 67 | parser.add_argument("command", type=str, help="command to execute") 68 | parser.add_argument( 69 | "args", type=str, nargs=argparse.REMAINDER, help="arguments for command" 70 | ) 71 | 72 | # Flag to pass additional options to SSH 73 | parser.add_argument( 74 | "--ssh", 75 | type=cls._parse_extra_args, 76 | help="additional arguments to pass to SSH", 77 | dest="extra_ssh_args", 78 | default=[], 79 | ) 80 | 81 | # Hidden flag to indicate that this is the outrun process on the remote side 82 | parser.add_argument("--remote", action="store_true", help=argparse.SUPPRESS) 83 | 84 | # Hidden flag to indicate that process has yet to be unshared 85 | parser.add_argument("--unshare", action="store_true", help=argparse.SUPPRESS) 86 | 87 | # Hidden flag to indicate expected protocol version 88 | parser.add_argument( 89 | "--protocol", 90 | type=cls._parse_version, 91 | default=semver.VersionInfo.parse(PROTOCOL_VERSION), 92 | help=argparse.SUPPRESS, 93 | ) 94 | 95 | # Hidden flag to indicate expected platform 96 | parser.add_argument( 97 | "--platform", type=str, default=platform.machine(), help=argparse.SUPPRESS, 98 | ) 99 | 100 | # Path to (optional) config file on remote 101 | parser.add_argument( 102 | "--config", 103 | type=str, 104 | help="path to config file (default is ~/.outrun/config)", 105 | default="~/.outrun/config", 106 | ) 107 | 108 | # Communication ports, default to a random choice 109 | parser.add_argument( 110 | "--environment-port", type=int, help="port to use for environment service" 111 | ) 112 | parser.add_argument( 113 | "--filesystem-port", type=int, help="port to use for file system service" 114 | ) 115 | parser.add_argument( 116 | "--cache-port", type=int, help="port to use for cache service" 117 | ) 118 | 119 | # Enable debug output for development 120 | parser.add_argument( 121 | "--debug", action="store_true", help="enable debug information" 122 | ) 123 | 124 | # Disable file system caching 125 | parser.add_argument( 126 | "--no-cache", 127 | action="store_false", 128 | help="disable file system caching", 129 | dest="cache", 130 | ) 131 | 132 | # Disable file system prefetching 133 | parser.add_argument( 134 | "--no-prefetch", 135 | action="store_false", 136 | help="disable file system prefetching", 137 | dest="prefetch", 138 | ) 139 | 140 | # Disable write-back cache on the remote 141 | parser.add_argument( 142 | "--sync-writes", 143 | action="store_false", 144 | help="disable write-back cache", 145 | dest="writeback_cache", 146 | ) 147 | 148 | # Configure network timeout 149 | parser.add_argument( 150 | "--timeout", 151 | type=cls._parse_timeout, 152 | help="timeout for network communications in milliseconds", 153 | default=5000, 154 | ) 155 | 156 | # Configure number of file system workers 157 | parser.add_argument( 158 | "--workers", type=int, help="number of local file system workers", default=4 159 | ) 160 | 161 | return parser 162 | 163 | @staticmethod 164 | def _parse_extra_args(arg: str) -> List[str]: 165 | return shlex.split(arg) 166 | 167 | @staticmethod 168 | def _parse_version(arg: str) -> semver.VersionInfo: 169 | try: 170 | return semver.VersionInfo.parse(arg) 171 | except (ValueError, TypeError): 172 | raise argparse.ArgumentTypeError("expected semantic version string") 173 | 174 | @staticmethod 175 | def _parse_timeout(arg: str) -> int: 176 | try: 177 | val = int(arg) 178 | assert val > 0 179 | return val 180 | except (ValueError, AssertionError): 181 | raise argparse.ArgumentTypeError("expected number > 0") 182 | -------------------------------------------------------------------------------- /outrun/config.py: -------------------------------------------------------------------------------- 1 | """Module for configuration variables with defaults that are overridable by a file.""" 2 | 3 | from __future__ import annotations 4 | 5 | from configparser import ConfigParser, SectionProxy 6 | from dataclasses import dataclass, field 7 | import os 8 | 9 | from outrun.logger import log 10 | 11 | 12 | @dataclass 13 | class CacheConfig: 14 | """Configuration variables related to file system caching.""" 15 | 16 | path: str = os.path.expanduser("~/.outrun/cache") 17 | 18 | max_entries: int = 1024 19 | max_size: int = 20 * 1024 * 1024 * 1024 # 20 GB 20 | 21 | @staticmethod 22 | def load(section: SectionProxy) -> CacheConfig: 23 | """Load overridden variables from a section within a config file.""" 24 | config = CacheConfig() 25 | 26 | config.path = os.path.expanduser(section.get("path", fallback=config.path)) 27 | 28 | config.max_entries = section.getint("max_entries", fallback=config.max_entries) 29 | config.max_size = section.getint("max_size", fallback=config.max_size) 30 | 31 | return config 32 | 33 | 34 | @dataclass 35 | class Config: 36 | """Configuration variables.""" 37 | 38 | cache: CacheConfig = field(default_factory=CacheConfig) 39 | 40 | @staticmethod 41 | def load(filename: str) -> Config: 42 | """Load overridden configuration variables from a config file.""" 43 | parser = ConfigParser() 44 | 45 | config = Config() 46 | 47 | try: 48 | with open(filename, "r") as f: 49 | parser.read_string(f.read(), filename) 50 | 51 | if "cache" in parser: 52 | config.cache = CacheConfig.load(parser["cache"]) 53 | except FileNotFoundError: 54 | log.info(f"no config file at {filename}") 55 | except Exception as e: 56 | # An unreadable config file is not considered a fatal error since we can 57 | # fall back to defaults. 58 | log.error(f"failed to read config file {filename}: {e}") 59 | else: 60 | log.info(f"loaded config: {config}") 61 | 62 | return config 63 | -------------------------------------------------------------------------------- /outrun/constants.py: -------------------------------------------------------------------------------- 1 | """Module defining various global constants.""" 2 | 3 | # outrun version 4 | VERSION = "1.0.0" 5 | 6 | # outrun protocol 7 | # The major version must be identical on local and remote. 8 | # 9 | # Note that many changes, like improved prefetching rules, can be implemented without 10 | # having to change the protocol version. 11 | PROTOCOL_VERSION = "1.0.0" 12 | 13 | # Special exit code for when outrun itself fails. 14 | OUTRUN_ERROR_CODE = 254 15 | 16 | # Application ID used to derive an app-specific machine identifier from /etc/machine-id. 17 | # See http://man7.org/linux/man-pages/man3/sd_id128_get_machine_app_specific.3.html. 18 | APP_ID = b"a4313318cef44e0ca7ecdca13fdc417a" 19 | 20 | # Name of the FUSE file system 21 | FILESYSTEM_NAME = "outrunfs" 22 | -------------------------------------------------------------------------------- /outrun/filesystem/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | Modules that take care of mirroring the local file system on the remote machine. 3 | 4 | Outrun comes with its own network file system to expose the local machine's file system 5 | on the remote machine. The local machine runs an RPC service that allows for calling 6 | common I/O functions like open(), stat(), and readdir(). The remote machine mounts a 7 | FUSE file system that mostly simply forwards all of its calls to that RPC service. 8 | 9 | So, why not use an existing solution like NFS or SSHFS? While these are very capable 10 | generic network file system solutions, they are unsuitable for use in outrun for a 11 | couple of reasons: 12 | 13 | * NFS 14 | * Designed for local networks and isn't easy to tunnel over SSH. 15 | * Designed for long term mounts rather than quick set up and tear down of sessions. 16 | * SSHFS 17 | * While it is designed to easily and quickly set up sessions, it would require the 18 | remote machine to be able to SSH back to the local machine. This would require 19 | complex solutions like having outrun host its own SSH server and tunneling that over 20 | the existing SSH session. 21 | 22 | The file system in this module tackles all of these issues by being very easy to tunnel 23 | (just one TCP port), all implemented in user mode (very easy to start and shutdown), and 24 | simple per-session tokens for authentication. 25 | 26 | Once a network file system is up and running, the next most important concern is 27 | performance. Outrun is designed to mount a file system over the internet and latency is 28 | significant bottleneck there for all I/O calls. A lot of this latency can be avoided 29 | using clever caching and prefetching and this is another key advantage of having a 30 | custom implementation. Outrun's file system is purpose built for running programs off of 31 | it and this enables it to have a caching layer on top of the base file system that can 32 | make a lot of assumptions about I/O operations. All of this logic is implemented in the 33 | 'caching' submodule. 34 | """ 35 | 36 | from .filesystem import RemoteFileSystem 37 | from .service import LocalFileSystemService 38 | 39 | __all__ = [ 40 | "RemoteFileSystem", 41 | "LocalFileSystemService", 42 | ] 43 | -------------------------------------------------------------------------------- /outrun/filesystem/caching/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | Modules that extend the mirrored file system with caching and prefetching logic. 3 | 4 | The network file system is the primary performance bottleneck of outrun, primarily 5 | because every I/O operation incurs significant latency overhead. Therefore this module 6 | aims to extend the file system with a caching and prefetching layer that significantly 7 | reduces the total number of I/O calls. 8 | 9 | It's based on the idea that a lot of the I/O calls are the result of the application and 10 | its dependencies being loaded from directories like /lib and /usr/bin. We can optimize 11 | the access of these directories in two dictinct ways: caching and prefetching. 12 | 13 | We consider these directories to be read-only and unchanging for the duration of an 14 | outrun session, which is a relatively safe assumptions because they are generally only 15 | modified because of system updates. Therefore all file system metadata and contents that 16 | are read from these directories are persistently cached on the remote machine. When a 17 | new session starts it will verify the freshness of the cache by doing a metadata check 18 | with the local machine. 19 | 20 | While caching makes subsequent runs of a program much faster, it doesn't do much more 21 | the initial startup time, and that is where prefetching comes into play. We assume that 22 | bandwidth is much cheaper than latency. In other words, we'd rather do a single I/O call 23 | that pulls in a bit too much data that we don't use, than make a lot of specific I/O 24 | calls that just pull in the data we need. 25 | 26 | For example, when a file is opened for reading from the aforementioned directories, we 27 | assume that all of its contents will eventually be read and transfer its entire 28 | compressed contents over the network immediately. This may waste a bit of bandwidth for 29 | the parts that aren't accessed (e.g. code in a shared library that isn't used), but it 30 | significantly reduces the number of read() calls. A more complex example is the opening 31 | of an ELF binary, where outrun will scan its shared library dependencies using ldd and 32 | will send those along with the binary, all in a single request. This massively reduces 33 | latency when running programs like ffmpeg and blender. 34 | 35 | The prefetching is implemented as a push model where the local machine decides upon an 36 | I/O call which additional data to push that it may think the remote machine will ask for 37 | next. This allows for making a lot of prefetching decisions without additional round 38 | trips. 39 | 40 | The optimizations are implemented by hosting an additional RPC service alongside the 41 | existing file system RPC service, and extending the FUSE file system to make use of 42 | these caching and prefetching operations. 43 | """ 44 | 45 | from .filesystem import RemoteCachedFileSystem 46 | from .service import LocalCacheService 47 | 48 | __all__ = [ 49 | "RemoteCachedFileSystem", 50 | "LocalCacheService", 51 | ] 52 | -------------------------------------------------------------------------------- /outrun/filesystem/caching/common.py: -------------------------------------------------------------------------------- 1 | """Data structures used by multiple file system caching and prefetching components.""" 2 | 3 | from __future__ import annotations 4 | 5 | import collections 6 | from contextlib import contextmanager 7 | from dataclasses import dataclass 8 | import hashlib 9 | import threading 10 | from typing import Any, Dict, Iterator, Optional 11 | 12 | import lz4.frame 13 | 14 | from outrun.filesystem.common import Attributes 15 | 16 | 17 | @dataclass 18 | class Metadata: 19 | """ 20 | Container of metadata related to accessing file system entries. 21 | 22 | Used to store all metadata required to answer calls like access() and readlink(). It 23 | may alternatively contain the I/O error that should be returned upon these calls. 24 | """ 25 | 26 | attr: Optional[Attributes] = None 27 | link: Optional[str] = None 28 | error: Optional[Exception] = None 29 | 30 | 31 | @dataclass 32 | class FileContents: 33 | """ 34 | Container for the full contents of a file. 35 | 36 | File contents are compressed to reduce bandwidth usage, which speeds up file access 37 | under the assumption that the entire file is read. LZ4 was chosen because it's fast 38 | enough to deliver a good trade-off between bandwidth reduction and 39 | compression/decompression latency. 40 | """ 41 | 42 | compressed_data: bytes 43 | checksum: str 44 | size: int 45 | 46 | @staticmethod 47 | def from_data(data: bytes) -> FileContents: 48 | """Wrap raw file data into a FileContents object.""" 49 | return FileContents( 50 | compressed_data=lz4.frame.compress(data), 51 | checksum=hashlib.sha256(data).hexdigest(), 52 | size=len(data), 53 | ) 54 | 55 | @property 56 | def data(self) -> bytes: 57 | """Retrieve and decompress the original file data.""" 58 | return lz4.frame.decompress(self.compressed_data) 59 | 60 | 61 | @dataclass 62 | class PrefetchEntry: 63 | """ 64 | Container for a prefetched file system entry. 65 | 66 | Used by the caching and prefetching RPC service to push additional entries back to 67 | the remote machine. 68 | """ 69 | 70 | path: str 71 | metadata: Metadata 72 | contents: Optional[FileContents] 73 | 74 | 75 | class LockIndex: 76 | """ 77 | Collection of mutexes to lock critical sections by arbitrary values. 78 | 79 | Its use case is to lock critical sections based on unpredictable input values, like 80 | arbitrary file descriptors. Locks are automatically garbage collected when no longer 81 | in use (no threads in the critical section and none waiting to enter). 82 | """ 83 | 84 | def __init__(self) -> None: 85 | """Instantiate a LockIndex.""" 86 | self._global_lock = threading.Lock() 87 | 88 | self._locks: Dict[Any, threading.Lock] = collections.defaultdict(threading.Lock) 89 | self._lock_users: Dict[Any, int] = collections.defaultdict(int) 90 | 91 | @contextmanager 92 | def lock(self, key: Any, blocking=True) -> Iterator[bool]: 93 | """Lock a critical section based on the specified key.""" 94 | # Retrieve lock and increment user count 95 | with self._global_lock: 96 | self._lock_users[key] += 1 97 | lock = self._locks[key] 98 | 99 | acquired = lock.acquire(blocking) 100 | 101 | try: 102 | yield acquired 103 | finally: 104 | if acquired: 105 | lock.release() 106 | 107 | # Decrement user count and delete lock if there are none left 108 | with self._global_lock: 109 | self._lock_users[key] -= 1 110 | 111 | if self._lock_users[key] == 0: 112 | del self._lock_users[key] 113 | del self._locks[key] 114 | 115 | @property 116 | def lock_count(self): 117 | """Return the number of locks currently in use.""" 118 | with self._global_lock: 119 | return len(self._locks) 120 | -------------------------------------------------------------------------------- /outrun/filesystem/caching/filesystem.py: -------------------------------------------------------------------------------- 1 | """ 2 | Module that extends the FUSE file system to use caching for file metadata and contents. 3 | 4 | It is designed to be used in conjunction with caching.LocalFileSystemService. 5 | """ 6 | 7 | from __future__ import annotations 8 | 9 | import errno 10 | import os 11 | from typing import Callable, Dict, Optional 12 | 13 | import outrun.filesystem as filesystem 14 | from outrun.filesystem.caching.cache import RemoteCache 15 | import outrun.rpc as rpc 16 | 17 | 18 | class RemoteCachedFileSystem(filesystem.RemoteFileSystem): 19 | """ 20 | Extension of RemoteFileSystem that selectively caches file system operations. 21 | 22 | Caching and prefetching are only used for directories that are considered safe to 23 | cache. A directory is safe to cache if it contains files that change only relatively 24 | infrequently, meaning that they can be assumed to be read-only during an outrun 25 | session. 26 | 27 | Good examples are /usr/lib and /usr/bin that contain large application dependencies 28 | like executables and shared libraries, and will generally only change as a result of 29 | system/package updates. 30 | 31 | Some calls like readdir() are not cached at all because they are infrequently used 32 | in practice. 33 | """ 34 | 35 | def __init__( 36 | self, 37 | client: rpc.Client, 38 | mount_callback: Optional[Callable], 39 | cache: RemoteCache, 40 | ) -> None: 41 | """Instantiate cached file system with its cache.""" 42 | super().__init__(client, mount_callback) 43 | 44 | self._cache = cache 45 | 46 | def destroy(self) -> None: 47 | """Persist the cache to disk as part of unmounting the file system.""" 48 | self._cache.save() 49 | 50 | def getattr(self, path: str, fh: Optional[int]) -> Dict: 51 | """Retrieve (cached) file system entry attributes.""" 52 | if not self._cache.is_cacheable(path): 53 | return super().getattr(path, fh) 54 | 55 | return self._cache.get_metadata(path).attr.__dict__ 56 | 57 | def readlink(self, path: str) -> str: 58 | """Read the (cached) path referenced by a symlink.""" 59 | if not self._cache.is_cacheable(path): 60 | return super().readlink(path) 61 | 62 | link = self._cache.get_metadata(path).link 63 | 64 | # Path does not point to a symlink 65 | if not link: 66 | raise OSError(errno.EINVAL) 67 | 68 | return link 69 | 70 | def open(self, path, flags) -> int: 71 | """ 72 | Open a (cached) file for reading or writing. 73 | 74 | Cached files can only be opened for reading. 75 | """ 76 | if not self._cache.is_cacheable(path): 77 | return super().open(path, flags) 78 | 79 | return self._cache.open_contents(path, flags) 80 | 81 | def read(self, path: str, fh: int, offset: int, size: int) -> bytes: 82 | """Read a chunk of bytes from a (cached) file.""" 83 | if not self._cache.is_cacheable(path): 84 | return super().read(path, fh, offset, size) 85 | 86 | return os.pread(fh, size, offset) 87 | 88 | def release(self, path: str, fh: int) -> None: 89 | """Close a (cached) file.""" 90 | if not self._cache.is_cacheable(path): 91 | super().release(path, fh) 92 | return 93 | 94 | os.close(fh) 95 | 96 | def flush(self, path: str, fh: int) -> None: 97 | """ 98 | Flush changes to a file to disk. 99 | 100 | This is a no-op if the file descriptor refers to a cached file. (It is still 101 | called by FUSE for read-only files in an attempt to update metadata.) 102 | """ 103 | if not self._cache.is_cacheable(path): 104 | super().flush(path, fh) 105 | -------------------------------------------------------------------------------- /outrun/filesystem/caching/prefetching.py: -------------------------------------------------------------------------------- 1 | """ 2 | Module that implements prefetching heuristics for the caching file system. 3 | 4 | The idea of prefetching is to send metadata and file contents for caching if we suspect 5 | that they'll be accessed in the future based on current file system operations. This is 6 | an extension of the cache's assumption that bandwidth is cheap and latency is expensive. 7 | We try to reduce the number of individual RPC calls as much as possible by predicting 8 | which data will be required next and sending it in bulk. 9 | 10 | One of the simplest examples is readlink() on a symlink that will very likely lead to a 11 | stat() for the entry targeted by that symlink. A slightly more complex example is the 12 | reading of an ELF binary that will likely lead to its shared library dependencies being 13 | accessed and read shortly afterwards. 14 | 15 | Each prefetch rule is either based on the event of a file being accessed 16 | (e.g. stat/readlink) or a file being read (open). These can be triggered by calling the 17 | file_access and file_read functions, respectively. Each rule returns suggestions of data 18 | to be prefetched where a suggestion is comprised of a path and the decision if only its 19 | metadata should be loaded or if its contents should be included as well. 20 | 21 | Rules are currently intentionally not recursive (e.g. a symlink with another symlink as 22 | target doesn't result in a chain of suggestions) to stop the prefetching from 23 | extrapolating too much. 24 | """ 25 | 26 | from dataclasses import dataclass 27 | import glob 28 | import os 29 | import re 30 | import subprocess 31 | from typing import List 32 | 33 | from outrun.logger import log 34 | 35 | 36 | @dataclass 37 | class PrefetchSuggestion: 38 | """A suggestion to prefetch the specified path's metadata and maybe its contents.""" 39 | 40 | path: str 41 | contents: bool 42 | 43 | 44 | def file_access(path: str) -> List[PrefetchSuggestion]: 45 | """Prefetch data based on the fact that the specified file is accessed.""" 46 | suggestions = [] 47 | 48 | # Prefetch the target when a symlink is accessed. 49 | suggestions += symlink_target(path) 50 | 51 | # Prefetch __pycache__ when a Python source file is accessed. 52 | suggestions += python_pycache(path) 53 | 54 | # Prefetch Perl module when its compiled version is accessed. 55 | suggestions += compiled_perl_module(path) 56 | 57 | return suggestions 58 | 59 | 60 | def file_read(path: str) -> List[PrefetchSuggestion]: 61 | """Prefetch data based on the fact that the specified file is read.""" 62 | suggestions = [] 63 | 64 | # Prefetch shared libraries when ELF executables are opened. 65 | suggestions += elf_dependencies(path) 66 | 67 | return suggestions 68 | 69 | 70 | def symlink_target(path: str) -> List[PrefetchSuggestion]: 71 | """ 72 | Prefetch the entry that a symlink points to. 73 | 74 | This rule is based on the idea that it is likely for the target of a symlink to be 75 | looked up after accessing the symlink itself. 76 | """ 77 | prefetches = [] 78 | 79 | if os.path.islink(path): 80 | link_path = os.path.normpath(os.path.join(path, "..", os.readlink(path))) 81 | prefetches.append(PrefetchSuggestion(path=link_path, contents=False)) 82 | 83 | return prefetches 84 | 85 | 86 | def python_pycache(path: str) -> List[PrefetchSuggestion]: 87 | """ 88 | Prefetch the associated __pycache__ file(s) when accessing a .py file. 89 | 90 | This rule is based on the way CPython looks for previously compiled bytecode when 91 | accessing a Python source file. 92 | """ 93 | prefetches = [] 94 | 95 | if path.endswith(".py") and os.path.isfile(path): 96 | # Prefetch Python source file itself. 97 | prefetches.append(PrefetchSuggestion(path=path, contents=True)) 98 | 99 | # Prefetch __pycache__ directory itself. 100 | pycache_path = os.path.normpath(os.path.join(path, "..", "__pycache__")) 101 | prefetches.append(PrefetchSuggestion(path=pycache_path, contents=False)) 102 | 103 | # Look for .pyc files matching the .py's filename and fully prefetch them. 104 | pyc_pattern = os.path.basename(path).replace(".py", "") + "*" 105 | cache_files = glob.glob(pycache_path + f"/{pyc_pattern}") 106 | 107 | for full_path in cache_files: 108 | prefetches.append(PrefetchSuggestion(path=full_path, contents=True)) 109 | 110 | return prefetches 111 | 112 | 113 | def compiled_perl_module(path: str) -> List[PrefetchSuggestion]: 114 | """ 115 | Prefetch .pm file when its compiled .pmc associate is accessed. 116 | 117 | Note that the .pmc file doesn't necessarily need to exist. This rule is based on 118 | observing Perl behaviour while running "cowsay". 119 | """ 120 | prefetches = [] 121 | 122 | if path.endswith(".pmc"): 123 | module_path = path.replace(".pmc", ".pm") 124 | prefetches.append(PrefetchSuggestion(path=module_path, contents=True)) 125 | 126 | return prefetches 127 | 128 | 129 | def elf_dependencies(path: str) -> List[PrefetchSuggestion]: 130 | """ 131 | Prefetch shared libraries dependencies of an ELF binary. 132 | 133 | This rule is based on the assumption that if an ELF binary is read then that's 134 | likely because it's being executed and its dependencies will soon be loaded as well. 135 | """ 136 | prefetches = [] 137 | 138 | if is_elf_binary(path): 139 | try: 140 | dependencies = read_elf_dependencies(path) 141 | except Exception as e: 142 | log.warning(f"failed to read elf dependencies of {path}: {e}") 143 | dependencies = [] 144 | 145 | # Dependencies may be symlinks, so prefetch those. 146 | prefetches += [ 147 | PrefetchSuggestion(path=dep, contents=False) for dep in dependencies 148 | ] 149 | 150 | # Prefetch contents of the final shared libraries. 151 | prefetches += [ 152 | PrefetchSuggestion(path=os.path.realpath(dep), contents=True) 153 | for dep in dependencies 154 | ] 155 | 156 | return prefetches 157 | 158 | 159 | def is_elf_binary(path: str) -> bool: 160 | """Check if the specified file is an ELF binary.""" 161 | try: 162 | output = subprocess.check_output(["file", path]).decode() 163 | return "ELF" in output 164 | except Exception as e: 165 | log.warning(f"failed to check if {path} is elf binary: {e}") 166 | return False 167 | 168 | 169 | def read_elf_dependencies(path: str) -> List[str]: 170 | """ 171 | Retrieve the shared library dependencies of an ELF binary. 172 | 173 | Dependencies with spaces in the name are ignored because they cannot easily be 174 | extracted from ldd output, especially when you consider that the name itself may 175 | include sequences like '=>'. 176 | """ 177 | output = subprocess.check_output(["ldd", path], stderr=subprocess.DEVNULL).decode() 178 | 179 | dependencies = [] 180 | 181 | for line in output.splitlines(): 182 | match = re.search(r"^[^ ]+ => ([^ ]+) \([0-9a-fx]+\)$", line.lstrip()) 183 | 184 | if match: 185 | dependencies.append(match.group(1)) 186 | 187 | return dependencies 188 | -------------------------------------------------------------------------------- /outrun/filesystem/caching/service.py: -------------------------------------------------------------------------------- 1 | """ 2 | Module with an RPC service that facilitates file system caching and prefetching. 3 | 4 | It is designed to be used in conjunction with caching.RemoteCache. 5 | """ 6 | 7 | import collections 8 | from dataclasses import replace 9 | import hashlib 10 | import os 11 | import stat 12 | import threading 13 | from typing import Any, Dict, List, Optional, Set, Tuple 14 | 15 | import outrun.constants as constants 16 | from outrun.filesystem.caching.cache import FileContents, Metadata, PrefetchEntry 17 | import outrun.filesystem.caching.prefetching as prefetching 18 | from outrun.filesystem.common import Attributes 19 | from outrun.logger import log 20 | 21 | 22 | class LocalCacheService: 23 | """RPC service that provides bulk I/O calls for caching and prefetching.""" 24 | 25 | def __init__(self): 26 | """Instantiate new caching and prefetching RPC service.""" 27 | super().__init__() 28 | 29 | self._fetched_lock = threading.Lock() 30 | self._fetched_metadata: Set[str] = set() 31 | self._fetched_contents: Set[str] = set() 32 | 33 | self._prefetchable_paths: Optional[List[str]] = None 34 | 35 | def get_metadata(self, path: str) -> Metadata: 36 | """Retrieve all metadata of a file system entry, or the resulting I/O error.""" 37 | metadata = Metadata() 38 | 39 | try: 40 | metadata.attr = Attributes.from_stat(os.lstat(path)) 41 | 42 | if stat.S_ISLNK(metadata.attr.st_mode): 43 | metadata.link = os.readlink(path) 44 | except Exception as e: 45 | metadata.error = e 46 | 47 | with self._fetched_lock: 48 | self._fetched_metadata.add(path) 49 | 50 | return metadata 51 | 52 | @staticmethod 53 | def _significant_meta(meta: Metadata) -> Any: 54 | """ 55 | Turn metadata into a derivative to check if it has changed significantly. 56 | 57 | Significance means that it warrants sending the change to the remote machine to 58 | update its cache. For example, a changed file access time is not very important 59 | in the grand scheme of things. 60 | """ 61 | significant_attribs = ( 62 | replace(meta.attr, st_atime_ns=None) if meta.attr else None 63 | ) 64 | comparable_error = (type(meta.error), meta.error.args) if meta.error else None 65 | 66 | return (significant_attribs, comparable_error, meta.link) 67 | 68 | def get_changed_metadata( 69 | self, cached_metadata: Dict[str, Metadata] 70 | ) -> Dict[str, Metadata]: 71 | """Retrieve file system metadata that has changed since it was last cached.""" 72 | changed_metadata = {} 73 | 74 | for path, metadata in cached_metadata.items(): 75 | new_metadata = self.get_metadata(path) 76 | 77 | if self._significant_meta(new_metadata) != self._significant_meta(metadata): 78 | changed_metadata[path] = new_metadata 79 | 80 | return changed_metadata 81 | 82 | def readfile(self, path: str) -> FileContents: 83 | """Read the contents of the specified file.""" 84 | with open(path, "rb") as f: 85 | data = f.read() 86 | 87 | with self._fetched_lock: 88 | self._fetched_contents.add(path) 89 | 90 | return FileContents.from_data(data) 91 | 92 | def readfile_conditional(self, path: str, checksum: str) -> Optional[FileContents]: 93 | """Read the contents of the specified file if the checksum has changed.""" 94 | new_contents = self.readfile(path) 95 | 96 | if checksum != new_contents.checksum: 97 | return new_contents 98 | else: 99 | return None 100 | 101 | @staticmethod 102 | def get_app_specific_machine_id() -> str: 103 | """Derive a unique machine/installation identifier.""" 104 | with open("/etc/machine-id", "rb") as f: 105 | confidential_id = f.read().strip() 106 | 107 | # http://man7.org/linux/man-pages/man5/machine-id.5.html 108 | # Based on sd_id128_get_machine_app_specific implementation 109 | return hashlib.sha256(confidential_id + constants.APP_ID).hexdigest()[:32] 110 | 111 | def mark_previously_fetched_contents(self, paths: List[str]) -> None: 112 | """ 113 | Mark contents of specified paths as having already been fetched. 114 | 115 | This is used by the remote to indicate that the specified files should not be 116 | prefetched as their contents are already in the remote cache. Previously fetched 117 | metadata is already marked separately by get_changed_metadata(). 118 | """ 119 | with self._fetched_lock: 120 | self._fetched_contents.update(paths) 121 | 122 | def set_prefetchable_paths(self, paths: Optional[List[str]]) -> None: 123 | """ 124 | Set the base paths that may be prefetched. 125 | 126 | If this function is not called or paths is set to None, then all paths may be 127 | prefetched. 128 | """ 129 | self._prefetchable_paths = paths 130 | 131 | def _is_prefetchable(self, path: str) -> bool: 132 | """Return whether the specified path is prefetchable.""" 133 | if self._prefetchable_paths is None: 134 | return True 135 | else: 136 | return any( 137 | os.path.commonpath([path, p]) == p for p in self._prefetchable_paths 138 | ) 139 | 140 | def get_metadata_prefetch(self, path: str) -> Tuple[Metadata, List[PrefetchEntry]]: 141 | """Retrieve metadata of an entry and prefetch related data.""" 142 | base = self.get_metadata(path) 143 | 144 | try: 145 | suggestions = prefetching.file_access(path) 146 | return base, self._resolve_prefetches(suggestions) 147 | except Exception as e: 148 | # Avoid complete I/O failure if prefetching breaks 149 | log.warning(f"prefetching for get_metadata({path}) failed: {e}") 150 | return base, [] 151 | 152 | def readfile_prefetch(self, path: str) -> Tuple[FileContents, List[PrefetchEntry]]: 153 | """Retrieve file contents and prefetch related data.""" 154 | base = self.readfile(path) 155 | 156 | try: 157 | suggestions = prefetching.file_read(path) 158 | return base, self._resolve_prefetches(suggestions) 159 | except Exception as e: 160 | # Avoid complete I/O failure if prefetching breaks 161 | log.warning(f"prefetching for readfile({path}) failed: {e}") 162 | return base, [] 163 | 164 | def _resolve_prefetches( 165 | self, suggestions: List[prefetching.PrefetchSuggestion] 166 | ) -> List[PrefetchEntry]: 167 | # Group prefetch suggestions by path 168 | suggestions_by_path = collections.defaultdict(list) 169 | 170 | for suggestion in suggestions: 171 | suggestions_by_path[suggestion.path].append(suggestion) 172 | 173 | # Resolve suggestions into actual prefetches 174 | prefetches: List[PrefetchEntry] = [] 175 | 176 | for path, path_suggestions in suggestions_by_path.items(): 177 | # Don't prefetch things outside the prefetchable paths 178 | if not self._is_prefetchable(path): 179 | continue 180 | 181 | prefetch_contents = any(s.contents for s in path_suggestions) 182 | 183 | # Don't prefetch contents that have already been fetched, or metadata that 184 | # has already been fetched. 185 | with self._fetched_lock: 186 | if prefetch_contents and path in self._fetched_contents: 187 | continue 188 | elif not prefetch_contents and path in self._fetched_metadata: 189 | continue 190 | 191 | entry = PrefetchEntry( 192 | path=path, metadata=self.get_metadata(path), contents=None 193 | ) 194 | 195 | # Try to prefetch contents if requested and available 196 | if prefetch_contents and not entry.metadata.error: 197 | try: 198 | entry.contents = self.readfile(path) 199 | except Exception as e: 200 | log.warning(f"failed to prefetch contents of {path}: {e}") 201 | 202 | prefetches.append(entry) 203 | 204 | return prefetches 205 | -------------------------------------------------------------------------------- /outrun/filesystem/common.py: -------------------------------------------------------------------------------- 1 | """Data structures used by multiple file system components.""" 2 | 3 | from __future__ import annotations 4 | 5 | import dataclasses 6 | from dataclasses import dataclass 7 | import os 8 | import stat 9 | from typing import Union 10 | 11 | 12 | @dataclass 13 | class Attributes: 14 | """Container of file system attributes (basically os.stat_result as a dataclass).""" 15 | 16 | st_mode: int 17 | st_ino: int 18 | st_dev: int 19 | st_nlink: int 20 | st_uid: int 21 | st_gid: int 22 | st_size: int 23 | st_atime_ns: int 24 | st_mtime_ns: int 25 | st_ctime_ns: int 26 | 27 | def __init__(self, **attribs: Union[int, float]) -> None: 28 | """ 29 | Instantiate with the specified file system attributes. 30 | 31 | You must specify the attributes declared in this class, but you may include any 32 | number of extra attributes, like st_atime_ns. 33 | """ 34 | for name, value in attribs.items(): 35 | setattr(self, name, value) 36 | 37 | @staticmethod 38 | def from_stat(st: os.stat_result) -> Attributes: 39 | """Instantiate from the attributes contained within an os.stat_result object.""" 40 | st_dict = {k: getattr(st, k) for k in dir(st) if k.startswith("st_")} 41 | return Attributes(**st_dict) 42 | 43 | def as_readonly(self) -> Attributes: 44 | """Copy the attributes with write permissions removed from the mode.""" 45 | readonly_mode = self.st_mode & ~(stat.S_IWUSR | stat.S_IWGRP | stat.S_IWOTH) 46 | 47 | return dataclasses.replace(self, st_mode=readonly_mode) 48 | -------------------------------------------------------------------------------- /outrun/filesystem/filesystem.py: -------------------------------------------------------------------------------- 1 | """Module that contains the remote file system that simply forwards all calls.""" 2 | 3 | from typing import Callable, List, Optional, Tuple 4 | 5 | from outrun.filesystem.fuse import Operations 6 | import outrun.rpc as rpc 7 | 8 | 9 | class RemoteFileSystem(Operations): 10 | """Class that implements a FUSE network file system that runs on RPC calls.""" 11 | 12 | def __init__(self, client: rpc.Client, mount_callback: Optional[Callable]): 13 | """Instantiate file system with RPC client.""" 14 | self._client = client 15 | self._mount_callback = mount_callback 16 | 17 | def init(self) -> None: 18 | """File system has been successfully mounted by FUSE.""" 19 | if self._mount_callback is not None: 20 | self._mount_callback() 21 | 22 | # 23 | # File operations 24 | # 25 | 26 | def open(self, path: str, flags: int) -> int: 27 | return self._client.open(path, flags) 28 | 29 | def create(self, path: str, flags: int, mode: int) -> int: 30 | return self._client.create(path, flags, mode) 31 | 32 | def read(self, path: str, fh: int, offset: int, size: int) -> bytes: 33 | return self._client.read(fh, offset, size) 34 | 35 | def write(self, path: str, fh: int, offset: int, data: bytes) -> int: 36 | return self._client.write(fh, offset, data) 37 | 38 | def lseek(self, path: str, fh: int, offset: int, whence: int) -> int: 39 | return self._client.lseek(fh, offset, whence) 40 | 41 | def fsync(self, path: str, fh: int, datasync: bool) -> None: 42 | self._client.fsync(fh, datasync) 43 | 44 | def flush(self, path: str, fh: int) -> None: 45 | self._client.flush(fh) 46 | 47 | def truncate(self, path: str, fh: Optional[int], size: int) -> None: 48 | self._client.truncate(path, fh, size) 49 | 50 | def release(self, path: str, fh: int) -> None: 51 | self._client.release(fh) 52 | 53 | # 54 | # Metadata access 55 | # 56 | 57 | def readdir(self, path: str) -> List[str]: 58 | return self._client.readdir(path) 59 | 60 | def readlink(self, path: str) -> str: 61 | return self._client.readlink(path) 62 | 63 | def getattr(self, path: str, fh: Optional[int]) -> dict: 64 | return self._client.getattr(path, fh).__dict__ 65 | 66 | # 67 | # Metadata modification 68 | # 69 | 70 | def chmod(self, path: str, fh: Optional[int], mode: int) -> None: 71 | self._client.chmod(path, fh, mode) 72 | 73 | def chown(self, path: str, fh: Optional[int], uid: int, gid: int) -> None: 74 | self._client.chown(path, fh, uid, gid) 75 | 76 | def utimens(self, path: str, fh: Optional[int], times: Tuple[int, int]) -> None: 77 | self._client.utimens(path, fh, times) 78 | 79 | # 80 | # File system structure 81 | # 82 | 83 | def link(self, path: str, target: str) -> None: 84 | self._client.link(path, target) 85 | 86 | def symlink(self, path: str, target: str) -> None: 87 | self._client.symlink(path, target) 88 | 89 | def mkdir(self, path: str, mode: int) -> None: 90 | self._client.mkdir(path, mode) 91 | 92 | def mknod(self, path: str, mode: int, rdev: int) -> None: 93 | self._client.mknod(path, mode, rdev) 94 | 95 | def rename(self, old: str, new: str) -> None: 96 | self._client.rename(old, new) 97 | 98 | def unlink(self, path: str) -> None: 99 | self._client.unlink(path) 100 | 101 | def rmdir(self, path: str) -> None: 102 | self._client.rmdir(path) 103 | 104 | # 105 | # Miscellaneous 106 | # 107 | 108 | def statfs(self, path: str) -> dict: 109 | return self._client.statfs(path) 110 | -------------------------------------------------------------------------------- /outrun/filesystem/fuse/fuse.py: -------------------------------------------------------------------------------- 1 | """ 2 | Module with ctypes bindings for the high-level FUSE 3.x API. 3 | 4 | These are based on the following libfuse headers: 5 | 6 | * fuse/fuse.h 7 | * fuse/fuse_common.h 8 | * fuse/fuse_opt.h 9 | 10 | Currently the implementation assumes an x86-64 system. 11 | """ 12 | 13 | import ctypes 14 | import ctypes.util 15 | from enum import Enum 16 | 17 | # 18 | # Helpers 19 | # 20 | 21 | 22 | def struct_to_dict(struct: ctypes.Structure) -> dict: 23 | """Create a dict from a struct's fields and their values.""" 24 | return {name: getattr(struct, name) for name, _ in getattr(struct, "_fields_")} 25 | 26 | 27 | # 28 | # Types 29 | # 30 | 31 | # sys/types.h 32 | off_t = ctypes.c_long 33 | mode_t = ctypes.c_uint 34 | dev_t = ctypes.c_ulong 35 | uid_t = ctypes.c_uint 36 | gid_t = ctypes.c_uint 37 | fsblkcnt_t = ctypes.c_ulong 38 | fsfilcnt_t = ctypes.c_ulong 39 | 40 | 41 | class fuse_args(ctypes.Structure): 42 | """fuse/fuse_opt.h: struct fuse_args.""" 43 | 44 | _fields_ = [ 45 | ("argc", ctypes.c_int), 46 | ("argv", ctypes.POINTER(ctypes.POINTER(ctypes.c_char))), 47 | ("allocated", ctypes.c_int), 48 | ] 49 | 50 | 51 | fuse_args_p = ctypes.POINTER(fuse_args) 52 | 53 | 54 | class fuse_opt(ctypes.Structure): 55 | """fuse/fuse_opt.h: struct fuse_args.""" 56 | 57 | _fields_ = [ 58 | ("templ", ctypes.c_char_p), 59 | ("offset", ctypes.c_ulong), 60 | ("value", ctypes.c_int), 61 | ] 62 | 63 | 64 | fuse_opt_p = ctypes.POINTER(fuse_opt) 65 | 66 | 67 | class fuse_conn_info(ctypes.Structure): 68 | """fuse/fuse_common.h: struct fuse_conn_info.""" 69 | 70 | _fields_ = [ 71 | ("proto_major", ctypes.c_uint), 72 | ("proto_minor", ctypes.c_uint), 73 | ("max_write", ctypes.c_uint), 74 | ("max_read", ctypes.c_uint), 75 | ("max_readahead", ctypes.c_uint), 76 | ("capable", ctypes.c_uint), 77 | ("want", ctypes.c_uint), 78 | ("max_background", ctypes.c_uint), 79 | ("congestion_threshold", ctypes.c_uint), 80 | ("time_gran", ctypes.c_uint), 81 | ] + [("reserved", ctypes.c_uint)] * 22 82 | 83 | 84 | fuse_conn_info_p = ctypes.POINTER(fuse_conn_info) 85 | 86 | 87 | class fuse_config(ctypes.Structure): 88 | """fuse/fuse.h: struct fuse_config.""" 89 | 90 | _fields_ = [ 91 | ("set_gid", ctypes.c_int), 92 | ("gid", ctypes.c_uint), 93 | ("set_uid", ctypes.c_int), 94 | ("uid", ctypes.c_uint), 95 | ("set_mode", ctypes.c_int), 96 | ("umask", ctypes.c_uint), 97 | ("entry_timeout", ctypes.c_double), 98 | ("negative_timeout", ctypes.c_double), 99 | ("attr_timeout", ctypes.c_double), 100 | ("intr", ctypes.c_int), 101 | ("intr_signal", ctypes.c_int), 102 | ("remember", ctypes.c_int), 103 | ("hard_remove", ctypes.c_int), 104 | ("use_ino", ctypes.c_int), 105 | ("readdir_ino", ctypes.c_int), 106 | ("direct_io", ctypes.c_int), 107 | ("kernel_cache", ctypes.c_int), 108 | ("auto_cache", ctypes.c_int), 109 | ("ac_attr_timeout_set", ctypes.c_int), 110 | ("ac_attr_timeout", ctypes.c_double), 111 | ("nullpath_ok", ctypes.c_int), 112 | ("show_help", ctypes.c_int), 113 | ("modules", ctypes.POINTER(ctypes.c_char)), 114 | ("debug", ctypes.c_int), 115 | ] 116 | 117 | 118 | fuse_config_p = ctypes.POINTER(fuse_config) 119 | 120 | 121 | class fuse_file_info(ctypes.Structure): 122 | """fuse/fuse_common.h: struct fuse_file_info.""" 123 | 124 | _fields_ = [ 125 | ("flags", ctypes.c_int), 126 | ("writepage", ctypes.c_uint, 1), 127 | ("direct_io", ctypes.c_uint, 1), 128 | ("keep_cache", ctypes.c_uint, 1), 129 | ("flush", ctypes.c_uint, 1), 130 | ("nonseekable", ctypes.c_uint, 1), 131 | ("flock_release", ctypes.c_uint, 1), 132 | ("cache_readdir", ctypes.c_uint, 1), 133 | ("padding", ctypes.c_uint, 25), 134 | ("padding2", ctypes.c_uint, 32), 135 | ("fh", ctypes.c_uint64), 136 | ("lock_owner", ctypes.c_uint64), 137 | ("poll_events", ctypes.c_uint32), 138 | ] 139 | 140 | 141 | fuse_file_info_p = ctypes.POINTER(fuse_file_info) 142 | 143 | 144 | class timespec(ctypes.Structure): 145 | """bits/types/struct_timespec.h: struct timespec.""" 146 | 147 | _fields_ = [ 148 | ("tv_sec", ctypes.c_long), 149 | ("tv_nsec", ctypes.c_long), 150 | ] 151 | 152 | 153 | timespec_p = ctypes.POINTER(timespec) 154 | 155 | 156 | class stat(ctypes.Structure): 157 | """bits/stat.h: struct stat.""" 158 | 159 | _fields_ = [ 160 | ("st_dev", ctypes.c_ulong), 161 | ("st_ino", ctypes.c_ulong), 162 | ("st_nlink", ctypes.c_ulong), 163 | ("st_mode", ctypes.c_uint), 164 | ("st_uid", ctypes.c_uint), 165 | ("st_gid", ctypes.c_uint), 166 | ("__pad0", ctypes.c_int), 167 | ("st_rdev", ctypes.c_ulong), 168 | ("st_size", ctypes.c_long), 169 | ("st_blksize", ctypes.c_long), 170 | ("st_blocks", ctypes.c_long), 171 | ("st_atim", timespec), 172 | ("st_mtim", timespec), 173 | ("st_ctim", timespec), 174 | ] + [("__glibc_reserved", ctypes.c_long)] * 3 175 | 176 | 177 | stat_p = ctypes.POINTER(stat) 178 | 179 | 180 | class statvfs_t(ctypes.Structure): 181 | """bits/statvfs.h: struct statvfs.""" 182 | 183 | _fields_ = [ 184 | ("f_bsize", ctypes.c_ulong), 185 | ("f_frsize", ctypes.c_ulong), 186 | ("f_blocks", fsblkcnt_t), 187 | ("f_bfree", fsblkcnt_t), 188 | ("f_bavail", fsblkcnt_t), 189 | ("f_files", fsfilcnt_t), 190 | ("f_ffree", fsfilcnt_t), 191 | ("f_favail", fsfilcnt_t), 192 | ("f_fsid", ctypes.c_ulong), 193 | ("f_flag", ctypes.c_ulong), 194 | ("f_namemax", ctypes.c_ulong), 195 | ("__f_spare", ctypes.c_int * 6), 196 | ] 197 | 198 | 199 | statvfs_t_p = ctypes.POINTER(statvfs_t) 200 | 201 | 202 | class fuse_readdir_flags(int, Enum): 203 | """fuse/fuse.h: enum fuse_readdir_flags.""" 204 | 205 | FUSE_READDIR_PLUS = 1 << 0 206 | 207 | 208 | class fuse_fill_dir_flags(int, Enum): 209 | """fuse/fuse.h: enum fuse_fill_dir_flags.""" 210 | 211 | FUSE_FILL_DIR_PLUS = 1 << 1 212 | 213 | 214 | # fuse/fuse_opt.h 215 | fuse_opt_proc_t = ctypes.CFUNCTYPE( 216 | ctypes.c_int, ctypes.c_void_p, ctypes.c_char_p, ctypes.c_int, fuse_args_p 217 | ) 218 | 219 | # fuse/fuse.h 220 | fuse_fill_dir_t = ctypes.CFUNCTYPE( 221 | ctypes.c_int, ctypes.c_void_p, ctypes.c_char_p, stat_p, off_t, ctypes.c_int 222 | ) 223 | 224 | 225 | class fuse_operations(ctypes.Structure): 226 | """fuse/fuse.h: struct fuse_operations.""" 227 | 228 | _fields_ = [ 229 | ( 230 | "getattr", 231 | ctypes.CFUNCTYPE(ctypes.c_int, ctypes.c_char_p, stat_p, fuse_file_info_p), 232 | ), 233 | ( 234 | "readlink", 235 | ctypes.CFUNCTYPE( 236 | ctypes.c_int, 237 | ctypes.c_char_p, 238 | ctypes.POINTER(ctypes.c_char), 239 | ctypes.c_size_t, 240 | ), 241 | ), 242 | ("mknod", ctypes.CFUNCTYPE(ctypes.c_int, ctypes.c_char_p, mode_t, dev_t)), 243 | ("mkdir", ctypes.CFUNCTYPE(ctypes.c_int, ctypes.c_char_p, mode_t)), 244 | ("unlink", ctypes.CFUNCTYPE(ctypes.c_int, ctypes.c_char_p)), 245 | ("rmdir", ctypes.CFUNCTYPE(ctypes.c_int, ctypes.c_char_p)), 246 | ("symlink", ctypes.CFUNCTYPE(ctypes.c_int, ctypes.c_char_p, ctypes.c_char_p)), 247 | ( 248 | "rename", 249 | ctypes.CFUNCTYPE( 250 | ctypes.c_int, ctypes.c_char_p, ctypes.c_char_p, ctypes.c_uint 251 | ), 252 | ), 253 | ("link", ctypes.CFUNCTYPE(ctypes.c_int, ctypes.c_char_p, ctypes.c_char_p)), 254 | ( 255 | "chmod", 256 | ctypes.CFUNCTYPE(ctypes.c_int, ctypes.c_char_p, mode_t, fuse_file_info_p), 257 | ), 258 | ( 259 | "chown", 260 | ctypes.CFUNCTYPE( 261 | ctypes.c_int, ctypes.c_char_p, uid_t, gid_t, fuse_file_info_p, 262 | ), 263 | ), 264 | ( 265 | "truncate", 266 | ctypes.CFUNCTYPE(ctypes.c_int, ctypes.c_char_p, off_t, fuse_file_info_p), 267 | ), 268 | ("open", ctypes.CFUNCTYPE(ctypes.c_int, ctypes.c_char_p, fuse_file_info_p)), 269 | ( 270 | "read", 271 | ctypes.CFUNCTYPE( 272 | ctypes.c_int, 273 | ctypes.c_char_p, 274 | ctypes.POINTER(ctypes.c_char), 275 | ctypes.c_size_t, 276 | off_t, 277 | fuse_file_info_p, 278 | ), 279 | ), 280 | ( 281 | "write", 282 | ctypes.CFUNCTYPE( 283 | ctypes.c_int, 284 | ctypes.c_char_p, 285 | ctypes.POINTER(ctypes.c_char), 286 | ctypes.c_size_t, 287 | off_t, 288 | fuse_file_info_p, 289 | ), 290 | ), 291 | ("statfs", ctypes.CFUNCTYPE(ctypes.c_int, ctypes.c_char_p, statvfs_t_p)), 292 | ("flush", ctypes.CFUNCTYPE(ctypes.c_int, ctypes.c_char_p, fuse_file_info_p)), 293 | ("release", ctypes.CFUNCTYPE(ctypes.c_int, ctypes.c_char_p, fuse_file_info_p)), 294 | ( 295 | "fsync", 296 | ctypes.CFUNCTYPE( 297 | ctypes.c_int, ctypes.c_char_p, ctypes.c_int, fuse_file_info_p 298 | ), 299 | ), 300 | ("setxattr", ctypes.c_void_p), 301 | ("getxattr", ctypes.c_void_p), 302 | ("listxattr", ctypes.c_void_p), 303 | ("removexattr", ctypes.c_void_p), 304 | ("opendir", ctypes.c_void_p), 305 | ( 306 | "readdir", 307 | ctypes.CFUNCTYPE( 308 | ctypes.c_int, 309 | ctypes.c_char_p, 310 | ctypes.c_void_p, 311 | fuse_fill_dir_t, 312 | off_t, 313 | fuse_file_info_p, 314 | ctypes.c_int, 315 | ), 316 | ), 317 | ("releasedir", ctypes.c_void_p), 318 | ("fsyncdir", ctypes.c_void_p), 319 | ("init", ctypes.CFUNCTYPE(None, fuse_conn_info_p, fuse_config_p)), 320 | ("destroy", ctypes.CFUNCTYPE(None, ctypes.c_void_p)), 321 | ("access", ctypes.CFUNCTYPE(ctypes.c_int, ctypes.c_char_p, ctypes.c_int)), 322 | ( 323 | "create", 324 | ctypes.CFUNCTYPE(ctypes.c_int, ctypes.c_char_p, mode_t, fuse_file_info_p), 325 | ), 326 | ("lock", ctypes.c_void_p), 327 | ( 328 | "utimens", 329 | ctypes.CFUNCTYPE( 330 | ctypes.c_int, 331 | ctypes.c_char_p, 332 | ctypes.POINTER(timespec), 333 | fuse_file_info_p, 334 | ), 335 | ), 336 | ("bmap", ctypes.c_void_p), 337 | ("ioctl", ctypes.c_void_p), 338 | ("poll", ctypes.c_void_p), 339 | ("write_buf", ctypes.c_void_p), 340 | ("read_buf", ctypes.c_void_p), 341 | ("flock", ctypes.c_void_p), 342 | ("fallocate", ctypes.c_void_p), 343 | ("copy_file_range", ctypes.c_void_p), 344 | ( 345 | "lseek", 346 | ctypes.CFUNCTYPE( 347 | off_t, ctypes.c_char_p, off_t, ctypes.c_int, fuse_file_info_p 348 | ), 349 | ), 350 | ] 351 | 352 | 353 | fuse_operations_p = ctypes.POINTER(fuse_operations) 354 | 355 | # 356 | # Macros 357 | # 358 | 359 | 360 | def FUSE_ARGS_INIT( 361 | argc: ctypes.c_int, argv: ctypes.POINTER(ctypes.POINTER(ctypes.c_char)) 362 | ) -> fuse_args: 363 | """fuse/fuse_opt.h: macro FUSE_ARGS_INIT.""" 364 | return fuse_args(argc=argc, argv=argv, allocated=0) 365 | 366 | 367 | # 368 | # Constants 369 | # 370 | 371 | # bits/stat.h 372 | UTIME_NOW = (1 << 30) - 1 373 | UTIME_OMIT = (1 << 30) - 2 374 | 375 | # fuse/fuse_common.h 376 | FUSE_CAP_ASYNC_READ = 1 << 0 377 | FUSE_CAP_POSIX_LOCKS = 1 << 1 378 | FUSE_CAP_ATOMIC_O_TRUNC = 1 << 3 379 | FUSE_CAP_EXPORT_SUPPORT = 1 << 4 380 | FUSE_CAP_DONT_MASK = 1 << 6 381 | FUSE_CAP_SPLICE_WRITE = 1 << 7 382 | FUSE_CAP_SPLICE_MOVE = 1 << 8 383 | FUSE_CAP_SPLICE_READ = 1 << 9 384 | FUSE_CAP_FLOCK_LOCKS = 1 << 10 385 | FUSE_CAP_IOCTL_DIR = 1 << 11 386 | FUSE_CAP_AUTO_INVAL_DATA = 1 << 12 387 | FUSE_CAP_READDIRPLUS = 1 << 13 388 | FUSE_CAP_READDIRPLUS_AUTO = 1 << 14 389 | FUSE_CAP_ASYNC_DIO = 1 << 15 390 | FUSE_CAP_WRITEBACK_CACHE = 1 << 16 391 | FUSE_CAP_NO_OPEN_SUPPORT = 1 << 17 392 | FUSE_CAP_PARALLEL_DIROPS = 1 << 18 393 | FUSE_CAP_POSIX_ACL = 1 << 19 394 | FUSE_CAP_HANDLE_KILLPRIV = 1 << 20 395 | FUSE_CAP_NO_OPENDIR_SUPPORT = 1 << 24 396 | FUSE_CAP_EXPLICIT_INVAL_DATA = 1 << 25 397 | 398 | # 399 | # Functions 400 | # 401 | 402 | fuse3_so = ctypes.util.find_library("fuse3") 403 | 404 | if not fuse3_so: 405 | raise RuntimeError("failed to find fuse3 library") 406 | 407 | fuse3 = ctypes.cdll.LoadLibrary(fuse3_so) 408 | 409 | # fuse/fuse.h 410 | fuse_main_real = fuse3.fuse_main_real 411 | fuse_main_real.restype = ctypes.c_int 412 | fuse_main_real.argtypes = [ 413 | ctypes.c_int, 414 | ctypes.POINTER(ctypes.POINTER(ctypes.c_char)), 415 | fuse_operations_p, 416 | ctypes.c_size_t, 417 | ctypes.c_void_p, 418 | ] 419 | 420 | # fuse/fuse_opt.h 421 | fuse_opt_parse = fuse3.fuse_opt_parse 422 | fuse_opt_parse.restype = ctypes.c_int 423 | fuse_opt_parse.argtypes = [ 424 | fuse_args_p, 425 | ctypes.c_void_p, 426 | fuse_opt_p, 427 | fuse_opt_proc_t, 428 | ] 429 | 430 | # fuse/fuse_opt.h 431 | fuse_opt_add_arg = fuse3.fuse_opt_add_arg 432 | fuse_opt_add_arg.restype = ctypes.c_int 433 | fuse_opt_add_arg.argtypes = [fuse_args_p, ctypes.c_char_p] 434 | -------------------------------------------------------------------------------- /outrun/filesystem/service.py: -------------------------------------------------------------------------------- 1 | """Module that exposes local file system calls as an RPC service.""" 2 | 3 | import os 4 | import os.path 5 | import stat 6 | from typing import List, Optional 7 | 8 | from outrun.filesystem.common import Attributes 9 | 10 | 11 | class LocalFileSystemService: 12 | """RPC service that exposes local file system operations.""" 13 | 14 | # 15 | # File operations 16 | # 17 | 18 | @staticmethod 19 | def open(path: str, flags: int) -> int: 20 | return os.open(path, flags) 21 | 22 | @staticmethod 23 | def create(path: str, flags: int, mode: int) -> int: 24 | return os.open(path, flags, mode) 25 | 26 | @staticmethod 27 | def read(fh: int, offset: int, size: int) -> bytes: 28 | return os.pread(fh, size, offset) 29 | 30 | @staticmethod 31 | def write(fh: int, offset: int, data: bytes) -> int: 32 | return os.pwrite(fh, data, offset) 33 | 34 | @staticmethod 35 | def lseek(fh: int, offset: int, whence: int) -> int: 36 | return os.lseek(fh, offset, whence) 37 | 38 | @staticmethod 39 | def fsync(fh: int, datasync: bool) -> None: 40 | if datasync: 41 | os.fdatasync(fh) 42 | else: 43 | os.fsync(fh) 44 | 45 | @staticmethod 46 | def flush(fh: int) -> None: 47 | os.close(os.dup(fh)) 48 | 49 | @staticmethod 50 | def truncate(path: str, fh: Optional[int], size: int) -> None: 51 | if fh and os.truncate in os.supports_fd: 52 | os.truncate(fh, size) 53 | else: 54 | os.truncate(path, size) 55 | 56 | @staticmethod 57 | def release(fh: int) -> None: 58 | os.close(fh) 59 | 60 | # 61 | # Metadata access 62 | # 63 | 64 | @staticmethod 65 | def readdir(path: str) -> List[str]: 66 | return [".", ".."] + os.listdir(path) 67 | 68 | @staticmethod 69 | def readlink(path: str) -> str: 70 | return os.readlink(path) 71 | 72 | @staticmethod 73 | def getattr(path: str, fh: Optional[int]) -> Attributes: 74 | if fh and os.stat in os.supports_fd: 75 | st = os.stat(fh) 76 | else: 77 | if os.stat in os.supports_follow_symlinks: 78 | st = os.stat(path, follow_symlinks=False) 79 | else: 80 | st = os.stat(path) 81 | 82 | return Attributes.from_stat(st) 83 | 84 | # 85 | # Metadata modification 86 | # 87 | 88 | @staticmethod 89 | def chmod(path: str, fh: Optional[int], mode: int) -> None: 90 | if fh and os.chmod in os.supports_fd: 91 | os.chmod(fh, mode) 92 | else: 93 | if os.chmod in os.supports_follow_symlinks: 94 | os.chmod(path, mode, follow_symlinks=False) 95 | else: 96 | os.chmod(path, mode) 97 | 98 | @staticmethod 99 | def chown(path: str, fh: Optional[int], uid: int, gid: int) -> None: 100 | if fh and os.chown in os.supports_fd: 101 | os.chown(fh, uid, gid) 102 | else: 103 | if os.chown in os.supports_follow_symlinks: 104 | os.chown(path, uid, gid, follow_symlinks=False) 105 | else: 106 | os.chown(path, uid, gid) 107 | 108 | @staticmethod 109 | def utimens(path: str, fh: Optional[int], times: List[int]) -> None: 110 | # times turns into a list due to the RPC encoding 111 | times_tpl = (times[0], times[1]) 112 | 113 | if fh and os.utime in os.supports_fd: 114 | os.utime(fh, ns=times_tpl) 115 | else: 116 | if os.utime in os.supports_follow_symlinks: 117 | os.utime(path, ns=times_tpl, follow_symlinks=False) 118 | else: 119 | os.utime(path, ns=times_tpl) 120 | 121 | # 122 | # File system structure 123 | # 124 | 125 | @staticmethod 126 | def link(path: str, target: str) -> None: 127 | os.link(target, path) 128 | 129 | @staticmethod 130 | def symlink(path: str, target: str) -> None: 131 | os.symlink(target, path) 132 | 133 | @staticmethod 134 | def mkdir(path: str, mode: int) -> None: 135 | os.mkdir(path, mode) 136 | 137 | @staticmethod 138 | def mknod(path: str, mode: int, rdev: int) -> None: 139 | if stat.S_ISFIFO(mode): 140 | os.mkfifo(path, mode) 141 | else: 142 | os.mknod(path, mode, rdev) 143 | 144 | @staticmethod 145 | def rename(old: str, new: str) -> None: 146 | os.rename(old, new) 147 | 148 | @staticmethod 149 | def unlink(path: str) -> None: 150 | os.unlink(path) 151 | 152 | @staticmethod 153 | def rmdir(path: str) -> None: 154 | os.rmdir(path) 155 | 156 | # 157 | # Miscellaneous 158 | # 159 | 160 | @staticmethod 161 | def statfs(path: str) -> dict: 162 | st = os.statvfs(path) 163 | return {name: getattr(st, name) for name in dir(st) if name.startswith("f_")} 164 | -------------------------------------------------------------------------------- /outrun/logger.py: -------------------------------------------------------------------------------- 1 | """Module containing utilities for logging, along with a standard logger.""" 2 | 3 | import logging 4 | from typing import Any, Optional 5 | 6 | 7 | def _get_logger(name: Optional[str] = "outrun") -> logging.Logger: 8 | stderrOutput = logging.StreamHandler() 9 | 10 | # Explicitly emit a carriage return to help properly combine output. 11 | stderrOutput.terminator = "\r\n" 12 | 13 | formatter = logging.Formatter("%(asctime)s - %(levelname)s - %(message)s") 14 | stderrOutput.setFormatter(formatter) 15 | 16 | logger = logging.getLogger(name) 17 | logger.addHandler(stderrOutput) 18 | 19 | return logger 20 | 21 | 22 | def summarize(obj: Any, max_length: int = 255) -> str: 23 | """Return a stringified representation of the object up to the given length.""" 24 | stringified_obj = str(obj) 25 | 26 | if len(stringified_obj) <= max_length: 27 | return stringified_obj 28 | else: 29 | return stringified_obj[: max_length - 3] + "..." 30 | 31 | 32 | # Default logger 33 | log = _get_logger() 34 | -------------------------------------------------------------------------------- /outrun/operations/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | Modules that implement the local and remote logic of outrun. 3 | 4 | The local instance starts an SSH session with the remote machine to launch the remote 5 | instance of outrun along with two TCP tunnels to later expose RPC services. The remote 6 | instance will first ensure that it runs as root (to access chroot), unshares its mounts, 7 | and enables the FUSE kernel module. 8 | 9 | To prevent any unwanted programs on either the local or remote machine from connecting 10 | to these services, the local and remote end need to agree on a shared authentication 11 | token. The token is generated by the remote instance and written to stdout, surrounded 12 | by SOH (Start of Header) and STX (Start of Text) control characters. The local instance 13 | proxies all of SSH's stdout to look for and filter this token. The token includes a 14 | checksum that provides some degree of confidence that the remote outrun instance has 15 | correctly started. 16 | 17 | Once this handshake has completed, the local instance starts two RPC services: 18 | 19 | * Environment service: provides the context for running the command, like working 20 | directory, environment variables, the command itself, and its arguments. 21 | * Local file system service: provides I/O functions to mirror the local file system on 22 | the remote machine. 23 | 24 | The remote instance waits for these services to be ready and connects to them through 25 | the previously set up SSH tunnels. 26 | 27 | The process of mirroring the local environment can now begin and starts by mounting a 28 | FUSE file system that, in essence, simply forwards all of its I/O calls to the local 29 | file system RPC service. The command to run and its context is then retrieved from the 30 | environment RPC service. A new process is started with the original environment 31 | variables and original command that chroots into the mirrored file system, sets the 32 | working directory and then begins running as normal. All of its stdin, stdout, and 33 | stderr is forwarded over SSH, and all of its file system operations are handled as if 34 | the program was running locally. 35 | 36 | Once the program has exited, the remote instance unmounts the file system and exits with 37 | the same exit code as the program. SSH's exit code further propagates this to the local 38 | instance and it too will exit with that same exit code to complete the illusion of the 39 | command having run as normally as possible, just on a different machine. 40 | 41 | Any unexpected turn of events, like RPC services failing or the mirrored file system 42 | unexpectedly unmounting before the program has completed, are picked up by the event 43 | queue and result in an immediate shutdown. 44 | """ 45 | 46 | from .common import Operations 47 | from .local import LocalOperations 48 | from .remote import RemoteOperations 49 | 50 | __all__ = [ 51 | "Operations", 52 | "LocalOperations", 53 | "RemoteOperations", 54 | ] 55 | -------------------------------------------------------------------------------- /outrun/operations/common.py: -------------------------------------------------------------------------------- 1 | """Shared functionality between local and remote operations.""" 2 | 3 | from abc import ABC 4 | import contextlib 5 | import ctypes 6 | import signal 7 | import threading 8 | from typing import Any, Callable 9 | 10 | 11 | class Operations(ABC): 12 | """Base class for local or remote operations logic.""" 13 | 14 | def run(self) -> int: 15 | """Run the operations and clean up properly in case of errors.""" 16 | with contextlib.ExitStack() as stack: 17 | return self._run(stack) 18 | 19 | # https://github.com/python/mypy/issues/7726 20 | assert False, "unreachable" 21 | 22 | def _run(self, stack: contextlib.ExitStack) -> int: 23 | """Run the actual operations.""" 24 | raise NotImplementedError() 25 | 26 | @staticmethod 27 | def _start_thread(target: Callable[..., None], *args: Any) -> threading.Thread: 28 | """ 29 | Start a thread with the specified function. 30 | 31 | It is still made a daemon just in case the thread fails to exit properly and 32 | blocks the shutting down of the program. 33 | """ 34 | t = threading.Thread(target=target, args=args, daemon=True) 35 | t.start() 36 | return t 37 | 38 | @staticmethod 39 | def _start_disposable_thread(target: Callable[..., None], *args: Any) -> None: 40 | """Start a disposable thread with the specified function.""" 41 | t = threading.Thread(target=target, args=args, daemon=True) 42 | t.start() 43 | 44 | # https://stackoverflow.com/a/19448096/238180 45 | @staticmethod 46 | def _set_death_signal(sig: signal.Signals) -> int: 47 | """Set the signal that the current process gets when its parent dies.""" 48 | libc = ctypes.CDLL("libc.so.6") 49 | 50 | # https://github.com/torvalds/linux/blob/master/include/uapi/linux/prctl.h#L9 51 | PR_SET_PDEATHSIG = 1 52 | 53 | return libc.prctl(PR_SET_PDEATHSIG, sig) 54 | 55 | @staticmethod 56 | def _ignore_process_error(call: Callable[[], Any]) -> Callable[[], None]: 57 | """ 58 | Workaround for race condition in Popen.terminate/Popen.kill. 59 | 60 | https://bugs.python.org/issue40550 61 | """ 62 | 63 | def wrapper() -> None: 64 | with contextlib.suppress(ProcessLookupError): 65 | call() 66 | 67 | return wrapper 68 | -------------------------------------------------------------------------------- /outrun/operations/environment.py: -------------------------------------------------------------------------------- 1 | """Module that contains the local environment RPC service.""" 2 | 3 | import os 4 | from typing import Dict, List 5 | 6 | 7 | class LocalEnvironmentService: 8 | """An RPC service with details about the local environment.""" 9 | 10 | def __init__(self, command: List[str]): 11 | """Construct the local environment service with the command to be executed.""" 12 | self._command = command 13 | 14 | def get_command(self) -> List[str]: 15 | """Get the command to be executed.""" 16 | return self._command 17 | 18 | @staticmethod 19 | def get_working_dir() -> str: 20 | """Get the working directory at the moment of execution.""" 21 | return os.getcwd() 22 | 23 | @staticmethod 24 | def get_environment() -> Dict[str, str]: 25 | """Get all environment variables at the moment of execution.""" 26 | return dict(os.environ) 27 | -------------------------------------------------------------------------------- /outrun/operations/events.py: -------------------------------------------------------------------------------- 1 | """ 2 | Module with utilities for coordinating a linear sequence of events across threads. 3 | 4 | outrun depends on multiple threads to asynchronously work together and it is important 5 | to assure that operations are executed in the right order and without errors to catch 6 | unexpected events and properly shut down. Therefore the main thread sets up a central 7 | event bus that the threads use to signal important events with the main thread asserting 8 | that these happen (are posted to the queue) one after another in the expected order. 9 | 10 | For example, consider a producer thread that provides work items to a consumer thread, 11 | where these consumer and producer threads also depend on a bunch of background services 12 | to function. Then we could enforce proper operations like this: 13 | 14 | def main(): 15 | q = EventQueue() 16 | 17 | # These threads signal HELPER_STOPPED when exiting. 18 | start_thread(helper_service_a, q) 19 | start_thread(helper_service_b, q) 20 | 21 | # These threads signal PRODUCER_FINISHED and CONSUMER_FINISHED, respectively. 22 | start_thread(producer, q) 23 | start_thread(consumer, q) 24 | 25 | q.expect(PRODUCER_FINISHED) 26 | q.expect(CONSUMER_FINISHED) 27 | 28 | stop_helpers() 29 | 30 | q.expect(HELPER_STOPPED) 31 | q.expect(HELPER_STOPPED) 32 | 33 | This code clearly describes that we expect the consumer to only finish once the producer 34 | has finished generating work, and that we don't expect helpers to stop in the meanwhile. 35 | Any deviation from this raises an exception that we can use to abort the program. 36 | """ 37 | 38 | from __future__ import annotations 39 | 40 | from enum import auto, Enum 41 | import queue 42 | from typing import Any, Tuple, Union 43 | 44 | 45 | class Event(Enum): 46 | """Types of events.""" 47 | 48 | # Local specific events 49 | SSH_START = auto() 50 | TOKEN_READ = auto() 51 | 52 | # Remote specific events 53 | PROCESS_START = auto() 54 | 55 | ROOT_FILESYSTEM_MOUNT = auto() 56 | ROOT_FILESYSTEM_UNMOUNT = auto() 57 | 58 | # Shared events 59 | PROGRAM_EXIT = auto() 60 | 61 | EXCEPTION = auto() 62 | 63 | 64 | class UnexpectedEvent(Exception): 65 | """Exception raised when an event occurs that is not being waited upon.""" 66 | 67 | def __init__( 68 | self, 69 | message: str, 70 | expected_event: Event, 71 | actual_event: Event, 72 | actual_value: Any, 73 | ) -> None: 74 | """Instantiate the exception with a description of what happened.""" 75 | super().__init__(message, expected_event, actual_event, actual_value) 76 | 77 | self.message = message 78 | 79 | self.expected_event = expected_event 80 | self.actual_event = actual_event 81 | self.actual_value = actual_value 82 | 83 | 84 | class EventQueue: 85 | """Thread-safe queue of events that can be notified of and waited upon.""" 86 | 87 | def __init__(self) -> None: 88 | """Instantiate a new EventQueue.""" 89 | self._queue: queue.Queue[Tuple[Event, Any]] = queue.Queue() 90 | 91 | def notify(self, event: Event, value: Any = None) -> None: 92 | """Post an event and any associated value to the queue.""" 93 | self._queue.put((event, value)) 94 | 95 | def exception(self, exception: Union[Exception, str]) -> None: 96 | """Post an exception event to the queue.""" 97 | if isinstance(exception, Exception): 98 | self.notify(Event.EXCEPTION, exception) 99 | else: 100 | self.notify(Event.EXCEPTION, RuntimeError(exception)) 101 | 102 | def expect(self, expected_event: Event) -> Any: 103 | """Wait for the next event on the queue and check if it matches.""" 104 | event, value = self._queue.get() 105 | 106 | if event == expected_event: 107 | return value 108 | elif event == Event.EXCEPTION: 109 | raise value 110 | else: 111 | raise UnexpectedEvent( 112 | f"expected {expected_event}, but got {event}", 113 | expected_event, 114 | event, 115 | value, 116 | ) 117 | -------------------------------------------------------------------------------- /outrun/operations/local.py: -------------------------------------------------------------------------------- 1 | """Module that implements the local logic of outrun and the environment RPC service.""" 2 | 3 | import contextlib 4 | import curses.ascii 5 | import hashlib 6 | import os 7 | import random 8 | import signal 9 | import subprocess 10 | import sys 11 | from typing import Any, Callable, Iterable, List 12 | 13 | from outrun.args import Arguments 14 | import outrun.filesystem as filesystem 15 | import outrun.filesystem.caching as caching 16 | from outrun.logger import log 17 | import outrun.rpc as rpc 18 | from .common import Operations 19 | from .environment import LocalEnvironmentService 20 | from .events import Event, EventQueue, UnexpectedEvent 21 | 22 | 23 | class LocalOperations(Operations): 24 | """Class that encapsulates all work on the local side.""" 25 | 26 | def __init__(self, args: Arguments): 27 | """Initialize local operations based on command-line arguments.""" 28 | self._args = args 29 | 30 | # Distinct random ports to support concurrent outrun sessions 31 | ports = random.sample(range(30000, 32000), 3) 32 | 33 | self._environment_port = args.environment_port or ports[0] 34 | self._filesystem_port = args.filesystem_port or ports[1] 35 | self._cache_port = args.cache_port or ports[2] 36 | 37 | def _run(self, stack: contextlib.ExitStack) -> int: 38 | """Run the operations on the local side.""" 39 | events = EventQueue() 40 | 41 | # Set up stdout redirection where RPC token can be read from remote outrun 42 | out_reader, out_writer = os.pipe() 43 | token_thread = self._start_thread(self._run_token_skimmer, events, out_reader) 44 | stack.callback(token_thread.join, timeout=5.0) 45 | stack.callback(os.close, out_writer) 46 | 47 | # Start SSH session with redirected stdout 48 | # stderr is suppressed in TTY mode for undesired output like "connection closed" 49 | ssh_proc = self._start_ssh(out_writer, self._is_tty()) 50 | ssh_thread = self._start_thread(self._watch_ssh, events, ssh_proc) 51 | stack.callback(ssh_thread.join, timeout=5.0) 52 | stack.callback(self._ignore_process_error(ssh_proc.terminate)) 53 | 54 | # Wait for RPC token to be read 55 | try: 56 | token: str = events.expect(Event.TOKEN_READ) 57 | except UnexpectedEvent as e: 58 | if e.actual_event == Event.PROGRAM_EXIT: 59 | raise RuntimeError("remote outrun failed to start") 60 | else: 61 | raise e 62 | 63 | # Start services to expose local environment 64 | self._start_disposable_thread(self._run_environment_service, events, token) 65 | self._start_disposable_thread(self._run_filesystem_service, events, token) 66 | 67 | if self._args.cache: 68 | self._start_disposable_thread(self._run_cache_service, events, token) 69 | 70 | # Wait for program on remote to finish executing 71 | exit_code: int = events.expect(Event.PROGRAM_EXIT) 72 | 73 | return exit_code 74 | 75 | @classmethod 76 | def _run_token_skimmer(cls, events: EventQueue, stdout_reader: int) -> None: 77 | """Forward SSH's stdout to the actual stdout while capturing the token.""" 78 | # Forward output until start marker of token 79 | cls._read_until_symbol(stdout_reader, curses.ascii.SOH, cls._write_stdout) 80 | 81 | # Read token 82 | buf_bytes: List[bytes] = [] 83 | cls._read_until_symbol(stdout_reader, curses.ascii.STX, buf_bytes.append) 84 | 85 | buf = b"".join(buf_bytes) 86 | 87 | token = buf[:32].decode() 88 | token_checksum = buf[32:].decode() 89 | 90 | token_expected_checksum = hashlib.sha256(token.encode()).hexdigest() 91 | 92 | if token_checksum != token_expected_checksum: 93 | events.exception("handshake failed (invalid token checksum)") 94 | 95 | # If the output was not a valid token then it should be forwarded as normal 96 | cls._write_stdout(buf) 97 | else: 98 | events.notify(Event.TOKEN_READ, token) 99 | 100 | # Simply pass through all other output from this point 101 | while True: 102 | chunk = os.read(stdout_reader, 1024) 103 | 104 | if len(chunk) > 0: 105 | cls._write_stdout(chunk) 106 | else: 107 | # End of stream 108 | break 109 | 110 | @staticmethod 111 | def _write_stdout(data: bytes): 112 | """Write text to stdout and immediately flush it.""" 113 | sys.stdout.buffer.write(data) 114 | sys.stdout.buffer.flush() 115 | 116 | @staticmethod 117 | def _read_until_symbol( 118 | fd: int, ascii_code: int, callback: Callable[[bytes], Any] 119 | ) -> None: 120 | """Read and forward bytes from the file descriptor until specific symbol.""" 121 | symbol = chr(ascii_code).encode() 122 | 123 | while True: 124 | c = os.read(fd, 1) 125 | 126 | if c == symbol or len(c) == 0: 127 | break 128 | else: 129 | callback(c) 130 | 131 | def _start_ssh(self, stdout_writer: int, suppress_stderr: bool) -> subprocess.Popen: 132 | """Start the SSH process to run the remote outrun instance.""" 133 | try: 134 | # Set up command to run to start remote session 135 | outrun_command = self._compose_remote_outrun_command() 136 | ssh_command = self._compose_ssh_command(outrun_command) 137 | 138 | def preexec_fn() -> None: 139 | # Terminate ssh if outrun is terminated 140 | self._set_death_signal(signal.SIGTERM) 141 | 142 | # Start SSH session that invokes outrun on the remote 143 | log.debug(f"running {ssh_command}") 144 | 145 | ssh = subprocess.Popen( 146 | ssh_command, 147 | # Proxy stdout to token skimmer 148 | stdout=stdout_writer, 149 | # Conditionally capture stderr 150 | stderr=subprocess.PIPE if suppress_stderr else None, 151 | preexec_fn=preexec_fn, 152 | ) 153 | 154 | return ssh 155 | except Exception as e: 156 | raise RuntimeError(f"failed to start ssh: {e}") 157 | 158 | @staticmethod 159 | def _watch_ssh(events: EventQueue, ssh: subprocess.Popen) -> None: 160 | """Wait for the SSH process to finish successfully or with an error.""" 161 | try: 162 | _, stderr = ssh.communicate() 163 | 164 | # SSH exits with the remote command's exit code or 255 in case of failure 165 | if ssh.returncode == 255: 166 | if stderr is not None: 167 | events.exception(f"ssh failed: {stderr.decode().strip()}") 168 | else: 169 | events.exception("ssh failed") 170 | else: 171 | events.notify(Event.PROGRAM_EXIT, ssh.returncode) 172 | except Exception as e: 173 | events.exception(f"ssh failed: {e}") 174 | 175 | def _run_environment_service(self, events: EventQueue, token: str) -> None: 176 | """Serve the environment RPC service.""" 177 | try: 178 | service = LocalEnvironmentService([self._args.command] + self._args.args) 179 | 180 | server = rpc.Server(service, token) 181 | server.serve(f"tcp://127.0.0.1:{self._environment_port}") 182 | 183 | # This service should never stop running 184 | events.exception("environment service unexpectedly stopped") 185 | except Exception as e: 186 | events.exception(f"environment service failed: {e}") 187 | 188 | def _run_filesystem_service(self, events: EventQueue, token: str) -> None: 189 | """Serve the local file system RPC service.""" 190 | try: 191 | service = filesystem.LocalFileSystemService() 192 | 193 | server = rpc.Server(service, token, self._args.workers) 194 | server.serve(f"tcp://127.0.0.1:{self._filesystem_port}") 195 | 196 | # This service should never stop running 197 | events.exception("file system service unexpectedly stopped") 198 | except Exception as e: 199 | events.exception(f"file system service failed: {e}") 200 | 201 | def _run_cache_service(self, events: EventQueue, token: str) -> None: 202 | """Serve the local file system cache RPC service.""" 203 | try: 204 | service = caching.LocalCacheService() 205 | 206 | server = rpc.Server(service, token, self._args.workers) 207 | server.serve(f"tcp://127.0.0.1:{self._cache_port}") 208 | 209 | # This service should never stop running 210 | events.exception("cache service unexpectedly stopped") 211 | except Exception as e: 212 | events.exception(f"cache service failed: {e}") 213 | 214 | def _compose_remote_outrun_command(self) -> List[str]: 215 | """Compose the command for invoking outrun on the remote host.""" 216 | outrun_command = ["outrun"] 217 | 218 | # Arguments 219 | outrun_command.extend( 220 | [ 221 | "--remote", 222 | "--unshare", 223 | f"--protocol={self._args.protocol}", 224 | f"--platform={self._args.platform}", 225 | f"--config={self._args.config}", 226 | f"--timeout={self._args.timeout}", 227 | f"--environment-port={self._environment_port}", 228 | f"--filesystem-port={self._filesystem_port}", 229 | ] 230 | ) 231 | if self._args.debug: 232 | outrun_command.append("--debug") 233 | 234 | if not self._args.cache: 235 | outrun_command.append("--no-cache") 236 | else: 237 | outrun_command.append(f"--cache-port={self._cache_port}") 238 | 239 | if not self._args.prefetch: 240 | outrun_command.append("--no-prefetch") 241 | 242 | if not self._args.writeback_cache: 243 | outrun_command.append("--sync-writes") 244 | 245 | # Pass dummy remote and command arguments 246 | outrun_command.extend([".", "."]) 247 | 248 | return outrun_command 249 | 250 | def _compose_ssh_command(self, outrun_command: Iterable[str]) -> List[str]: 251 | """ 252 | Compose the full command for starting the SSH session on the remote. 253 | 254 | This includes the SSH tunnels for the RPC services, and the command to start the 255 | remote outrun instance. 256 | """ 257 | ssh_command = ["ssh"] 258 | 259 | # Disable SSH INFO messages like the TCP tunnel not being able to connect yet 260 | ssh_command.extend(["-o", "LogLevel=error"]) 261 | 262 | # Configure the port forwards for the communication channels 263 | ssh_command.extend( 264 | ["-R", f"{self._environment_port}:localhost:{self._environment_port}"] 265 | ) 266 | 267 | ssh_command.extend( 268 | ["-R", f"{self._filesystem_port}:localhost:{self._filesystem_port}"] 269 | ) 270 | 271 | if self._args.cache: 272 | ssh_command.extend( 273 | ["-R", f"{self._cache_port}:localhost:{self._cache_port}"] 274 | ) 275 | 276 | # Enable/disable interactive terminal based on whether outrun itself 277 | # is interacting with an interactive terminal (rather than being piped 278 | # for example) 279 | if self._is_tty(): 280 | ssh_command.append("-tt") 281 | else: 282 | ssh_command.append("-T") 283 | 284 | # Append any additional arguments 285 | ssh_command.extend(self._args.extra_ssh_args) 286 | 287 | # Specify the remote host 288 | ssh_command.append(self._args.destination) 289 | 290 | # Invoke outrun command on remote 291 | ssh_command.extend(outrun_command) 292 | 293 | return ssh_command 294 | 295 | @staticmethod 296 | def _is_tty(): 297 | """Check if outrun is being executed in an interactive terminal.""" 298 | # stderr is not considered because it's not used for primary I/O 299 | # For example, 2>/dev/null should not affect TTY status 300 | return sys.stdout.isatty() and sys.stdin.isatty() 301 | -------------------------------------------------------------------------------- /outrun/operations/remote.py: -------------------------------------------------------------------------------- 1 | """Module that implements the remote logic of outrun.""" 2 | 3 | import contextlib 4 | import curses.ascii 5 | import hashlib 6 | import os 7 | import secrets 8 | import shlex 9 | import signal 10 | import subprocess 11 | import sys 12 | import tempfile 13 | 14 | from outrun.args import Arguments 15 | from outrun.config import Config 16 | import outrun.constants as constants 17 | import outrun.filesystem as filesystem 18 | import outrun.filesystem.caching as caching 19 | import outrun.filesystem.fuse as fuse 20 | from outrun.filesystem.fuse import FUSE, FuseConfig 21 | from outrun.logger import log 22 | import outrun.operations.local as local 23 | import outrun.rpc as rpc 24 | from .common import Operations 25 | from .events import Event, EventQueue, UnexpectedEvent 26 | 27 | 28 | class RemoteOperations(Operations): 29 | """Class that encapsulates all work on the remote side.""" 30 | 31 | def __init__(self, args: Arguments): 32 | """Initialize remote operations based on the command-line arguments.""" 33 | self._args = args 34 | self._config = Config() 35 | 36 | def _run(self, stack: contextlib.ExitStack) -> int: 37 | """Run the operations on the remote side.""" 38 | self._setup() 39 | 40 | # Generate RPC token and communicate it over stdout 41 | token = self._token_handshake() 42 | 43 | events = EventQueue() 44 | 45 | # Mount local file system and overlay special file systems 46 | fs_thread = self._start_thread(self._mount_root_filesystem, events, token) 47 | stack.callback(fs_thread.join, timeout=5.0) 48 | 49 | fs_path: str = events.expect(Event.ROOT_FILESYSTEM_MOUNT) 50 | stack.callback(self._unmount_all_filesystems, fs_path) 51 | 52 | self._mount_special_filesystems(fs_path) 53 | 54 | # Start running command 55 | cmd_proc = self._start_command(token, fs_path) 56 | cmd_thread = self._start_thread(self._watch_command, events, cmd_proc) 57 | stack.callback(cmd_thread.join, timeout=5.0) 58 | stack.callback(self._ignore_process_error(cmd_proc.terminate)) 59 | 60 | # Forward SIGINT (Ctrl-C) to command process 61 | signal.signal(signal.SIGINT, lambda *_: cmd_proc.send_signal(signal.SIGINT)) 62 | 63 | # Wait for program to finish or error 64 | try: 65 | exit_code: int = events.expect(Event.PROGRAM_EXIT) 66 | 67 | return exit_code 68 | except UnexpectedEvent as e: 69 | # If something unexpected happened then the program is likely not able 70 | # to exit normally 71 | self._ignore_process_error(cmd_proc.kill) 72 | 73 | if e.actual_event == Event.ROOT_FILESYSTEM_UNMOUNT: 74 | raise RuntimeError("root file system unexpectedly unmounted") 75 | else: 76 | raise e 77 | 78 | def _mount_root_filesystem(self, events: EventQueue, token: str) -> None: 79 | """Mount the mirrored local file system.""" 80 | try: 81 | # No timeout is applied since it would interfere with slow I/O operations. 82 | client = rpc.Client( 83 | filesystem.LocalFileSystemService, 84 | f"tcp://localhost:{self._args.filesystem_port}", 85 | token, 86 | ) 87 | 88 | # Ensure availability of the file system service. 89 | client.ping(self._args.timeout) 90 | 91 | # Pick a random temporary directory to enable concurrent sessions. 92 | # This directory will be cleaned up automatically. 93 | mount_dir = tempfile.TemporaryDirectory(prefix="outrun_fs_") 94 | 95 | def mount_callback() -> None: 96 | events.notify(Event.ROOT_FILESYSTEM_MOUNT, mount_dir.name) 97 | 98 | fs: fuse.Operations 99 | 100 | if self._args.cache: 101 | cache = self._init_filesystem_cache(token) 102 | fs = caching.RemoteCachedFileSystem(client, mount_callback, cache) 103 | else: 104 | fs = filesystem.RemoteFileSystem(client, mount_callback) 105 | 106 | # Mount 107 | config = FuseConfig() 108 | config.auto_cache = True 109 | config.use_ino = True 110 | config.writeback_cache = self._args.writeback_cache 111 | 112 | instance = FUSE(fs, config) 113 | instance.mount(constants.FILESYSTEM_NAME, mount_dir.name) 114 | 115 | events.notify(Event.ROOT_FILESYSTEM_UNMOUNT) 116 | except Exception as e: 117 | events.exception(f"root file system mount failed: {e}") 118 | 119 | def _init_filesystem_cache(self, token: str) -> caching.cache.RemoteCache: 120 | """Initialize the cache for the caching remote file system.""" 121 | # No timeout is applied since it would interfere with slow I/O operations. 122 | client = rpc.Client( 123 | caching.LocalCacheService, 124 | f"tcp://localhost:{self._args.cache_port}", 125 | token, 126 | ) 127 | 128 | # Ensure availability of the cache service. 129 | client.ping(self._args.timeout) 130 | 131 | cache = caching.cache.RemoteCache( 132 | base_path=self._config.cache.path, 133 | machine_id=client.get_app_specific_machine_id(), 134 | client=client, 135 | prefetch=self._args.prefetch, 136 | max_entries=self._config.cache.max_entries, 137 | max_size=self._config.cache.max_size, 138 | ) 139 | 140 | # Load the disk cache. 141 | try: 142 | cache.load() 143 | except FileNotFoundError: 144 | log.debug("starting with fresh cache") 145 | except Exception as e: 146 | log.error(f"failed to load cache: {e}") 147 | 148 | # Synchronize changed files with the local machine. 149 | cache.sync() 150 | 151 | return cache 152 | 153 | @staticmethod 154 | def _mount_special_filesystems(root_path: str) -> None: 155 | """Mount special remote file systems over the mirrored file system.""" 156 | # File systems that should be overlayed from the remote rather than being 157 | # serviced by the local machine 158 | special_filesystems = ["dev", "proc", "sys", "run"] 159 | 160 | for name in special_filesystems: 161 | try: 162 | # Only overlay file systems that exist on both systems 163 | os.stat(f"/{name}") 164 | os.stat(os.path.join(root_path, name)) 165 | 166 | mount_path = os.path.join(root_path, name) 167 | subprocess.check_output(["mount", "--rbind", f"/{name}", mount_path]) 168 | subprocess.check_output(["mount", "--make-rslave", mount_path]) 169 | except FileNotFoundError: 170 | continue 171 | except Exception as e: 172 | raise RuntimeError(f"failed to mount special file system {name}: {e}") 173 | 174 | @staticmethod 175 | def _unmount_all_filesystems(root_path: str) -> None: 176 | """ 177 | Find and undo all mounts under the root path tree. 178 | 179 | We can't simply do the reverse of _mount_special_filesystems() because things 180 | like /sys result in a lot of nested mounts. 181 | """ 182 | # Find all mounts from the root directory 183 | with open("/proc/mounts", "rb") as f: 184 | mount_lines = f.readlines() 185 | 186 | mount_paths = [] 187 | 188 | for line in mount_lines: 189 | path = line.split(b" ")[1] 190 | if path.startswith(root_path.encode()): 191 | # Unescape paths with spaces and other strange characters 192 | mount_paths.append(path.decode("unicode-escape")) 193 | 194 | # Unmount paths from most nested all the way up to the root directory 195 | for mount_path in reversed(sorted(mount_paths)): 196 | subprocess.call(["umount", "-f", mount_path], stderr=subprocess.DEVNULL) 197 | 198 | def _start_command(self, token: str, root_path: str) -> subprocess.Popen: 199 | """Start the command in an environment mirrored from the local machine.""" 200 | try: 201 | # Gather info about the command and local machine environment 202 | environment_client = rpc.Client( 203 | local.LocalEnvironmentService, 204 | f"tcp://localhost:{self._args.environment_port}", 205 | token, 206 | self._args.timeout, 207 | ) 208 | command = environment_client.get_command() 209 | working_dir = environment_client.get_working_dir() 210 | environment = environment_client.get_environment() 211 | 212 | # Escape command into shell execution 213 | shell_command = " ".join(map(shlex.quote, command)) 214 | 215 | def preexec_fn() -> None: 216 | # Terminate the command if outrun is terminated 217 | self._set_death_signal(signal.SIGTERM) 218 | 219 | # Chroot into mounted local file system 220 | os.chroot(root_path) 221 | 222 | # Set working directory 223 | os.chdir(working_dir) 224 | 225 | # Start the actual command 226 | proc = subprocess.Popen( 227 | shell_command, 228 | env=environment, 229 | # Make it possible to run shell commands 230 | shell=True, 231 | preexec_fn=preexec_fn, 232 | ) 233 | 234 | return proc 235 | except Exception as e: 236 | raise RuntimeError(f"failed to start command: {e}") 237 | 238 | @staticmethod 239 | def _watch_command(events: EventQueue, proc: subprocess.Popen) -> None: 240 | """Wait for the command process to finish successfully or with an error.""" 241 | try: 242 | # Wait for process to exit 243 | proc.communicate() 244 | 245 | if proc.returncode >= 0: 246 | events.notify(Event.PROGRAM_EXIT, proc.returncode) 247 | else: 248 | # Killed by a signal 249 | # https://www.tldp.org/LDP/abs/html/exitcodes.html 250 | events.notify(Event.PROGRAM_EXIT, 128 - proc.returncode) 251 | except Exception as e: 252 | events.exception(f"command failed: {e}") 253 | 254 | @staticmethod 255 | def _token_handshake() -> str: 256 | """Generate and communicate a random authentication token over stdout.""" 257 | token = secrets.token_hex(16) 258 | token_signature = hashlib.sha256(token.encode()).hexdigest() 259 | 260 | # Output token and its checksum as in-band signal 261 | sys.stdout.buffer.write(chr(curses.ascii.SOH).encode()) 262 | sys.stdout.buffer.write(f"{token}{token_signature}".encode()) 263 | sys.stdout.buffer.write(chr(curses.ascii.STX).encode()) 264 | 265 | sys.stdout.buffer.flush() 266 | 267 | return token 268 | 269 | def _setup(self) -> None: 270 | """Ensure that the remote server is set up correctly.""" 271 | self._become_root() 272 | self._unshare_mounts() 273 | self._enable_fuse() 274 | 275 | # Ensure that outrun files are only accessibly by root 276 | os.umask(0o077) 277 | 278 | # Load config 279 | self._config = Config.load(os.path.expanduser(self._args.config)) 280 | 281 | @staticmethod 282 | def _become_root() -> None: 283 | """Ensure that we're running as root to be able to use chroot.""" 284 | if os.geteuid() != 0: 285 | try: 286 | home_env = f"HOME={os.getenv('HOME')}" 287 | os.execvp("sudo", ["sudo", home_env, sys.executable] + sys.argv) 288 | except OSError as e: 289 | raise RuntimeError(f"failed to become root using sudo: {e}") 290 | 291 | def _unshare_mounts(self) -> None: 292 | """Unshare mount namespace to hide file system mount from other processes.""" 293 | if self._args.unshare: 294 | try: 295 | new_args = [arg for arg in sys.argv if arg != "--unshare"] 296 | os.execvp("unshare", ["unshare", "-m", sys.executable] + new_args) 297 | except OSError as e: 298 | raise RuntimeError(f"failed to unshare mount namespace: {e}") 299 | 300 | @staticmethod 301 | def _enable_fuse() -> None: 302 | """Ensure that the FUSE kernel module is installed and enabled.""" 303 | try: 304 | output = subprocess.check_output(["lsmod"], stderr=subprocess.PIPE).decode() 305 | except (OSError, subprocess.CalledProcessError) as e: 306 | raise RuntimeError(f"failed to look for FUSE kernel module: {e}") 307 | 308 | if "fuse" not in output: 309 | try: 310 | subprocess.check_output(["modprobe", "fuse"], stderr=subprocess.PIPE) 311 | except (OSError, subprocess.CalledProcessError) as e: 312 | raise RuntimeError(f"failed to enable FUSE kernel module: {e}") 313 | -------------------------------------------------------------------------------- /outrun/tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Overv/outrun/20af1136060ecb0a53f464b73e5cdac913a097c3/outrun/tests/__init__.py -------------------------------------------------------------------------------- /outrun/tests/conftest.py: -------------------------------------------------------------------------------- 1 | """Module that adds flags to pytest to enable certain extra tests.""" 2 | 3 | import pytest 4 | 5 | 6 | def pytest_addoption(parser): 7 | parser.addoption( 8 | "--vagrant", action="store_true", default=False, help="Run Vagrant tests" 9 | ) 10 | 11 | parser.addoption( 12 | "--fuse", action="store_true", default=False, help="Run FUSE tests" 13 | ) 14 | 15 | 16 | def pytest_configure(config): 17 | config.addinivalue_line("markers", "vagrant: mark test as requiring Vagrant to run") 18 | config.addinivalue_line("markers", "fuse: mark test as requiring FUSE to run") 19 | 20 | 21 | def pytest_collection_modifyitems(config, items): 22 | if not config.getoption("--vagrant"): 23 | skip_vagrant = pytest.mark.skip(reason="only runs with --vagrant option") 24 | 25 | for item in items: 26 | if "vagrant" in item.keywords: 27 | item.add_marker(skip_vagrant) 28 | 29 | if not config.getoption("--fuse"): 30 | skip_fuse = pytest.mark.skip(reason="only runs with --fuse option") 31 | 32 | for item in items: 33 | if "fuse" in item.keywords: 34 | item.add_marker(skip_fuse) 35 | -------------------------------------------------------------------------------- /outrun/tests/test_args.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from outrun.args import Arguments 4 | 5 | 6 | def test_no_args(): 7 | with pytest.raises(SystemExit): 8 | Arguments.parse() 9 | 10 | 11 | def test_basic_usage(): 12 | args = Arguments.parse(["hostname", "command", "arg1", "arg2"]) 13 | 14 | assert args.destination == "hostname" 15 | assert args.command == "command" 16 | assert args.args == ["arg1", "arg2"] 17 | 18 | assert not args.remote 19 | 20 | 21 | def test_command_without_args(): 22 | args = Arguments.parse(["hostname", "command"]) 23 | 24 | assert args.command == "command" 25 | assert args.args == [] 26 | 27 | 28 | def test_protocol_parsing(): 29 | args = Arguments.parse(["--protocol=1.2.3", "hostname", "command", "arg"]) 30 | 31 | assert args.protocol.major == 1 32 | assert args.protocol.minor == 2 33 | assert args.protocol.patch == 3 34 | 35 | with pytest.raises(SystemExit): 36 | Arguments.parse(["--protocol=abc", "hostname", "command", "arg"]) 37 | 38 | 39 | def test_cache(): 40 | args = Arguments.parse(["hostname", "command", "arg"]) 41 | assert args.cache 42 | 43 | args = Arguments.parse(["--no-cache", "hostname", "command", "arg"]) 44 | assert not args.cache 45 | 46 | 47 | def test_timeout(): 48 | args = Arguments.parse(["--timeout=1234", "hostname", "command", "arg"]) 49 | assert args.timeout == 1234 50 | 51 | with pytest.raises(SystemExit): 52 | Arguments.parse(["--timeout=-1", "hostname", "command", "arg"]) 53 | 54 | 55 | def test_command_arguments_that_resemble_flags(): 56 | args = Arguments.parse(["hostname", "command", "args", "--debug"]) 57 | 58 | assert not args.debug 59 | assert args.args == ["args", "--debug"] 60 | 61 | 62 | def test_extra_ssh_args(): 63 | args = Arguments.parse(["--ssh=-4 -E logfile", "hostname", "command"]) 64 | 65 | assert args.extra_ssh_args == ["-4", "-E", "logfile"] 66 | -------------------------------------------------------------------------------- /outrun/tests/test_config.py: -------------------------------------------------------------------------------- 1 | import os.path 2 | 3 | from configparser import ConfigParser 4 | 5 | from outrun.config import Config, CacheConfig 6 | 7 | 8 | def test_cache_config_defaults(): 9 | parser = ConfigParser() 10 | parser.read_string("[cache]") 11 | 12 | cfg = CacheConfig.load(parser["cache"]) 13 | 14 | assert cfg.path is not None 15 | assert cfg.max_entries is not None 16 | assert cfg.max_size is not None 17 | 18 | 19 | def test_cache_config_load(): 20 | parser = ConfigParser() 21 | parser.read_string( 22 | """ 23 | [cache] 24 | path = ~/test 25 | max_entries = 123 26 | max_size = 456 27 | """ 28 | ) 29 | 30 | cfg = CacheConfig.load(parser["cache"]) 31 | 32 | assert cfg.path == os.path.expanduser("~/test") 33 | assert cfg.max_entries == 123 34 | assert cfg.max_size == 456 35 | 36 | 37 | def test_config_defaults(tmpdir): 38 | cfg = Config.load(str(tmpdir / "nonexistent")) 39 | 40 | assert cfg.cache is not None 41 | 42 | 43 | def test_config_load(tmp_path): 44 | (tmp_path / "config").write_text( 45 | """ 46 | [cache] 47 | path = ~/test 48 | max_entries = 123 49 | max_size = 456 50 | """ 51 | ) 52 | 53 | cfg = Config.load(str(tmp_path / "config")) 54 | 55 | assert cfg.cache.path == os.path.expanduser("~/test") 56 | assert cfg.cache.max_entries == 123 57 | assert cfg.cache.max_size == 456 58 | 59 | 60 | def test_config_load_failure_nonfatal(tmp_path): 61 | (tmp_path / "config").write_text("blabla") 62 | 63 | cfg = Config.load(str(tmp_path / "config")) 64 | 65 | assert cfg.cache is not None 66 | -------------------------------------------------------------------------------- /outrun/tests/test_filesystem/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Overv/outrun/20af1136060ecb0a53f464b73e5cdac913a097c3/outrun/tests/test_filesystem/__init__.py -------------------------------------------------------------------------------- /outrun/tests/test_filesystem/test_caching/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Overv/outrun/20af1136060ecb0a53f464b73e5cdac913a097c3/outrun/tests/test_filesystem/test_caching/__init__.py -------------------------------------------------------------------------------- /outrun/tests/test_filesystem/test_caching/test_cache.py: -------------------------------------------------------------------------------- 1 | import contextlib 2 | import dataclasses 3 | import os 4 | import time 5 | from unittest import mock 6 | 7 | from outrun.filesystem.caching.service import LocalCacheService 8 | from outrun.filesystem.caching.cache import CacheEntry, RemoteCache 9 | 10 | 11 | def create_cache(tmp_path, **override_args): 12 | base_args = dict( 13 | base_path=str(tmp_path / "cache"), 14 | machine_id="machine", 15 | client=LocalCacheService(), 16 | prefetch=False, 17 | max_entries=1024, 18 | max_size=1024 * 1024, 19 | cacheable_paths=["/"], 20 | ) 21 | 22 | final_args = {**base_args, **override_args} 23 | 24 | return RemoteCache(**final_args) 25 | 26 | 27 | def test_cache_entry_newer_than(): 28 | entry_a = CacheEntry("a", LocalCacheService().get_metadata("/")) 29 | entry_b = CacheEntry("b", LocalCacheService().get_metadata("/")) 30 | 31 | assert entry_b.newer_than(entry_a) 32 | 33 | entry_a.last_update = time.time() 34 | 35 | assert not entry_b.newer_than(entry_a) 36 | 37 | 38 | def test_concurrent_cache_get_metadata(tmp_path): 39 | meta = LocalCacheService().get_metadata("/") 40 | meta = dataclasses.replace(meta, attr=meta.attr.as_readonly()) 41 | 42 | mock_client = mock.Mock() 43 | mock_client.get_metadata.return_value = meta 44 | 45 | (tmp_path / "cache").mkdir() 46 | cache = create_cache(tmp_path, client=mock_client) 47 | 48 | for _ in range(10): 49 | assert cache.get_metadata("/") == meta 50 | 51 | assert mock_client.get_metadata.call_count == 1 52 | 53 | 54 | def test_concurrent_cache_open_content(tmp_path): 55 | fs = LocalCacheService() 56 | 57 | (tmp_path / "cache").mkdir() 58 | (tmp_path / "hello").write_text("world") 59 | 60 | mock_client = mock.Mock() 61 | mock_client.readfile.side_effect = fs.readfile 62 | 63 | cache = create_cache(tmp_path, client=mock_client) 64 | 65 | for _ in range(10): 66 | fd = cache.open_contents(str(tmp_path / "hello"), os.O_RDONLY) 67 | try: 68 | os.lseek(fd, 0, 0) 69 | assert os.read(fd, 1024) == b"world" 70 | finally: 71 | os.close(fd) 72 | 73 | assert mock_client.readfile.call_count == 1 74 | 75 | 76 | def test_concurrent_cache_load_save(tmp_path): 77 | meta = LocalCacheService().get_metadata("/") 78 | meta = dataclasses.replace(meta, attr=meta.attr.as_readonly()) 79 | 80 | mock_client = mock.Mock() 81 | mock_client.get_metadata.return_value = meta 82 | 83 | (tmp_path / "cache").mkdir() 84 | 85 | cache_a = create_cache(tmp_path, client=mock_client) 86 | assert cache_a.get_metadata("/") == meta 87 | cache_a.save() 88 | 89 | cache_b = create_cache(tmp_path, client=mock_client) 90 | cache_b.load() 91 | assert cache_b.get_metadata("/") == meta 92 | 93 | assert mock_client.get_metadata.call_count == 1 94 | 95 | 96 | def test_concurrent_cache_per_machine(tmp_path): 97 | meta = LocalCacheService().get_metadata("/") 98 | meta = dataclasses.replace(meta, attr=meta.attr.as_readonly()) 99 | 100 | mock_client = mock.Mock() 101 | mock_client.get_metadata.return_value = meta 102 | 103 | (tmp_path / "cache").mkdir() 104 | 105 | cache_a = create_cache(tmp_path, machine_id="machine_a", client=mock_client) 106 | assert cache_a.get_metadata("/") == meta 107 | cache_a.save() 108 | 109 | cache_b = create_cache(tmp_path, machine_id="machine_b", client=mock_client) 110 | cache_b.load() 111 | assert cache_b.get_metadata("/") == meta 112 | 113 | assert mock_client.get_metadata.call_count == 2 114 | 115 | 116 | def test_concurrent_cache_lru_entries(tmp_path): 117 | (tmp_path / "cache").mkdir() 118 | 119 | cache = create_cache(tmp_path, max_entries=3) 120 | 121 | for x in ["a", "b", "c", "d"]: 122 | with contextlib.suppress(OSError): 123 | cache.get_metadata(f"/{x}") 124 | 125 | cache.save() 126 | cache.load() 127 | 128 | assert cache.count() == 3 129 | assert cache.size() == 0 130 | 131 | 132 | def test_concurrent_cache_lru_size(tmp_path): 133 | (tmp_path / "cache").mkdir() 134 | 135 | cache = create_cache(tmp_path, max_size=3) 136 | 137 | for x in ["a", "b", "c", "d"]: 138 | (tmp_path / x).write_text(" ") 139 | 140 | for x in ["a", "b", "c", "d"]: 141 | fd = cache.open_contents(str(tmp_path / x), os.O_RDONLY) 142 | os.close(fd) 143 | 144 | cache.save() 145 | cache.load() 146 | 147 | assert cache.count() == 4 148 | assert cache.size() == 3 149 | 150 | 151 | def test_concurrent_cache_content_cleanup(tmp_path): 152 | (tmp_path / "cache").mkdir() 153 | cache = create_cache(tmp_path, max_size=3) 154 | 155 | for x in ["a", "b", "c", "d"]: 156 | (tmp_path / x).write_text("123") 157 | 158 | for x in ["a", "b", "c", "d"]: 159 | fd = cache.open_contents(str(tmp_path / x), os.O_RDONLY) 160 | os.close(fd) 161 | 162 | cache.save() 163 | 164 | assert len(os.listdir(tmp_path / "cache" / "contents")) == 1 165 | 166 | cache = RemoteCache( 167 | str(tmp_path / "cache"), 168 | "machine", 169 | LocalCacheService(), 170 | prefetch=False, 171 | max_entries=1024, 172 | max_size=1024 * 1024, 173 | cacheable_paths=["/"], 174 | ) 175 | cache.save(merge_disk_cache=False) 176 | 177 | assert len(os.listdir(tmp_path / "cache" / "contents")) == 0 178 | 179 | 180 | def test_concurrent_cache_refresh_metadata(tmp_path): 181 | (tmp_path / "file").write_text("foo") 182 | (tmp_path / "cache").mkdir() 183 | 184 | cache = create_cache(tmp_path) 185 | 186 | meta_1 = cache.get_metadata(str(tmp_path / "file")) 187 | 188 | os.truncate(tmp_path / "file", 20) 189 | 190 | meta_2 = cache.get_metadata(str(tmp_path / "file")) 191 | 192 | assert meta_2 == meta_1 193 | 194 | cache.sync() 195 | 196 | meta_3 = cache.get_metadata(str(tmp_path / "file")) 197 | 198 | assert meta_3 != meta_1 199 | 200 | 201 | def test_concurrent_cache_refresh_contents(tmp_path): 202 | (tmp_path / "file").write_text("foo") 203 | (tmp_path / "cache").mkdir() 204 | 205 | cache = create_cache(tmp_path) 206 | 207 | fd = cache.open_contents(str(tmp_path / "file"), os.O_RDONLY) 208 | try: 209 | os.lseek(fd, 0, 0) 210 | assert os.read(fd, 1024) == b"foo" 211 | finally: 212 | os.close(fd) 213 | 214 | (tmp_path / "file").write_text("foobar") 215 | 216 | fd = cache.open_contents(str(tmp_path / "file"), os.O_RDONLY) 217 | try: 218 | os.lseek(fd, 0, 0) 219 | assert os.read(fd, 1024) == b"foo" 220 | finally: 221 | os.close(fd) 222 | 223 | cache.sync() 224 | 225 | fd = cache.open_contents(str(tmp_path / "file"), os.O_RDONLY) 226 | try: 227 | os.lseek(fd, 0, 0) 228 | assert os.read(fd, 1024) == b"foobar" 229 | finally: 230 | os.close(fd) 231 | 232 | 233 | def test_concurrent_cache_disk_merge(tmp_path): 234 | (tmp_path / "foo").touch() 235 | (tmp_path / "bar").touch() 236 | 237 | cache_a = create_cache(tmp_path) 238 | cache_b = create_cache(tmp_path) 239 | 240 | cache_a.get_metadata(str(tmp_path / "foo")) 241 | cache_b.get_metadata(str(tmp_path / "bar")) 242 | 243 | cache_a.save() 244 | cache_b.save() 245 | 246 | cache_c = RemoteCache( 247 | str(tmp_path / "cache"), 248 | "machine", 249 | LocalCacheService(), 250 | prefetch=False, 251 | max_entries=1024, 252 | max_size=1024 * 1024, 253 | cacheable_paths=["/"], 254 | ) 255 | cache_c.load() 256 | 257 | assert cache_c.count() == 2 258 | 259 | 260 | def test_concurrent_cache_prefetch_symlink(tmp_path): 261 | os.symlink("bar", tmp_path / "foo") 262 | 263 | cache = create_cache(tmp_path, prefetch=True) 264 | 265 | cache.get_metadata(str(tmp_path / "foo")) 266 | 267 | assert cache.count() == 2 268 | 269 | 270 | def test_concurrent_cache_prefetch_contents_upon_access(tmp_path): 271 | (tmp_path / "test.py").write_text("abc") 272 | 273 | cache = create_cache(tmp_path, prefetch=True) 274 | 275 | cache.get_metadata(str(tmp_path / "test.py")) 276 | 277 | assert cache.size() == 3 278 | 279 | 280 | def test_concurrent_cache_mark_fetched_contents(tmp_path): 281 | (tmp_path / "file").touch() 282 | 283 | # Cache contents of a file 284 | cache_a = create_cache(tmp_path, prefetch=True) 285 | 286 | fd = cache_a.open_contents(str(tmp_path / "file"), os.O_RDONLY) 287 | os.close(fd) 288 | 289 | cache_a.save() 290 | 291 | # Reload cache and expect that local is informed about cached contents 292 | mock_client = mock.Mock() 293 | mock_client.get_changed_metadata.return_value = {} 294 | 295 | cache_b = create_cache(tmp_path, client=mock_client, prefetch=True) 296 | cache_b.load() 297 | cache_b.sync() 298 | 299 | mock_client.mark_previously_fetched_contents.assert_called_with( 300 | [str(tmp_path / "file")] 301 | ) 302 | -------------------------------------------------------------------------------- /outrun/tests/test_filesystem/test_caching/test_common.py: -------------------------------------------------------------------------------- 1 | import hashlib 2 | 3 | from outrun.filesystem.caching.common import FileContents, LockIndex 4 | 5 | 6 | def test_file_contents_from_data(): 7 | contents = FileContents.from_data(b"abc") 8 | 9 | assert contents.size == 3 10 | assert contents.checksum == hashlib.sha256(b"abc").hexdigest() 11 | 12 | 13 | def test_file_contents_data(): 14 | contents = FileContents.from_data(b"abc") 15 | 16 | assert contents.data == b"abc" 17 | 18 | 19 | def test_lock_index(): 20 | index = LockIndex() 21 | 22 | with index.lock("a"): 23 | with index.lock("b"): 24 | with index.lock("c"): 25 | assert index.lock_count == 3 26 | 27 | assert index.lock_count == 1 28 | 29 | assert index.lock_count == 0 30 | -------------------------------------------------------------------------------- /outrun/tests/test_filesystem/test_caching/test_filesystem.py: -------------------------------------------------------------------------------- 1 | import errno 2 | import os 3 | import stat 4 | from unittest import mock 5 | 6 | import pytest 7 | 8 | from outrun.filesystem.service import LocalFileSystemService 9 | from outrun.filesystem.caching.service import LocalCacheService 10 | from outrun.filesystem.caching.filesystem import RemoteCachedFileSystem 11 | from outrun.filesystem.caching.cache import RemoteCache 12 | 13 | 14 | def create_cache(tmp_path, **override_args): 15 | base_args = dict( 16 | base_path=str(tmp_path / "cache"), 17 | machine_id="machine", 18 | client=LocalCacheService(), 19 | prefetch=False, 20 | max_entries=1024, 21 | max_size=1024 * 1024, 22 | cacheable_paths=["/"], 23 | ) 24 | 25 | final_args = {**base_args, **override_args} 26 | 27 | return RemoteCache(**final_args) 28 | 29 | 30 | def create_remote_file_system(tmp_path, **override_args): 31 | base_args = dict( 32 | client=LocalFileSystemService(), 33 | mount_callback=None, 34 | cache=create_cache(tmp_path), 35 | ) 36 | 37 | final_args = {**base_args, **override_args} 38 | 39 | return RemoteCachedFileSystem(**final_args) 40 | 41 | 42 | def test_cacheable_paths(tmp_path): 43 | (tmp_path / "cache").mkdir() 44 | 45 | (tmp_path / "cached").mkdir() 46 | (tmp_path / "notcached").mkdir() 47 | 48 | mock_client = mock.Mock() 49 | mock_client.get_metadata.return_value = LocalCacheService().get_metadata("/") 50 | 51 | fs = create_remote_file_system( 52 | tmp_path, 53 | client=mock_client, 54 | cache=create_cache( 55 | tmp_path, client=mock_client, cacheable_paths=[str(tmp_path / "cached")] 56 | ), 57 | ) 58 | 59 | fs.getattr(str(tmp_path / "cached" / "a"), None) 60 | fs.getattr(str(tmp_path / "cached" / "b"), None) 61 | fs.getattr(str(tmp_path / "cached" / "a"), None) 62 | 63 | assert mock_client.get_metadata.call_count == 2 64 | 65 | # Should not even be retrieved through cache 66 | fs.getattr(str(tmp_path / "notcached" / "a"), 123) 67 | fs.getattr(str(tmp_path / "notcached" / "b"), 456) 68 | fs.getattr(str(tmp_path / "notcached" / "a"), 789) 69 | 70 | assert mock_client.get_metadata.call_count == 2 71 | 72 | 73 | def test_cached_readlink(tmp_path): 74 | (tmp_path / "cache").mkdir() 75 | (tmp_path / "cached").mkdir() 76 | 77 | os.symlink("a", tmp_path / "cached/b") 78 | 79 | mock_client = mock.Mock() 80 | mock_client.get_metadata.side_effect = LocalCacheService().get_metadata 81 | 82 | fs = create_remote_file_system( 83 | tmp_path, 84 | client=mock_client, 85 | cache=create_cache( 86 | tmp_path, client=mock_client, cacheable_paths=[str(tmp_path / "cached")] 87 | ), 88 | ) 89 | 90 | with pytest.raises(FileNotFoundError): 91 | fs.getattr(str(tmp_path / "cached" / "a"), None) 92 | 93 | with pytest.raises(FileNotFoundError): 94 | fs.readlink(str(tmp_path / "cached" / "a")) 95 | 96 | assert mock_client.get_metadata.call_count == 1 97 | 98 | with pytest.raises(OSError) as e: 99 | fs.readlink(str(tmp_path / "cached")) 100 | 101 | assert e.value.args == (errno.EINVAL,) 102 | 103 | assert fs.readlink(str(tmp_path / "cached/b")) == "a" 104 | 105 | 106 | def test_uncached_readlink(tmp_path): 107 | (tmp_path / "cache").mkdir() 108 | os.symlink("bar", tmp_path / "foo") 109 | 110 | fs = create_remote_file_system( 111 | tmp_path, cache=create_cache(tmp_path, cacheable_paths=[]), 112 | ) 113 | 114 | assert fs.readlink(str(tmp_path / "foo")) == "bar" 115 | 116 | 117 | def test_cached_read(tmp_path): 118 | (tmp_path / "cache").mkdir() 119 | (tmp_path / "file").write_text("abcd") 120 | 121 | fs = create_remote_file_system(tmp_path) 122 | 123 | fd = fs.open(str(tmp_path / "file"), os.O_RDONLY) 124 | 125 | try: 126 | assert fs.read(str(tmp_path / "file"), fd, 1, 2) == b"bc" 127 | assert fs.read(str(tmp_path / "file"), fd, 0, 2) == b"ab" 128 | finally: 129 | fs.release(str(tmp_path / "file"), fd) 130 | 131 | 132 | def test_uncached_read(tmp_path): 133 | (tmp_path / "cache").mkdir() 134 | (tmp_path / "file").write_text("abcd") 135 | 136 | fs = create_remote_file_system( 137 | tmp_path, cache=create_cache(tmp_path, cacheable_paths=[]), 138 | ) 139 | 140 | fd = fs.open(str(tmp_path / "file"), os.O_RDONLY) 141 | 142 | try: 143 | assert fs.read(str(tmp_path / "file"), fd, 1, 2) == b"bc" 144 | assert fs.read(str(tmp_path / "file"), fd, 0, 2) == b"ab" 145 | finally: 146 | fs.release(str(tmp_path / "file"), fd) 147 | 148 | 149 | def test_cache_not_writable(tmp_path): 150 | (tmp_path / "cache").mkdir() 151 | (tmp_path / "file").write_text("abcd") 152 | 153 | fs = create_remote_file_system( 154 | tmp_path, cache=create_cache(tmp_path, cacheable_paths=[]), 155 | ) 156 | 157 | assert ( 158 | fs.getattr(str(tmp_path / "file"), None)["st_mode"] 159 | & (stat.S_IWUSR | stat.S_IWGRP | stat.S_IWOTH) 160 | != 0 161 | ) 162 | 163 | fs = create_remote_file_system(tmp_path) 164 | 165 | assert ( 166 | fs.getattr(str(tmp_path / "file"), None)["st_mode"] 167 | & (stat.S_IWUSR | stat.S_IWGRP | stat.S_IWOTH) 168 | == 0 169 | ) 170 | 171 | 172 | def test_cache_flush_file(tmp_path): 173 | (tmp_path / "cache").mkdir() 174 | 175 | fs = create_remote_file_system(tmp_path) 176 | 177 | # Ensure that flush is a no-op for cached files 178 | fs.flush(str(tmp_path / "file"), 0) 179 | -------------------------------------------------------------------------------- /outrun/tests/test_filesystem/test_caching/test_prefetching.py: -------------------------------------------------------------------------------- 1 | import os 2 | import shutil 3 | from unittest import mock 4 | 5 | from outrun.filesystem.caching.prefetching import PrefetchSuggestion 6 | import outrun.filesystem.caching.prefetching as prefetching 7 | 8 | 9 | def test_symlink_target_on_symlink(tmp_path): 10 | os.symlink(tmp_path / "bar", tmp_path / "foo") 11 | 12 | suggestions = prefetching.symlink_target(str(tmp_path / "foo")) 13 | assert suggestions == [PrefetchSuggestion(str(tmp_path / "bar"), contents=False)] 14 | 15 | 16 | def test_symlink_target_on_non_symlink(tmp_path): 17 | suggestions = prefetching.symlink_target(str(tmp_path)) 18 | assert len(suggestions) == 0 19 | 20 | 21 | def test_python_pycache(tmp_path): 22 | (tmp_path / "sample.py").touch() 23 | (tmp_path / "__pycache__").mkdir() 24 | (tmp_path / "__pycache__" / "sample.cpython-38.pyc").touch() 25 | 26 | suggestions = prefetching.python_pycache(str(tmp_path / "sample.py")) 27 | 28 | assert ( 29 | PrefetchSuggestion(path=str(tmp_path / "sample.py"), contents=True) 30 | in suggestions 31 | ) 32 | 33 | assert ( 34 | PrefetchSuggestion(path=str(tmp_path / "__pycache__"), contents=False) 35 | in suggestions 36 | ) 37 | 38 | assert ( 39 | PrefetchSuggestion( 40 | path=str(tmp_path / "__pycache__" / "sample.cpython-38.pyc"), contents=True 41 | ) 42 | in suggestions 43 | ) 44 | 45 | 46 | def test_python_pycache_non_python_file(tmp_path): 47 | suggestions = prefetching.python_pycache(str(tmp_path / "nonexistent")) 48 | assert len(suggestions) == 0 49 | 50 | 51 | def test_compiled_perl_module(tmp_path): 52 | suggestions = prefetching.compiled_perl_module(str(tmp_path / "sample.pmc")) 53 | assert PrefetchSuggestion(str(tmp_path / "sample.pm"), contents=True) in suggestions 54 | 55 | 56 | def test_compiled_perl_module_non_perl_file(tmp_path): 57 | suggestions = prefetching.compiled_perl_module(str(tmp_path / "nonexistent")) 58 | assert len(suggestions) == 0 59 | 60 | 61 | def test_elf_dependencies(): 62 | sh_path = shutil.which("ssh") 63 | 64 | suggestions = prefetching.elf_dependencies(sh_path) 65 | 66 | assert len(suggestions) > 0 67 | assert all(".so" in s.path for s in suggestions) 68 | 69 | 70 | def test_elf_dependencies_non_executable(tmp_path): 71 | (tmp_path / "non_elf_executable").touch() 72 | (tmp_path / "non_elf_executable").chmod(0o777) 73 | 74 | suggestions = prefetching.elf_dependencies(str(tmp_path / "non_elf_executable")) 75 | assert len(suggestions) == 0 76 | 77 | 78 | def test_elf_dependencies_symlinks(tmp_path): 79 | os.symlink("bar.so", tmp_path / "foo.so") 80 | 81 | with mock.patch( 82 | "outrun.filesystem.caching.prefetching.is_elf_binary" 83 | ) as mock_is_elf_binary: 84 | mock_is_elf_binary.return_value = True 85 | 86 | with mock.patch( 87 | "outrun.filesystem.caching.prefetching.read_elf_dependencies" 88 | ) as mock_read_elf_dependencies: 89 | mock_read_elf_dependencies.return_value = [str(tmp_path / "foo.so")] 90 | 91 | suggestions = prefetching.elf_dependencies("dummy") 92 | 93 | assert ( 94 | PrefetchSuggestion(str(tmp_path / "foo.so"), contents=False) 95 | in suggestions 96 | ) 97 | assert ( 98 | PrefetchSuggestion(str(tmp_path / "bar.so"), contents=True) 99 | in suggestions 100 | ) 101 | 102 | 103 | def test_elf_dependencies_with_weird_characters(): 104 | with mock.patch("subprocess.check_output") as mock_check_output: 105 | mock_check_output.return_value = b""" 106 | linux-vdso.so.1 (0x123abc) 107 | foo bar.so => /usr/lib/foo bar.so (0x123abc) 108 | foo=>bar.so => /usr/lib/foo=>bar.so (0x123abc) 109 | """ 110 | 111 | dependencies = prefetching.read_elf_dependencies("dummy") 112 | 113 | assert dependencies == ["/usr/lib/foo=>bar.so"] 114 | -------------------------------------------------------------------------------- /outrun/tests/test_filesystem/test_caching/test_service.py: -------------------------------------------------------------------------------- 1 | import os 2 | from pathlib import Path 3 | import shutil 4 | import stat 5 | from unittest import mock 6 | 7 | import pytest 8 | 9 | from outrun.filesystem.common import Attributes 10 | from outrun.filesystem.caching.common import Metadata 11 | from outrun.filesystem.caching.prefetching import PrefetchSuggestion 12 | from outrun.filesystem.caching.service import LocalCacheService 13 | 14 | 15 | @pytest.fixture 16 | def service(): 17 | return LocalCacheService() 18 | 19 | 20 | def test_get_metadata_error(service, tmp_path): 21 | meta = service.get_metadata(str(tmp_path / "nonexistent")) 22 | 23 | assert meta.attr is None 24 | assert meta.link is None 25 | assert isinstance(meta.error, FileNotFoundError) 26 | 27 | 28 | def test_get_metadata_dir(service, tmp_path): 29 | (tmp_path / "dir").mkdir() 30 | 31 | meta = service.get_metadata(str(tmp_path / "dir")) 32 | 33 | assert meta.error is None 34 | assert meta.link is None 35 | assert stat.S_ISDIR(meta.attr.st_mode) 36 | 37 | 38 | def test_get_metadata_file(service, tmp_path): 39 | (tmp_path / "file").write_text("") 40 | 41 | meta = service.get_metadata(str(tmp_path / "file")) 42 | 43 | assert meta.error is None 44 | assert meta.link is None 45 | assert stat.S_ISREG(meta.attr.st_mode) 46 | 47 | 48 | def test_get_metadata_symlink(service, tmp_path): 49 | os.symlink(tmp_path / "nonexistent", tmp_path / "link") 50 | 51 | meta = service.get_metadata(str(tmp_path / "link")) 52 | 53 | assert meta.error is None 54 | assert meta.link == str(tmp_path / "nonexistent") 55 | assert stat.S_ISLNK(meta.attr.st_mode) 56 | 57 | 58 | def test_changed_metadata(service, tmp_path): 59 | (tmp_path / "a").mkdir() 60 | (tmp_path / "b").mkdir() 61 | 62 | meta = { 63 | str(tmp_path / "a"): service.get_metadata(str(tmp_path / "a")), 64 | str(tmp_path / "b"): service.get_metadata(str(tmp_path / "b")), 65 | str(tmp_path / "c"): service.get_metadata(str(tmp_path / "c")), 66 | } 67 | 68 | # No changes yet 69 | assert list(service.get_changed_metadata(meta).keys()) == [] 70 | 71 | # Make changes to metadata 72 | os.utime(tmp_path / "a", (0, 0)) 73 | os.makedirs(tmp_path / "c") 74 | 75 | # Expect to receive changes 76 | changed_meta = service.get_changed_metadata(meta) 77 | 78 | assert list(changed_meta.keys()) == [ 79 | str(tmp_path / "a"), 80 | str(tmp_path / "c"), 81 | ] 82 | 83 | assert changed_meta[str(tmp_path / "a")] == service.get_metadata(tmp_path / "a") 84 | assert changed_meta[str(tmp_path / "c")] == service.get_metadata(tmp_path / "c") 85 | 86 | 87 | def test_access_time_changes_ignored(service): 88 | metadata = service.get_metadata("/") 89 | cached_metadata = {"/": metadata} 90 | 91 | with mock.patch( 92 | "outrun.filesystem.caching.service.LocalCacheService.get_metadata" 93 | ) as mock_meta: 94 | new_attr = Attributes(**metadata.attr.__dict__) 95 | new_attr.st_atime = 0.0 96 | new_metadata = Metadata(attr=new_attr) 97 | 98 | mock_meta.return_value = new_metadata 99 | 100 | assert service.get_changed_metadata(cached_metadata) == {} 101 | 102 | 103 | def test_readfile(service, tmp_path): 104 | (tmp_path / "file").write_text("abc") 105 | contents = service.readfile(str(tmp_path / "file")) 106 | 107 | assert contents.data == b"abc" 108 | 109 | (tmp_path / "file").write_text("def") 110 | new_contents = service.readfile(str(tmp_path / "file")) 111 | 112 | assert new_contents.data == b"def" 113 | assert contents.checksum != new_contents.checksum 114 | 115 | 116 | def test_readfile_conditional(service, tmp_path): 117 | (tmp_path / "file").write_text("abc") 118 | 119 | contents = service.readfile_conditional(str(tmp_path / "file"), "") 120 | assert contents.data == b"abc" 121 | 122 | new_contents = service.readfile_conditional( 123 | str(tmp_path / "file"), contents.checksum 124 | ) 125 | assert new_contents is None 126 | 127 | (tmp_path / "file").write_text("def") 128 | new_contents = service.readfile_conditional( 129 | str(tmp_path / "file"), contents.checksum 130 | ) 131 | 132 | assert new_contents.data == b"def" 133 | assert contents.checksum != new_contents.checksum 134 | 135 | 136 | def test_machine_id_consistent(service): 137 | machine_id_1 = service.get_app_specific_machine_id() 138 | machine_id_2 = service.get_app_specific_machine_id() 139 | 140 | assert machine_id_1 == machine_id_2 141 | 142 | 143 | def test_original_machine_id_not_being_exposed(service): 144 | machine_id = service.get_app_specific_machine_id() 145 | 146 | assert machine_id.strip() != Path("/etc/machine-id").read_text().strip() 147 | 148 | 149 | def test_get_metadata_prefetch_symlink(service, tmp_path): 150 | os.symlink("foo", tmp_path / "link") 151 | metadata, prefetches = service.get_metadata_prefetch(str(tmp_path / "link")) 152 | 153 | assert metadata.link is not None 154 | 155 | assert len(prefetches) == 1 156 | assert prefetches[0].path == str(tmp_path / "foo") 157 | assert isinstance(prefetches[0].metadata.error, FileNotFoundError) 158 | 159 | 160 | def test_get_metadata_prefetch_symlink_with_previously_fetched_target( 161 | service, tmp_path 162 | ): 163 | os.symlink("foo", tmp_path / "link") 164 | 165 | service.get_metadata(str(tmp_path / "foo")) 166 | _metadata, prefetches = service.get_metadata_prefetch(str(tmp_path / "link")) 167 | 168 | assert len(prefetches) == 0 169 | 170 | 171 | def test_readfile_prefetch_executable(service): 172 | sh_path = shutil.which("ssh") 173 | _metadata, prefetches = service.readfile_prefetch(sh_path) 174 | 175 | assert len(prefetches) > 0 176 | assert all(".so" in p.path for p in prefetches) 177 | 178 | 179 | def test_readfile_prefetch_executable_with_previously_fetched_contents(): 180 | sh_path = shutil.which("ssh") 181 | 182 | service = LocalCacheService() 183 | _metadata, prefetches = service.readfile_prefetch(sh_path) 184 | 185 | assert any(p.contents for p in prefetches) 186 | 187 | service = LocalCacheService() 188 | service.mark_previously_fetched_contents([p.path for p in prefetches]) 189 | _metadata, prefetches = service.readfile_prefetch(sh_path) 190 | 191 | assert not any(p.contents for p in prefetches) 192 | 193 | 194 | def test_prefetch_inside_prefetchable_paths(service, tmp_path): 195 | os.symlink("foo", tmp_path / "link") 196 | 197 | service.set_prefetchable_paths([str(tmp_path)]) 198 | _metadata, prefetches = service.get_metadata_prefetch(str(tmp_path / "link")) 199 | 200 | assert len(prefetches) != 0 201 | 202 | 203 | def test_prefetch_outside_prefetchable_paths(service, tmp_path): 204 | os.symlink("foo", tmp_path / "link") 205 | 206 | service.set_prefetchable_paths(["/nonexistent"]) 207 | _metadata, prefetches = service.get_metadata_prefetch(str(tmp_path / "link")) 208 | 209 | assert len(prefetches) == 0 210 | 211 | 212 | def test_get_metadata_prefetch_failure_handling(service, tmp_path): 213 | with mock.patch( 214 | "outrun.filesystem.caching.prefetching.file_access" 215 | ) as mock_prefetch: 216 | mock_prefetch.side_effect = Exception() 217 | 218 | service.get_metadata_prefetch(str(tmp_path)) 219 | 220 | 221 | def test_readfile_prefetch_failure_handling(service, tmp_path): 222 | (tmp_path / "foo").write_text("bar") 223 | 224 | with mock.patch("outrun.filesystem.caching.prefetching.file_read") as mock_prefetch: 225 | mock_prefetch.side_effect = Exception() 226 | 227 | service.readfile_prefetch(str(tmp_path / "foo")) 228 | 229 | 230 | def test_prefetching_unreadable_file(service, tmp_path): 231 | with mock.patch( 232 | "outrun.filesystem.caching.prefetching.file_access" 233 | ) as mock_prefetch: 234 | mock_prefetch.return_value = [ 235 | PrefetchSuggestion(str(tmp_path / "nonexistent"), contents=True) 236 | ] 237 | 238 | _metadata, prefetches = service.get_metadata_prefetch("/") 239 | 240 | # Assert that the metadata (non-contents) are still successfully prefetched 241 | assert len(prefetches) == 1 242 | assert isinstance(prefetches[0].metadata.error, FileNotFoundError) 243 | assert prefetches[0].path == str(tmp_path / "nonexistent") 244 | -------------------------------------------------------------------------------- /outrun/tests/test_filesystem/test_common.py: -------------------------------------------------------------------------------- 1 | import os 2 | import stat 3 | 4 | from outrun.filesystem.common import Attributes 5 | 6 | 7 | def test_base_attributes(): 8 | st = os.lstat(__file__) 9 | attribs = Attributes.from_stat(st) 10 | 11 | assert st.st_atime == attribs.st_atime 12 | assert st.st_size == attribs.st_size 13 | 14 | assert all(value is not None for value in attribs.__dict__.values()) 15 | 16 | 17 | def test_extra_attributes(): 18 | st = os.lstat(__file__) 19 | attribs = Attributes.from_stat(st) 20 | 21 | assert st.st_atime_ns == getattr(attribs, "st_atime_ns") 22 | 23 | 24 | def test_readonly(): 25 | st = os.lstat(__file__) 26 | 27 | attribs = Attributes.from_stat(st) 28 | readonly_attribs = attribs.as_readonly() 29 | 30 | assert attribs.st_mode & (stat.S_IWUSR | stat.S_IWGRP | stat.S_IWOTH) != 0 31 | assert readonly_attribs.st_mode & (stat.S_IWUSR | stat.S_IWGRP | stat.S_IWOTH) == 0 32 | 33 | -------------------------------------------------------------------------------- /outrun/tests/test_filesystem/test_filesystem.py: -------------------------------------------------------------------------------- 1 | import os 2 | from unittest import mock 3 | 4 | from outrun.filesystem.filesystem import RemoteFileSystem 5 | from outrun.filesystem.common import Attributes 6 | 7 | 8 | def test_mount_callback(): 9 | callback = mock.Mock() 10 | 11 | mock_client = mock.Mock() 12 | fs = RemoteFileSystem(mock_client, callback) 13 | 14 | assert not callback.called 15 | fs.init() 16 | assert callback.called 17 | 18 | 19 | def test_open(): 20 | mock_client = mock.Mock() 21 | fs = RemoteFileSystem(mock_client, None) 22 | 23 | mock_client.open.return_value = 9 24 | assert fs.open("a/b/c", 123) == 9 25 | mock_client.open.assert_called_with("a/b/c", 123) 26 | 27 | 28 | def test_create(): 29 | mock_client = mock.Mock() 30 | fs = RemoteFileSystem(mock_client, None) 31 | 32 | mock_client.create.return_value = 9 33 | assert fs.create("a/b/c", os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 567) == 9 34 | mock_client.create.assert_called_with( 35 | "a/b/c", os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 567 36 | ) 37 | 38 | 39 | def test_read(): 40 | mock_client = mock.Mock() 41 | fs = RemoteFileSystem(mock_client, None) 42 | 43 | mock_client.read.return_value = b"abc" 44 | fs.read("file", 1234, 5678, 9) 45 | mock_client.read.assert_called_with(1234, 5678, 9) 46 | 47 | 48 | def test_write(): 49 | mock_client = mock.Mock() 50 | fs = RemoteFileSystem(mock_client, None) 51 | 52 | mock_client.write.return_value = 3 53 | assert fs.write("file", b"abc", 123, 456) == 3 54 | mock_client.write.assert_called_with(b"abc", 123, 456) 55 | 56 | 57 | def test_lseek(): 58 | if not hasattr(os, "SEEK_DATA"): 59 | return 60 | 61 | mock_client = mock.Mock() 62 | fs = RemoteFileSystem(mock_client, None) 63 | 64 | mock_client.lseek.return_value = 3 65 | assert fs.lseek("file", 123, 456, os.SEEK_DATA) == 3 66 | mock_client.lseek.assert_called_with(123, 456, os.SEEK_DATA) 67 | 68 | 69 | def test_fsync(): 70 | mock_client = mock.Mock() 71 | fs = RemoteFileSystem(mock_client, None) 72 | 73 | fs.fsync("path", 123, False) 74 | mock_client.fsync.assert_called_with(123, False) 75 | fs.fsync("path", 123, True) 76 | mock_client.fsync.assert_called_with(123, True) 77 | 78 | 79 | def test_flush(): 80 | mock_client = mock.Mock() 81 | fs = RemoteFileSystem(mock_client, None) 82 | 83 | fs.flush("path", 1337) 84 | mock_client.flush.assert_called_with(1337) 85 | 86 | 87 | def test_truncate(): 88 | mock_client = mock.Mock() 89 | fs = RemoteFileSystem(mock_client, None) 90 | 91 | fs.truncate("path", 123, 456) 92 | mock_client.truncate.assert_called_with("path", 123, 456) 93 | 94 | 95 | def test_release(): 96 | mock_client = mock.Mock() 97 | fs = RemoteFileSystem(mock_client, None) 98 | 99 | fs.release("path", 123) 100 | mock_client.release.assert_called_with(123) 101 | 102 | 103 | def test_readdir(): 104 | mock_client = mock.Mock() 105 | fs = RemoteFileSystem(mock_client, None) 106 | 107 | mock_client.readdir.return_value = [".", "..", "foo", "bar"] 108 | assert fs.readdir("dir") == [".", "..", "foo", "bar"] 109 | mock_client.readdir.assert_called_with("dir") 110 | 111 | 112 | def test_readlink(): 113 | mock_client = mock.Mock() 114 | fs = RemoteFileSystem(mock_client, None) 115 | 116 | mock_client.readlink.return_value = "foo" 117 | assert fs.readlink("link") == "foo" 118 | mock_client.readlink.assert_called_with("link") 119 | 120 | 121 | def test_getattr(): 122 | mock_client = mock.Mock() 123 | fs = RemoteFileSystem(mock_client, None) 124 | 125 | attribs = Attributes.from_stat(os.lstat(__file__)) 126 | mock_client.getattr.return_value = attribs 127 | assert fs.getattr("foo", 123) == attribs.__dict__ 128 | mock_client.getattr.assert_called_with("foo", 123) 129 | 130 | 131 | def test_chmod(): 132 | mock_client = mock.Mock() 133 | fs = RemoteFileSystem(mock_client, None) 134 | 135 | fs.chmod("path", 123, 456) 136 | mock_client.chmod.assert_called_with("path", 123, 456) 137 | 138 | 139 | def test_chown(): 140 | mock_client = mock.Mock() 141 | fs = RemoteFileSystem(mock_client, None) 142 | 143 | fs.chown("path", 123, 456, 789) 144 | mock_client.chown.assert_called_with("path", 123, 456, 789) 145 | 146 | 147 | def test_utimens(): 148 | mock_client = mock.Mock() 149 | fs = RemoteFileSystem(mock_client, None) 150 | 151 | # RPC serialization turns the tuple into a list, but we're not testing that here 152 | fs.utimens("path", 123, (1, 2)) 153 | mock_client.utimens.assert_called_with("path", 123, (1, 2)) 154 | 155 | 156 | def test_link(): 157 | mock_client = mock.Mock() 158 | fs = RemoteFileSystem(mock_client, None) 159 | 160 | fs.link("target", "source") 161 | mock_client.link.assert_called_with("target", "source") 162 | 163 | 164 | def test_symlink(): 165 | mock_client = mock.Mock() 166 | fs = RemoteFileSystem(mock_client, None) 167 | 168 | fs.symlink("target", "source") 169 | mock_client.symlink.assert_called_with("target", "source") 170 | 171 | 172 | def test_mkdir(): 173 | mock_client = mock.Mock() 174 | fs = RemoteFileSystem(mock_client, None) 175 | 176 | fs.mkdir("path", 123) 177 | mock_client.mkdir.assert_called_with("path", 123) 178 | 179 | 180 | def test_mknod(): 181 | mock_client = mock.Mock() 182 | fs = RemoteFileSystem(mock_client, None) 183 | 184 | fs.mknod("path", 123, 456) 185 | mock_client.mknod.assert_called_with("path", 123, 456) 186 | 187 | 188 | def test_rename(): 189 | mock_client = mock.Mock() 190 | fs = RemoteFileSystem(mock_client, None) 191 | 192 | fs.rename("old", "new") 193 | mock_client.rename.assert_called_with("old", "new") 194 | 195 | 196 | def test_unlink(): 197 | mock_client = mock.Mock() 198 | fs = RemoteFileSystem(mock_client, None) 199 | 200 | fs.unlink("path") 201 | mock_client.unlink.assert_called_with("path") 202 | 203 | 204 | def test_rmdir(): 205 | mock_client = mock.Mock() 206 | fs = RemoteFileSystem(mock_client, None) 207 | 208 | fs.rmdir("path") 209 | mock_client.rmdir.assert_called_with("path") 210 | 211 | 212 | def test_statfs(): 213 | mock_client = mock.Mock() 214 | fs = RemoteFileSystem(mock_client, None) 215 | 216 | stfs = {"a": 1, "b": 2, "c": 3} 217 | 218 | mock_client.statfs.return_value = stfs 219 | assert fs.statfs("/") == stfs 220 | mock_client.statfs.assert_called_with("/") 221 | -------------------------------------------------------------------------------- /outrun/tests/test_filesystem/test_fuse/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Overv/outrun/20af1136060ecb0a53f464b73e5cdac913a097c3/outrun/tests/test_filesystem/test_fuse/__init__.py -------------------------------------------------------------------------------- /outrun/tests/test_filesystem/test_fuse/test_operations.py: -------------------------------------------------------------------------------- 1 | from contextlib import contextmanager 2 | import errno 3 | import os 4 | from unittest import mock 5 | import subprocess 6 | import stat 7 | import threading 8 | import time 9 | from typing import List, Optional, Tuple 10 | 11 | import pytest 12 | 13 | from outrun.filesystem.fuse import FuseConfig, FUSE, Operations 14 | 15 | 16 | class LoopbackFS(Operations): 17 | def getattr(self, path: str, fh: Optional[int]) -> dict: 18 | if fh and os.stat in os.supports_fd: 19 | st = os.stat(fh) 20 | else: 21 | if os.stat in os.supports_follow_symlinks: 22 | st = os.stat(path, follow_symlinks=False) 23 | else: 24 | st = os.stat(path) 25 | 26 | return {k: getattr(st, k) for k in dir(st) if k.startswith("st_")} 27 | 28 | def readlink(self, path: str) -> str: 29 | return os.readlink(path) 30 | 31 | def readdir(self, path: str) -> List[str]: 32 | return [".", ".."] + os.listdir(path) 33 | 34 | def mknod(self, path: str, mode: int, rdev: int) -> None: 35 | if stat.S_ISFIFO(mode): 36 | os.mkfifo(path, mode) 37 | else: 38 | os.mknod(path, mode, rdev) 39 | 40 | def mkdir(self, path: str, mode: int) -> None: 41 | os.mkdir(path, mode) 42 | 43 | def symlink(self, path: str, target: str) -> None: 44 | os.symlink(target, path) 45 | 46 | def unlink(self, path: str) -> None: 47 | os.unlink(path) 48 | 49 | def rmdir(self, path: str) -> None: 50 | os.rmdir(path) 51 | 52 | def rename(self, old: str, new: str) -> None: 53 | os.rename(old, new) 54 | 55 | def link(self, path: str, target: str) -> None: 56 | os.link(target, path) 57 | 58 | def chmod(self, path: str, fh: Optional[int], mode: int) -> None: 59 | if fh and os.chmod in os.supports_fd: 60 | os.chmod(fh, mode) 61 | else: 62 | if os.chmod in os.supports_follow_symlinks: 63 | os.chmod(path, mode, follow_symlinks=False) 64 | else: 65 | os.chmod(path, mode) 66 | 67 | def chown(self, path: str, fh: Optional[int], uid: int, gid: int) -> None: 68 | if fh and os.chown in os.supports_fd: 69 | os.chown(fh, uid, gid) 70 | else: 71 | if os.chown in os.supports_follow_symlinks: 72 | os.chown(path, uid, gid, follow_symlinks=False) 73 | else: 74 | os.chown(path, uid, gid) 75 | 76 | def truncate(self, path: str, fh: Optional[int], size: int) -> None: 77 | if fh and os.truncate in os.supports_fd: 78 | os.truncate(fh, size) 79 | else: 80 | os.truncate(path, size) 81 | 82 | def utimens(self, path: str, fh: Optional[int], times: Tuple[int, int]) -> None: 83 | if fh and os.utime in os.supports_fd: 84 | os.utime(fh, ns=times) 85 | else: 86 | if os.utime in os.supports_follow_symlinks: 87 | os.utime(path, ns=times, follow_symlinks=False) 88 | else: 89 | os.utime(path, ns=times) 90 | 91 | def open(self, path: str, flags: int) -> int: 92 | return os.open(path, flags) 93 | 94 | def create(self, path: str, flags: int, mode: int) -> int: 95 | return os.open(path, flags, mode) 96 | 97 | def read(self, path: str, fh: int, offset: int, size: int) -> bytes: 98 | return os.pread(fh, size, offset) 99 | 100 | def write(self, path: str, fh: int, offset: int, data: bytes) -> int: 101 | return os.pwrite(fh, data, offset) 102 | 103 | def statfs(self, path: str) -> dict: 104 | st = os.statvfs(path) 105 | return {k: getattr(st, k) for k in dir(st) if k.startswith("f_")} 106 | 107 | def release(self, path: str, fh: int) -> None: 108 | os.close(fh) 109 | 110 | def flush(self, path: str, fh: int) -> None: 111 | os.close(os.dup(fh)) 112 | 113 | def fsync(self, path: str, fh: int, datasync: bool) -> None: 114 | if datasync: 115 | os.fdatasync(fh) 116 | else: 117 | os.fsync(fh) 118 | 119 | def lseek(self, path: str, fh: int, offset: int, whence: int) -> int: 120 | return os.lseek(fh, offset, whence) 121 | 122 | 123 | @contextmanager 124 | def mount_fs(mount_path, fs, config): 125 | def do_mount(): 126 | fuse = FUSE(fs, config) 127 | fuse.mount("test", str(mount_path)) 128 | 129 | t = threading.Thread(target=do_mount) 130 | t.start() 131 | 132 | while True: 133 | # Command will fail until mount has completed. 134 | try: 135 | subprocess.check_call(["mountpoint", mount_path]) 136 | except subprocess.CalledProcessError: 137 | pass 138 | else: 139 | break 140 | 141 | if not t.is_alive(): 142 | raise RuntimeError("file system mount failed") 143 | 144 | try: 145 | yield 146 | finally: 147 | subprocess.check_output(["fusermount", "-u", str(mount_path)]) 148 | 149 | # Wait for unmount to really finish 150 | while True: 151 | try: 152 | subprocess.check_call(["mountpoint", mount_path]) 153 | except Exception: 154 | break 155 | 156 | 157 | @pytest.fixture 158 | def loopback_fs(): 159 | return mock.Mock(wraps=LoopbackFS()) 160 | 161 | 162 | @pytest.fixture 163 | def loopback_fs_root(tmp_path): 164 | mount_path = tmp_path / "mount" 165 | root_path = tmp_path / "root" 166 | 167 | os.makedirs(mount_path) 168 | os.makedirs(root_path) 169 | 170 | config = FuseConfig() 171 | config.writeback_cache = True 172 | 173 | with mount_fs(mount_path, LoopbackFS(), config): 174 | yield mount_path / root_path.relative_to("/") 175 | 176 | 177 | @pytest.mark.fuse 178 | def test_init(loopback_fs, tmp_path): 179 | with mount_fs(tmp_path, loopback_fs, FuseConfig()): 180 | loopback_fs.init.assert_called_once() 181 | 182 | 183 | @pytest.mark.fuse 184 | def test_destroy(loopback_fs, tmp_path): 185 | with mount_fs(tmp_path, loopback_fs, FuseConfig()): 186 | pass 187 | 188 | loopback_fs.destroy.assert_called_once() 189 | 190 | 191 | @pytest.mark.fuse 192 | def test_os_error(loopback_fs, tmp_path): 193 | loopback_fs.readdir.side_effect = OSError( 194 | errno.EADDRINUSE, os.strerror(errno.EADDRINUSE) 195 | ) 196 | 197 | with mount_fs(tmp_path, loopback_fs, FuseConfig()): 198 | with pytest.raises(OSError) as e: 199 | os.listdir(tmp_path) 200 | 201 | assert e.value.errno == errno.EADDRINUSE 202 | 203 | 204 | @pytest.mark.fuse 205 | def test_unimplemented(loopback_fs, tmp_path): 206 | loopback_fs.readdir.side_effect = NotImplementedError() 207 | 208 | with mount_fs(tmp_path, loopback_fs, FuseConfig()): 209 | with pytest.raises(OSError) as e: 210 | os.listdir(tmp_path) 211 | 212 | assert e.value.errno == errno.ENOSYS 213 | 214 | 215 | @pytest.mark.fuse 216 | def test_unknown_exception(loopback_fs, tmp_path): 217 | loopback_fs.readdir.side_effect = RuntimeError() 218 | 219 | with mount_fs(tmp_path, loopback_fs, FuseConfig()): 220 | with pytest.raises(OSError) as e: 221 | os.listdir(tmp_path) 222 | 223 | assert e.value.errno == errno.EIO 224 | 225 | 226 | @pytest.mark.fuse 227 | def test_getattr(loopback_fs_root): 228 | (loopback_fs_root / "dir").mkdir() 229 | (loopback_fs_root / "file").touch() 230 | 231 | with pytest.raises(FileNotFoundError): 232 | os.lstat(loopback_fs_root / "nonexistent") 233 | 234 | (loopback_fs_root / "dir").is_dir() 235 | (loopback_fs_root / "file").is_file() 236 | 237 | 238 | @pytest.mark.fuse 239 | def test_getattr_fd(loopback_fs_root): 240 | fd = os.open(loopback_fs_root / "file", os.O_RDWR | os.O_CREAT) 241 | 242 | try: 243 | os.unlink(loopback_fs_root / "file") 244 | 245 | os.write(fd, b"abc") 246 | os.fsync(fd) 247 | 248 | assert os.fstat(fd).st_size == 3 249 | 250 | with pytest.raises(FileNotFoundError): 251 | os.lstat(loopback_fs_root / "file") 252 | finally: 253 | os.close(fd) 254 | 255 | 256 | @pytest.mark.fuse 257 | def test_readlink(loopback_fs_root): 258 | os.symlink("nonexistent", loopback_fs_root / "link") 259 | assert os.readlink(loopback_fs_root / "link") == "nonexistent" 260 | 261 | 262 | @pytest.mark.fuse 263 | def test_readdir(loopback_fs_root): 264 | assert os.listdir(loopback_fs_root) == [] 265 | 266 | (loopback_fs_root / "foo").touch() 267 | (loopback_fs_root / "bar").mkdir() 268 | 269 | assert set(os.listdir(loopback_fs_root)) == set(["foo", "bar"]) 270 | 271 | 272 | @pytest.mark.fuse 273 | def test_mknod(loopback_fs_root): 274 | os.mkfifo(loopback_fs_root / "foo", 0o600) 275 | assert (loopback_fs_root / "foo").is_fifo() 276 | 277 | 278 | @pytest.mark.fuse 279 | def test_mkdir(loopback_fs_root): 280 | (loopback_fs_root / "dir").mkdir() 281 | 282 | assert (loopback_fs_root / "dir").is_dir() 283 | 284 | with pytest.raises(FileExistsError): 285 | (loopback_fs_root / "dir").mkdir() 286 | 287 | 288 | @pytest.mark.fuse 289 | def test_symlink(loopback_fs_root): 290 | (loopback_fs_root / "link").symlink_to("nonexistent") 291 | assert os.readlink(loopback_fs_root / "link") == "nonexistent" 292 | 293 | 294 | @pytest.mark.fuse 295 | def test_unlink(loopback_fs_root): 296 | with pytest.raises(FileNotFoundError): 297 | (loopback_fs_root / "file").unlink() 298 | 299 | (loopback_fs_root / "file").touch() 300 | (loopback_fs_root / "file").unlink() 301 | 302 | 303 | @pytest.mark.fuse 304 | def test_rmdir(loopback_fs_root): 305 | (loopback_fs_root / "file").touch() 306 | 307 | with pytest.raises(NotADirectoryError): 308 | (loopback_fs_root / "file").rmdir() 309 | 310 | (loopback_fs_root / "dir").mkdir() 311 | (loopback_fs_root / "dir").rmdir() 312 | 313 | 314 | @pytest.mark.fuse 315 | def test_rename(loopback_fs_root): 316 | (loopback_fs_root / "file").touch() 317 | (loopback_fs_root / "file").rename(loopback_fs_root / "file2") 318 | assert (loopback_fs_root / "file2").is_file() 319 | 320 | 321 | @pytest.mark.fuse 322 | def test_link(loopback_fs_root): 323 | (loopback_fs_root / "file").touch() 324 | (loopback_fs_root / "file").link_to(loopback_fs_root / "link") 325 | 326 | with pytest.raises(FileNotFoundError): 327 | (loopback_fs_root / "nonexistent").link_to(loopback_fs_root / "link") 328 | 329 | 330 | @pytest.mark.fuse 331 | def test_chmod(loopback_fs_root): 332 | (loopback_fs_root / "file").touch() 333 | (loopback_fs_root / "file").chmod(0o600) 334 | 335 | 336 | @pytest.mark.fuse 337 | def test_chown(loopback_fs_root): 338 | (loopback_fs_root / "file").touch() 339 | os.chown(loopback_fs_root / "file", os.getuid(), os.getgid()) 340 | 341 | 342 | @pytest.mark.fuse 343 | def test_truncate(loopback_fs_root): 344 | (loopback_fs_root / "file").touch() 345 | os.truncate(loopback_fs_root / "file", 123) 346 | assert (loopback_fs_root / "file").stat().st_size == 123 347 | 348 | 349 | @pytest.mark.fuse 350 | def test_utimens(loopback_fs_root): 351 | (loopback_fs_root / "file").touch() 352 | 353 | os.utime(loopback_fs_root / "file", ns=(123, 456)) 354 | assert (loopback_fs_root / "file").stat().st_atime_ns == 123 355 | assert (loopback_fs_root / "file").stat().st_mtime_ns == 456 356 | 357 | (loopback_fs_root / "file").touch() 358 | 359 | assert (loopback_fs_root / "file").stat().st_mtime >= time.time() - 1.0 360 | 361 | 362 | @pytest.mark.fuse 363 | def test_read_write(loopback_fs_root): 364 | (loopback_fs_root / "file").write_bytes(b"abcdef") 365 | 366 | fd = os.open(loopback_fs_root / "file", os.O_RDWR) 367 | 368 | try: 369 | os.pwrite(fd, b"xxx", 2) 370 | assert os.pread(fd, 3, 1) == b"bxx" 371 | os.fsync(fd) 372 | finally: 373 | os.close(fd) 374 | 375 | 376 | @pytest.mark.fuse 377 | def test_statfs(loopback_fs_root): 378 | subprocess.check_call(["df", loopback_fs_root]) 379 | 380 | 381 | @pytest.mark.fuse 382 | def test_lseek(loopback_fs_root): 383 | if not hasattr(os, "SEEK_DATA"): 384 | return 385 | 386 | fd = os.open(loopback_fs_root / "file", os.O_WRONLY | os.O_CREAT) 387 | try: 388 | os.pwrite(fd, b"abc", 1024 * 1024) 389 | finally: 390 | os.close(fd) 391 | 392 | # Note that the file must be reopened like this for lseek to work when the writeback 393 | # cache is enabled. 394 | 395 | fd = os.open(loopback_fs_root / "file", os.O_RDONLY) 396 | try: 397 | assert os.lseek(fd, 0, os.SEEK_DATA) > 0 398 | assert os.lseek(fd, 0, os.SEEK_HOLE) == 0 399 | finally: 400 | os.close(fd) 401 | -------------------------------------------------------------------------------- /outrun/tests/test_filesystem/test_service.py: -------------------------------------------------------------------------------- 1 | import os 2 | import stat 3 | 4 | import pytest 5 | 6 | from outrun.filesystem.service import LocalFileSystemService 7 | 8 | 9 | @pytest.fixture 10 | def fs(): 11 | return LocalFileSystemService() 12 | 13 | 14 | def test_open_modes(fs, tmp_path): 15 | with pytest.raises(FileNotFoundError): 16 | fs.open(str(tmp_path / "write.txt"), os.O_WRONLY) 17 | 18 | fh = fs.open(str(tmp_path / "write.txt"), os.O_CREAT | os.O_WRONLY) 19 | 20 | try: 21 | with pytest.raises(OSError): 22 | fs.read(fh, 0, 2) 23 | finally: 24 | fs.release(fh) 25 | 26 | 27 | def test_file_reads(fs, tmp_path): 28 | with open(tmp_path / "read.txt", "wb") as f: 29 | f.write(b"abcdef") 30 | 31 | fh = fs.open(str(tmp_path / "read.txt"), os.O_RDONLY) 32 | 33 | try: 34 | assert fs.read(fh, 0, 2) == b"ab" 35 | assert fs.read(fh, 0, 3) == b"abc" 36 | assert fs.read(fh, 1, 2) == b"bc" 37 | assert fs.read(fh, 0, 1024) == b"abcdef" 38 | finally: 39 | fs.release(fh) 40 | 41 | 42 | def test_file_writes(fs, tmp_path): 43 | fh = fs.create( 44 | str(tmp_path / "write.txt"), os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o777 45 | ) 46 | 47 | fs.write(fh, 1, b"bc") 48 | fs.write(fh, 0, b"a") 49 | 50 | fs.release(fh) 51 | 52 | with open(tmp_path / "write.txt", "rb") as f: 53 | assert f.read() == b"abc" 54 | 55 | 56 | def test_file_truncate(fs, tmp_path): 57 | with open(tmp_path / "small", "wb"): 58 | pass 59 | 60 | with open(tmp_path / "large", "wb"): 61 | pass 62 | 63 | fs.truncate(str(tmp_path / "small"), None, 10) 64 | fs.truncate(str(tmp_path / "large"), None, 100) 65 | 66 | assert os.lstat(tmp_path / "small").st_size == 10 67 | assert os.lstat(tmp_path / "large").st_size == 100 68 | 69 | 70 | def test_lseek(fs, tmp_path): 71 | if not hasattr(os, "SEEK_DATA"): 72 | return 73 | 74 | fd = fs.open(tmp_path / "file", os.O_RDWR | os.O_CREAT) 75 | try: 76 | fs.write(fd, 1024 * 1024, b"abc") 77 | assert fs.lseek(fd, 0, os.SEEK_DATA) > 0 78 | assert fs.lseek(fd, 0, os.SEEK_HOLE) == 0 79 | finally: 80 | fs.release(fd) 81 | 82 | 83 | def test_readdir(fs, tmp_path): 84 | assert sorted(fs.readdir(str(tmp_path))) == sorted([".", ".."]) 85 | 86 | with open(tmp_path / "a", "wb"): 87 | pass 88 | 89 | os.link(tmp_path / "a", tmp_path / "b") 90 | 91 | os.makedirs(tmp_path / "c" / "d") 92 | 93 | assert sorted(fs.readdir(str(tmp_path))) == sorted([".", "..", "a", "b", "c"]) 94 | 95 | 96 | def test_readlink(fs, tmp_path): 97 | os.symlink(tmp_path / "a", tmp_path / "b") 98 | 99 | assert fs.readlink(str(tmp_path / "b")) == str(tmp_path / "a") 100 | 101 | 102 | def test_getattr(fs, tmp_path): 103 | with pytest.raises(FileNotFoundError): 104 | fs.getattr(str(tmp_path / "link"), None) 105 | 106 | with open(tmp_path / "file", "wb"): 107 | pass 108 | 109 | os.symlink(tmp_path / "file", tmp_path / "link") 110 | 111 | assert os.path.islink(tmp_path / "link") 112 | assert os.path.isfile(tmp_path / "file") 113 | 114 | assert fs.getattr(__file__, None).st_mtime == os.stat(__file__).st_mtime 115 | 116 | 117 | def test_chmod(fs, tmp_path): 118 | with open(tmp_path / "file", "wb"): 119 | pass 120 | 121 | fs.chmod(str(tmp_path / "file"), None, 0o000) 122 | 123 | assert stat.S_IMODE(os.lstat(tmp_path / "file").st_mode) == 0o000 124 | 125 | fs.chmod(str(tmp_path / "file"), None, 0o777) 126 | 127 | assert stat.S_IMODE(os.lstat(tmp_path / "file").st_mode) == 0o777 128 | 129 | 130 | def test_utimens_explicit(fs, tmp_path): 131 | with open(tmp_path / "file", "wb"): 132 | pass 133 | 134 | fs.utimens(str(tmp_path / "file"), None, [1, 2]) 135 | 136 | st = os.stat(tmp_path / "file") 137 | assert st.st_atime_ns == 1 138 | assert st.st_mtime_ns == 2 139 | 140 | 141 | def test_link(fs, tmp_path): 142 | with open(tmp_path / "file", "wb"): 143 | pass 144 | 145 | with pytest.raises(FileNotFoundError): 146 | fs.link(str(tmp_path / "link"), str(tmp_path / "nonexistent")) 147 | 148 | fs.link(str(tmp_path / "link"), str(tmp_path / "file")) 149 | 150 | assert os.path.isfile(tmp_path / "link") 151 | 152 | 153 | def test_symlink(fs, tmp_path): 154 | fs.symlink(str(tmp_path / "link"), str(tmp_path / "nonexistent")) 155 | 156 | assert os.path.islink(tmp_path / "link") 157 | assert os.readlink(tmp_path / "link") == str(tmp_path / "nonexistent") 158 | 159 | 160 | def test_mkdir(fs, tmp_path): 161 | fs.mkdir(str(tmp_path / "dir"), 0o777) 162 | 163 | with pytest.raises(FileExistsError): 164 | fs.mkdir(str(tmp_path / "dir"), 0o777) 165 | 166 | assert os.path.isdir(tmp_path / "dir") 167 | 168 | 169 | def test_rename(fs, tmp_path): 170 | with open(tmp_path / "file", "wb") as f: 171 | f.write(b"abc") 172 | 173 | st = os.lstat(tmp_path / "file") 174 | 175 | fs.rename(str(tmp_path / "file"), str(tmp_path / "file2")) 176 | 177 | assert not os.path.exists(tmp_path / "file") 178 | assert os.path.isfile(tmp_path / "file2") 179 | 180 | with open(tmp_path / "file2", "rb") as f: 181 | assert f.read() == b"abc" 182 | 183 | assert os.lstat(tmp_path / "file2").st_mtime == st.st_mtime 184 | 185 | 186 | def test_unlink(fs, tmp_path): 187 | with pytest.raises(FileNotFoundError): 188 | fs.unlink(str(tmp_path / "nonexistent")) 189 | 190 | os.makedirs(tmp_path / "dir") 191 | 192 | with pytest.raises(IsADirectoryError): 193 | fs.unlink(str(tmp_path / "dir")) 194 | 195 | with open(tmp_path / "file", "wb"): 196 | pass 197 | 198 | fs.unlink(str(tmp_path / "file")) 199 | assert not os.path.exists(tmp_path / "file") 200 | 201 | 202 | def test_rmdir(fs, tmp_path): 203 | with pytest.raises(FileNotFoundError): 204 | fs.rmdir(str(tmp_path / "nonexistent")) 205 | 206 | with open(tmp_path / "file", "wb"): 207 | pass 208 | 209 | with pytest.raises(NotADirectoryError): 210 | fs.rmdir(str(tmp_path / "file")) 211 | 212 | os.makedirs(tmp_path / "dir") 213 | fs.rmdir(str(tmp_path / "dir")) 214 | assert not os.path.exists(tmp_path / "dir") 215 | 216 | 217 | def test_statfs(fs): 218 | statvfs = fs.statfs("/") 219 | 220 | assert isinstance(statvfs, dict) 221 | assert "f_blocks" in statvfs 222 | -------------------------------------------------------------------------------- /outrun/tests/test_logger.py: -------------------------------------------------------------------------------- 1 | import outrun.logger as logger 2 | 3 | 4 | def test_summarize_matching_length(): 5 | assert logger.summarize("abc", max_length=3) == "abc" 6 | 7 | 8 | def test_summarize_exceeding_length(): 9 | assert logger.summarize("abcdef", max_length=5) == "ab..." 10 | 11 | 12 | def test_summarize_list(): 13 | x = [1, 2, 3, 4, 5] 14 | assert logger.summarize(x, max_length=6) == "[1,..." 15 | -------------------------------------------------------------------------------- /outrun/tests/test_main.py: -------------------------------------------------------------------------------- 1 | from unittest import mock 2 | import logging 3 | 4 | import semver 5 | import pytest 6 | 7 | from outrun.__main__ import main 8 | from outrun.logger import log 9 | 10 | 11 | def test_no_args(): 12 | with pytest.raises(SystemExit): 13 | main(["outrun"]) 14 | 15 | 16 | def test_protocol_check(caplog): 17 | mismatching_major = semver.VersionInfo.parse("0.0.0") 18 | 19 | with pytest.raises(SystemExit): 20 | main( 21 | ["outrun", f"--protocol={mismatching_major}", "hostname", "command", "arg",] 22 | ) 23 | 24 | assert "incompatible protocol" in caplog.text 25 | 26 | 27 | def test_debug_flag_set(): 28 | with mock.patch("outrun.operations.LocalOperations"): 29 | with pytest.raises(SystemExit): 30 | main(["outrun", "--debug", "host", "command"]) 31 | 32 | assert log.getEffectiveLevel() == logging.DEBUG 33 | 34 | 35 | def test_debug_flag_not_set(): 36 | with mock.patch("outrun.operations.LocalOperations"): 37 | with pytest.raises(SystemExit): 38 | main(["outrun", "host", "command"]) 39 | 40 | assert log.getEffectiveLevel() == logging.ERROR 41 | 42 | 43 | def test_local_operations(): 44 | with mock.patch("outrun.operations.LocalOperations") as mock_operations: 45 | with pytest.raises(SystemExit): 46 | main(["outrun", "host", "command"]) 47 | 48 | assert mock_operations().run.called 49 | 50 | 51 | def test_remote_operations(): 52 | with mock.patch("outrun.operations.RemoteOperations") as mock_operations: 53 | with pytest.raises(SystemExit): 54 | main(["outrun", "--remote", "host", "command"]) 55 | 56 | assert mock_operations().run.called 57 | 58 | 59 | def test_command_failure(caplog): 60 | with mock.patch("outrun.operations.LocalOperations") as mock_operations: 61 | mock_operations().run.side_effect = Exception("foo") 62 | 63 | with pytest.raises(SystemExit): 64 | main(["outrun", "host", "command"]) 65 | 66 | assert "failed to run command: foo" in caplog.text 67 | -------------------------------------------------------------------------------- /outrun/tests/test_operations/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Overv/outrun/20af1136060ecb0a53f464b73e5cdac913a097c3/outrun/tests/test_operations/__init__.py -------------------------------------------------------------------------------- /outrun/tests/test_operations/test_environment.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | import pytest 4 | 5 | from outrun.operations.local import LocalEnvironmentService 6 | 7 | 8 | @pytest.fixture 9 | def service(): 10 | return LocalEnvironmentService(["a", "b", "c"]) 11 | 12 | 13 | def test_get_command(service): 14 | assert service.get_command() == ["a", "b", "c"] 15 | 16 | 17 | def test_get_working_dir(service): 18 | assert service.get_working_dir() == os.getcwd() 19 | 20 | 21 | def test_get_environment(service): 22 | assert service.get_environment() == dict(os.environ) 23 | -------------------------------------------------------------------------------- /outrun/tests/test_operations/test_events.py: -------------------------------------------------------------------------------- 1 | import pytest 2 | 3 | from outrun.operations.events import Event, EventQueue, UnexpectedEvent 4 | 5 | 6 | def test_expect_expected(): 7 | q = EventQueue() 8 | q.notify(Event.SSH_START) 9 | q.expect(Event.SSH_START) 10 | 11 | 12 | def test_expect_unexpected(): 13 | q = EventQueue() 14 | q.notify(Event.SSH_START, 1234) 15 | 16 | with pytest.raises(UnexpectedEvent) as e: 17 | q.expect(Event.TOKEN_READ) 18 | 19 | assert e.value.expected_event == Event.TOKEN_READ 20 | assert e.value.actual_event == Event.SSH_START 21 | assert e.value.actual_value == 1234 22 | 23 | 24 | def test_exception_from_string(): 25 | q = EventQueue() 26 | q.exception("foo") 27 | 28 | with pytest.raises(RuntimeError) as e: 29 | q.expect(Event.SSH_START) 30 | assert e.value.args == ("foo",) 31 | 32 | 33 | def test_builtin_exception(): 34 | q = EventQueue() 35 | q.exception(OSError(1)) 36 | 37 | with pytest.raises(OSError) as e: 38 | q.expect(Event.SSH_START) 39 | assert e.value.args == (1,) 40 | -------------------------------------------------------------------------------- /outrun/tests/test_operations/test_local_ops.py: -------------------------------------------------------------------------------- 1 | import hashlib 2 | from unittest import mock 3 | import secrets 4 | import subprocess 5 | 6 | import pytest 7 | 8 | from outrun.operations.local import LocalOperations 9 | from outrun.args import Arguments 10 | from outrun.constants import PROTOCOL_VERSION 11 | 12 | 13 | def mock_ssh(substitute_command: str): 14 | realPopen = subprocess.Popen 15 | 16 | def wrapper(_command, *args, **kwargs): 17 | return realPopen(substitute_command, shell=True, *args, **kwargs) 18 | 19 | return wrapper 20 | 21 | 22 | def test_missing_ssh(): 23 | args = Arguments.parse(["dest", "cmd"]) 24 | ops = LocalOperations(args) 25 | 26 | with mock.patch("subprocess.Popen") as mock_popen: 27 | mock_popen.side_effect = FileNotFoundError() 28 | 29 | with pytest.raises(RuntimeError) as e: 30 | ops.run() 31 | 32 | assert "failed to start ssh" in str(e.value) 33 | 34 | 35 | def test_ssh_error_tty(): 36 | args = Arguments.parse(["dest", "cmd"]) 37 | ops = LocalOperations(args) 38 | 39 | callback = mock_ssh("echo 'could not resolve hostname' >&2; exit 255") 40 | 41 | with mock.patch("outrun.operations.local.LocalOperations._is_tty") as m: 42 | m.return_value = True 43 | 44 | with mock.patch("subprocess.Popen", side_effect=callback): 45 | with pytest.raises(RuntimeError) as e: 46 | ops.run() 47 | 48 | assert "ssh failed" in str(e.value) 49 | assert "could not resolve hostname" in str(e.value) 50 | 51 | 52 | def test_ssh_error_no_tty(): 53 | args = Arguments.parse(["dest", "cmd"]) 54 | ops = LocalOperations(args) 55 | 56 | callback = mock_ssh("echo 'could not resolve hostname' >&2; exit 255") 57 | 58 | with mock.patch("subprocess.Popen", side_effect=callback): 59 | with pytest.raises(RuntimeError) as e: 60 | ops.run() 61 | 62 | assert "ssh failed" in str(e.value) 63 | assert "could not resolve hostname" not in str(e.value) 64 | 65 | 66 | def test_remote_outrun_missing(): 67 | args = Arguments.parse(["dest", "cmd"]) 68 | ops = LocalOperations(args) 69 | 70 | callback = mock_ssh("echo 'outrun not found'; exit 127") 71 | 72 | with mock.patch("subprocess.Popen", side_effect=callback): 73 | with pytest.raises(RuntimeError) as e: 74 | ops.run() 75 | 76 | assert "remote outrun failed to start" in str(e.value) 77 | 78 | 79 | def test_remote_outrun_bad_checksum(): 80 | args = Arguments.parse(["dest", "cmd"]) 81 | ops = LocalOperations(args) 82 | 83 | # Sleep simulates the time taken to connect to the RPC services after the failed 84 | # handshake. This is to prevent the test from triggering an unexpected control flow. 85 | callback = mock_ssh("echo '\x01abc\x02'; sleep 0.1s; echo 'success!'") 86 | 87 | with mock.patch("subprocess.Popen", side_effect=callback): 88 | with pytest.raises(RuntimeError) as e: 89 | ops.run() 90 | 91 | assert "handshake failed" in str(e.value) 92 | 93 | 94 | def test_command_success(capsys): 95 | args = Arguments.parse(["dest", "cmd"]) 96 | ops = LocalOperations(args) 97 | 98 | token = secrets.token_hex(16) 99 | token_signature = hashlib.sha256(token.encode()).hexdigest() 100 | handshake = f"\x01{token}{token_signature}\x02" 101 | 102 | # Sleep simulates the time taken to connect to the RPC services after handshake. 103 | # This is to prevent the test from triggering an unexpected control flow. 104 | callback = mock_ssh(f"echo 'foobar'; echo '{handshake}'; sleep 0.1s; echo 'ok!'") 105 | 106 | with mock.patch("subprocess.Popen", side_effect=callback): 107 | exit_code = ops.run() 108 | assert exit_code == 0 109 | 110 | assert "ok" in capsys.readouterr().out 111 | 112 | 113 | def test_command_nonzero_exit(capsys): 114 | args = Arguments.parse(["dest", "cmd"]) 115 | ops = LocalOperations(args) 116 | 117 | token = secrets.token_hex(16) 118 | token_signature = hashlib.sha256(token.encode()).hexdigest() 119 | handshake = f"\x01{token}{token_signature}\x02" 120 | 121 | # See test_command_success() for why the sleep is there 122 | callback = mock_ssh(f"echo '{handshake}'; sleep 0.1s; echo 'failed!'; exit 123") 123 | 124 | with mock.patch("subprocess.Popen", side_effect=callback): 125 | exit_code = ops.run() 126 | assert exit_code == 123 127 | 128 | assert "failed" in capsys.readouterr().out 129 | 130 | 131 | def test_without_flags(): 132 | args = Arguments.parse(["dest", "cmd"]) 133 | ops = LocalOperations(args) 134 | 135 | with mock.patch("subprocess.Popen") as mock_popen: 136 | mock_popen.side_effect = Exception() 137 | 138 | with pytest.raises(RuntimeError): 139 | ops.run() 140 | 141 | assert "--debug" not in mock_popen.call_args[0][0] 142 | assert "--no-cache" not in mock_popen.call_args[0][0] 143 | assert "--no-prefetch" not in mock_popen.call_args[0][0] 144 | 145 | assert f"--protocol={PROTOCOL_VERSION}" in mock_popen.call_args[0][0] 146 | assert "--remote" in mock_popen.call_args[0][0] 147 | 148 | 149 | def test_with_flags(): 150 | args = Arguments.parse( 151 | [ 152 | "--debug", 153 | "--no-cache", 154 | "--no-prefetch", 155 | "--sync-writes", 156 | "--timeout=1234", 157 | "dest", 158 | "cmd", 159 | ] 160 | ) 161 | ops = LocalOperations(args) 162 | 163 | with mock.patch("subprocess.Popen") as mock_popen: 164 | mock_popen.side_effect = Exception() 165 | 166 | with pytest.raises(RuntimeError): 167 | ops.run() 168 | 169 | assert "--debug" in mock_popen.call_args[0][0] 170 | assert "--no-cache" in mock_popen.call_args[0][0] 171 | assert "--no-prefetch" in mock_popen.call_args[0][0] 172 | assert "--sync-writes" in mock_popen.call_args[0][0] 173 | assert "--timeout=1234" in mock_popen.call_args[0][0] 174 | 175 | 176 | def test_ssh_port_forwarding(): 177 | args = Arguments.parse( 178 | [ 179 | "--environment-port=1234", 180 | "--filesystem-port=5678", 181 | "--cache-port=4321", 182 | "dest", 183 | "cmd", 184 | ] 185 | ) 186 | ops = LocalOperations(args) 187 | 188 | with mock.patch("subprocess.Popen") as mock_popen: 189 | mock_popen.side_effect = Exception() 190 | 191 | with pytest.raises(RuntimeError): 192 | ops.run() 193 | 194 | assert "1234:localhost:1234" in mock_popen.call_args[0][0] 195 | assert "5678:localhost:5678" in mock_popen.call_args[0][0] 196 | assert "4321:localhost:4321" in mock_popen.call_args[0][0] 197 | assert "--environment-port=1234" in mock_popen.call_args[0][0] 198 | assert "--filesystem-port=5678" in mock_popen.call_args[0][0] 199 | assert "--cache-port=4321" in mock_popen.call_args[0][0] 200 | 201 | 202 | def test_service_failure_detection(): 203 | args = Arguments.parse(["--environment-port=-1", "dest", "cmd"]) 204 | ops = LocalOperations(args) 205 | 206 | token = secrets.token_hex(16) 207 | token_signature = hashlib.sha256(token.encode()).hexdigest() 208 | handshake = f"\x01{token}{token_signature}\x02" 209 | 210 | # exec is necessary to ensure that sleep receives the termination signal 211 | callback = mock_ssh(f"echo '{handshake}'; exec sleep 10s") 212 | 213 | with mock.patch("subprocess.Popen", side_effect=callback): 214 | with mock.patch("outrun.rpc.Server") as mock_rpc: 215 | mock_rpc().serve.return_value = None 216 | 217 | with pytest.raises(RuntimeError) as e: 218 | ops.run() 219 | 220 | assert "service unexpectedly stopped" in str(e.value) 221 | -------------------------------------------------------------------------------- /outrun/tests/test_operations/test_remote_ops.py: -------------------------------------------------------------------------------- 1 | # This module is not tested because it is very dependent on interaction with the OS. 2 | # It is intended to be covered by the Vagrant tests instead. 3 | -------------------------------------------------------------------------------- /outrun/tests/test_rpc.py: -------------------------------------------------------------------------------- 1 | from dataclasses import dataclass 2 | import multiprocessing 3 | from io import StringIO 4 | from unittest import mock 5 | from typing import List 6 | 7 | from pytest_cov.embed import cleanup_on_sigterm 8 | import pytest 9 | 10 | from outrun.rpc import Client, Encoding, InvalidTokenError, Server 11 | 12 | 13 | def start_server_process(server: Server) -> multiprocessing.Process: 14 | def run_server(): 15 | cleanup_on_sigterm() 16 | server.serve("tcp://127.0.0.1:8000") 17 | 18 | proc = multiprocessing.Process(target=run_server) 19 | proc.start() 20 | 21 | return proc 22 | 23 | 24 | def test_call(): 25 | class Service: 26 | @staticmethod 27 | def add(a, b): 28 | return a + b 29 | 30 | server = Server(Service()) 31 | server_process = start_server_process(server) 32 | 33 | try: 34 | client = Client(Service, "tcp://127.0.0.1:8000", timeout_ms=1000) 35 | assert client.add(3, 5) == 8 36 | finally: 37 | server_process.terminate() 38 | server_process.join() 39 | 40 | 41 | def test_nonexistent_call(): 42 | class Service: 43 | pass 44 | 45 | server = Server(Service()) 46 | server_process = start_server_process(server) 47 | 48 | try: 49 | client = Client(Service, "tcp://127.0.0.1:8000", timeout_ms=1000) 50 | 51 | with pytest.raises(AttributeError): 52 | client.foo() 53 | finally: 54 | server_process.terminate() 55 | server_process.join() 56 | 57 | 58 | def test_successful_ping(): 59 | class Service: 60 | pass 61 | 62 | server = Server(Service()) 63 | server_process = start_server_process(server) 64 | 65 | try: 66 | client = Client(Service, "tcp://127.0.0.1:8000", timeout_ms=1000) 67 | client.ping() 68 | finally: 69 | server_process.terminate() 70 | server_process.join() 71 | 72 | 73 | def test_failing_ping_with_custom_timeout(): 74 | class Service: 75 | pass 76 | 77 | client = Client(Service, "tcp://127.0.0.1:8000", timeout_ms=-1) 78 | 79 | with pytest.raises(IOError): 80 | client.ping(timeout_ms=1) 81 | 82 | 83 | def test_timeout(): 84 | class Service: 85 | pass 86 | 87 | client = Client(Service, "tcp://127.0.0.1:8000", timeout_ms=1) 88 | 89 | with pytest.raises(IOError): 90 | client.foo() 91 | 92 | 93 | def test_socket_per_thread(): 94 | class Service: 95 | pass 96 | 97 | server = Server(Service()) 98 | server_process = start_server_process(server) 99 | 100 | try: 101 | client = Client(Service, "tcp://127.0.0.1:8000", timeout_ms=1000) 102 | 103 | with mock.patch("threading.current_thread") as m: 104 | m.return_value = 1 105 | client.ping() 106 | m.return_value = 2 107 | client.ping() 108 | 109 | assert client.socket_count == 2 110 | finally: 111 | server_process.terminate() 112 | server_process.join() 113 | 114 | 115 | def test_tuple_serialization(): 116 | class Service: 117 | @staticmethod 118 | def get_tuple(): 119 | return (1, 2, 3) 120 | 121 | server = Server(Service()) 122 | server_process = start_server_process(server) 123 | 124 | try: 125 | client = Client(Service, "tcp://127.0.0.1:8000", timeout_ms=1000) 126 | 127 | # tuples are serialized as lists 128 | assert client.get_tuple() == [1, 2, 3] 129 | finally: 130 | server_process.terminate() 131 | server_process.join() 132 | 133 | 134 | def test_dataclasses(): 135 | @dataclass 136 | class Point: 137 | x: int 138 | y: int 139 | 140 | @dataclass 141 | class Line: 142 | p1: Point 143 | p2: Point 144 | 145 | class Service: 146 | @staticmethod 147 | def make_line(p1: Point, p2: Point) -> Line: 148 | return Line(p1, p2) 149 | 150 | server = Server(Service()) 151 | server_process = start_server_process(server) 152 | 153 | try: 154 | client = Client(Service, "tcp://127.0.0.1:8000", timeout_ms=1000) 155 | 156 | p1 = Point(1, 2) 157 | p2 = Point(3, 4) 158 | 159 | assert client.make_line(p1, p2) == Line(p1, p2) 160 | finally: 161 | server_process.terminate() 162 | server_process.join() 163 | 164 | 165 | def test_dataclass_in_container_type(): 166 | @dataclass 167 | class Point: 168 | x: int 169 | y: int 170 | 171 | class Service: 172 | @staticmethod 173 | def make_point_list(x: int, y: int) -> List[Point]: 174 | return [Point(x, y)] 175 | 176 | server = Server(Service()) 177 | server_process = start_server_process(server) 178 | 179 | try: 180 | client = Client(Service, "tcp://127.0.0.1:8000", timeout_ms=1000) 181 | 182 | assert client.make_point_list(1, 2) == [Point(1, 2)] 183 | finally: 184 | server_process.terminate() 185 | server_process.join() 186 | 187 | 188 | def test_builtin_exceptions(): 189 | class Service: 190 | @staticmethod 191 | def os_failure(): 192 | raise OSError("foo") 193 | 194 | @staticmethod 195 | def value_failure(): 196 | raise ValueError("bar") 197 | 198 | server = Server(Service()) 199 | server_process = start_server_process(server) 200 | 201 | try: 202 | client = Client(Service, "tcp://127.0.0.1:8000", timeout_ms=1000) 203 | 204 | with pytest.raises(OSError) as e: 205 | client.os_failure() 206 | assert e.value.args == ("foo",) 207 | 208 | with pytest.raises(ValueError) as e: 209 | client.value_failure() 210 | assert e.value.args == ("bar",) 211 | finally: 212 | server_process.terminate() 213 | server_process.join() 214 | 215 | 216 | def test_custom_exception(): 217 | class CustomException(Exception): 218 | pass 219 | 220 | class Service: 221 | @staticmethod 222 | def custom_failure(): 223 | raise CustomException("a", "b", "c") 224 | 225 | server = Server(Service()) 226 | server_process = start_server_process(server) 227 | 228 | try: 229 | client = Client(Service, "tcp://127.0.0.1:8000", timeout_ms=1000) 230 | 231 | with pytest.raises(Exception) as e: 232 | client.custom_failure() 233 | assert e.value.args == ("a", "b", "c") 234 | finally: 235 | server_process.terminate() 236 | server_process.join() 237 | 238 | 239 | def test_missing_token(): 240 | class Service: 241 | pass 242 | 243 | server = Server(Service(), token="1234") 244 | server_process = start_server_process(server) 245 | 246 | try: 247 | client = Client(Service, "tcp://127.0.0.1:8000", timeout_ms=1000) 248 | with pytest.raises(InvalidTokenError): 249 | client.ping() 250 | finally: 251 | server_process.terminate() 252 | server_process.join() 253 | 254 | 255 | def test_invalid_token(): 256 | class Service: 257 | pass 258 | 259 | server = Server(Service(), token="1234") 260 | server_process = start_server_process(server) 261 | 262 | try: 263 | client = Client(Service, "tcp://127.0.0.1:8000", token="5678", timeout_ms=1000) 264 | with pytest.raises(InvalidTokenError): 265 | client.ping() 266 | finally: 267 | server_process.terminate() 268 | server_process.join() 269 | 270 | 271 | def test_valid_token(): 272 | class Service: 273 | pass 274 | 275 | server = Server(Service(), token="1234") 276 | server_process = start_server_process(server) 277 | 278 | try: 279 | client = Client(Service, "tcp://127.0.0.1:8000", token="1234", timeout_ms=1000) 280 | client.ping() 281 | finally: 282 | server_process.terminate() 283 | server_process.join() 284 | 285 | 286 | def test_json_encoding_dataclasses(): 287 | @dataclass 288 | class Point: 289 | x: int 290 | y: int 291 | 292 | @dataclass 293 | class Line: 294 | p1: Point 295 | p2: Point 296 | 297 | encoding = Encoding(Line) 298 | 299 | obj_in = ["abc", True, Line(Point(1, 2), Point(3, 4)), Point(5, 6)] 300 | 301 | io = StringIO() 302 | encoding.dump_json(obj_in, io) 303 | 304 | io.seek(0) 305 | obj_out = encoding.load_json(io) 306 | 307 | assert obj_in == obj_out 308 | 309 | 310 | def test_json_encoding_exceptions(): 311 | encoding = Encoding() 312 | 313 | exceptions_in = [OSError("a", "b"), TypeError("c"), NotImplementedError()] 314 | 315 | io = StringIO() 316 | encoding.dump_json(exceptions_in, io) 317 | 318 | io.seek(0) 319 | exceptions_out = encoding.load_json(io) 320 | 321 | with pytest.raises(OSError) as e: 322 | raise exceptions_out[0] 323 | assert e.value.args == ("a", "b") 324 | 325 | with pytest.raises(TypeError) as e: 326 | raise exceptions_out[1] 327 | assert e.value.args == ("c",) 328 | 329 | with pytest.raises(NotImplementedError) as e: 330 | raise exceptions_out[2] 331 | assert e.value.args == () 332 | 333 | 334 | def test_unserializable_object(): 335 | encoding = Encoding() 336 | 337 | with pytest.raises(ValueError): 338 | encoding.serialize_obj(set()) 339 | 340 | 341 | def test_deserialize_unknown_dataclass(): 342 | @dataclass 343 | class Point: 344 | x: int 345 | y: int 346 | 347 | encoding = Encoding(Point) 348 | serialized = encoding.serialize_obj(Point(1, 2)) 349 | 350 | with pytest.raises(TypeError): 351 | encoding = Encoding() 352 | encoding.deserialize_obj(serialized) 353 | -------------------------------------------------------------------------------- /outrun/tests/vagrant/.gitignore: -------------------------------------------------------------------------------- 1 | .vagrant 2 | -------------------------------------------------------------------------------- /outrun/tests/vagrant/Vagrantfile: -------------------------------------------------------------------------------- 1 | Vagrant.configure("2") do |config| 2 | # Local machine that will invoke outrun 3 | config.vm.define "local" do |local| 4 | local.vm.box = "generic/ubuntu2004" 5 | local.vm.network "private_network", ip: "10.0.0.200" 6 | 7 | # Install outrun 8 | local.vm.provision "shell", inline: "apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -yq fuse3 python3 python3-pip sshpass" 9 | 10 | local.vm.provision "file", source: "../../../outrun", destination: "$HOME/outrun/outrun", run: "always" 11 | local.vm.provision "file", source: "../../../setup.py", destination: "$HOME/outrun/setup.py", run: "always" 12 | local.vm.provision "file", source: "../../../README.md", destination: "$HOME/outrun/README.md", run: "always" 13 | 14 | local.vm.provision "shell", privileged: true, inline: "pip3 install -e ./outrun", run: "always" 15 | 16 | # Fix file permissions that may change as a result of command above 17 | local.vm.provision "shell", privileged: true, inline: "chown vagrant:vagrant -R ./outrun", run: "always" 18 | 19 | # Set up SSH for connection with remote 20 | local.vm.provision "shell", privileged: false, inline: "echo -e 'Host remote\n\tHostname 10.0.0.201\n\tUser vagrant' > .ssh/config", run: "always" 21 | local.vm.provision "shell", privileged: false, inline: "yes | ssh-keygen -q -f .ssh/id_rsa -q -N '' && ssh-keyscan 10.0.0.201 > ~/.ssh/known_hosts && sshpass -p vagrant ssh-copy-id vagrant@10.0.0.201", run: "always" 22 | 23 | # Install programs to test with outrun 24 | local.vm.provision "shell", inline: "DEBIAN_FRONTEND=noninteractive apt-get install -yq ffmpeg lua5.3" 25 | end 26 | 27 | # Remote machine that will run commands 28 | # 29 | # Alpine has specifically been chosen as distro that significantly differs from 30 | # Ubuntu (different package manager, very minimalistic, musl vs. glibc) 31 | config.vm.define "remote" do |remote| 32 | remote.vm.box = "generic/alpine310" 33 | remote.vm.network "private_network", ip: "10.0.0.201" 34 | 35 | # Install outrun 36 | remote.vm.provision "shell", inline: "apk update && apk add python3 build-base python3-dev zeromq-dev fuse3 sshpass" 37 | 38 | remote.vm.provision "file", source: "../../../outrun", destination: "$HOME/outrun/outrun", run: "always" 39 | remote.vm.provision "file", source: "../../../setup.py", destination: "$HOME/outrun/setup.py", run: "always" 40 | remote.vm.provision "file", source: "../../../README.md", destination: "$HOME/outrun/README.md", run: "always" 41 | 42 | remote.vm.provision "shell", privileged: true, inline: "pip3 install -e ./outrun", run: "always" 43 | 44 | # Fix file permissions that may change as a result of command above 45 | remote.vm.provision "shell", privileged: true, inline: "chown vagrant:vagrant -R ./outrun", run: "always" 46 | 47 | # Ensure that outrun can be found 48 | remote.vm.provision "shell", privileged: false, inline: "echo 'PATH=/home/vagrant/.local/bin:$PATH' >> ~/.profile" 49 | end 50 | end 51 | -------------------------------------------------------------------------------- /outrun/tests/vagrant/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Overv/outrun/20af1136060ecb0a53f464b73e5cdac913a097c3/outrun/tests/vagrant/__init__.py -------------------------------------------------------------------------------- /outrun/tests/vagrant/test_vagrant.py: -------------------------------------------------------------------------------- 1 | from base64 import b64encode 2 | import os 3 | import subprocess 4 | 5 | import pytest 6 | 7 | from vagrant import Vagrant 8 | 9 | 10 | @pytest.fixture(scope="module") 11 | def vagrant(): 12 | v = Vagrant(os.path.dirname(__file__)) 13 | 14 | # Start remote VM first so local VM can copy its SSH key to it afterwards 15 | v.up(vm_name="remote") 16 | v.up(vm_name="local") 17 | 18 | yield v 19 | 20 | v.halt() 21 | 22 | 23 | @pytest.mark.vagrant 24 | def test_basic_command(vagrant): 25 | output = vagrant.ssh(vm_name="local", command="outrun remote echo hi").strip() 26 | assert output == "hi" 27 | 28 | 29 | @pytest.mark.vagrant 30 | def test_ffmpeg_help(vagrant): 31 | output = vagrant.ssh(vm_name="local", command="outrun remote ffmpeg -h").strip() 32 | assert "Hyper fast Audio and Video encoder" in output 33 | 34 | 35 | @pytest.mark.vagrant 36 | def test_stdin(vagrant): 37 | output = vagrant.ssh( 38 | vm_name="local", command="printf foobar | outrun remote base64" 39 | ).strip() 40 | assert output == b64encode(b"foobar").strip().decode() 41 | 42 | 43 | @pytest.mark.vagrant 44 | def test_file_manipulation(vagrant): 45 | output = vagrant.ssh( 46 | vm_name="local", command="rm -f foo && outrun remote touch foo && stat foo" 47 | ).strip() 48 | assert "No such file or directory" not in output 49 | 50 | 51 | @pytest.mark.vagrant 52 | def test_lua(vagrant): 53 | output = vagrant.ssh( 54 | vm_name="local", 55 | command="echo 'print(123)' > script.lua && outrun remote lua script.lua", 56 | ).strip() 57 | assert "123" in output 58 | 59 | 60 | @pytest.mark.vagrant 61 | def test_interruption(vagrant): 62 | with pytest.raises(subprocess.CalledProcessError) as e: 63 | vagrant.ssh( 64 | vm_name="local", 65 | command="timeout --signal=INT --kill-after=10s 5s outrun remote sleep 10s", 66 | ) 67 | 68 | # Timeout exits with code 124 if the signal was used successfully 69 | # See also 'man timeout' 70 | assert e.value.args[0] == 124 71 | 72 | 73 | @pytest.mark.vagrant 74 | def test_ssh_error(vagrant): 75 | output = vagrant.ssh( 76 | vm_name="local", command="outrun nonexistent_host echo hi 2>&1 || true" 77 | ) 78 | 79 | assert "Could not resolve hostname" in output 80 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | """Packaging information for outrun.""" 2 | 3 | import sys 4 | 5 | import setuptools 6 | 7 | from outrun.constants import VERSION 8 | 9 | if sys.version_info[:3] < (3, 7, 0): 10 | print("outrun requires Python 3.7 to run.") 11 | sys.exit(1) 12 | 13 | install_requires = [ 14 | "msgpack>=1.0.0", 15 | "pyzmq>=19.0.0", 16 | "lz4>=3.0.2", 17 | "fasteners>=0.15", 18 | "semver>=2.9.1", 19 | ] 20 | 21 | extras_require = { 22 | "dev": [ 23 | "rope>=0.14.0", 24 | "flake8>=3.7.9", 25 | "flake8-docstrings>=1.5.0", 26 | "flake8-import-order>=0.18.1", 27 | "black>=19.10b0", 28 | "pylint>=2.4.4", 29 | "mypy>=0.770", 30 | "pytest>=5.4.1", 31 | "pytest-cov>=2.8.1", 32 | "python-vagrant>=0.5.15", 33 | ] 34 | } 35 | 36 | 37 | def _long_description(): 38 | with open("README.md") as f: 39 | return f.read() 40 | 41 | 42 | setuptools.setup( 43 | name="outrun", 44 | version=VERSION, 45 | description="Delegate execution of a local command to a remote machine.", 46 | long_description=_long_description(), 47 | long_description_content_type="text/markdown", 48 | url="https://github.com/Overv/outrun", 49 | download_url="https://github.com/Overv/outrun", 50 | author="Alexander Overvoorde", 51 | author_email="overv161@gmail.com", 52 | license="Apache", 53 | packages=setuptools.find_packages(exclude=["outrun.tests*"]), 54 | entry_points={"console_scripts": ["outrun = outrun.__main__:main"]}, 55 | install_requires=install_requires, 56 | extras_require=extras_require, 57 | classifiers=[ 58 | "Development Status :: 3 - Alpha", 59 | "Environment :: Console", 60 | "Programming Language :: Python :: 3", 61 | "Programming Language :: Python :: 3.7", 62 | "Intended Audience :: Information Technology", 63 | "License :: OSI Approved :: Apache Software License", 64 | "Operating System :: POSIX :: Linux", 65 | "Topic :: System :: Clustering", 66 | ], 67 | python_requires=">=3.7", 68 | ) 69 | --------------------------------------------------------------------------------