├── .bumpversion.cfg ├── .circleci └── config.yml ├── .clabot ├── .coveragerc ├── .github ├── issue_template.md └── pull_request_template.md ├── .gitignore ├── CLA.md ├── CONTRIBUTING.md ├── LICENSE.md ├── MANIFEST.in ├── README.md ├── conftest.py ├── development-requirements.txt ├── media ├── htop-demo.gif └── ycombinator.png ├── setup.cfg ├── setup.py ├── src └── exodus_bundler │ ├── __init__.py │ ├── bundling.py │ ├── cli.py │ ├── dependency_detection.py │ ├── errors.py │ ├── input_parsing.py │ ├── launchers.py │ ├── templates │ ├── install-bundle-noninteractive.sh │ ├── install-bundle.sh │ ├── launcher.c │ └── launcher.sh │ └── templating.py ├── tests ├── data │ ├── binaries │ │ ├── GLIB_LICENSES │ │ ├── MUSL-COPYRIGHT │ │ ├── README.md │ │ ├── chroot │ │ │ ├── bin │ │ │ │ ├── echo-args-glibc-32 │ │ │ │ ├── echo-proc-self-exe-glibc-32 │ │ │ │ ├── fizz-buzz-glibc-32 │ │ │ │ ├── fizz-buzz-glibc-32-exe │ │ │ │ ├── fizz-buzz-glibc-64 │ │ │ │ ├── fizz-buzz-musl-64 │ │ │ │ └── ldd │ │ │ ├── lib │ │ │ │ ├── ld-linux.so.2 │ │ │ │ └── ld-musl-x86_64.so.1 │ │ │ ├── lib64 │ │ │ │ └── ld-linux-x86-64.so.2 │ │ │ └── usr │ │ │ │ ├── lib │ │ │ │ ├── ld-linux-x86-64.so.2 │ │ │ │ └── libc.so.6 │ │ │ │ └── lib32 │ │ │ │ ├── ld-linux.so.2 │ │ │ │ └── libc.so.6 │ │ ├── echo-args.c │ │ ├── echo-proc-self-exe.c │ │ └── fizz-buzz.c │ ├── ldd-output │ │ ├── htop-amazon-linux-dependencies.txt │ │ ├── htop-amazon-linux.txt │ │ ├── htop-arch-dependencies.txt │ │ ├── htop-arch.txt │ │ ├── htop-ubuntu-14.04-dependencies.txt │ │ └── htop-ubuntu-14.04.txt │ ├── strace-output │ │ └── exodus-output.txt │ ├── template-result.txt │ └── template.txt ├── test_bundling.py ├── test_cli.py ├── test_dependency_detection.py ├── test_input_parsing.py ├── test_launchers.py ├── test_pytest.py └── test_templating.py └── tox.ini /.bumpversion.cfg: -------------------------------------------------------------------------------- 1 | [bumpversion] 2 | current_version = 2.0.4 3 | commit = True 4 | tag = True 5 | 6 | [bumpversion:file:setup.py] 7 | 8 | [bumpversion:file:src/exodus_bundler/__init__.py] 9 | 10 | -------------------------------------------------------------------------------- /.circleci/config.yml: -------------------------------------------------------------------------------- 1 | defaults: &defaults 2 | working_directory: ~/exodus 3 | docker: 4 | - image: circleci/python:3.9-buster 5 | 6 | 7 | version: 2 8 | jobs: 9 | build: 10 | <<: *defaults 11 | steps: 12 | - checkout 13 | 14 | - restore_cache: 15 | keys: 16 | - v1-dependencies-{{ arch }}-{{ .Branch }}-{{ checksum "development-requirements.txt" }} 17 | - v1-dependencies- 18 | 19 | - run: 20 | name: Install Dependencies 21 | command: | 22 | python3 -m venv .env 23 | . .env/bin/activate 24 | pip install -r development-requirements.txt 25 | sudo apt-get install dietlibc-dev gcc musl musl-tools 26 | - save_cache: 27 | paths: 28 | - ./.env 29 | key: v1-dependencies-{{ arch }}-{{ .Branch }}-{{ checksum "development-requirements.txt" }} 30 | 31 | - run: 32 | name: Run the tests. 33 | command: | 34 | . .env/bin/activate 35 | tox 36 | echo "{ \"coverage\": \"$(coverage report | tail -n 1 | awk '{print $6}')\" }" > htmlcov/total-coverage.json 37 | 38 | - run: 39 | name: Build the package. 40 | command: | 41 | python setup.py sdist bdist_wheel 42 | 43 | - run: 44 | name: Build Self-Hosted Bundle 45 | command: | 46 | sudo apt-get install python-pip strace 47 | sudo pip2 install . 48 | 49 | (cat < ./exodus.c 50 | #include 51 | #include 52 | #include 53 | #include 54 | 55 | int main(int argc, char *argv[]) { 56 | char buffer[4096] = { 0 }; 57 | if (readlink("/proc/self/exe", buffer, sizeof(buffer) - 25)) { 58 | char *current_directory = dirname(buffer); 59 | int current_directory_length = strlen(current_directory); 60 | 61 | char python[4096] = { 0 }; 62 | strcpy(python, current_directory); 63 | strcat(python, "/usr/bin/python"); 64 | 65 | char exodus[4096] = { 0 }; 66 | strcpy(exodus, current_directory); 67 | strcat(exodus, "/usr/local/bin/exodus"); 68 | 69 | char **combined_args = malloc(sizeof(char*) * (argc + 2)); 70 | combined_args[0] = python; 71 | combined_args[1] = exodus; 72 | memcpy(combined_args + 2, argv + 1, sizeof(char*) * (argc - 1)); 73 | combined_args[argc + 1] = NULL; 74 | 75 | char *envp[2]; 76 | char pythonpath[4096] = { 0 }; 77 | strcpy(pythonpath, "PYTHONPATH="); 78 | strcat(pythonpath, current_directory); 79 | strcat(pythonpath, "/usr/local/lib/python2.7/"); 80 | envp[0] = pythonpath; 81 | 82 | envp[1] = NULL; 83 | 84 | execve(python, combined_args, envp); 85 | } 86 | 87 | return 1; 88 | } 89 | EOF 90 | ) 91 | sudo cp exodus.c / 92 | cd / 93 | sudo gcc -O3 exodus.c -o exodus 94 | sudo chmod a+x /exodus 95 | 96 | sudo mv /etc/ld.so.cache /tmp/ld.so.cache.bck 97 | 98 | export LD_LIBRARY_PATH=/usr/local/lib/:/lib/x86_64-linux-gnu/:/usr/lib/x86_64-linux-gnu/:${LD_LIBRARY_PATH} 99 | strace -f /usr/bin/python /usr/local/bin/exodus --shell-launchers -q /usr/bin/python -o /dev/null 2>&1 | exodus ./exodus --add /usr/local/lib/python2.7/dist-packages/exodus_bundler/ --no-symlink /usr/local/lib/python2.7/dist-packages/exodus_bundler/templating.py --no-symlink /usr/local/lib/python2.7/dist-packages/exodus_bundler/launchers.py --tar --output /home/circleci/exodus/exodus-x64.tgz 100 | strace -f /usr/bin/python /usr/local/bin/exodus --shell-launchers -q /usr/bin/python -o /dev/null 2>&1 | exodus ./exodus --add /usr/local/lib/python2.7/dist-packages/exodus_bundler/ --no-symlink /usr/local/lib/python2.7/dist-packages/exodus_bundler/templating.py --no-symlink /usr/local/lib/python2.7/dist-packages/exodus_bundler/launchers.py > /home/circleci/exodus/exodus-x64.sh 101 | 102 | sudo mv /tmp/ld.so.cache.bck /etc/ld.so.cache 103 | - store_artifacts: 104 | path: htmlcov 105 | destination: coverage-report 106 | - store_artifacts: 107 | path: exodus-x64.sh 108 | destination: exodus-x64.sh 109 | - store_artifacts: 110 | path: exodus-x64.tgz 111 | destination: exodus-x64.tgz 112 | 113 | - persist_to_workspace: 114 | root: ~/exodus 115 | paths: 116 | - .env 117 | - dist 118 | 119 | deploy: 120 | <<: *defaults 121 | steps: 122 | - attach_workspace: 123 | at: ~/exodus 124 | - run: 125 | name: Upload package to PyPI. 126 | command: | 127 | . .env/bin/activate 128 | twine upload --skip-existing dist/* 129 | 130 | 131 | workflows: 132 | version: 2 133 | 134 | build: 135 | jobs: 136 | - build: 137 | filters: 138 | tags: 139 | ignore: /v[0-9]+(\.[0-9]+)*/ 140 | 141 | release: 142 | jobs: 143 | - build: 144 | filters: 145 | tags: 146 | only: /v[0-9]+(\.[0-9]+)*/ 147 | branches: 148 | ignore: /.*/ 149 | - deploy: 150 | filters: 151 | tags: 152 | only: /v[0-9]+(\.[0-9]+)*/ 153 | branches: 154 | ignore: /.*/ 155 | requires: 156 | - build 157 | -------------------------------------------------------------------------------- /.clabot: -------------------------------------------------------------------------------- 1 | { 2 | "contributors": [ 3 | "sangaline" 4 | ], 5 | "message": "Thank you for making a contribution! We require new contributors to sign our [Contributor License Agreement (CLA)](https://github.com/intoli/exodus/blob/master/CLA.md). Please review our [Contributing Guide](https://github.com/intoli/exodus/blob/master/CONTRIBUTING.md) and make sure that you've followed the steps listed there." 6 | } 7 | -------------------------------------------------------------------------------- /.coveragerc: -------------------------------------------------------------------------------- 1 | [paths] 2 | source = 3 | src/exodus_bundler 4 | */site-packages/exodus_bundler 5 | 6 | [run] 7 | branch = true 8 | source = 9 | exodus_bundler 10 | tests 11 | parallel = true 12 | 13 | [report] 14 | show_missing = true 15 | precision = 2 16 | -------------------------------------------------------------------------------- /.github/issue_template.md: -------------------------------------------------------------------------------- 1 | Please include the following information if you're reporting an issue. 2 | 3 | - Which operating system you're using. 4 | - Which Python version. 5 | - Which exodus package version. 6 | - The command that you're running (please include the `--verbose` flag so that the stack trace is included). 7 | - The full output of the command. 8 | - Where `$BINARY` is the name of the executable that you're trying to bundle, include the outputs of the following: 9 | - `which $BINARY` 10 | - `readelf -a $BINARY` 11 | - `echo $PATH` 12 | - `echo $LD_LIBRARY_PATH` 13 | -------------------------------------------------------------------------------- /.github/pull_request_template.md: -------------------------------------------------------------------------------- 1 | *Replace this with the pull request description...* 2 | 3 | 4 | **Please check the boxes below to confirm that you have completed these steps:** 5 | 6 | * [ ] I have read and followed the [Contribution Agreement](https://github.com/intoli/exodus/blob/master/CONTRIBUTING.md). 7 | * [ ] I have signed the project [Contributor License Agreement](https://github.com/intoli/exodus/blob/master/CONTRIBUTING.md). 8 | 9 | 10 | *Replace this text with either "Closes #N" or "Connects #N," where N is the corresponding GitHub issue number.* 11 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Virtualenvs. 2 | .env 3 | .tox 4 | 5 | # Python stuff. 6 | *.pyc 7 | __pycache__ 8 | *.egg-info 9 | 10 | 11 | # Coverage reports. 12 | .coverage 13 | .coverage.* 14 | htmlcov 15 | -------------------------------------------------------------------------------- /CLA.md: -------------------------------------------------------------------------------- 1 | ## Contributor Agreement 2 | 3 | ## Individual Contributor Exclusive License Agreement 4 | 5 | ## (including the Traditional Patent License OPTION) 6 | 7 | Thank you for your interest in contributing to Intoli, LLC's Exodus ("We" or "Us"). 8 | 9 | The purpose of this contributor agreement ("Agreement") is to clarify and document the rights granted by contributors to Us. To make this document effective, please follow the instructions at https://github.com/intoli/exodus/blob/master/CONTRIBUTING.md. 10 | 11 | ### How to use this Contributor Agreement 12 | 13 | If You are an employee and have created the Contribution as part of your employment, You need to have Your employer approve this Agreement or sign the Entity version of this document. If You do not own the Copyright in the entire work of authorship, any other author of the Contribution should also sign this – in any event, please contact Us at open-source@intoli.com 14 | 15 | ### 1\. Definitions 16 | 17 | **"You"** means the individual Copyright owner who Submits a Contribution to Us. 18 | 19 | **"Legal Entity"** means an entity that is not a natural person. 20 | 21 | **"Affiliate"** means any other Legal Entity that controls, is controlled by, or under common control with that Legal Entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such Legal Entity, whether by contract or otherwise, (ii) ownership of fifty percent (50%) or more of the outstanding shares or securities that vote to elect the management or other persons who direct such Legal Entity or (iii) beneficial ownership of such entity. 22 | 23 | **"Contribution"** means any original work of authorship, including any original modifications or additions to an existing work of authorship, Submitted by You to Us, in which You own the Copyright. 24 | 25 | **"Copyright"** means all rights protecting works of authorship, including copyright, moral and neighboring rights, as appropriate, for the full term of their existence. 26 | 27 | **"Material"** means the software or documentation made available by Us to third parties. When this Agreement covers more than one software project, the Material means the software or documentation to which the Contribution was Submitted. After You Submit the Contribution, it may be included in the Material. 28 | 29 | **"Submit"** means any act by which a Contribution is transferred to Us by You by means of tangible or intangible media, including but not limited to electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, Us, but excluding any transfer that is conspicuously marked or otherwise designated in writing by You as "Not a Contribution." 30 | 31 | **"Documentation"** means any non-software portion of a Contribution. 32 | 33 | ### 2\. License grant 34 | 35 | #### 2.1 Copyright license to Us 36 | 37 | Subject to the terms and conditions of this Agreement, You hereby grant to Us a worldwide, royalty-free, Exclusive, perpetual and irrevocable (except as stated in Section 8.2) license, with the right to transfer an unlimited number of non-exclusive licenses or to grant sublicenses to third parties, under the Copyright covering the Contribution to use the Contribution by all means, including, but not limited to: 38 | 39 | * publish the Contribution, 40 | * modify the Contribution, 41 | * prepare derivative works based upon or containing the Contribution and/or to combine the Contribution with other Materials, 42 | * reproduce the Contribution in original or modified form, 43 | * distribute, to make the Contribution available to the public, display and publicly perform the Contribution in original or modified form. 44 | 45 | #### 2.2 Moral rights 46 | 47 | Moral Rights remain unaffected to the extent they are recognized and not waivable by applicable law. Notwithstanding, You may add your name to the attribution mechanism customary used in the Materials you Contribute to, such as the header of the source code files of Your Contribution, and We will respect this attribution when using Your Contribution. 48 | 49 | #### 2.3 Copyright license back to You 50 | 51 | Upon such grant of rights to Us, We immediately grant to You a worldwide, royalty-free, non-exclusive, perpetual and irrevocable license, with the right to transfer an unlimited number of non-exclusive licenses or to grant sublicenses to third parties, under the Copyright covering the Contribution to use the Contribution by all means, including, but not limited to: 52 | 53 | * publish the Contribution, 54 | * modify the Contribution, 55 | * prepare derivative works based upon or containing the Contribution and/or to combine the Contribution with other Materials, 56 | * reproduce the Contribution in original or modified form, 57 | * distribute, to make the Contribution available to the public, display and publicly perform the Contribution in original or modified form. 58 | 59 | This license back is limited to the Contribution and does not provide any rights to the Material. 60 | 61 | ### 3\. Patents 62 | 63 | #### 3.1 Patent license 64 | 65 | Subject to the terms and conditions of this Agreement You hereby grant to Us and to recipients of Materials distributed by Us a worldwide, royalty-free, non-exclusive, perpetual and irrevocable (except as stated in Section 3.2) patent license, with the right to transfer an unlimited number of non-exclusive licenses or to grant sublicenses to third parties, to make, have made, use, sell, offer for sale, import and otherwise transfer the Contribution and the Contribution in combination with any Material (and portions of such combination). This license applies to all patents owned or controlled by You, whether already acquired or hereafter acquired, that would be infringed by making, having made, using, selling, offering for sale, importing or otherwise transferring of Your Contribution(s) alone or by combination of Your Contribution(s) with any Material. 66 | 67 | #### 3.2 Revocation of patent license 68 | 69 | You reserve the right to revoke the patent license stated in section 3.1 if We make any infringement claim that is targeted at your Contribution and not asserted for a Defensive Purpose. An assertion of claims of the Patents shall be considered for a "Defensive Purpose" if the claims are asserted against an entity that has filed, maintained, threatened, or voluntarily participated in a patent infringement lawsuit against Us or any of Our licensees. 70 | 71 | ### 4. Disclaimer 72 | 73 | THE CONTRIBUTION IS PROVIDED "AS IS". MORE PARTICULARLY, ALL EXPRESS OR IMPLIED WARRANTIES INCLUDING, WITHOUT LIMITATION, ANY IMPLIED WARRANTY OF SATISFACTORY QUALITY, FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT ARE EXPRESSLY DISCLAIMED BY YOU TO US AND BY US TO YOU. TO THE EXTENT THAT ANY SUCH WARRANTIES CANNOT BE DISCLAIMED, SUCH WARRANTY IS LIMITED IN DURATION AND EXTENT TO THE MINIMUM PERIOD AND EXTENT PERMITTED BY LAW. 74 | 75 | ### 5. Consequential damage waiver 76 | 77 | TO THE MAXIMUM EXTENT PERMITTED BY APPLICABLE LAW, IN NO EVENT WILL YOU OR WE BE LIABLE FOR ANY LOSS OF PROFITS, LOSS OF ANTICIPATED SAVINGS, LOSS OF DATA, INDIRECT, SPECIAL, INCIDENTAL, CONSEQUENTIAL AND EXEMPLARY DAMAGES ARISING OUT OF THIS AGREEMENT REGARDLESS OF THE LEGAL OR EQUITABLE THEORY (CONTRACT, TORT OR OTHERWISE) UPON WHICH THE CLAIM IS BASED. 78 | 79 | ### 6. Approximation of disclaimer and damage waiver 80 | 81 | IF THE DISCLAIMER AND DAMAGE WAIVER MENTIONED IN SECTION 4. AND SECTION 5. CANNOT BE GIVEN LEGAL EFFECT UNDER APPLICABLE LOCAL LAW, REVIEWING COURTS SHALL APPLY LOCAL LAW THAT MOST CLOSELY APPROXIMATES AN ABSOLUTE WAIVER OF ALL CIVIL OR CONTRACTUAL LIABILITY IN CONNECTION WITH THE CONTRIBUTION. 82 | 83 | ### 7. Term 84 | 85 | 7.1 This Agreement shall come into effect upon Your acceptance of the terms and conditions. 86 | 87 | 7.3 In the event of a termination of this Agreement Sections 4, 5, 6, 7 and 8 shall survive such termination and shall remain in full force thereafter. For the avoidance of doubt, Free and Open Source Software (sub)licenses that have already been granted for Contributions at the date of the termination shall remain in full force after the termination of this Agreement. 88 | 89 | ### 8 Miscellaneous 90 | 91 | 8.1 This Agreement and all disputes, claims, actions, suits or other proceedings arising out of this agreement or relating in any way to it shall be governed by the laws of United States excluding its private international law provisions. 92 | 93 | 8.2 This Agreement sets out the entire agreement between You and Us for Your Contributions to Us and overrides all other agreements or understandings. 94 | 95 | 8.3 In case of Your death, this agreement shall continue with Your heirs. In case of more than one heir, all heirs must exercise their rights through a commonly authorized person. 96 | 97 | 8.4 If any provision of this Agreement is found void and unenforceable, such provision will be replaced to the extent possible with a provision that comes closest to the meaning of the original provision and that is enforceable. The terms and conditions set forth in this Agreement shall apply notwithstanding any failure of essential purpose of this Agreement or any limited remedy to the maximum extent possible under law. 98 | 99 | 8.5 You agree to notify Us of any facts or circumstances of which you become aware that would make this Agreement inaccurate in any respect. 100 | 101 | ### You 102 | 103 | Date: 104 | 105 | Name: 106 | 107 | Title: 108 | 109 | Address: 110 | 111 | ### Us 112 | 113 | Date: 114 | 115 | Name: 116 | 117 | Title: 118 | 119 | Address: 120 | 121 | #### Recreate this Contributor License Agreement 122 | 123 | [https://contributoragreements.org/ca-cla-chooser/?beneficiary-name=Intoli%2C+LLC&project-name=Exodus&project-website=https%3A%2F%2Fgithub.com%2Fintoli%2Fexodus&project-email=open-source%40intoli.com&process-url=https%3A%2F%2Fgithub.com%2Fintoli%2Fexodus%2Fblob%2Fmaster%2FCONTRIBUTING.md&project-jurisdiction=United+States&agreement-exclusivity=exclusive&fsfe-compliance=&fsfe-fla=&outbound-option=no-commitment&outboundlist=&outboundlist-custom=&medialist=\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_\_&patent-option=Traditional&your-date=&your-name=&your-title=&your-address=&your-patents=&pos=apply&action=](https://contributoragreements.org/ca-cla-chooser/?beneficiary-name=Intoli%2C+LLC&project-name=Exodus&project-website=https%3A%2F%2Fgithub.com%2Fintoli%2Fexodus&project-email=open-source%40intoli.com&process-url=https%3A%2F%2Fgithub.com%2Fintoli%2Fexodus%2Fblob%2Fmaster%2FCONTRIBUTING.md&project-jurisdiction=United+States&agreement-exclusivity=exclusive&fsfe-compliance=&fsfe-fla=&outbound-option=no-commitment&outboundlist=&outboundlist-custom=&medialist=____________________&patent-option=Traditional&your-date=&your-name=&your-title=&your-address=&your-patents=&pos=apply&action=) 124 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | ## Contributing 2 | 3 | Contributions are welcome, but please follow these contributor guidelines: 4 | 5 | - Create an issue on [the issue tracker](https://github.com/intoli/exodus/issues/new) to discuss potential changes before submitting a pull request. 6 | - Include at least one test to cover any new functionality or bug fixes. 7 | - Make sure that all of your tests are passing and that there are no merge conflicts. 8 | - Print, sign, and email the [Contributor License Agreement](https://github.com/intoli/exodus/blob/master/CLA.md) to [open-source@intoli.com](mailto:open-source@intoli.com). 9 | -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | 2-Clause BSD License 2 | ====================== 3 | 4 | *Copyright © `2017-present`, `Intoli, LLC`* 5 | 6 | *https://github.com/intoli/exodus/* 7 | 8 | *https://intoli.com/* 9 | 10 | *All rights reserved.* 11 | 12 | Redistribution and use in source and binary forms, with or without 13 | modification, are permitted provided that the following conditions are met: 14 | 15 | 1. Redistributions of source code must retain the above copyright notice, this 16 | list of conditions and the following disclaimer. 17 | 2. Redistributions in binary form must reproduce the above copyright notice, 18 | this list of conditions and the following disclaimer in the documentation 19 | and/or other materials provided with the distribution. 20 | 21 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS “AS IS” AND 22 | ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 23 | WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 24 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR 25 | ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 26 | (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 27 | LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND 28 | ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 29 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 30 | SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 31 | 32 | The views and conclusions contained in the software and documentation are those 33 | of the authors and should not be interpreted as representing official policies, 34 | either expressed or implied, of the FreeBSD Project. 35 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | graft media 2 | graft src 3 | graft tests 4 | graft .circleci 5 | graft .github 6 | 7 | include .bumpversion.cfg 8 | include .clabot 9 | include .coveragerc 10 | include conftest.py 11 | include development-requirements.txt 12 | 13 | include *.md 14 | 15 | include tox.ini 16 | 17 | global-exclude *.py[cod] __pycache__ *.so *.dylib .DS_Store 18 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 |

Exodus 2 | 3 | Tweet 5 | 6 | Share on Facebook 8 | 9 | Share on Reddit 11 | 12 | Share on Hacker News 14 |

15 | 16 |

17 | 18 | Build Status 20 | 21 | Coverage 23 | 24 | License 26 | 27 | PyPI Version 29 |

30 | 31 | 32 | Exodus is a tool that makes it easy to successfully relocate Linux ELF binaries from one system to another. 33 | This is useful in situations where you don't have root access on a machine or where a package simply isn't available for a given Linux distribution. 34 | For example, CentOS 6.X and Amazon Linux don't have packages for [Google Chrome](https://www.google.com/chrome/browser/desktop/index.html) or [aria2](https://aria2.github.io/). 35 | Server-oriented distributions tend to have more limited and outdated packages than desktop distributions, so it's fairly common that one might have a piece of software installed on their laptop that they can't easily install on a remote machine. 36 | 37 | With exodus, transferring a piece of software that's working on one computer to another is as simple as this. 38 | 39 | ```bash 40 | exodus aria2c | ssh intoli.com 41 | ``` 42 | 43 | Exodus handles bundling all of the binary's dependencies, compiling a statically linked wrapper for the executable that invokes the relocated linker directly, and installing the bundle in `~/.exodus/` on the remote machine. 44 | You can see it in action here. 45 | 46 | ![Demonstration of usage with htop](media/htop-demo.gif) 47 | 48 | 49 | ## Table of Contents 50 | 51 | - [The Problem Being Solved](#the-problem-being-solved) - An overview of some of the challenges that arise when relocating binaries. 52 | - [Installation](#installation) - Instructions for installing exodus. 53 | - [Usage](#usage) 54 | - [The Command-Line Interface](#command-line-interface) - The options supported by the command-line utility. 55 | - [Usage Examples](#examples) - Common usage patterns, helpful for getting started quickly. 56 | - [How It Works](#how-it-works) - An overview of how exodus works. 57 | - [Known Limitations](#known-limitations) - Situations that are currently outside the scope of what exodus can handle. 58 | - [Development](#development) - Instructions for setting up the development environment. 59 | - [Contributing](#contributing) - Guidelines for contributing. 60 | - [License](#license) - License details for the project. 61 | 62 | 63 | ## The Problem Being Solved 64 | 65 | If you simply copy an executable file from one system to another, then you're very likely going to run into problems. 66 | Most binaries available on Linux are dynamically linked and depend on a number of external library files. 67 | You'll get an error like this when running a relocated binary when it has a missing dependency. 68 | 69 | ``` 70 | aria2c: error while loading shared libraries: libgnutls.so.30: cannot open shared object file: No such file or directory 71 | ``` 72 | 73 | You can try to install these libraries manually, or to relocate them and set `LD_LIBRARY_PATH` to wherever you put them, but it turns out that the locations of the [ld-linux](https://linux.die.net/man/8/ld-linux) linker and the [glibc](https://www.gnu.org/software/libc/) libraries are hardcoded. 74 | Things can very quickly turn into a mess of relocation errors, 75 | 76 | ``` 77 | aria2c: relocation error: /lib/libpthread.so.0: symbol __getrlimit, version 78 | GLIBC_PRIVATE not defined in file libc.so.6 with link time reference 79 | ``` 80 | 81 | segmentation faults, 82 | 83 | ``` 84 | Segmentation fault (core dumped) 85 | ``` 86 | 87 | or, if you're really unlucky, this very confusing symptom of a missing linker. 88 | 89 | ``` 90 | $ ./aria2c 91 | bash: ./aria2c: No such file or directory 92 | $ ls -lha ./aria2c 93 | -rwxr-xr-x 1 sangaline sangaline 2.8M Jan 30 21:18 ./aria2c 94 | ``` 95 | 96 | Exodus works around these issues by compiling a small statically linked launcher binary that invokes the relocated linker directly with any hardcoded `RPATH` library paths overridden. 97 | The relocated binary will run with the exact same linker and libraries that it ran with on its origin machine. 98 | 99 | 100 | ## Installation 101 | 102 | The package can be installed from [the package on pypi](https://pypi.python.org/pypi/exodus_bundler). 103 | Running the following will install `exodus` locally for your current user. 104 | 105 | ```bash 106 | pip install --user exodus-bundler 107 | ``` 108 | 109 | You will then need to add `~/.local/bin/` to your `PATH` variable in order to run the `exodus` executable (if you haven't already done so). 110 | This can be done by adding 111 | 112 | ``` 113 | export PATH="~/.local/bin/:${PATH}" 114 | ``` 115 | 116 | to your `~/.bashrc` file. 117 | 118 | 119 | ### Optional/Recommended Dependencies 120 | 121 | It is also highly recommended that you install [gcc](https://gcc.gnu.org/) and one of either [musl libc](https://www.musl-libc.org/) or [diet libc](https://www.fefe.de/dietlibc/) on the machine where you'll be packaging binaries. 122 | If present, these small C libraries will be used to compile small statically linked launchers for the bundled applications. 123 | An equivalent shell script will be used as a fallback, but it carries significant overhead compared to the compiled launchers. 124 | 125 | 126 | ## Usage 127 | 128 | ### Command-Line Interface 129 | 130 | The command-line interface supports the following options. 131 | 132 | ``` 133 | usage: exodus [-h] [-c CHROOT_PATH] [-a DEPENDENCY] [-d] [--no-symlink FILE] 134 | [-o OUTPUT_FILE] [-q] [-r [NEW_NAME]] [--shell-launchers] [-t] 135 | [-v] 136 | EXECUTABLE [EXECUTABLE ...] 137 | 138 | Bundle ELF binary executables with all of their runtime dependencies so that 139 | they can be relocated to other systems with incompatible system libraries. 140 | 141 | positional arguments: 142 | EXECUTABLE One or more ELF executables to include in the exodus 143 | bundle. 144 | 145 | optional arguments: 146 | -h, --help show this help message and exit 147 | -c CHROOT_PATH, --chroot CHROOT_PATH 148 | A directory that will be treated as the root during 149 | linking. Useful for testing and bundling extracted 150 | packages that won't run without a chroot. (default: 151 | None) 152 | -a DEPENDENCY, --add DEPENDENCY, --additional-file DEPENDENCY 153 | Specifies an additional file to include in the bundle, 154 | useful for adding programatically loaded libraries and 155 | other non-library dependencies. The argument can be 156 | used more than once to include multiple files, and 157 | directories will be included recursively. (default: 158 | []) 159 | -d, --detect Attempt to autodetect direct dependencies using the 160 | system package manager. Operating system support is 161 | limited. (default: False) 162 | --no-symlink FILE Signifies that a file must not be symlinked to the 163 | deduplicated data directory. This is useful if a file 164 | looks for other resources based on paths relative its 165 | own location. This is enabled by default for 166 | executables. (default: []) 167 | -o OUTPUT_FILE, --output OUTPUT_FILE 168 | The file where the bundle will be written out to. The 169 | extension depends on the output type. The 170 | "{{executables}}" and "{{extension}}" template strings 171 | can be used in the provided filename. If omitted, the 172 | output will go to stdout when it is being piped, or to 173 | "./exodus-{{executables}}-bundle.{{extension}}" 174 | otherwise. (default: None) 175 | -q, --quiet Suppress warning messages. (default: False) 176 | -r [NEW_NAME], --rename [NEW_NAME] 177 | Renames the binary executable(s) before packaging. The 178 | order of rename tags must match the order of 179 | positional executable arguments. (default: []) 180 | --shell-launchers Force the use of shell launchers instead of attempting 181 | to compile statically linked ones. (default: False) 182 | -t, --tarball Creates a tarball for manual extraction instead of an 183 | installation script. Note that this will change the 184 | output extension from ".sh" to ".tgz". (default: 185 | False) 186 | -v, --verbose Output additional informational messages. (default: 187 | False) 188 | ``` 189 | 190 | 191 | ### Examples 192 | 193 | #### Piping Over SSH 194 | 195 | The easiest way to install an executable bundle on a remote machine is to pipe the `exodus` command output over SSH. 196 | For example, the following will install the `aria2c` command on the `intoli.com` server. 197 | 198 | ```bash 199 | exodus aria2c | ssh intoli.com 200 | ``` 201 | 202 | This requires that the default shell for the remote user be set to `bash` (or a compatible shell). 203 | If you use `csh`, then you need to additionally run `bash` on the remote server like this. 204 | 205 | ```bash 206 | exodus aria2c | ssh intoli.com bash 207 | ``` 208 | 209 | #### Explicitly Adding Extra Files 210 | 211 | Additional files can be added to bundles in a number of different ways. 212 | If there is a specific file or directory that you would like to include, then you can use the `--add` option. 213 | For example, the following command will bundle `nmap` and include the contents of `/usr/share/nmap` in the bundle. 214 | 215 | ```bash 216 | exodus --add /usr/share/nmap nmap 217 | ``` 218 | 219 | You can also pipe a list of dependencies into `exodus`. 220 | This allows you to use standard Linux utilities to find and filter dependencies as you see fit. 221 | The following command sequence uses `find` to include all of the Lua scripts under `/usr/share/nmap`. 222 | 223 | ```bash 224 | find /usr/share/nmap/ -iname '*.lua' | exodus nmap 225 | ``` 226 | 227 | These two approaches can be used together, and the `--add` flag can also be used multiple times in one command. 228 | 229 | 230 | #### Auto-Detecting Extra Files 231 | 232 | If you're not sure which extra dependencies are necessary, you can use the `--detect` option to query your system's package manager and automatically include any files in the corresponding packages. 233 | Running 234 | 235 | ```bash 236 | exodus --detect nmap 237 | ``` 238 | 239 | will include the contents of `/usr/share/nmap` as well as its man pages and the contents of `/usr/share/zenmap/`. 240 | If you ever try to relocate a binary that doesn't work with the default configuration, the `--detect` option is a good first thing to try. 241 | 242 | You can also pipe the output of `strace` into `exodus` and all of the files that are accessed will be automatically included. 243 | This is particularly useful in situations where shared libraries are loaded programmatically, but it can also be used to determine which files are necessary to run a specific command. 244 | The following command will determine all of the files that `nmap` accesses while running the set of default scripts. 245 | 246 | ```bash 247 | strace -f nmap --script default 127.0.0.1 2>&1 | exodus nmap 248 | ``` 249 | 250 | The output of `strace` is then parsed by `exodus` and all of the files are included. 251 | It's generally more robust to use `--detect` to find the non-library dependencies, but the `strace` pattern can be indispensable when a program uses `dlopen()` to load libraries programmatically. 252 | Also, note that *any* files that a program accesses will be included in a bundle when following this approach. 253 | Never distribute a bundle without being certain that you haven't accidentally included a file that you don't want to make public. 254 | 255 | 256 | #### Renaming Binaries 257 | 258 | Multiple binaries that have the same name can be installed in parallel through the use of the `--rename`/`-r` option. 259 | Say that you have two different versions of `grep` on your local machine: one at `/bin/grep` and one at `/usr/local/bin/grep`. 260 | In that situation, using the `-r` flag allows you to assign aliases for each binary. 261 | 262 | ```bash 263 | exodus -r grep-1 -r grep-2 /bin/grep /usr/local/bin/grep 264 | ``` 265 | 266 | The above command would install the two `grep` versions in parallel with `/bin/grep` called `grep-1` and `/usr/local/bin/grep` called `grep-2`. 267 | 268 | 269 | #### Manual Extraction 270 | 271 | You can create a compressed tarball directly instead of the default script by specifying the `--tarball` option. 272 | To create a tarball, copy it to a remote server, and then extract it in `~/custom-location`, you could run the following. 273 | 274 | ```bash 275 | # Create the tarball. 276 | exodus --tarball aria2c --output aria2c.tgz 277 | 278 | # Copy it to the remote server and remove it locally. 279 | scp aria2c.tgz intoli.com:/tmp/aria2c.tgz 280 | rm aria2c.tgz 281 | 282 | # Make sure that `~/custom-location` exists. 283 | ssh intoli.com "mkdir -p ~/custom-location" 284 | 285 | # Extract the tarball remotely. 286 | ssh intoli.com "tar --strip 1 -C ~/custom-location -zxf /tmp/aria2c.tgz" 287 | 288 | # Remove the remote tarball. 289 | ssh intoli.com "rm /tmp/aria2c.tgz" 290 | ``` 291 | 292 | You will additionally need to add `~/custom-location/bin` to your `PATH` variable on the remote server. 293 | This can be done by adding the following to `~/.bashrc` on the remote server. 294 | 295 | ```bash 296 | export PATH="~/custom-location/bin:${PATH}" 297 | ``` 298 | 299 | 300 | #### Adding to a Docker Image 301 | 302 | Tarball formatted exodus bundles can easily be included in Docker images by using the [ADD](https://docs.docker.com/engine/reference/builder/#add) instruction. 303 | You must first create a bundle using the `--tarball` option 304 | 305 | ```bash 306 | # Create and enter a directory for the Docker image. 307 | mkdir jq 308 | cd jq 309 | 310 | # Generate the `exodus-jq-bundle.tgz` bundle. 311 | exodus --tarball jq 312 | ``` 313 | 314 | and then create a `Dockerfile` file inside of the `jq` directory with the following contents. 315 | 316 | ``` 317 | FROM scratch 318 | ADD exodus-jq-bundle.tgz /opt/ 319 | ENTRYPOINT ["/opt/exodus/bin/jq"] 320 | ``` 321 | 322 | The Docker image can then be built by running 323 | 324 | ```bash 325 | docker build -t jq . 326 | ``` 327 | 328 | and `jq` can be run inside of the container. 329 | 330 | ```bash 331 | docker run jq 332 | ``` 333 | 334 | This simple image will include only the `jq` binary and dependencies, but the bundles can be included in existing docker images in the same way. 335 | For example, adding 336 | 337 | ```bash 338 | ENV PATH="/opt/exodus/bin:${PATH}" 339 | ADD exodus-jq-bundle.tgz /opt/ 340 | ``` 341 | 342 | to an existing `Dockerfile` will make the `jq` binary available for use inside the container. 343 | 344 | 345 | ## How It Works 346 | 347 | There are two main components to how exodus works: 348 | 349 | 1. Finding and bundling all of a binary's dependencies. 350 | 351 | 2. Launching the binary in such a way that the proper dependencies are used without any potential interaction from system libraries on the destination machine. 352 | 353 | The first component is actually fairly simple. 354 | You can invoke [ld-linux](https://linux.die.net/man/8/ld-linux) with the `LD_TRACE_LOADED_OBJECTS` environment variable set to `1` and it will list all of the resolved library dependencies for a binary. 355 | For example, running 356 | 357 | ```bash 358 | LD_TRACE_LOADED_OBJECTS=1 /lib64/ld-linux-x86-64.so.2 /bin/grep 359 | ``` 360 | 361 | will output the following. 362 | 363 | ``` 364 | linux-vdso.so.1 => (0x00007ffc7495c000) 365 | libpcre.so.0 => /lib64/libpcre.so.0 (0x00007f89b2f3e000) 366 | libc.so.6 => /lib64/libc.so.6 (0x00007f89b2b7a000) 367 | libpthread.so.0 => /usr/lib/libpthread.so.0 (0x00007f0e95e8c000) 368 | /lib64/ld-linux-x86-64.so.2 (0x00007f89b3196000) 369 | ``` 370 | 371 | The `linus-vdso.so.1` dependency refers to kernel space routines that are exported to user space, but the other four are shared library files on disk that are required in order to run `grep`. 372 | Notably, one of these dependencies is the `/lib64/ld-linux-x86-64.so.2` linker itself. 373 | The location of this file is typically hardcoded into an ELF binary's `INTERP` header and the linker is invoked by the kernel when you run the program. 374 | We'll come back to that in a minute, but for now the main point is that we can find a binary's direct dependencies using the linker. 375 | 376 | Of course, these direct dependencies might have additional dependencies of their own. 377 | We can iteratively find all of the necessary dependencies by following the same approach of invoking the linker again for each of the library dependencies. 378 | This isn't actually necessary for `grep`, but exodus does handle finding the full set of dependencies for you. 379 | 380 | After all of the dependencies are found, exodus puts them together with the binary in a tarball that can be extracted (typically into either `/opt/exodus/` or `~/.exodus`). 381 | We can explore the structure of the `grep` bundle by using [tree](https://linux.die.net/man/1/tree) combined with a `sed` one-liner to truncate long SHA-256 hashes to 8 digits. 382 | Running 383 | 384 | ```bash 385 | alias truncate-hashes="sed -r 's/([a-f0-9]{8})[a-f0-9]{56}/\1.../g'" 386 | tree ~/.exodus/ | truncate-hashes 387 | ``` 388 | 389 | will show us all of the files and folders included in the `grep` bundle. 390 | 391 | ``` 392 | /home/sangaline/.exodus/ 393 | ├── bin 394 | │   └── grep -> ../bundles/3124cd96.../usr/bin/grep 395 | ├── bundles 396 | │   └── 3124cd96... 397 | │   ├── lib64 398 | │   │   └── ld-linux-x86-64.so.2 -> ../../../data/dfd5de26... 399 | │   └── usr 400 | │   ├── bin 401 | │   │   ├── grep 402 | │   │   ├── grep-x -> ../../../../data/7477c1a7... 403 | │   │   └── linker-dfd5de26... 404 | │   └── lib 405 | │   ├── libc.so.6 -> ../../../../data/6d0e1d45... 406 | │   ├── libpcre.so.1 -> ../../../../data/a0862ebc... 407 | │   └── libpthread.so.0 -> ../../../../data/85cb56a5... 408 | └── data 409 | ├── 6d0e1d45... 410 | ├── 7477c1a7... 411 | ├── 85cb56a5... 412 | ├── a0862ebc... 413 | └── dfd5de26... 414 | 415 | 8 directories, 13 files 416 | ``` 417 | 418 | You can see that there are three top-level directories within `~/.exodus/`: `bin`, `bundles`, and `data`. 419 | Let's cover these in reverse-alphabetical order, starting with the `data` directory. 420 | 421 | The `data` directory contains the actual files from the bundles with names corresponding to SHA-256 hashes of their content. 422 | This is done so that multiple versions of a file with the same filename can be extracted in the `data` directory without overwriting each other. 423 | On the other hand, files that do have the same content *will* overwrite each other. 424 | This avoids the need to store multiple copies of the same data, even if the identical files appear in different bundles or directories. 425 | 426 | Next, we have the `bundles` directory, which is full of subfolders that also have SHA-256 hashes as names. 427 | The hashes this time are determined based on the combined directory structure and content of everything included in the bundle. 428 | The hash provides a unique fingerprint for the bundle and allows multiple bundles to be extracted without their directory contents mixing. 429 | 430 | Inside of each bundle subdirectory, the original directory structure of the bundle's contents on the host machine is mirrored. 431 | For this particular `grep` bundle, there are `lib64`, `usr/bin`, and `usr/lib` directories. 432 | A more complicated bundle could include additional files from `/usr/share`, `/opt/local`, a user's home directory, or really anywhere on the system (see the `--add` and `--detect` options). 433 | The files in both `lib64` and `usr/lib` simply consist of symlinks to the actual library files in the top-level `data/` directory. 434 | The `usr/bin` directory is a little more complicated. 435 | 436 | The `grep` file isn't actually the original `grep` binary, it's a special executable that `exodus` constructs called a "launcher." 437 | A launcher is a tiny program that invokes the linker and overrides the library search path in such a way that our original binary can run without any system libraries being used and causing issues due to incompatibilities. 438 | The linker in this case is the `linker-dfd5de26...` file. 439 | This is located in the same directory so that resource paths can be resolved relative to the running executable. 440 | Finally, the `grep-x` symlink points to the actual `grep` binary that was bundled and extracted in the top-level `data/` directory (this is the ELF file that the linker interprets). 441 | 442 | When a C compiler and either [musl libc](https://www.musl-libc.org/) or [diet libc](https://www.fefe.de/dietlibc/) are available, exodus will compile a statically linked binary launcher. 443 | If neither of these are present, it will fall back to using a shell script to perform the task of the launcher. 444 | This adds a little bit of overhead relative to the binary launchers, but they are helpful for understanding what the launchers do. 445 | Here's the shell script version of the `grep-launcher`, for example. 446 | 447 | ```bash 448 | #! /bin/bash 449 | 450 | current_directory="$(dirname "$(readlink -f "$0")")" 451 | executable="${current_directory}/./grep-x" 452 | library_path="../../lib64:../lib64:../../lib:../lib:../../lib32:../lib32" 453 | library_path="${current_directory}/${library_path//:/:${current_directory}/}" 454 | linker="${current_directory}/./linker-dfd5de2638cea087685b67786050dcdc33aac7b67f5f8c2753b7da538517880a" 455 | exec "${linker}" --library-path "${library_path}" --inhibit-rpath "" "${executable}" "$@" 456 | ``` 457 | 458 | You can see that the launcher first constructs the full paths for all of the `LD_LIBRARY_PATH` directories, the executable, and the linker based on its own location. 459 | It then executes the linker with a set of arguments that allow it to search the proper library directories, ignore the hardcoded `RPATH`, and run the binary with any command-line arguments passed along. 460 | This serves a similar purpose to something like [patchelf](https://github.com/NixOS/patchelf) that would modify the `INTERP` and `RPATH` of the binary, but it additionally allows for both the linker and library locations to be specified based *solely on their relative locations*. 461 | This is what allows for the exodus bundles to be extracted in `~/.exodus`, `/opt/exodus/`, or any other location, as long as the internal bundle structure is preserved. 462 | 463 | Continuing on with our reverse-alphabetical order, we finally get to the top-level `bin` directory. 464 | The top-level `bin` directory consists of symlinks of the binary names to their corresponding launchers. 465 | This allows for the addition of a single directory to a user's `PATH` variable in order to make the migrated exodus binaries accessible. 466 | For example, adding `export PATH="~/.exodus/bin:${PATH}"` to a `~/.bashrc` file will add all of these entry points to a user's `PATH` and allow them to be run without specifying their full path. 467 | 468 | 469 | ## Known Limitations 470 | 471 | There are several scenarios under which bundling an application with exodus will fail. 472 | Many of these are things that we're working on and hope to improve in the future, but some are fundamentally by design and are unlikely to change. 473 | Here you can see an overview of situations where exodus will not be able to successfully relocate executables. 474 | 475 | - **Non-ELF Binaries** - Exodus currently only supports completely bundling ELF binaries. 476 | Interpreted executable files, like shell scripts, can be included in bundles, but their shebang interpreter directives will not be changed. 477 | This generally means that they will be interpreted using the system version of `bash`, `python`, `perl`, or whatever else. 478 | The problem that exodus aims to solve is largely centered around the dynamic linking of ELF binaries, so this is unlikely to change in the foreseeable future. 479 | - **Incompatible CPU Architectures** - Binaries compiled for one CPU architecture will generally not be able to run on a CPU of another architecture. 480 | There are some exceptions to this, for example x64 processors are backwards compatible with x86 instruction sets, but you will not be able to migrate x64 binaries to an x86 or an ARM machine. 481 | Doing so would require processor emulation, and this is definitely outside the scope of the exodus project. 482 | If you find yourself looking for a solution to this problem, then you might want to check out [QEMU](https://www.qemu.org/). 483 | - **Incompatible Glibc and Kernel Versions** - When glibc is compiled, it is configured to target a specific kernel version. 484 | Trying to run any software that was compiled against glibc on a system using an older kernel version than glibc's target version will result in a `FATAL: kernel too old` error. 485 | You can check the oldest supported kernel version for a binary by running `file /path/to/binary`. 486 | The output should include a string like `for GNU/Linux 2.6.32` which signifies the oldest kernel version that the binary is compatible with. 487 | As a workaround, you can create exodus bundles in a Docker image using an operating system image which supports older kernels (*e.g.* use an outdated version of the operating system). 488 | - **Driver Dependent Libraries** - Unlike some other application bundlers, exodus aims to include all of the required libraries when the bundle is created and to completely isolate the transported binary from the destination machine's system libraries. 489 | This means that any libraries which are compiled for specific hardware drivers will only work on machines with the same drivers. 490 | A key example of this is the `libGLX_indirect.so` library which can link to either `libGLX_mesa.so` or `libGLX_nvidia.so` depending on which graphics card drivers are used on a given system. 491 | Bundling dependencies that are not locally available on the source machine is fundamentally outside the scope of what exodus is designed to do, and this will never change. 492 | 493 | 494 | ## Development 495 | 496 | The development environment can be setup by running the following. 497 | 498 | ```bash 499 | # Clone the repository. 500 | git clone https://github.com/intoli/exodus.git 501 | cd exodus 502 | 503 | # Create and enter a virtualenv. 504 | virtualenv .env 505 | . .env/bin/activate 506 | 507 | # Install the development requirements. 508 | pip install -r development-requirements.txt 509 | 510 | # Install the exodus package in editable mode. 511 | pip install -e . 512 | ``` 513 | 514 | The test suite can then be run using [tox](https://tox.readthedocs.io/en/latest/). 515 | 516 | ```bash 517 | tox 518 | ``` 519 | 520 | ## Contributing 521 | 522 | Contributions are welcome, but please follow these contributor guidelines outlined in [CONTRIBUTING.md](CONTRIBUTING.md). 523 | 524 | 525 | ## License 526 | 527 | Exodus is licensed under a [BSD 2-Clause License](LICENSE.md) and is copyright [Intoli, LLC](https://intoli.com). 528 | -------------------------------------------------------------------------------- /conftest.py: -------------------------------------------------------------------------------- 1 | collect_ignore = ['setup.py'] 2 | -------------------------------------------------------------------------------- /development-requirements.txt: -------------------------------------------------------------------------------- 1 | bumpversion==0.5.3 2 | coverage>=5.4 3 | m2r>=0.1.12 4 | pluggy==0.5.2 5 | py==1.4.34 6 | pytest==3.2.3 7 | pytest-sugar==0.9.0 8 | pytest-watch==4.1.0 9 | six==1.11.0 10 | tox==2.9.1 11 | twine==1.9.1 12 | virtualenv==15.1.0 13 | -------------------------------------------------------------------------------- /media/htop-demo.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/intoli/exodus/ef3d5e92c1b604b09cf0a57baff0f4d0b421b8da/media/htop-demo.gif -------------------------------------------------------------------------------- /media/ycombinator.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/intoli/exodus/ef3d5e92c1b604b09cf0a57baff0f4d0b421b8da/media/ycombinator.png -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [bdist_wheel] 2 | universal = 1 3 | 4 | [metadata] 5 | license_file = LICENSE.md 6 | 7 | [flake8] 8 | ignore = E128 9 | max-line-length = 100 10 | 11 | [tool:pytest] 12 | norecursedirs = 13 | .git 14 | .tox 15 | .env 16 | dist 17 | build 18 | python_files = 19 | test_*.py 20 | *_test.py 21 | tests.py 22 | addopts = 23 | -rxEfsw 24 | --strict 25 | --doctest-modules 26 | --doctest-glob=\*.rst 27 | --tb=short 28 | 29 | [isort] 30 | force_single_line = True 31 | line_length = 100 32 | lines_after_imports = 2 33 | known_first_party = exodus_bundler 34 | default_section = THIRDPARTY 35 | not_skip = __init__.py 36 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | # -*- encoding: utf-8 -*- 3 | from __future__ import absolute_import 4 | from __future__ import print_function 5 | 6 | from glob import glob 7 | from os.path import basename 8 | from os.path import splitext 9 | 10 | from setuptools import find_packages 11 | from setuptools import setup 12 | 13 | 14 | setup( 15 | name='exodus-bundler', 16 | version='2.0.4', 17 | license='BSD', 18 | platforms=['Linux'], 19 | description='The exodus application bundler.', 20 | long_description='See the documentation for details.', 21 | author='Intoli', 22 | author_email='contact@intoli.com', 23 | url='https://github.com/intoli/exodus', 24 | packages=find_packages('src'), 25 | package_dir={'': 'src'}, 26 | py_modules=[splitext(basename(path))[0] for path in glob('src/*.py')], 27 | include_package_data=True, 28 | zip_safe=False, 29 | classifiers=[ 30 | # complete classifier list: http://pypi.python.org/pypi?%3Aaction=list_classifiers 31 | 'Development Status :: 5 - Production/Stable', 32 | 'Intended Audience :: Developers', 33 | 'Intended Audience :: End Users/Desktop', 34 | 'Intended Audience :: Information Technology', 35 | 'Intended Audience :: Science/Research', 36 | 'Intended Audience :: System Administrators', 37 | 'License :: OSI Approved :: BSD License', 38 | 'Operating System :: POSIX :: Linux', 39 | 'Programming Language :: Python', 40 | 'Programming Language :: Python :: 2.7', 41 | 'Programming Language :: Python :: 3', 42 | 'Programming Language :: Python :: 3.0', 43 | 'Programming Language :: Python :: 3.1', 44 | 'Programming Language :: Python :: 3.2', 45 | 'Programming Language :: Python :: 3.3', 46 | 'Programming Language :: Python :: 3.4', 47 | 'Programming Language :: Python :: 3.5', 48 | 'Programming Language :: Python :: 3.6', 49 | 'Programming Language :: Python :: 3.7', 50 | 'Programming Language :: Python :: 3.8', 51 | 'Programming Language :: Python :: 3.9', 52 | 'Programming Language :: Python :: Implementation :: CPython', 53 | 'Programming Language :: Python :: Implementation :: PyPy', 54 | 'Topic :: System :: Archiving :: Packaging', 55 | 'Topic :: Utilities', 56 | ], 57 | keywords=[ 58 | 'linux', 'executable', 'elf', 'binaries', 59 | ], 60 | install_requires=[ 61 | ], 62 | entry_points={ 63 | 'console_scripts': [ 64 | 'exodus = exodus_bundler.cli:main', 65 | ], 66 | }, 67 | ) 68 | -------------------------------------------------------------------------------- /src/exodus_bundler/__init__.py: -------------------------------------------------------------------------------- 1 | import logging 2 | 3 | 4 | __version__ = '2.0.4' 5 | 6 | root_logger = logging.getLogger(__name__) 7 | root_logger.handlers = [logging.NullHandler()] 8 | -------------------------------------------------------------------------------- /src/exodus_bundler/bundling.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | import base64 3 | import filecmp 4 | import hashlib 5 | import io 6 | import logging 7 | import os 8 | import re 9 | import shutil 10 | import stat 11 | import struct 12 | import sys 13 | import tarfile 14 | import tempfile 15 | from collections import defaultdict 16 | from subprocess import PIPE 17 | from subprocess import Popen 18 | 19 | from exodus_bundler.dependency_detection import detect_dependencies 20 | from exodus_bundler.errors import DependencyDetectionError 21 | from exodus_bundler.errors import InvalidElfBinaryError 22 | from exodus_bundler.errors import MissingFileError 23 | from exodus_bundler.errors import UnexpectedDirectoryError 24 | from exodus_bundler.errors import UnsupportedArchitectureError 25 | from exodus_bundler.launchers import CompilerNotFoundError 26 | from exodus_bundler.launchers import construct_bash_launcher 27 | from exodus_bundler.launchers import construct_binary_launcher 28 | from exodus_bundler.templating import render_template 29 | from exodus_bundler.templating import render_template_file 30 | 31 | 32 | logger = logging.getLogger(__name__) 33 | 34 | 35 | def bytes_to_int(bytes, byteorder='big'): 36 | """Simple helper function to convert byte strings into integers.""" 37 | endian = {'big': '>', 'little': '<'}[byteorder] 38 | chars = struct.unpack(endian + ('B' * len(bytes)), bytes) 39 | if byteorder == 'big': 40 | chars = chars[::-1] 41 | return sum(int(char) * 256 ** i for (i, char) in enumerate(chars)) 42 | 43 | 44 | def create_bundle(executables, output, tarball=False, rename=[], chroot=None, add=[], 45 | no_symlink=[], shell_launchers=False, detect=False): 46 | """Handles the creation of the full bundle.""" 47 | # Initialize these ahead of time so they're always available for error handling. 48 | output_filename, output_file, root_directory = None, None, None 49 | try: 50 | 51 | # Create a temporary unpackaged bundle for the executables. 52 | root_directory = create_unpackaged_bundle( 53 | executables, rename=rename, chroot=chroot, add=add, no_symlink=no_symlink, 54 | shell_launchers=shell_launchers, detect=detect, 55 | ) 56 | 57 | # Populate the filename template. 58 | output_filename = render_template(output, 59 | executables=('-'.join(os.path.basename(executable) for executable in executables)), 60 | extension=('tgz' if tarball else 'sh'), 61 | ) 62 | 63 | # Store a gzipped tarball of the bundle in memory. 64 | tar_stream = io.BytesIO() 65 | with tarfile.open(fileobj=tar_stream, mode='w:gz') as tar: 66 | tar.add(root_directory, arcname='exodus') 67 | 68 | # Configure the appropriate output mechanism. 69 | if output_filename == '-': 70 | output_file = getattr(sys.stdout, 'buffer', sys.stdout) 71 | else: 72 | output_file = open(output_filename, 'wb') 73 | 74 | # Construct the installation script and write it out. 75 | if not tarball: 76 | if output_filename == '-': 77 | base64_encoded_tarball = base64.b64encode(tar_stream.getvalue()).decode('utf-8') 78 | script_content = render_template_file('install-bundle-noninteractive.sh', 79 | base64_encoded_tarball=base64_encoded_tarball) 80 | output_file.write(script_content.encode('utf-8')) 81 | else: 82 | output_file.write(render_template_file('install-bundle.sh').encode('utf-8')) 83 | output_file.write(tar_stream.getvalue()) 84 | else: 85 | # Or just write out the tarball. 86 | output_file.write(tar_stream.getvalue()) 87 | 88 | # Write out the success message. 89 | logger.info('Successfully created "%s".' % output_filename) 90 | return True 91 | except: # noqa: E722 92 | raise 93 | finally: 94 | if root_directory: 95 | shutil.rmtree(root_directory) 96 | if output_file and output_filename: 97 | output_file.close() 98 | if not tarball and output_filename not in ['-', '/dev/null']: 99 | st = os.stat(output_filename) 100 | os.chmod(output_filename, st.st_mode | stat.S_IEXEC) 101 | 102 | 103 | def create_unpackaged_bundle(executables, rename=[], chroot=None, add=[], no_symlink=[], 104 | shell_launchers=False, detect=False): 105 | """Creates a temporary directory containing the unpackaged contents of the bundle.""" 106 | bundle = Bundle(chroot=chroot, working_directory=True) 107 | try: 108 | # Sanitize the inputs. 109 | assert len(executables), 'No executables were specified.' 110 | assert len(executables) >= len(rename), \ 111 | 'More renamed options were included than executables.' 112 | # Pad the rename's with `True` so that `entry_point` can be specified. 113 | entry_points = rename + [True for i in range(len(executables) - len(rename))] 114 | 115 | # Populate the bundle with main executable files and their dependencies. 116 | for (executable, entry_point) in zip(executables, entry_points): 117 | file = bundle.add_file(executable, entry_point=entry_point) 118 | 119 | # We'll only auto-detect dependencies for these entry points as well. 120 | # If we did this later, it would practically bring in the whole system... 121 | if detect: 122 | dependency_paths = detect_dependencies(file.path) 123 | if not dependency_paths: 124 | raise DependencyDetectionError( 125 | ('Automatic dependency detection failed. Either "%s" ' % file.path) + 126 | 'is not tracked by your package manager, or your operating system ' 127 | 'is not currently compatible with the `--detect` option. If not, please ' 128 | "create an issue at https://github.com/intoli/exodus and we'll try our " 129 | ' to add support for it in the future.', 130 | ) 131 | 132 | for path in dependency_paths: 133 | bundle.add_file(path) 134 | 135 | # Add "additional files" specified with the `--add` option. 136 | for filename in add: 137 | bundle.add_file(filename) 138 | 139 | # Mark the required files as `no_symlink=True`. 140 | for path in no_symlink: 141 | path = resolve_file_path(path) 142 | file = next(iter(file for file in bundle.files if file.path == path), None) 143 | if file: 144 | file.no_symlink = True 145 | 146 | bundle.create_bundle(shell_launchers=shell_launchers) 147 | 148 | return bundle.working_directory 149 | except: # noqa: E722 150 | bundle.delete_working_directory() 151 | raise 152 | 153 | 154 | def detect_elf_binary(filename): 155 | """Returns `True` if a file has an ELF header.""" 156 | if not os.path.exists(filename): 157 | raise MissingFileError('The "%s" file was not found.' % filename) 158 | 159 | with open(filename, 'rb') as f: 160 | first_four_bytes = f.read(4) 161 | 162 | return first_four_bytes == b'\x7fELF' 163 | 164 | 165 | def parse_dependencies_from_ldd_output(content): 166 | """Takes the output of `ldd` as a string or list of lines and parses the dependencies.""" 167 | if type(content) == str: 168 | content = content.split('\n') 169 | 170 | dependencies = [] 171 | for line in content: 172 | # This first one is a special case of invoking the linker as `ldd`. 173 | if re.search(r'^\s*(/.*?)\s*=>\s*ldd\s*\(', line): 174 | # We'll exclude this because it's the hardcoded INTERP path, and it would be 175 | # impossible to get the full path from this command output. 176 | continue 177 | match = re.search(r'=>\s*(/.*?)\s*\(', line) 178 | match = match or re.search(r'\s*(/.*?)\s*\(', line) 179 | if match: 180 | dependencies.append(match.group(1)) 181 | 182 | return dependencies 183 | 184 | 185 | def resolve_binary(binary): 186 | """Attempts to find the absolute path to the binary.""" 187 | absolute_binary_path = os.path.normpath(os.path.abspath(binary)) 188 | if not os.path.exists(absolute_binary_path): 189 | for path in os.getenv('PATH', '/bin/:/usr/bin/').split(os.pathsep): 190 | absolute_binary_path = os.path.normpath(os.path.abspath(os.path.join(path, binary))) 191 | if os.path.exists(absolute_binary_path): 192 | break 193 | else: 194 | raise MissingFileError('The "%s" binary could not be found in $PATH.' % binary) 195 | return absolute_binary_path 196 | 197 | 198 | def resolve_file_path(path, search_environment_path=False): 199 | """Attempts to find a normalized path to a file. 200 | 201 | If the file is not found, or if it is a directory, appropriate exceptions will be thrown. 202 | 203 | Args: 204 | path (str): Either a relative or absolute path to a file, or the name of an 205 | executable if `search_environment_path` is `True`. 206 | search_environment_path (bool): Whether PATH should be used to resolve the file. 207 | """ 208 | if search_environment_path: 209 | path = resolve_binary(path) 210 | if not os.path.exists(path): 211 | raise MissingFileError('The "%s" file was not found.' % path) 212 | if os.path.isdir(path): 213 | raise UnexpectedDirectoryError('"%s" is a directory, not a file.' % path) 214 | return os.path.normpath(os.path.abspath(path)) 215 | 216 | 217 | def run_ldd(ldd, binary): 218 | """Runs `ldd` and gets the combined stdout/stderr output as a list of lines.""" 219 | if not detect_elf_binary(resolve_binary(binary)): 220 | raise InvalidElfBinaryError('The "%s" file is not a binary ELF file.' % binary) 221 | 222 | process = Popen([ldd, binary], stdout=PIPE, stderr=PIPE) 223 | stdout, stderr = process.communicate() 224 | return stdout.decode('utf-8').split('\n') + stderr.decode('utf-8').split('\n') 225 | 226 | 227 | class stored_property(object): 228 | """Simple decorator for a class property that will be cached indefinitely.""" 229 | def __init__(self, function): 230 | self.__doc__ = getattr(function, '__doc__') 231 | self.function = function 232 | 233 | def __get__(self, instance, type): 234 | result = instance.__dict__[self.function.__name__] = self.function(instance) 235 | return result 236 | 237 | 238 | class Elf(object): 239 | """Parses basic attributes from the ELF header of a file. 240 | 241 | Attributes: 242 | bits (int): The number of bits for an ELF binary, either 32 or 64. 243 | chroot (str): The root directory used when invoking the linker (or `None`). 244 | file_factory (function): A function used to create new `File` instances. 245 | linker_file (File): The linker/interpreter specified in the program header. 246 | path (str): The path to the file. 247 | type (str): The binary type, one of 'relocatable', 'executable', 'shared', or 'core'. 248 | """ 249 | def __init__(self, path, chroot=None, file_factory=None): 250 | """Constructs the `Elf` instance. 251 | 252 | Args: 253 | path (str): The full path to the ELF binary. 254 | chroot (str, optional): If specified, all dependency and linker paths will be considered 255 | relative to this directory (mainly useful for testing). 256 | file_factory (function, optional): A function to use when creating new `File` instances. 257 | """ 258 | if not os.path.exists(path): 259 | raise MissingFileError('The "%s" file was not found.' % path) 260 | self.path = path 261 | self.chroot = chroot 262 | self.file_factory = file_factory or File 263 | 264 | with open(path, 'rb') as f: 265 | # Make sure that this is actually an ELF binary. 266 | first_four_bytes = f.read(4) 267 | if first_four_bytes != b'\x7fELF': 268 | raise InvalidElfBinaryError('The "%s" file is not a binary ELF file.' % path) 269 | 270 | # Determine whether this is a 32-bit or 64-bit file. 271 | format_byte = f.read(1) 272 | self.bits = {b'\x01': 32, b'\x02': 64}.get(format_byte) 273 | if not self.bits: 274 | raise UnsupportedArchitectureError( 275 | ('The "%s" file does not appear to be either 32 or 64 bits. ' % path) + 276 | 'Other architectures are not currently supported, but you can open an ' 277 | 'issue at https://github.com/intoli/exodus stating your use-case and ' 278 | 'support might get extended in the future.', 279 | ) 280 | 281 | # Determine whether it's big or little endian and construct an integer parsing function. 282 | endian_byte = f.read(1) 283 | byteorder = {b'\x01': 'little', b'\x02': 'big'}[endian_byte] 284 | assert byteorder == 'little', 'Big endian is not supported right now.' 285 | if not byteorder: 286 | raise UnsupportedArchitectureError( 287 | ('The "%s" file does not appear to be little endian, ' % path) + 288 | 'and big endian binaries are not currently supported. You can open an ' 289 | 'issue at https://github.com/intoli/exodus stating your use-case and ' 290 | 'support might get extended in the future.', 291 | ) 292 | 293 | def hex(bytes): 294 | return bytes_to_int(bytes, byteorder=byteorder) 295 | 296 | # Determine the type of the binary. 297 | f.seek(hex(b'\x10')) 298 | e_type = hex(f.read(2)) 299 | self.type = {1: 'relocatable', 2: 'executable', 3: 'shared', 4: 'core'}[e_type] 300 | 301 | # Find the program header offset. 302 | e_phoff_start = {32: hex(b'\x1c'), 64: hex(b'\x20')}[self.bits] 303 | e_phoff_length = {32: 4, 64: 8}[self.bits] 304 | f.seek(e_phoff_start) 305 | e_phoff = hex(f.read(e_phoff_length)) 306 | 307 | # Determine the size of a program header entry. 308 | e_phentsize_start = {32: hex(b'\x2a'), 64: hex(b'\x36')}[self.bits] 309 | f.seek(e_phentsize_start) 310 | e_phentsize = hex(f.read(2)) 311 | 312 | # Determine the number of program header entries. 313 | e_phnum_start = {32: hex(b'\x2c'), 64: hex(b'\x38')}[self.bits] 314 | f.seek(e_phnum_start) 315 | e_phnum = hex(f.read(2)) 316 | 317 | # Loop through each program header. 318 | self.linker_file = None 319 | for header_index in range(e_phnum): 320 | header_start = e_phoff + header_index * e_phentsize 321 | f.seek(header_start) 322 | p_type = f.read(4) 323 | # A p_type of \x03 corresponds to a PT_INTERP header (e.g. the linker). 324 | if len(p_type) == 0: 325 | break 326 | if not p_type == b'\x03\x00\x00\x00': 327 | continue 328 | 329 | # Determine the offset for the segment. 330 | p_offset_start = header_start + {32: hex(b'\04'), 64: hex(b'\x08')}[self.bits] 331 | p_offset_length = {32: 4, 64: 8}[self.bits] 332 | f.seek(p_offset_start) 333 | p_offset = hex(f.read(p_offset_length)) 334 | 335 | # Determine the size of the segment. 336 | p_filesz_start = header_start + {32: hex(b'\x10'), 64: hex(b'\x20')}[self.bits] 337 | p_filesz_length = {32: 4, 64: 8}[self.bits] 338 | f.seek(p_filesz_start) 339 | p_filesz = hex(f.read(p_filesz_length)) 340 | 341 | # Read in the segment. 342 | f.seek(p_offset) 343 | segment = f.read(p_filesz) 344 | # It should be null-terminated (b'\x00' in Python 2, 0 in Python 3). 345 | assert segment[-1] in [b'\x00', 0], 'The string should be null terminated.' 346 | assert self.linker_file is None, 'More than one linker found.' 347 | linker_path = segment[:-1].decode('ascii') 348 | if chroot: 349 | linker_path = os.path.join(chroot, os.path.relpath(linker_path, '/')) 350 | self.linker_file = self.file_factory(linker_path, chroot=self.chroot) 351 | 352 | def __eq__(self, other): 353 | return isinstance(other, Elf) and self.path == self.path 354 | 355 | def __hash__(self): 356 | """Defines a hash for the object so it can be used in sets.""" 357 | return hash(self.path) 358 | 359 | def __repr__(self): 360 | return '' % self.path 361 | 362 | def find_direct_dependencies(self, linker_file=None): 363 | """Runs the specified linker and returns a set of the dependencies as `File` instances.""" 364 | linker_file = linker_file or self.linker_file 365 | if not linker_file: 366 | return set() 367 | linker_path = linker_file.path 368 | environment = {} 369 | environment.update(os.environ) 370 | environment['LD_TRACE_LOADED_OBJECTS'] = '1' 371 | extra_ldd_arguments = [] 372 | if self.chroot: 373 | ld_library_path = '/lib64:/usr/lib64:/lib/:/usr/lib:/lib32/:/usr/lib32/:' 374 | ld_library_path += environment.get('LD_LIBRARY_PATH', '') 375 | directories = [] 376 | for directory in ld_library_path.split(':'): 377 | if os.path.isabs(directory): 378 | directory = os.path.join(self.chroot, os.path.relpath(directory, '/')) 379 | directories.append(directory) 380 | ld_library_path = ':'.join(directories) 381 | environment['LD_LIBRARY_PATH'] = ld_library_path 382 | # We only need to avoid including system dependencies if there's a chroot set. 383 | extra_ldd_arguments += ['--inhibit-cache', '--inhibit-rpath', ''] 384 | 385 | process = Popen(['ldd'] + extra_ldd_arguments + [self.path], 386 | executable=linker_path, stdout=PIPE, stderr=PIPE, env=environment) 387 | stdout, stderr = process.communicate() 388 | combined_output = stdout.decode('utf-8').split('\n') + stderr.decode('utf-8').split('\n') 389 | # Note that we're explicitly adding the linker because when we invoke it as `ldd` we can't 390 | # extract the real path from the trace output. Even if it were here twice, it would be 391 | # deduplicated though the use of a set. 392 | filenames = parse_dependencies_from_ldd_output(combined_output) + [linker_path] 393 | return set(self.file_factory(filename, chroot=self.chroot, library=True) 394 | for filename in filenames) 395 | 396 | @stored_property 397 | def dependencies(self): 398 | """Run's the files' linker iteratively and returns a set of all library dependencies.""" 399 | all_dependencies = set() 400 | unprocessed_dependencies = set(self.direct_dependencies) 401 | while len(unprocessed_dependencies): 402 | all_dependencies |= unprocessed_dependencies 403 | new_dependencies = set() 404 | for dependency in unprocessed_dependencies: 405 | if dependency.elf: 406 | new_dependencies |= set( 407 | dependency.elf.find_direct_dependencies(self.linker_file)) 408 | unprocessed_dependencies = new_dependencies - all_dependencies 409 | return all_dependencies 410 | 411 | @stored_property 412 | def direct_dependencies(self): 413 | """Runs the file's linker and returns a set of the dependencies as `File` instances.""" 414 | return self.find_direct_dependencies() 415 | 416 | 417 | class File(object): 418 | """Represents a file on disk and provides access to relevant properties and actions. 419 | 420 | Note: 421 | The `File` class is tied to the bundling format. For example, the `destination` property 422 | will correspond to a path like 'data/{hash}' which is then used in bundling. 423 | 424 | Attributes: 425 | chroot (str): A location to treat as the root during dependency linking (or `None`). 426 | elf (Elf): A corresponding `Elf` object, or `None` if it is not an ELF formatted file. 427 | entry_point (str): The name of the bundle entry point for an executable binary (or `None`). 428 | file_factory (function): A function used to create new `File` instances. 429 | library (bool): Specifies that this file is explicitly a shared library. 430 | no_symlink (bool): Specifies that a file must not be symlinked to the common data directory. 431 | path (str): The absolute normalized path to the file on disk. 432 | """ 433 | 434 | def __init__(self, path, entry_point=None, chroot=None, library=False, file_factory=None): 435 | """Constructor for the `File` class. 436 | 437 | Note: 438 | A `MissingFileError` will be thrown if a matching file cannot be found. 439 | 440 | Args: 441 | path (str): Can be either an absolute path, relative path, or a binary name in `PATH`. 442 | entry_point (string, optional): The name of the bundle entry point for an executable. 443 | If `True`, the executable's basename will be used. 444 | chroot (str, optional): If specified, all dependency and linker paths will be considered 445 | relative to this directory (mainly useful for testing). 446 | file_factory (function, optional): A function to use when creating new `File` instances. 447 | """ 448 | # Find the full path to the file. 449 | self.path = resolve_file_path(path, search_environment_path=(entry_point is not None)) 450 | 451 | # Set the entry point for the file. 452 | if entry_point is True: 453 | self.entry_point = os.path.basename(self.path).replace(os.sep, '') 454 | else: 455 | self.entry_point = entry_point or None 456 | 457 | # Parse an `Elf` object from the file. 458 | try: 459 | self.elf = Elf(path, chroot=chroot, file_factory=file_factory) 460 | except InvalidElfBinaryError: 461 | self.elf = None 462 | 463 | self.chroot = chroot 464 | self.file_factory = file_factory or File 465 | self.library = library 466 | self.no_symlink = self.entry_point and not self.requires_launcher 467 | 468 | def __eq__(self, other): 469 | return isinstance(other, File) and self.path == self.path and \ 470 | self.entry_point == self.entry_point 471 | 472 | def __hash__(self): 473 | """Computes a hash for the instance unique up to the file path and entry point.""" 474 | return hash((self.path, self.entry_point)) 475 | 476 | def __repr__(self): 477 | return '' % self.path 478 | 479 | def copy(self, working_directory): 480 | """Copies the file to a location based on its `destination` property. 481 | 482 | Args: 483 | working_directory (str): The root that the `destination` will be joined with. 484 | Returns: 485 | str: The normalized and absolute destination path. 486 | """ 487 | full_destination = os.path.join(working_directory, self.destination) 488 | full_destination = os.path.normpath(os.path.abspath(full_destination)) 489 | 490 | # The filenames are based on content hashes, so there's no need to copy it twice. 491 | if os.path.exists(full_destination): 492 | return full_destination 493 | 494 | parent_directory = os.path.dirname(full_destination) 495 | if not os.path.exists(parent_directory): 496 | os.makedirs(parent_directory) 497 | 498 | shutil.copy(self.path, full_destination) 499 | 500 | return full_destination 501 | 502 | def create_entry_point(self, working_directory, bundle_root): 503 | """Creates a symlink in `bin/` to the executable or its launcher. 504 | 505 | Note: 506 | The destination must already exist. 507 | Args: 508 | working_directory (str): The root that the `destination` will be joined with. 509 | bundle_root (str): The root that `source` will be joined with. 510 | """ 511 | source_path = os.path.join(bundle_root, self.source) 512 | bin_directory = os.path.join(working_directory, 'bin') 513 | if not os.path.exists(bin_directory): 514 | os.makedirs(bin_directory) 515 | entry_point_path = os.path.join(bin_directory, self.entry_point) 516 | relative_destination_path = os.path.relpath(source_path, bin_directory) 517 | os.symlink(relative_destination_path, entry_point_path) 518 | 519 | def create_launcher(self, working_directory, bundle_root, linker_basename, symlink_basename, 520 | shell_launcher=False): 521 | """Creates a launcher at `source` for `destination`. 522 | 523 | Note: 524 | If an `entry_point` has been specified, it will also be created. 525 | Args: 526 | working_directory (str): The root that the `destination` will be joined with. 527 | bundle_root (str): The root that `source` will be joined with. 528 | linker_basename (str): The basename of the linker to place in the same directory. 529 | symlink_basename (str): The basename of the symlink to the actual executable. 530 | shell_launcher (bool, optional): Forces the use of shell script launcher instead of 531 | attempting to compile first using musl or diet c. 532 | Returns: 533 | str: The normalized and absolute path to the launcher. 534 | """ 535 | destination_path = os.path.join(working_directory, self.destination) 536 | source_path = os.path.join(bundle_root, self.source) 537 | 538 | # Create the symlink. 539 | source_parent = os.path.dirname(source_path) 540 | if not os.path.exists(source_parent): 541 | os.makedirs(source_parent) 542 | relative_destination_path = os.path.relpath(destination_path, source_parent) 543 | symlink_path = os.path.join(source_parent, symlink_basename) 544 | os.symlink(relative_destination_path, symlink_path) 545 | executable = os.path.join('.', symlink_basename) 546 | 547 | # Copy over the linker. 548 | linker_path = os.path.join(source_parent, linker_basename) 549 | if not os.path.exists(linker_path): 550 | shutil.copy(self.elf.linker_file.path, linker_path) 551 | else: 552 | assert filecmp.cmp(self.elf.linker_file.path, linker_path), \ 553 | 'The "%s" linker file already exists and has differing contents.' % linker_path 554 | linker = os.path.join('.', linker_basename) 555 | 556 | # Construct the library path 557 | original_file_parent = os.path.dirname(self.path) 558 | library_paths = os.environ.get('LD_LIBRARY_PATH', '').split(':') 559 | library_paths += ['/lib64', '/usr/lib64', '/lib', '/usr/lib', '/lib32', '/usr/lib32'] 560 | for dependency in self.elf.dependencies: 561 | library_paths.append(os.path.dirname(dependency.path)) 562 | relative_library_paths = [] 563 | for directory in library_paths: 564 | if not len(directory): 565 | continue 566 | 567 | # Get the actual absolute path for the library directory. 568 | directory = os.path.normpath(os.path.abspath(directory)) 569 | if self.chroot: 570 | directory = os.path.join(self.chroot, os.path.relpath(directory, '/')) 571 | 572 | # Convert it into a path relative to the launcher/source. 573 | relative_library_path = os.path.relpath(directory, original_file_parent) 574 | if relative_library_path not in relative_library_paths: 575 | relative_library_paths.append(relative_library_path) 576 | library_path = ':'.join(relative_library_paths) 577 | 578 | # Determine whether this is a "full" linker (*e.g.* GNU linker). 579 | with open(self.elf.linker_file.path, 'rb') as f: 580 | linker_content = f.read() 581 | full_linker = (linker_content.find(b'inhibit-rpath') > -1) 582 | 583 | # Try a c launcher first and fallback. 584 | try: 585 | if shell_launcher: 586 | raise CompilerNotFoundError() 587 | 588 | launcher_content = construct_binary_launcher( 589 | linker=linker, library_path=library_path, executable=executable, 590 | full_linker=full_linker) 591 | with open(source_path, 'wb') as f: 592 | f.write(launcher_content) 593 | except CompilerNotFoundError: 594 | if not shell_launcher: 595 | logger.warning(( 596 | 'Installing either the musl or diet C libraries will result in more efficient ' 597 | 'launchers (currently using bash fallbacks instead).' 598 | )) 599 | launcher_content = construct_bash_launcher( 600 | linker=linker, library_path=library_path, executable=executable, 601 | full_linker=full_linker) 602 | with open(source_path, 'w') as f: 603 | f.write(launcher_content) 604 | shutil.copymode(self.path, source_path) 605 | 606 | return os.path.normpath(os.path.abspath(source_path)) 607 | 608 | def symlink(self, working_directory, bundle_root): 609 | """Creates a relative symlink from the `source` to the `destination`. 610 | 611 | Args: 612 | working_directory (str): The root that `destination` will be joined with. 613 | bundle_root (str): The root that `source` will be joined with. 614 | Returns: 615 | str: The normalized and absolute path to the symlink. 616 | """ 617 | destination_path = os.path.join(working_directory, self.destination) 618 | source_path = os.path.join(bundle_root, self.source) 619 | 620 | source_parent = os.path.dirname(source_path) 621 | if not os.path.exists(source_parent): 622 | os.makedirs(source_parent) 623 | relative_destination_path = os.path.relpath(destination_path, source_parent) 624 | if os.path.exists(source_path): 625 | assert os.path.islink(source_path) 626 | assert os.path.realpath(source_path) == relative_destination_path 627 | else: 628 | os.symlink(relative_destination_path, source_path) 629 | 630 | return os.path.normpath(os.path.abspath(source_path)) 631 | 632 | @stored_property 633 | def destination(self): 634 | """str: The relative path for the destination of the actual file contents.""" 635 | return os.path.join('.', 'data', self.hash) 636 | 637 | @stored_property 638 | def executable(self): 639 | return os.access(self.path, os.X_OK) 640 | 641 | @stored_property 642 | def elf(self): 643 | """bool: Determines whether a file is a file is an ELF binary.""" 644 | return detect_elf_binary(self.path) 645 | 646 | @stored_property 647 | def hash(self): 648 | """str: Computes a hash based on the file content, useful for file deduplication.""" 649 | with open(self.path, 'rb') as f: 650 | return hashlib.sha256(f.read()).hexdigest() 651 | 652 | @stored_property 653 | def requires_launcher(self): 654 | """bool: Whether a launcher is necessary for this file.""" 655 | # This is unfortunately a heuristic approach because many executables are compiled 656 | # as shared libraries, and many mostly-libraries are executable (*e.g.* glibc). 657 | 658 | # The easy ones. 659 | if self.library or not self.elf or not self.elf.linker_file or not self.executable: 660 | return False 661 | if self.elf.type == 'executable': 662 | return True 663 | if self.entry_point: 664 | return True 665 | 666 | # These will hopefully do more good than harm. 667 | bin_directories = ['/bin/', '/bin32/', '/bin64/'] 668 | lib_directories = ['/lib/', '/lib32/', '/lib64/'] 669 | in_bin_directory = any(directory in self.path for directory in bin_directories) 670 | in_lib_directory = any(directory in self.path for directory in lib_directories) 671 | if in_bin_directory and not in_lib_directory: 672 | return True 673 | if in_lib_directory and not in_bin_directory: 674 | return False 675 | 676 | # Most libraries will include `.so` in the filename. 677 | return re.search(r'\.so(?:\.|$)', self.path) 678 | 679 | @stored_property 680 | def source(self): 681 | """str: The relative path for the source of the actual file contents.""" 682 | return os.path.relpath(self.path, '/') 683 | 684 | 685 | class Bundle(object): 686 | """A collection of files to be included in a bundle and utilities for creating bundles. 687 | 688 | Attributes: 689 | chroot (str): The root directory used when invoking the linker (or `None` for `/`). 690 | files (:obj:`set` of :obj:`File`): The files to be included in the bundle. 691 | linker_files (:obj:`set` of :obj:`File`): A list of observed linker files. 692 | working_directory (str): The root directory where the bundles will be written and packaged. 693 | """ 694 | def __init__(self, working_directory=None, chroot=None): 695 | """Constructor for the `Bundle` class. 696 | 697 | Args: 698 | working_directory (string, optional): The location where the bundle will be created on 699 | disk. A temporary directory will be constructed if specified as `True`. If left as 700 | `None`, some methods and properties will raise errors. 701 | chroot (str, optional): If specified, all absolute paths will be treated as being 702 | relative to this root (mainly useful for testing). 703 | """ 704 | self.working_directory = working_directory 705 | if working_directory is True: 706 | self.working_directory = tempfile.mkdtemp(prefix='exodus-bundle-') 707 | # The permissions on the `mkdtemp()` directory will be extremely restricted by default, 708 | # so we'll modify them to to reflect the current umask. 709 | umask = os.umask(0) 710 | os.umask(umask) 711 | os.chmod(self.working_directory, 0o777 & ~umask) 712 | self.chroot = chroot 713 | self.files = set() 714 | self.linker_files = set() 715 | 716 | def add_file(self, path, entry_point=None): 717 | """Adds an additional file to the bundle. 718 | 719 | Note: 720 | All of the file's dependencies will additionally be pulled into the bundle if the file 721 | corresponds to a an ELF binary. This is true regardless of whether or not an entry point 722 | is specified for the binary. 723 | 724 | Args: 725 | path (str): Can be either an absolute path, relative path, or a binary name in `PATH`. 726 | Directories will be included recursively for non-entry point dependencies. 727 | entry_point (string, optional): The name of the bundle entry point for an executable. 728 | If `True`, the executable's basename will be used. 729 | Returns: 730 | The `File` that was added, or `None` if it was a directory that was added recursively. 731 | """ 732 | try: 733 | file = self.file_factory(path, entry_point=entry_point, chroot=self.chroot) 734 | except UnexpectedDirectoryError: 735 | assert entry_point is None, "Directories can't have entry points." 736 | for root, directories, files in os.walk(path): 737 | for file in files: 738 | file_path = os.path.join(root, file) 739 | self.add_file(file_path) 740 | return 741 | 742 | self.files.add(file) 743 | if file.elf: 744 | if file.elf.linker_file: 745 | self.linker_files.add(file.elf.linker_file) 746 | self.files |= file.elf.dependencies 747 | else: 748 | # Manually set the linker if there isn't one in the program header, 749 | # and we've only seen one in all of the files that have been added. 750 | if len(self.linker_files) == 1: 751 | [file.elf.linker_file] = self.linker_files 752 | self.files |= file.elf.dependencies 753 | # We definitely don't want a launcher for this file, so clear the linker. 754 | file.elf.linker_file = None 755 | else: 756 | logger.warning(( 757 | 'An ELF binary without a suitable linker candidate was encountered. ' 758 | 'Either no linker was found or there are multiple conflicting linkers.' 759 | )) 760 | 761 | return file 762 | 763 | def create_bundle(self, shell_launchers=False): 764 | """Creates the unpackaged bundle in `working_directory`. 765 | 766 | Args: 767 | shell_launchers (bool, optional): Forces the use of shell script launchers instead of 768 | attempting to compile first using musl or diet c. 769 | """ 770 | file_paths = set() 771 | files_needing_launchers = defaultdict(set) 772 | for file in self.files: 773 | # Store the file path to avoid collisions later. 774 | file_path = os.path.join(self.bundle_root, file.source) 775 | file_paths.add(file_path) 776 | 777 | # Create a symlink in `./bin/` if an entry point is specified. 778 | if file.entry_point: 779 | file.create_entry_point(self.working_directory, self.bundle_root) 780 | 781 | if file.no_symlink: 782 | # We'll need to copy the actual file into the bundle subdirectory in this 783 | # case so that it can locate resources using paths relative to the executable. 784 | parent_directory = os.path.dirname(file_path) 785 | if not os.path.exists(parent_directory): 786 | os.makedirs(parent_directory) 787 | shutil.copy(file.path, file_path) 788 | continue 789 | 790 | # Copy over the actual file. 791 | file.copy(self.working_directory) 792 | 793 | if file.requires_launcher: 794 | # These are kind of complicated, we'll just store the requirements for now. 795 | directory_and_linker = (os.path.dirname(file_path), file.elf.linker_file) 796 | files_needing_launchers[directory_and_linker].add(file) 797 | else: 798 | file.symlink(working_directory=self.working_directory, bundle_root=self.bundle_root) 799 | 800 | # Now we need to write out one unique copy of each linker in each directory where it's 801 | # required. This is necessary so that `readlink("/proc/self/exe")` will return the correct 802 | # directory when programs use that to construct relative paths to resources. 803 | for ((directory, linker), executable_files) in files_needing_launchers.items(): 804 | # First, we'll find a unique name for the linker in this directory and write it out. 805 | desired_linker_path = os.path.join(directory, 'linker-%s' % linker.hash) 806 | linker_path = desired_linker_path 807 | iteration = 2 808 | while linker_path in file_paths: 809 | linker_path = '%s-%d' % (desired_linker_path, iteration) 810 | iteration += 1 811 | file_paths.add(linker_path) 812 | linker_dirname, linker_basename = os.path.split(linker_path) 813 | if not os.path.exists(linker_dirname): 814 | os.makedirs(linker_dirname) 815 | shutil.copy(linker.path, linker_path) 816 | 817 | # Now we need to construct a launcher for each executable that depends on this linker. 818 | for file in executable_files: 819 | # We'll again attempt to find a unique available name, this time for the symlink 820 | # to the executable. 821 | file_basename = file.entry_point or os.path.basename(file.path) 822 | desired_symlink_path = os.path.join(directory, '%s-x' % file_basename) 823 | symlink_path = desired_symlink_path 824 | iteration = 2 825 | while symlink_path in file_paths: 826 | symlink_path = '%s-%d' % (desired_symlink_path, iteration) 827 | iteration += 1 828 | file_paths.add(symlink_path) 829 | symlink_basename = os.path.basename(symlink_path) 830 | file.create_launcher(self.working_directory, self.bundle_root, 831 | linker_basename, symlink_basename, 832 | shell_launcher=shell_launchers) 833 | 834 | def delete_working_directory(self): 835 | """Recursively deletes the working directory.""" 836 | shutil.rmtree(self.working_directory) 837 | self.working_directory = None 838 | 839 | def file_factory(self, path, entry_point=None, chroot=None, library=False, file_factory=None): 840 | """Either creates a new `File`, or updates and returns one from `files`. 841 | 842 | This method can be used in place of `File.__init__()` when it is known that the `File` 843 | is going to end up being added to the `Bundle.files` set. The construction of a `File` is 844 | quite expensive due to the ELF parsing, so this allows avoiding the construction of `File` 845 | objects when an equivalent ones are already present in the set. Additionally, this allows 846 | for intelligent merging of properties between `File` objects. For example, a `File` with 847 | an entry point should always preserve that entry point, even if the file also gets added 848 | using `--add` or some other method without one. 849 | 850 | See the `File.__init__()` method for documentation of the arguments, they're identical. 851 | """ 852 | # Attempt to find an existing file with the same normalized path in `self.files`. 853 | path = resolve_file_path(path, search_environment_path=entry_point is not None) 854 | file = next((file for file in self.files if file.path == path), None) 855 | if file is not None: 856 | assert entry_point == file.entry_point or not entry_point or not file.entry_point, \ 857 | "The entry point property should always persist, but can't conflict." 858 | file.entry_point = file.entry_point or entry_point 859 | assert chroot == file.chroot, 'The chroot must match.' 860 | file.library = file.library or library 861 | assert not file.entry_point or not file.library, \ 862 | "A file can't be both an entry point and a library." 863 | return file 864 | 865 | return File(path, entry_point, chroot, library, file_factory) 866 | 867 | @property 868 | def bundle_root(self): 869 | """str: The root directory of the bundle where the original file structure is mirrored.""" 870 | path = os.path.join(self.working_directory, 'bundles', self.hash) 871 | return os.path.normpath(os.path.abspath(path)) 872 | 873 | @property 874 | def hash(self): 875 | """str: Computes a hash based on the current contents of the bundle.""" 876 | file_hashes = sorted(file.hash for file in self.files) 877 | combined_hashes = '\n'.join(file_hashes).encode('utf-8') 878 | return hashlib.sha256(combined_hashes).hexdigest() 879 | -------------------------------------------------------------------------------- /src/exodus_bundler/cli.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | import argparse 3 | import logging 4 | import sys 5 | 6 | from exodus_bundler import root_logger 7 | from exodus_bundler.bundling import create_bundle 8 | from exodus_bundler.errors import FatalError 9 | from exodus_bundler.input_parsing import extract_paths 10 | 11 | 12 | logger = logging.getLogger(__name__) 13 | 14 | 15 | def parse_args(args=None, namespace=None): 16 | """Constructs an argument parser and parses the arguments. The default behavior is 17 | to parse the arguments from `sys.argv`. A dictionary is returned rather than the 18 | typical namespace produced by `argparse`.""" 19 | formatter = argparse.ArgumentDefaultsHelpFormatter 20 | parser = argparse.ArgumentParser(formatter_class=formatter, description=( 21 | 'Bundle ELF binary executables with all of their runtime dependencies ' 22 | 'so that they can be relocated to other systems with incompatible system ' 23 | 'libraries.' 24 | )) 25 | 26 | parser.add_argument('executables', metavar='EXECUTABLE', nargs='+', help=( 27 | 'One or more ELF executables to include in the exodus bundle.' 28 | )) 29 | 30 | parser.add_argument('-c', '--chroot', metavar='CHROOT_PATH', 31 | default=None, 32 | help=( 33 | 'A directory that will be treated as the root during linking. Useful for testing and ' 34 | 'bundling extracted packages that won\t run without a chroot.' 35 | ), 36 | ) 37 | 38 | parser.add_argument('-a', '--add', '--additional-file', metavar='DEPENDENCY', action='append', 39 | default=[], 40 | help=( 41 | 'Specifies an additional file to include in the bundle, useful for adding ' 42 | 'programatically loaded libraries and other non-library dependencies. ' 43 | 'The argument can be used more than once to include multiple files, and ' 44 | 'directories will be included recursively.' 45 | ), 46 | ) 47 | 48 | parser.add_argument('-d', '--detect', action='store_true', help=( 49 | 'Attempt to autodetect direct dependencies using the system package manager. ' 50 | 'Operating system support is limited.' 51 | )) 52 | 53 | parser.add_argument('--no-symlink', metavar='FILE', action='append', 54 | default=[], 55 | help=( 56 | 'Signifies that a file must not be symlinked to the deduplicated data directory. This ' 57 | 'is useful if a file looks for other resources based on paths relative its own ' 58 | 'location. This is enabled by default for executables.' 59 | ), 60 | ) 61 | 62 | parser.add_argument('-o', '--output', metavar='OUTPUT_FILE', 63 | default=None, 64 | help=( 65 | 'The file where the bundle will be written out to. The extension depends on the ' 66 | 'output type. The "{{executables}}" and "{{extension}}" template strings can be ' 67 | ' used in the provided filename. If omitted, the output will go to stdout when ' 68 | 'it is being piped, or to "./exodus-{{executables}}-bundle.{{extension}}" otherwise.' 69 | ), 70 | ) 71 | 72 | parser.add_argument('-q', '--quiet', action='store_true', help=( 73 | 'Suppress warning messages.' 74 | )) 75 | 76 | parser.add_argument('-r', '--rename', metavar='NEW_NAME', nargs='?', action='append', 77 | default=[], help=( 78 | 'Renames the binary executable(s) before packaging. The order of rename tags must ' 79 | 'match the order of positional executable arguments.' 80 | ), 81 | ) 82 | 83 | parser.add_argument('--shell-launchers', action='store_true', help=( 84 | 'Force the use of shell launchers instead of attempting to compile statically linked ones.' 85 | )) 86 | 87 | parser.add_argument('-t', '--tarball', action='store_true', help=( 88 | 'Creates a tarball for manual extraction instead of an installation script. ' 89 | 'Note that this will change the output extension from ".sh" to ".tgz".' 90 | )) 91 | 92 | parser.add_argument('-v', '--verbose', action='store_true', help=( 93 | 'Output additional informational messages.' 94 | )) 95 | 96 | return vars(parser.parse_args(args, namespace)) 97 | 98 | 99 | def configure_logging(quiet, verbose, suppress_stdout=False): 100 | # Set the level. 101 | log_level = logging.WARN 102 | if quiet and not verbose: 103 | log_level = logging.ERROR 104 | elif verbose and not quiet: 105 | log_level = logging.INFO 106 | root_logger.setLevel(log_level) 107 | 108 | class StderrFilter(logging.Filter): 109 | def filter(self, record): 110 | return record.levelno in (logging.WARN, logging.ERROR) 111 | 112 | stderr_handler = logging.StreamHandler(sys.stderr) 113 | stderr_formatter = logging.Formatter('%(levelname)s: %(message)s') 114 | stderr_handler.setFormatter(stderr_formatter) 115 | stderr_handler.addFilter(StderrFilter()) 116 | root_logger.addHandler(stderr_handler) 117 | 118 | # We won't even configure/add the stdout handler if this is specified. 119 | if suppress_stdout: 120 | return 121 | 122 | class StdoutFilter(logging.Filter): 123 | def filter(self, record): 124 | return record.levelno in (logging.DEBUG, logging.INFO) 125 | 126 | stdout_formatter = logging.Formatter('%(message)s') 127 | stdout_handler = logging.StreamHandler(sys.stdout) 128 | stdout_handler.setFormatter(stdout_formatter) 129 | stdout_handler.addFilter(StdoutFilter()) 130 | root_logger.addHandler(stdout_handler) 131 | 132 | 133 | def main(args=None, namespace=None): 134 | args = parse_args(args, namespace) 135 | 136 | # Dynamically set the default output to stdout if it is being piped. 137 | if args['output'] is None: 138 | if sys.stdout.isatty(): 139 | args['output'] = './exodus-{{executables}}-bundle.{{extension}}' 140 | else: 141 | args['output'] = '-' 142 | 143 | # Handle the CLI specific options here, removing them from `args` in the process. 144 | quiet, verbose = args.pop('quiet'), args.pop('verbose') 145 | suppress_stdout = args['output'] == '-' 146 | configure_logging(quiet=quiet, verbose=verbose, suppress_stdout=suppress_stdout) 147 | 148 | # Allow piping in additional files. 149 | if not sys.stdin.isatty(): 150 | args['add'] += extract_paths(sys.stdin.read()) 151 | 152 | # Create the bundle with all of the arguments. 153 | try: 154 | create_bundle(**args) 155 | except FatalError as fatal_error: 156 | logger.error('Fatal error encountered, exiting.') 157 | logger.error(fatal_error, exc_info=verbose) 158 | sys.exit(1) 159 | -------------------------------------------------------------------------------- /src/exodus_bundler/dependency_detection.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | import os 3 | import re 4 | import subprocess 5 | 6 | from exodus_bundler.launchers import find_executable 7 | 8 | 9 | class PackageManager(object): 10 | """Base class representing a package manager. 11 | 12 | The class level attributes can be overwritten in derived classes to customize the behavior. 13 | 14 | Attributes: 15 | cache_directory (str): The location of the system's package cache. 16 | list_command (:obj:`list` of :obj:`str`): The command and arguments to list the 17 | dependencies of a package . 18 | list_regex (str): A regex to extract the file path from a single line of the output of the 19 | list command. 20 | owner_command (:obj:`owner` of :obj:`str`): The command and arguments to determine the 21 | package that owns a specific file. 22 | owner_regex (str): A regex to extract the package name from the output of the owner command. 23 | """ 24 | cache_directory = None 25 | list_command = None 26 | list_regex = '(.*)' 27 | owner_command = None 28 | owner_regex = '(.*)' 29 | 30 | def find_dependencies(self, path): 31 | """Finds a list of all of the files contained with the package containing a file.""" 32 | owner = self.find_owner(path) 33 | if not owner: 34 | return None 35 | 36 | args = self.list_command + [owner] 37 | process = subprocess.Popen(args, stdout=subprocess.PIPE) 38 | stdout, stderr = process.communicate() 39 | dependencies = [] 40 | for line in stdout.decode('utf-8').split('\n'): 41 | match = re.search(self.list_regex, line.strip()) 42 | if match: 43 | dependency_path = match.groups()[0] 44 | if os.path.exists(dependency_path) and not os.path.isdir(dependency_path): 45 | dependencies.append(dependency_path) 46 | 47 | return dependencies 48 | 49 | def find_owner(self, path): 50 | """Finds the package that owns the specified file path.""" 51 | if not self.cache_exists or not self.commands_exist: 52 | return None 53 | args = self.owner_command + [path] 54 | env = os.environ.copy() 55 | env['LC_ALL'] = 'C' 56 | process = subprocess.Popen(args, stdout=subprocess.PIPE, env=env) 57 | stdout, stderr = process.communicate() 58 | output = stdout.decode('utf-8').strip() 59 | match = re.search(self.owner_regex, output) 60 | if match: 61 | return match.groups()[0].strip() 62 | 63 | @property 64 | def cache_exists(self): 65 | """Whether or not the expected package cache directory exists.""" 66 | return os.path.exists(self.cache_directory) and os.path.isdir(self.cache_directory) 67 | 68 | @property 69 | def commands_exist(self): 70 | """Whether or not the list and owner package manager commands can be resolved.""" 71 | commands = {self.list_command[0], self.owner_command[0]} 72 | return all(find_executable(command) is not None for command in commands) 73 | 74 | 75 | class Apt(PackageManager): 76 | cache_directory = '/var/cache/apt' 77 | list_command = ['dpkg-query', '-L'] 78 | list_regex = '(.+)' 79 | owner_command = ['dpkg', '-S'] 80 | owner_regex = '(.+): ' 81 | 82 | 83 | class Pacman(PackageManager): 84 | cache_directory = '/var/cache/pacman' 85 | list_command = ['pacman', '-Ql'] 86 | list_regex = r'.*\s+(\/.+)' 87 | owner_command = ['pacman', '-Qo'] 88 | owner_regex = r' is owned by (.*)\s+.*' 89 | 90 | 91 | class Yum(PackageManager): 92 | cache_directory = '/var/cache/yum' 93 | list_command = ['rpm', '-ql'] 94 | list_regex = r'(.+)' 95 | owner_command = ['rpm', '-qf'] 96 | owner_regex = r'(.+)' 97 | 98 | 99 | package_managers = [ 100 | Apt(), 101 | Pacman(), 102 | Yum(), 103 | ] 104 | 105 | 106 | def detect_dependencies(path): 107 | # We'll go through the supported systems one by one. 108 | for package_manager in package_managers: 109 | dependencies = package_manager.find_dependencies(path) 110 | if dependencies: 111 | return dependencies 112 | 113 | return None 114 | -------------------------------------------------------------------------------- /src/exodus_bundler/errors.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | class FatalError(Exception): 3 | """Base class for exceptions that should terminate program execution.""" 4 | pass 5 | 6 | 7 | class DependencyDetectionError(FatalError): 8 | """Signifies that the dependency detection process failed.""" 9 | pass 10 | 11 | 12 | class InvalidElfBinaryError(FatalError): 13 | """Signifies that a file was expected to be an ELF binary, but wasn't.""" 14 | pass 15 | 16 | 17 | class MissingFileError(FatalError): 18 | """Signifies that a file was not found.""" 19 | pass 20 | 21 | 22 | class UnexpectedDirectoryError(FatalError): 23 | """Signifies that a path was unexpectedly a directory.""" 24 | pass 25 | 26 | 27 | class UnsupportedArchitectureError(FatalError): 28 | """Signifies that a binary has an unexpected architecture.""" 29 | pass 30 | -------------------------------------------------------------------------------- /src/exodus_bundler/input_parsing.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | import os 3 | import re 4 | 5 | 6 | # We don't actually want to include anything in these directories in bundles. 7 | blacklisted_directories = [ 8 | '/dev/', 9 | '/proc/', 10 | '/run/', 11 | '/sys/', 12 | # This isn't a directory exactly, but it will filter out active bundling. 13 | '/tmp/exodus-bundle-', 14 | ] 15 | 16 | exec_methods = [ 17 | 'execve', 18 | 'exec', 19 | 'execl', 20 | 'execlp', 21 | 'execle', 22 | 'execv', 23 | 'execvp', 24 | 'execvpe', 25 | ] 26 | 27 | 28 | def extract_exec_path(line): 29 | """Parse a line of strace output and returns the file being executed.""" 30 | line = strip_pid_prefix(line) 31 | for method in exec_methods: 32 | prefix = method + '("' 33 | if line.startswith(prefix): 34 | line = line[len(prefix):] 35 | parts = line.split('", ') 36 | if len(parts) > 1: 37 | return parts[0] 38 | return None 39 | 40 | 41 | def extract_open_path(line): 42 | """Parse a line of strace output and returns the file being opened.""" 43 | line = strip_pid_prefix(line) 44 | for prefix in ['openat(AT_FDCWD, "', 'open("']: 45 | if line.startswith(prefix): 46 | parts = line[len(prefix):].split('", ') 47 | if len(parts) != 2: 48 | continue 49 | if 'ENOENT' in parts[1]: 50 | continue 51 | if 'O_RDONLY' not in parts[1]: 52 | continue 53 | if 'O_DIRECTORY' in parts[1]: 54 | continue 55 | return parts[0] 56 | return None 57 | 58 | 59 | def extract_stat_path(line): 60 | """Parse a line of strace output and return the file that stat was called on.""" 61 | line = strip_pid_prefix(line) 62 | prefix = 'stat("' 63 | if line.startswith(prefix): 64 | parts = line[len(prefix):].split('", ') 65 | if len(parts) == 2 and 'ENOENT' not in parts[1]: 66 | return parts[0] 67 | return None 68 | 69 | 70 | def extract_paths(content, existing_only=True): 71 | """Parses paths from a piped input. 72 | 73 | Args: 74 | content (str): The raw input, can be either a list of files, 75 | or the output of the strace command. 76 | existing_only (bool, optional): Requires that files actually exist and aren't directories. 77 | Returns: 78 | A list of paths. 79 | """ 80 | lines = [line.strip() for line in content.splitlines() if len(line.strip())] 81 | if not len(lines): 82 | return lines 83 | 84 | # The strace output will start with the exec call of its argument. 85 | strace_mode = extract_exec_path(lines[0]) is not None 86 | if not strace_mode: 87 | return lines 88 | 89 | # Extract files from `open()`, `openat()`, and `exec()` calls. 90 | paths = set() 91 | for line in lines: 92 | path = extract_exec_path(line) or extract_open_path(line) or extract_stat_path(line) 93 | if path: 94 | blacklisted = any(path.startswith(directory) for directory in blacklisted_directories) 95 | if not blacklisted: 96 | if not existing_only: 97 | paths.add(path) 98 | continue 99 | if os.path.exists(path) and os.access(path, os.R_OK) and not os.path.isdir(path): 100 | paths.add(path) 101 | 102 | return list(paths) 103 | 104 | 105 | def strip_pid_prefix(line): 106 | """Strips out the `[pid XXX] ` prefix if present.""" 107 | match = re.match(r'\[pid\s+\d+\]\s*', line) 108 | if match: 109 | return line[len(match.group()):] 110 | return line 111 | -------------------------------------------------------------------------------- /src/exodus_bundler/launchers.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """Methods to produce launchers that will invoke the relocated executables with 3 | the proper linker and library paths.""" 4 | import os 5 | import re 6 | import tempfile 7 | from distutils.spawn import find_executable as find_executable_original 8 | from subprocess import PIPE 9 | from subprocess import Popen 10 | 11 | from exodus_bundler.templating import render_template_file 12 | 13 | 14 | parent_directory = os.path.dirname(os.path.realpath(__file__)) 15 | 16 | 17 | class CompilerNotFoundError(Exception): 18 | pass 19 | 20 | 21 | # This is kind of a hack to find things in PATH inside of bundles. 22 | def find_executable(binary_name, skip_original_for_testing=False): 23 | # This won't be set on Alpine Linux, but it's required for the `find_executable()` calls. 24 | if 'PATH' not in os.environ: 25 | os.environ['PATH'] = '/bin/:/usr/bin/' 26 | executable = find_executable_original(binary_name) 27 | if executable and not skip_original_for_testing: 28 | return executable 29 | # Try to find it within the same bundle if it's not actually in the PATH. 30 | directory = parent_directory 31 | while True: 32 | directory, basename = os.path.split(directory) 33 | if not len(basename): 34 | break 35 | # The bundle directory. 36 | if re.match('[A-Fa-f0-9]{64}', basename): 37 | for bin_directory in os.environ['PATH'].split(':'): 38 | if os.path.isabs(bin_directory): 39 | bin_directory = os.path.relpath(bin_directory, '/') 40 | candidate_executable = os.path.join(directory, basename, 41 | bin_directory, binary_name) 42 | if os.path.exists(candidate_executable): 43 | return candidate_executable 44 | 45 | 46 | def compile(code): 47 | try: 48 | return compile_musl(code) 49 | except CompilerNotFoundError: 50 | try: 51 | return compile_diet(code) 52 | except CompilerNotFoundError: 53 | raise CompilerNotFoundError('No suiteable C compiler was found.') 54 | 55 | 56 | def compile_diet(code): 57 | diet = find_executable('diet') 58 | gcc = find_executable('gcc') 59 | if diet is None or gcc is None: 60 | raise CompilerNotFoundError('The diet compiler was not found.') 61 | return compile_helper(code, [diet, 'gcc']) 62 | 63 | 64 | def compile_helper(code, initial_args): 65 | f, input_filename = tempfile.mkstemp(prefix='exodus-bundle-', suffix='.c') 66 | os.close(f) 67 | f, output_filename = tempfile.mkstemp(prefix='exodus-bundle-') 68 | os.close(f) 69 | try: 70 | with open(input_filename, 'w') as input_file: 71 | input_file.write(code) 72 | 73 | args = initial_args + ['-static', '-O3', input_filename, '-o', output_filename] 74 | process = Popen(args, stdout=PIPE, stderr=PIPE) 75 | stdout, stderr = process.communicate() 76 | assert process.returncode == 0, \ 77 | 'There was an error compiling: %s' % stderr.decode('utf-8') 78 | 79 | with open(output_filename, 'rb') as output_file: 80 | return output_file.read() 81 | finally: 82 | os.remove(input_filename) 83 | os.remove(output_filename) 84 | 85 | 86 | def compile_musl(code): 87 | musl = find_executable('musl-gcc') 88 | if musl is None: 89 | raise CompilerNotFoundError('The musl compiler was not found.') 90 | return compile_helper(code, [musl]) 91 | 92 | 93 | def construct_bash_launcher(linker, library_path, executable, full_linker=True): 94 | linker_dirname, linker_basename = os.path.split(linker) 95 | full_linker = 'true' if full_linker else 'false' 96 | return render_template_file('launcher.sh', linker_basename=linker_basename, 97 | linker_dirname=linker_dirname, library_path=library_path, 98 | executable=executable, full_linker=full_linker) 99 | 100 | 101 | def construct_binary_launcher(linker, library_path, executable, full_linker=True): 102 | linker_dirname, linker_basename = os.path.split(linker) 103 | full_linker = '1' if full_linker else '0' 104 | code = render_template_file('launcher.c', linker_basename=linker_basename, 105 | linker_dirname=linker_dirname, library_path=library_path, 106 | executable=executable, full_linker=full_linker) 107 | return compile(code) 108 | -------------------------------------------------------------------------------- /src/exodus_bundler/templates/install-bundle-noninteractive.sh: -------------------------------------------------------------------------------- 1 | #! /bin/sh 2 | 3 | user_directory="${HOME}/.exodus" 4 | output_directory=/opt/exodus 5 | mkdir -p "${output_directory}" 2> /dev/null 6 | if [ ! $? ] || [ ! -w ${output_directory} ] ; then 7 | output_directory=${user_directory} 8 | fi 9 | 10 | echo "Installing executable bundle in \"${output_directory}\"..." 11 | mkdir -p ${output_directory} 2> /dev/null 12 | 13 | # Actually perform the extraction. 14 | base64 -d << "END_OF_FILE" | tar -C "${output_directory}" --strip-components 1 --no-same-owner -p -zvxf - > /dev/null 15 | {{base64_encoded_tarball}} 16 | END_OF_FILE 17 | if [ $? -eq 0 ]; then 18 | echo "Successfully installed, be sure to add ${output_directory}/bin to your \$PATH." 19 | exit 0 20 | else 21 | echo "Something went wrong, please send an email to contact@intoli.com with details about the bundle." 22 | exit 1 23 | fi 24 | -------------------------------------------------------------------------------- /src/exodus_bundler/templates/install-bundle.sh: -------------------------------------------------------------------------------- 1 | #! /bin/bash 2 | 3 | user_directory="${HOME}/.exodus" 4 | if [ "$1" != "--user" ]; then 5 | output_directory=${1:-/opt/exodus} 6 | mkdir -p ${output_directory} 2> /dev/null 7 | if [ ! $? ] || [ ! -w ${output_directory} ] ; then 8 | echo "You don't have write access to "${output_directory}"." 9 | read -r -p "Would you like to install in ${user_directory} instead? [Y/n] " response 10 | if [ -z "${response}" ] || [ "${response:0:1}" == "Y" ] || [ "${response:0:1}" == "y" ]; then 11 | output_directory="${user_directory}" 12 | else 13 | echo "Ok, exiting. You can specify a different installation directory as an argument or run again as root." 14 | exit 1 15 | fi 16 | fi 17 | else 18 | output_directory=${user_directory} 19 | fi 20 | 21 | echo "Installing executable bundle in \"${output_directory}\"..." 22 | mkdir -p ${output_directory} 2> /dev/null 23 | 24 | # Actually perform the extraction. 25 | begin_tarball_line=$((1 + $(grep --text --line-number '^BEGIN-TARBALL$' $0 | cut -d ':' -f 1))) 26 | tail -n +$begin_tarball_line "$0" | tar -C "${output_directory}" --strip-components 1 --no-same-owner -p -zvxf - > /dev/null 27 | if [ $? -eq 0 ]; then 28 | echo "Successfully installed, be sure to add "${output_directory}/bin" to your \$PATH." 29 | exit 0 30 | else 31 | echo "Something went wrong, please send an email to contact@intoli.com with details about the bundle." 32 | exit 1 33 | fi 34 | 35 | # The tarball data will go here. 36 | BEGIN-TARBALL 37 | -------------------------------------------------------------------------------- /src/exodus_bundler/templates/launcher.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | 6 | int main(int argc, char *argv[]) { 7 | char *original_library_path = "{{library_path}}"; 8 | char *executable = "{{executable}}"; 9 | char *linker_basename = "{{linker_basename}}"; 10 | char *linker_dirname = "{{linker_dirname}}/"; 11 | 12 | char buffer[4096] = { 0 }; 13 | if (readlink("/proc/self/exe", buffer, sizeof(buffer) - strlen(linker_basename) - strlen(linker_dirname) - strlen(executable))) { 14 | // Determine the location of this launcher executable. 15 | char *current_directory = dirname(buffer); 16 | int current_directory_length = strlen(current_directory); 17 | current_directory[current_directory_length++] = '/'; 18 | current_directory[current_directory_length] = '\0'; 19 | 20 | // Prefix each segment with the current working directory so it's an absolute path. 21 | int library_segments = 1; 22 | int i; 23 | for (i = 0; original_library_path[i]; i++) { 24 | library_segments += (original_library_path[i] == ':'); 25 | } 26 | char *library_path = malloc( 27 | (strlen(original_library_path) + library_segments * strlen(current_directory) + 1) * sizeof(char)); 28 | strcpy(library_path, current_directory); 29 | int character_offset = current_directory_length; 30 | for (i = 0; original_library_path[i]; i++) { 31 | library_path[character_offset] = original_library_path[i]; 32 | character_offset++; 33 | if (original_library_path[i] == ':') { 34 | strcpy(library_path + character_offset, current_directory); 35 | character_offset += current_directory_length; 36 | } 37 | } 38 | library_path[character_offset] = '\0'; 39 | 40 | // Construct an absolute path to the linker. 41 | char full_linker_path[4096] = { 0 }; 42 | strcpy(full_linker_path, current_directory); 43 | strcat(full_linker_path, "/"); 44 | strcat(full_linker_path, linker_dirname); 45 | strcat(full_linker_path, linker_basename); 46 | 47 | // Construct an absolute path to the executable that we're trying to launch. 48 | char full_executable_path[4096] = { 0 }; 49 | strcpy(full_executable_path, current_directory); 50 | strcat(full_executable_path, "/"); 51 | strcat(full_executable_path, executable); 52 | 53 | // Construct all of the arguments for the linker. 54 | char *linker_args[] = { "--library-path", library_path, "--inhibit-rpath", "", "--inhibit-cache" }; 55 | char **combined_args = malloc(sizeof(linker_args) + sizeof(char*) * (argc + 1)); 56 | combined_args[0] = linker_basename; 57 | memcpy(combined_args + 1, linker_args, sizeof(linker_args)); 58 | // We can't use `--inhinit-rpath` or `--inhibit-cache` with the musl linker. 59 | int offset = (sizeof(linker_args) / sizeof(char*)) + 1 - ({{full_linker}} ? 0 : 3); 60 | combined_args[offset++] = full_executable_path; 61 | memcpy(combined_args + offset, argv + 1, sizeof(char*)*(argc - 1)); 62 | offset += argc - 1; 63 | combined_args[offset] = NULL; 64 | 65 | // Execute the linker. 66 | execv(full_linker_path, combined_args); 67 | } 68 | return 1; 69 | } 70 | -------------------------------------------------------------------------------- /src/exodus_bundler/templates/launcher.sh: -------------------------------------------------------------------------------- 1 | #! /bin/bash 2 | 3 | current_directory="$(dirname "$(readlink -f "$0")")" 4 | executable="${current_directory}/{{executable}}" 5 | library_path="{{library_path}}" 6 | library_path="${current_directory}/${library_path//:/:${current_directory}/}" 7 | linker="${current_directory}/{{linker_dirname}}/{{linker_basename}}" 8 | if [ "{{full_linker}}" == "true" ]; then 9 | exec "${linker}" --library-path "${library_path}" --inhibit-rpath "" "${executable}" "$@" 10 | else 11 | exec "${linker}" --library-path "${library_path}" "${executable}" "$@" 12 | fi 13 | -------------------------------------------------------------------------------- /src/exodus_bundler/templating.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | """Contains a couple of fairly trivial templating methods used for constructing 3 | the loaders and the output filename. Any instances of {{variable_name}} will be 4 | replaced by the corresponding values.""" 5 | 6 | import os 7 | 8 | 9 | parent_directory = os.path.dirname(os.path.realpath(__file__)) 10 | template_directory = os.path.join(parent_directory, 'templates') 11 | 12 | 13 | def render_template(string, **context): 14 | for key, value in context.items(): 15 | string = string.replace('{{%s}}' % key, value) 16 | return string 17 | 18 | 19 | def render_template_file(filename, **context): 20 | if not os.path.isabs(filename): 21 | filename = os.path.join(template_directory, filename) 22 | with open(filename, 'r') as f: 23 | return render_template(f.read(), **context) 24 | -------------------------------------------------------------------------------- /tests/data/binaries/GLIB_LICENSES: -------------------------------------------------------------------------------- 1 | This file contains the copying permission notices for various files in the 2 | GNU C Library distribution that have copyright owners other than the Free 3 | Software Foundation. These notices all require that a copy of the notice 4 | be included in the accompanying documentation and be distributed with 5 | binary distributions of the code, so be sure to include this file along 6 | with any binary distributions derived from the GNU C Library. 7 | 8 | 9 | All code incorporated from 4.4 BSD is distributed under the following 10 | license: 11 | 12 | Copyright (C) 1991 Regents of the University of California. 13 | All rights reserved. 14 | 15 | Redistribution and use in source and binary forms, with or without 16 | modification, are permitted provided that the following conditions 17 | are met: 18 | 19 | 1. Redistributions of source code must retain the above copyright 20 | notice, this list of conditions and the following disclaimer. 21 | 2. Redistributions in binary form must reproduce the above copyright 22 | notice, this list of conditions and the following disclaimer in the 23 | documentation and/or other materials provided with the distribution. 24 | 3. [This condition was removed.] 25 | 4. Neither the name of the University nor the names of its contributors 26 | may be used to endorse or promote products derived from this software 27 | without specific prior written permission. 28 | 29 | THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 30 | ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 31 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 32 | ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 33 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 34 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 35 | OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 36 | HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 37 | LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 38 | OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 39 | SUCH DAMAGE. 40 | 41 | The DNS resolver code, taken from BIND 4.9.5, is copyrighted by UC 42 | Berkeley, by Digital Equipment Corporation and by Internet Software 43 | Consortium. The DEC portions are under the following license: 44 | 45 | Portions Copyright (C) 1993 by Digital Equipment Corporation. 46 | 47 | Permission to use, copy, modify, and distribute this software for any 48 | purpose with or without fee is hereby granted, provided that the above 49 | copyright notice and this permission notice appear in all copies, and 50 | that the name of Digital Equipment Corporation not be used in 51 | advertising or publicity pertaining to distribution of the document or 52 | software without specific, written prior permission. 53 | 54 | THE SOFTWARE IS PROVIDED ``AS IS'' AND DIGITAL EQUIPMENT CORP. 55 | DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, INCLUDING ALL 56 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL 57 | DIGITAL EQUIPMENT CORPORATION BE LIABLE FOR ANY SPECIAL, DIRECT, 58 | INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING 59 | FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, 60 | NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION 61 | WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 62 | 63 | The ISC portions are under the following license: 64 | 65 | Portions Copyright (c) 1996-1999 by Internet Software Consortium. 66 | 67 | Permission to use, copy, modify, and distribute this software for any 68 | purpose with or without fee is hereby granted, provided that the above 69 | copyright notice and this permission notice appear in all copies. 70 | 71 | THE SOFTWARE IS PROVIDED "AS IS" AND INTERNET SOFTWARE CONSORTIUM DISCLAIMS 72 | ALL WARRANTIES WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES 73 | OF MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL INTERNET SOFTWARE 74 | CONSORTIUM BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL 75 | DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR 76 | PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS 77 | ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS 78 | SOFTWARE. 79 | 80 | The Sun RPC support (from rpcsrc-4.0) is covered by the following 81 | license: 82 | 83 | Copyright (c) 2010, Oracle America, Inc. 84 | 85 | Redistribution and use in source and binary forms, with or without 86 | modification, are permitted provided that the following conditions are 87 | met: 88 | 89 | * Redistributions of source code must retain the above copyright 90 | notice, this list of conditions and the following disclaimer. 91 | * Redistributions in binary form must reproduce the above 92 | copyright notice, this list of conditions and the following 93 | disclaimer in the documentation and/or other materials 94 | provided with the distribution. 95 | * Neither the name of the "Oracle America, Inc." nor the names of its 96 | contributors may be used to endorse or promote products derived 97 | from this software without specific prior written permission. 98 | 99 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 100 | "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 101 | LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 102 | FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 103 | COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, 104 | INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 105 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE 106 | GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 107 | INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 108 | WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING 109 | NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 110 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 111 | 112 | 113 | The following CMU license covers some of the support code for Mach, 114 | derived from Mach 3.0: 115 | 116 | Mach Operating System 117 | Copyright (C) 1991,1990,1989 Carnegie Mellon University 118 | All Rights Reserved. 119 | 120 | Permission to use, copy, modify and distribute this software and its 121 | documentation is hereby granted, provided that both the copyright 122 | notice and this permission notice appear in all copies of the 123 | software, derivative works or modified versions, and any portions 124 | thereof, and that both notices appear in supporting documentation. 125 | 126 | CARNEGIE MELLON ALLOWS FREE USE OF THIS SOFTWARE IN ITS ``AS IS'' 127 | CONDITION. CARNEGIE MELLON DISCLAIMS ANY LIABILITY OF ANY KIND FOR 128 | ANY DAMAGES WHATSOEVER RESULTING FROM THE USE OF THIS SOFTWARE. 129 | 130 | Carnegie Mellon requests users of this software to return to 131 | 132 | Software Distribution Coordinator 133 | School of Computer Science 134 | Carnegie Mellon University 135 | Pittsburgh PA 15213-3890 136 | 137 | or Software.Distribution@CS.CMU.EDU any improvements or 138 | extensions that they make and grant Carnegie Mellon the rights to 139 | redistribute these changes. 140 | 141 | The file if_ppp.h is under the following CMU license: 142 | 143 | Redistribution and use in source and binary forms, with or without 144 | modification, are permitted provided that the following conditions 145 | are met: 146 | 1. Redistributions of source code must retain the above copyright 147 | notice, this list of conditions and the following disclaimer. 148 | 2. Redistributions in binary form must reproduce the above copyright 149 | notice, this list of conditions and the following disclaimer in the 150 | documentation and/or other materials provided with the distribution. 151 | 3. Neither the name of the University nor the names of its contributors 152 | may be used to endorse or promote products derived from this software 153 | without specific prior written permission. 154 | 155 | THIS SOFTWARE IS PROVIDED BY CARNEGIE MELLON UNIVERSITY AND 156 | CONTRIBUTORS ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, 157 | INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 158 | MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 159 | IN NO EVENT SHALL THE UNIVERSITY OR CONTRIBUTORS BE LIABLE FOR ANY 160 | DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 161 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE 162 | GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 163 | INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER 164 | IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 165 | OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN 166 | IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 167 | 168 | The following license covers the files from Intel's "Highly Optimized 169 | Mathematical Functions for Itanium" collection: 170 | 171 | Intel License Agreement 172 | 173 | Copyright (c) 2000, Intel Corporation 174 | 175 | All rights reserved. 176 | 177 | Redistribution and use in source and binary forms, with or without 178 | modification, are permitted provided that the following conditions are 179 | met: 180 | 181 | * Redistributions of source code must retain the above copyright 182 | notice, this list of conditions and the following disclaimer. 183 | 184 | * Redistributions in binary form must reproduce the above copyright 185 | notice, this list of conditions and the following disclaimer in the 186 | documentation and/or other materials provided with the distribution. 187 | 188 | * The name of Intel Corporation may not be used to endorse or promote 189 | products derived from this software without specific prior written 190 | permission. 191 | 192 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 193 | "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 194 | LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 195 | A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL INTEL OR 196 | CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, 197 | EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 198 | PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 199 | PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF 200 | LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING 201 | NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 202 | SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 203 | 204 | The files inet/getnameinfo.c and sysdeps/posix/getaddrinfo.c are copyright 205 | (C) by Craig Metz and are distributed under the following license: 206 | 207 | /* The Inner Net License, Version 2.00 208 | 209 | The author(s) grant permission for redistribution and use in source and 210 | binary forms, with or without modification, of the software and documentation 211 | provided that the following conditions are met: 212 | 213 | 0. If you receive a version of the software that is specifically labelled 214 | as not being for redistribution (check the version message and/or README), 215 | you are not permitted to redistribute that version of the software in any 216 | way or form. 217 | 1. All terms of the all other applicable copyrights and licenses must be 218 | followed. 219 | 2. Redistributions of source code must retain the authors' copyright 220 | notice(s), this list of conditions, and the following disclaimer. 221 | 3. Redistributions in binary form must reproduce the authors' copyright 222 | notice(s), this list of conditions, and the following disclaimer in the 223 | documentation and/or other materials provided with the distribution. 224 | 4. [The copyright holder has authorized the removal of this clause.] 225 | 5. Neither the name(s) of the author(s) nor the names of its contributors 226 | may be used to endorse or promote products derived from this software 227 | without specific prior written permission. 228 | 229 | THIS SOFTWARE IS PROVIDED BY ITS AUTHORS AND CONTRIBUTORS ``AS IS'' AND ANY 230 | EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 231 | WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 232 | DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE FOR ANY 233 | DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES 234 | (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 235 | LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON 236 | ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 237 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS 238 | SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 239 | 240 | If these license terms cause you a real problem, contact the author. */ 241 | 242 | The file sunrpc/des_impl.c is copyright Eric Young: 243 | 244 | Copyright (C) 1992 Eric Young 245 | Collected from libdes and modified for SECURE RPC by Martin Kuck 1994 246 | This file is distributed under the terms of the GNU Lesser General 247 | Public License, version 2.1 or later - see the file COPYING.LIB for details. 248 | If you did not receive a copy of the license with this program, please 249 | see to obtain a copy. 250 | 251 | The libidn code is copyright Simon Josefsson, with portions copyright 252 | The Internet Society, Tom Tromey and Red Hat, Inc.: 253 | 254 | Copyright (C) 2002, 2003, 2004, 2011 Simon Josefsson 255 | 256 | This file is part of GNU Libidn. 257 | 258 | GNU Libidn is free software; you can redistribute it and/or 259 | modify it under the terms of the GNU Lesser General Public 260 | License as published by the Free Software Foundation; either 261 | version 2.1 of the License, or (at your option) any later version. 262 | 263 | GNU Libidn is distributed in the hope that it will be useful, 264 | but WITHOUT ANY WARRANTY; without even the implied warranty of 265 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 266 | Lesser General Public License for more details. 267 | 268 | You should have received a copy of the GNU Lesser General Public 269 | License along with GNU Libidn; if not, see . 270 | 271 | The following notice applies to portions of libidn/nfkc.c: 272 | 273 | This file contains functions from GLIB, including gutf8.c and 274 | gunidecomp.c, all licensed under LGPL and copyright hold by: 275 | 276 | Copyright (C) 1999, 2000 Tom Tromey 277 | Copyright 2000 Red Hat, Inc. 278 | 279 | The following applies to portions of libidn/punycode.c and 280 | libidn/punycode.h: 281 | 282 | This file is derived from RFC 3492bis written by Adam M. Costello. 283 | 284 | Disclaimer and license: Regarding this entire document or any 285 | portion of it (including the pseudocode and C code), the author 286 | makes no guarantees and is not responsible for any damage resulting 287 | from its use. The author grants irrevocable permission to anyone 288 | to use, modify, and distribute it in any way that does not diminish 289 | the rights of anyone else to use, modify, and distribute it, 290 | provided that redistributed derivative works do not contain 291 | misleading author or version information. Derivative works need 292 | not be licensed under similar terms. 293 | 294 | Copyright (C) The Internet Society (2003). All Rights Reserved. 295 | 296 | This document and translations of it may be copied and furnished to 297 | others, and derivative works that comment on or otherwise explain it 298 | or assist in its implementation may be prepared, copied, published 299 | and distributed, in whole or in part, without restriction of any 300 | kind, provided that the above copyright notice and this paragraph are 301 | included on all such copies and derivative works. However, this 302 | document itself may not be modified in any way, such as by removing 303 | the copyright notice or references to the Internet Society or other 304 | Internet organizations, except as needed for the purpose of 305 | developing Internet standards in which case the procedures for 306 | copyrights defined in the Internet Standards process must be 307 | followed, or as required to translate it into languages other than 308 | English. 309 | 310 | The limited permissions granted above are perpetual and will not be 311 | revoked by the Internet Society or its successors or assigns. 312 | 313 | This document and the information contained herein is provided on an 314 | "AS IS" basis and THE INTERNET SOCIETY AND THE INTERNET ENGINEERING 315 | TASK FORCE DISCLAIMS ALL WARRANTIES, EXPRESS OR IMPLIED, INCLUDING 316 | BUT NOT LIMITED TO ANY WARRANTY THAT THE USE OF THE INFORMATION 317 | HEREIN WILL NOT INFRINGE ANY RIGHTS OR ANY IMPLIED WARRANTIES OF 318 | MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE. 319 | 320 | The file inet/rcmd.c is under a UCB copyright and the following: 321 | 322 | Copyright (C) 1998 WIDE Project. 323 | All rights reserved. 324 | 325 | Redistribution and use in source and binary forms, with or without 326 | modification, are permitted provided that the following conditions 327 | are met: 328 | 1. Redistributions of source code must retain the above copyright 329 | notice, this list of conditions and the following disclaimer. 330 | 2. Redistributions in binary form must reproduce the above copyright 331 | notice, this list of conditions and the following disclaimer in the 332 | documentation and/or other materials provided with the distribution. 333 | 3. Neither the name of the project nor the names of its contributors 334 | may be used to endorse or promote products derived from this software 335 | without specific prior written permission. 336 | 337 | THIS SOFTWARE IS PROVIDED BY THE PROJECT AND CONTRIBUTORS ``AS IS'' AND 338 | ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 339 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 340 | ARE DISCLAIMED. IN NO EVENT SHALL THE PROJECT OR CONTRIBUTORS BE LIABLE 341 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 342 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 343 | OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 344 | HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 345 | LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 346 | OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 347 | SUCH DAMAGE. 348 | 349 | The file posix/runtests.c is copyright Tom Lord: 350 | 351 | Copyright 1995 by Tom Lord 352 | 353 | All Rights Reserved 354 | 355 | Permission to use, copy, modify, and distribute this software and its 356 | documentation for any purpose and without fee is hereby granted, 357 | provided that the above copyright notice appear in all copies and that 358 | both that copyright notice and this permission notice appear in 359 | supporting documentation, and that the name of the copyright holder not be 360 | used in advertising or publicity pertaining to distribution of the 361 | software without specific, written prior permission. 362 | 363 | Tom Lord DISCLAIMS ALL WARRANTIES WITH REGARD TO THIS SOFTWARE, 364 | INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS, IN NO 365 | EVENT SHALL TOM LORD BE LIABLE FOR ANY SPECIAL, INDIRECT OR 366 | CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF 367 | USE, DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR 368 | OTHER TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR 369 | PERFORMANCE OF THIS SOFTWARE. 370 | 371 | The posix/rxspencer tests are copyright Henry Spencer: 372 | 373 | Copyright 1992, 1993, 1994, 1997 Henry Spencer. All rights reserved. 374 | This software is not subject to any license of the American Telephone 375 | and Telegraph Company or of the Regents of the University of California. 376 | 377 | Permission is granted to anyone to use this software for any purpose on 378 | any computer system, and to alter it and redistribute it, subject 379 | to the following restrictions: 380 | 381 | 1. The author is not responsible for the consequences of use of this 382 | software, no matter how awful, even if they arise from flaws in it. 383 | 384 | 2. The origin of this software must not be misrepresented, either by 385 | explicit claim or by omission. Since few users ever read sources, 386 | credits must appear in the documentation. 387 | 388 | 3. Altered versions must be plainly marked as such, and must not be 389 | misrepresented as being the original software. Since few users 390 | ever read sources, credits must appear in the documentation. 391 | 392 | 4. This notice may not be removed or altered. 393 | 394 | The file posix/PCRE.tests is copyright University of Cambridge: 395 | 396 | Copyright (c) 1997-2003 University of Cambridge 397 | 398 | Permission is granted to anyone to use this software for any purpose on any 399 | computer system, and to redistribute it freely, subject to the following 400 | restrictions: 401 | 402 | 1. This software is distributed in the hope that it will be useful, 403 | but WITHOUT ANY WARRANTY; without even the implied warranty of 404 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 405 | 406 | 2. The origin of this software must not be misrepresented, either by 407 | explicit claim or by omission. In practice, this means that if you use 408 | PCRE in software that you distribute to others, commercially or 409 | otherwise, you must put a sentence like this 410 | 411 | Regular expression support is provided by the PCRE library package, 412 | which is open source software, written by Philip Hazel, and copyright 413 | by the University of Cambridge, England. 414 | 415 | somewhere reasonably visible in your documentation and in any relevant 416 | files or online help data or similar. A reference to the ftp site for 417 | the source, that is, to 418 | 419 | ftp://ftp.csx.cam.ac.uk/pub/software/programming/pcre/ 420 | 421 | should also be given in the documentation. However, this condition is not 422 | intended to apply to whole chains of software. If package A includes PCRE, 423 | it must acknowledge it, but if package B is software that includes package 424 | A, the condition is not imposed on package B (unless it uses PCRE 425 | independently). 426 | 427 | 3. Altered versions must be plainly marked as such, and must not be 428 | misrepresented as being the original software. 429 | 430 | 4. If PCRE is embedded in any software that is released under the GNU 431 | General Purpose Licence (GPL), or Lesser General Purpose Licence (LGPL), 432 | then the terms of that licence shall supersede any condition above with 433 | which it is incompatible. 434 | 435 | Files from Sun fdlibm are copyright Sun Microsystems, Inc.: 436 | 437 | Copyright (C) 1993 by Sun Microsystems, Inc. All rights reserved. 438 | 439 | Developed at SunPro, a Sun Microsystems, Inc. business. 440 | Permission to use, copy, modify, and distribute this 441 | software is freely granted, provided that this notice 442 | is preserved. 443 | 444 | Part of stdio-common/tst-printf.c is copyright C E Chew: 445 | 446 | (C) Copyright C E Chew 447 | 448 | Feel free to copy, use and distribute this software provided: 449 | 450 | 1. you do not pretend that you wrote it 451 | 2. you leave this copyright notice intact. 452 | 453 | Various long double libm functions are copyright Stephen L. Moshier: 454 | 455 | Copyright 2001 by Stephen L. Moshier 456 | 457 | This library is free software; you can redistribute it and/or 458 | modify it under the terms of the GNU Lesser General Public 459 | License as published by the Free Software Foundation; either 460 | version 2.1 of the License, or (at your option) any later version. 461 | 462 | This library is distributed in the hope that it will be useful, 463 | but WITHOUT ANY WARRANTY; without even the implied warranty of 464 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 465 | Lesser General Public License for more details. 466 | 467 | You should have received a copy of the GNU Lesser General Public 468 | License along with this library; if not, see 469 | . */ 470 | -------------------------------------------------------------------------------- /tests/data/binaries/MUSL-COPYRIGHT: -------------------------------------------------------------------------------- 1 | musl as a whole is licensed under the following standard MIT license: 2 | 3 | ---------------------------------------------------------------------- 4 | Copyright © 2005-2014 Rich Felker, et al. 5 | 6 | Permission is hereby granted, free of charge, to any person obtaining 7 | a copy of this software and associated documentation files (the 8 | "Software"), to deal in the Software without restriction, including 9 | without limitation the rights to use, copy, modify, merge, publish, 10 | distribute, sublicense, and/or sell copies of the Software, and to 11 | permit persons to whom the Software is furnished to do so, subject to 12 | the following conditions: 13 | 14 | The above copyright notice and this permission notice shall be 15 | included in all copies or substantial portions of the Software. 16 | 17 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 18 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 19 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. 20 | IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY 21 | CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, 22 | TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE 23 | SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 24 | ---------------------------------------------------------------------- 25 | 26 | Authors/contributors include: 27 | 28 | A. Wilcox 29 | Alex Dowad 30 | Alexander Monakov 31 | Andrew Kelley 32 | Anthony G. Basile 33 | Arvid Picciani 34 | Bartosz Brachaczek 35 | Bobby Bingham 36 | Boris Brezillon 37 | Brent Cook 38 | Chris Spiegel 39 | Clément Vasseur 40 | Daniel Micay 41 | Daniel Sabogal 42 | Daurnimator 43 | David Edelsohn 44 | Denys Vlasenko 45 | Dmitry Ivanov 46 | Dmitry V. Levin 47 | Emil Renner Berthing 48 | Felix Fietkau 49 | Felix Janda 50 | Gianluca Anzolin 51 | Hauke Mehrtens 52 | He X 53 | Hiltjo Posthuma 54 | Isaac Dunham 55 | Jaydeep Patil 56 | Jens Gustedt 57 | Jeremy Huntwork 58 | Jo-Philipp Wich 59 | Joakim Sindholt 60 | John Spencer 61 | Josiah Worcester 62 | Julien Ramseier 63 | Justin Cormack 64 | Khem Raj 65 | Kylie McClain 66 | Leah Neukirchen 67 | Luca Barbato 68 | Luka Perkov 69 | M Farkas-Dyck (Strake) 70 | Mahesh Bodapati 71 | Masanori Ogino 72 | Michael Forney 73 | Mikhail Kremnyov 74 | Natanael Copa 75 | Nicholas J. Kain 76 | orc 77 | Pascal Cuoq 78 | Petr Hosek 79 | Petr Skocik 80 | Pierre Carrier 81 | Reini Urban 82 | Rich Felker 83 | Richard Pennington 84 | Samuel Holland 85 | Shiz 86 | sin 87 | Solar Designer 88 | Stefan Kristiansson 89 | Szabolcs Nagy 90 | Timo Teräs 91 | Trutz Behn 92 | Valentin Ochs 93 | William Haddon 94 | William Pitcock 95 | 96 | Portions of this software are derived from third-party works licensed 97 | under terms compatible with the above MIT license: 98 | 99 | The TRE regular expression implementation (src/regex/reg* and 100 | src/regex/tre*) is Copyright © 2001-2008 Ville Laurikari and licensed 101 | under a 2-clause BSD license (license text in the source files). The 102 | included version has been heavily modified by Rich Felker in 2012, in 103 | the interests of size, simplicity, and namespace cleanliness. 104 | 105 | Much of the math library code (src/math/* and src/complex/*) is 106 | Copyright © 1993,2004 Sun Microsystems or 107 | Copyright © 2003-2011 David Schultz or 108 | Copyright © 2003-2009 Steven G. Kargl or 109 | Copyright © 2003-2009 Bruce D. Evans or 110 | Copyright © 2008 Stephen L. Moshier 111 | and labelled as such in comments in the individual source files. All 112 | have been licensed under extremely permissive terms. 113 | 114 | The ARM memcpy code (src/string/arm/memcpy_el.S) is Copyright © 2008 115 | The Android Open Source Project and is licensed under a two-clause BSD 116 | license. It was taken from Bionic libc, used on Android. 117 | 118 | The implementation of DES for crypt (src/crypt/crypt_des.c) is 119 | Copyright © 1994 David Burren. It is licensed under a BSD license. 120 | 121 | The implementation of blowfish crypt (src/crypt/crypt_blowfish.c) was 122 | originally written by Solar Designer and placed into the public 123 | domain. The code also comes with a fallback permissive license for use 124 | in jurisdictions that may not recognize the public domain. 125 | 126 | The smoothsort implementation (src/stdlib/qsort.c) is Copyright © 2011 127 | Valentin Ochs and is licensed under an MIT-style license. 128 | 129 | The BSD PRNG implementation (src/prng/random.c) and XSI search API 130 | (src/search/*.c) functions are Copyright © 2011 Szabolcs Nagy and 131 | licensed under following terms: "Permission to use, copy, modify, 132 | and/or distribute this code for any purpose with or without fee is 133 | hereby granted. There is no warranty." 134 | 135 | The x86_64 port was written by Nicholas J. Kain and is licensed under 136 | the standard MIT terms. 137 | 138 | The mips and microblaze ports were originally written by Richard 139 | Pennington for use in the ellcc project. The original code was adapted 140 | by Rich Felker for build system and code conventions during upstream 141 | integration. It is licensed under the standard MIT terms. 142 | 143 | The mips64 port was contributed by Imagination Technologies and is 144 | licensed under the standard MIT terms. 145 | 146 | The powerpc port was also originally written by Richard Pennington, 147 | and later supplemented and integrated by John Spencer. It is licensed 148 | under the standard MIT terms. 149 | 150 | All other files which have no copyright comments are original works 151 | produced specifically for use as part of this library, written either 152 | by Rich Felker, the main author of the library, or by one or more 153 | contibutors listed above. Details on authorship of individual files 154 | can be found in the git version control history of the project. The 155 | omission of copyright and license comments in each file is in the 156 | interest of source tree size. 157 | 158 | In addition, permission is hereby granted for all public header files 159 | (include/* and arch/*/bits/*) and crt files intended to be linked into 160 | applications (crt/*, ldso/dlstart.c, and arch/*/crt_arch.h) to omit 161 | the copyright notice and permission notice otherwise required by the 162 | license, and to use these files without any requirement of 163 | attribution. These files include substantial contributions from: 164 | 165 | Bobby Bingham 166 | John Spencer 167 | Nicholas J. Kain 168 | Rich Felker 169 | Richard Pennington 170 | Stefan Kristiansson 171 | Szabolcs Nagy 172 | 173 | all of whom have explicitly granted such permission. 174 | 175 | This file previously contained text expressing a belief that most of 176 | the files covered by the above exception were sufficiently trivial not 177 | to be subject to copyright, resulting in confusion over whether it 178 | negated the permissions granted in the license. In the spirit of 179 | permissive licensing, and of not having licensing issues being an 180 | obstacle to adoption, that text has been removed. 181 | -------------------------------------------------------------------------------- /tests/data/binaries/README.md: -------------------------------------------------------------------------------- 1 | # Test Programs 2 | 3 | This is where small test programs and their dependencies can be places for use in tests. 4 | The `chroot/` subdirectory should be treated as the root directory for placement of any runtime dependencies. 5 | The most commonly used test is `fizz-buzz` which can be compiled with `gcc` by running: 6 | 7 | ```bash 8 | gcc fizz-buzz.c -no-pie -m32 -o ./chroot/bin/fizz-buzz-glibc-32-exe 9 | gcc fizz-buzz.c -m32 -o ./chroot/bin/fizz-buzz-glibc-32 10 | gcc fizz-buzz.c -m64 -o ./chroot/bin/fizz-buzz-glibc-64 11 | ``` 12 | 13 | There is also a musl version which can be compiled similarly with `clang` (`gcc` gives an error for some reason). 14 | 15 | ```bash 16 | musl-clang fizz-buzz.c -m64 -o ./chroot/bin/fizz-buzz-musl-64 17 | ``` 18 | 19 | Additionally, there are two small utilities for echoing command arguments and the destination of `/proc/self/exe`. 20 | These can be compiled by running 21 | 22 | ```bash 23 | gcc echo-args.c -m32 -o ./chroot/bin/echo-args-glibc-32 24 | gcc echo-proc-self-exe.c -m32 -o ./chroot/bin/echo-proc-self-exe-glibc-32 25 | ``` 26 | 27 | ## Linking 28 | 29 | There is a script in [./chroot/bin/ldd](./chroot/bin/ldd) that attempts to invoke the linker with library paths set in such a way that the results would be comparable to running in actual chroot. 30 | The purpose of this is so that these binaries can be transported across systems for testing. 31 | 32 | 33 | ## License 34 | 35 | Components of the [GNU C Library (glibc)](https://www.gnu.org/software/libc/) are in included in this subdirectory in a binary form for the purpose of testing dependency resolution. 36 | These are licensed under a mixture of licenses. 37 | The most recent version of the licenses can be found [in the git repository for the project](https://sourceware.org/git/?p=glibc.git;a=blob_plain;f=LICENSES;hb=HEAD). 38 | Additionally, a copy has been duplicated in this directory in [GLIBC-LICENSES](./GLIBC-LICENSES). 39 | 40 | Components of the [MUSL C Library](https://www.musl-libc.org/) are similarly included for the same reasons. 41 | These are licensed under [a standard MIT license](https://git.musl-libc.org/cgit/musl/tree/COPYRIGHT). 42 | This copyright notice is reproduced here in [MUSL-COPYRIGHT](MUSL-COPYRIGHT). 43 | 44 | Any third-party dependencies that are included here were taken from the [Arch Linux Package Repositories](https://www.archlinux.org/packages/) without modification. 45 | They provide the source code for these packages for GPL compliance, and their offer is passed along here for any GPLed binaries in this test directory. 46 | 47 | If you're involved with one of these projects and feel that there's an issue with the licensing or attribution here, please open an issue on the repository and we'll make whatever changes are necessary. 48 | -------------------------------------------------------------------------------- /tests/data/binaries/chroot/bin/echo-args-glibc-32: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/intoli/exodus/ef3d5e92c1b604b09cf0a57baff0f4d0b421b8da/tests/data/binaries/chroot/bin/echo-args-glibc-32 -------------------------------------------------------------------------------- /tests/data/binaries/chroot/bin/echo-proc-self-exe-glibc-32: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/intoli/exodus/ef3d5e92c1b604b09cf0a57baff0f4d0b421b8da/tests/data/binaries/chroot/bin/echo-proc-self-exe-glibc-32 -------------------------------------------------------------------------------- /tests/data/binaries/chroot/bin/fizz-buzz-glibc-32: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/intoli/exodus/ef3d5e92c1b604b09cf0a57baff0f4d0b421b8da/tests/data/binaries/chroot/bin/fizz-buzz-glibc-32 -------------------------------------------------------------------------------- /tests/data/binaries/chroot/bin/fizz-buzz-glibc-32-exe: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/intoli/exodus/ef3d5e92c1b604b09cf0a57baff0f4d0b421b8da/tests/data/binaries/chroot/bin/fizz-buzz-glibc-32-exe -------------------------------------------------------------------------------- /tests/data/binaries/chroot/bin/fizz-buzz-glibc-64: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/intoli/exodus/ef3d5e92c1b604b09cf0a57baff0f4d0b421b8da/tests/data/binaries/chroot/bin/fizz-buzz-glibc-64 -------------------------------------------------------------------------------- /tests/data/binaries/chroot/bin/fizz-buzz-musl-64: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/intoli/exodus/ef3d5e92c1b604b09cf0a57baff0f4d0b421b8da/tests/data/binaries/chroot/bin/fizz-buzz-musl-64 -------------------------------------------------------------------------------- /tests/data/binaries/chroot/bin/ldd: -------------------------------------------------------------------------------- 1 | #! /bin/bash 2 | 3 | # This is sort of a poor man's ldd. It tries to set the libary path sufficiently for the test 4 | # programs to link in a pseudo-chroot environment. 5 | 6 | DIR="$(dirname "$(readlink -f "$0")")" 7 | LD_TRACE_LOADED_OBJECTS=1 ${DIR}/../lib/ld-linux.so.2 --library-path "\$ORIGIN/../lib/:\$ORIGIN/../usr/lib32/" --inhibit-rpath "" "$@" 8 | -------------------------------------------------------------------------------- /tests/data/binaries/chroot/lib/ld-linux.so.2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/intoli/exodus/ef3d5e92c1b604b09cf0a57baff0f4d0b421b8da/tests/data/binaries/chroot/lib/ld-linux.so.2 -------------------------------------------------------------------------------- /tests/data/binaries/chroot/lib/ld-musl-x86_64.so.1: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/intoli/exodus/ef3d5e92c1b604b09cf0a57baff0f4d0b421b8da/tests/data/binaries/chroot/lib/ld-musl-x86_64.so.1 -------------------------------------------------------------------------------- /tests/data/binaries/chroot/lib64/ld-linux-x86-64.so.2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/intoli/exodus/ef3d5e92c1b604b09cf0a57baff0f4d0b421b8da/tests/data/binaries/chroot/lib64/ld-linux-x86-64.so.2 -------------------------------------------------------------------------------- /tests/data/binaries/chroot/usr/lib/ld-linux-x86-64.so.2: -------------------------------------------------------------------------------- 1 | ../../lib64/ld-linux-x86-64.so.2 -------------------------------------------------------------------------------- /tests/data/binaries/chroot/usr/lib/libc.so.6: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/intoli/exodus/ef3d5e92c1b604b09cf0a57baff0f4d0b421b8da/tests/data/binaries/chroot/usr/lib/libc.so.6 -------------------------------------------------------------------------------- /tests/data/binaries/chroot/usr/lib32/ld-linux.so.2: -------------------------------------------------------------------------------- 1 | ../../lib/ld-linux.so.2 -------------------------------------------------------------------------------- /tests/data/binaries/chroot/usr/lib32/libc.so.6: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/intoli/exodus/ef3d5e92c1b604b09cf0a57baff0f4d0b421b8da/tests/data/binaries/chroot/usr/lib32/libc.so.6 -------------------------------------------------------------------------------- /tests/data/binaries/echo-args.c: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | int main(int argc, char **argv) { 4 | for (int i = 0; i < argc; i++) { 5 | printf("%s\n", argv[i]); 6 | } 7 | return 0; 8 | } 9 | -------------------------------------------------------------------------------- /tests/data/binaries/echo-proc-self-exe.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | 4 | int main(void) { 5 | char buffer[4096]; 6 | readlink("/proc/self/exe", buffer, sizeof(buffer) - 1); 7 | printf("%s\n", buffer); 8 | return 0; 9 | } 10 | -------------------------------------------------------------------------------- /tests/data/binaries/fizz-buzz.c: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | 4 | int main() { 5 | int i; 6 | for(i = 0; i <= 100; i++) { 7 | if (i % 3 == 0) { 8 | printf("FIZZ"); 9 | } 10 | 11 | if (i % 5 == 0) { 12 | printf("BUZZ"); 13 | } 14 | 15 | if ((i % 3 != 0) && (i % 5 != 0)) { 16 | printf("%d", i); 17 | } 18 | 19 | printf("\n"); 20 | } 21 | 22 | return 0; 23 | } 24 | -------------------------------------------------------------------------------- /tests/data/ldd-output/htop-amazon-linux-dependencies.txt: -------------------------------------------------------------------------------- 1 | /lib64/libncursesw.so.5 2 | /lib64/libtinfo.so.5 3 | /lib64/libm.so.6 4 | /lib64/libgcc_s.so.1 5 | /lib64/libc.so.6 6 | /lib64/libdl.so.2 7 | /lib64/ld-linux-x86-64.so.2 8 | -------------------------------------------------------------------------------- /tests/data/ldd-output/htop-amazon-linux.txt: -------------------------------------------------------------------------------- 1 | linux-vdso.so.1 => (0x00007ffc7ff9e000) 2 | libncursesw.so.5 => /lib64/libncursesw.so.5 (0x00007fce6f039000) 3 | libtinfo.so.5 => /lib64/libtinfo.so.5 (0x00007fce6ee18000) 4 | libm.so.6 => /lib64/libm.so.6 (0x00007fce6eb16000) 5 | libgcc_s.so.1 => /lib64/libgcc_s.so.1 (0x00007fce6e900000) 6 | libc.so.6 => /lib64/libc.so.6 (0x00007fce6e53c000) 7 | libdl.so.2 => /lib64/libdl.so.2 (0x00007fce6e338000) 8 | /lib64/ld-linux-x86-64.so.2 (0x00007fce6f26e000) 9 | -------------------------------------------------------------------------------- /tests/data/ldd-output/htop-arch-dependencies.txt: -------------------------------------------------------------------------------- 1 | /usr/lib/libncursesw.so.6 2 | /usr/lib/libm.so.6 3 | /usr/lib/libc.so.6 4 | /usr/lib64/ld-linux-x86-64.so.2 5 | -------------------------------------------------------------------------------- /tests/data/ldd-output/htop-arch.txt: -------------------------------------------------------------------------------- 1 | linux-vdso.so.1 (0x00007ffd7abf3000) 2 | libncursesw.so.6 => /usr/lib/libncursesw.so.6 (0x00007fbfc45be000) 3 | libm.so.6 => /usr/lib/libm.so.6 (0x00007fbfc4272000) 4 | libc.so.6 => /usr/lib/libc.so.6 (0x00007fbfc3ebb000) 5 | /lib64/ld-linux-x86-64.so.2 => /usr/lib64/ld-linux-x86-64.so.2 (0x00007fbfc4a58000) 6 | -------------------------------------------------------------------------------- /tests/data/ldd-output/htop-ubuntu-14.04-dependencies.txt: -------------------------------------------------------------------------------- 1 | /lib/x86_64-linux-gnu/libncursesw.so.5 2 | /lib/x86_64-linux-gnu/libtinfo.so.5 3 | /lib/x86_64-linux-gnu/libm.so.6 4 | /lib/x86_64-linux-gnu/libc.so.6 5 | /lib/x86_64-linux-gnu/libdl.so.2 6 | /lib64/ld-linux-x86-64.so.2 7 | -------------------------------------------------------------------------------- /tests/data/ldd-output/htop-ubuntu-14.04.txt: -------------------------------------------------------------------------------- 1 | linux-vdso.so.1 => (0x00007ffe625eb000) 2 | libncursesw.so.5 => /lib/x86_64-linux-gnu/libncursesw.so.5 (0x00007f18b88a9000) 3 | libtinfo.so.5 => /lib/x86_64-linux-gnu/libtinfo.so.5 (0x00007f18b8680000) 4 | libm.so.6 => /lib/x86_64-linux-gnu/libm.so.6 (0x00007f18b837a000) 5 | libc.so.6 => /lib/x86_64-linux-gnu/libc.so.6 (0x00007f18b7fb2000) 6 | libdl.so.2 => /lib/x86_64-linux-gnu/libdl.so.2 (0x00007f18b7dae000) 7 | /lib64/ld-linux-x86-64.so.2 (0x00007f18b8add000) 8 | -------------------------------------------------------------------------------- /tests/data/template-result.txt: -------------------------------------------------------------------------------- 1 | This is a file template. 2 | We need to subsitute words in here. 3 | -------------------------------------------------------------------------------- /tests/data/template.txt: -------------------------------------------------------------------------------- 1 | This is a file template. 2 | We need to subsitute {{noun}}s in {{location}}. 3 | -------------------------------------------------------------------------------- /tests/test_bundling.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | import os 3 | import shutil 4 | from subprocess import PIPE 5 | from subprocess import Popen 6 | 7 | import pytest 8 | 9 | from exodus_bundler.bundling import Bundle 10 | from exodus_bundler.bundling import Elf 11 | from exodus_bundler.bundling import File 12 | from exodus_bundler.bundling import bytes_to_int 13 | from exodus_bundler.bundling import create_unpackaged_bundle 14 | from exodus_bundler.bundling import detect_elf_binary 15 | from exodus_bundler.bundling import parse_dependencies_from_ldd_output 16 | from exodus_bundler.bundling import resolve_binary 17 | from exodus_bundler.bundling import resolve_file_path 18 | from exodus_bundler.bundling import run_ldd 19 | from exodus_bundler.bundling import stored_property 20 | 21 | 22 | parent_directory = os.path.dirname(os.path.realpath(__file__)) 23 | ldd_output_directory = os.path.join(parent_directory, 'data', 'ldd-output') 24 | chroot = os.path.join(parent_directory, 'data', 'binaries', 'chroot') 25 | ldd = os.path.join(chroot, 'bin', 'ldd') 26 | echo_args_glibc_32 = os.path.join(chroot, 'bin', 'echo-args-glibc-32') 27 | echo_proc_self_exe_glibc_32 = os.path.join(chroot, 'bin', 'echo-proc-self-exe-glibc-32') 28 | fizz_buzz_glibc_32 = os.path.join(chroot, 'bin', 'fizz-buzz-glibc-32') 29 | fizz_buzz_glibc_32_exe = os.path.join(chroot, 'bin', 'fizz-buzz-glibc-32-exe') 30 | fizz_buzz_glibc_64 = os.path.join(chroot, 'bin', 'fizz-buzz-glibc-64') 31 | fizz_buzz_musl_64 = os.path.join(chroot, 'bin', 'fizz-buzz-musl-64') 32 | 33 | 34 | @pytest.mark.parametrize('path,expected_file_count', [ 35 | (fizz_buzz_glibc_32, 3), 36 | (fizz_buzz_glibc_64, 3), 37 | (fizz_buzz_musl_64, 2), 38 | (ldd, 1), 39 | ]) 40 | def test_bundle_add_file(path, expected_file_count): 41 | bundle = Bundle(chroot=chroot) 42 | assert len(bundle.files) == 0, 'The initial bundle should contain no files.' 43 | bundle.add_file(path) 44 | assert len(bundle.files) == expected_file_count, \ 45 | 'The bundle should include %d files.' % expected_file_count 46 | 47 | 48 | def test_bundle_add_file_recursively(): 49 | bundle = Bundle(chroot=chroot) 50 | assert len(bundle.files) == 0, 'The initial bundle should contain no files.' 51 | bundle.add_file(chroot) 52 | second_bundle = Bundle(chroot=chroot) 53 | for path in [ldd, fizz_buzz_glibc_32, fizz_buzz_glibc_32_exe, fizz_buzz_musl_64]: 54 | second_bundle.add_file(path) 55 | assert second_bundle.files.issubset(bundle.files), \ 56 | 'All of the executables and their dependencies should be in the first bundle.' 57 | 58 | 59 | def test_bundle_delete_working_directory(): 60 | bundle = Bundle() 61 | assert bundle.working_directory is None, \ 62 | 'A directory should only be created if passed `working_directory=True`.' 63 | bundle = Bundle(working_directory=True) 64 | working_directory = bundle.working_directory 65 | assert os.path.exists(working_directory), \ 66 | 'A working directory should have been created.' 67 | bundle.delete_working_directory() 68 | assert not os.path.exists(working_directory), \ 69 | 'The working directory should have been deleted.' 70 | assert bundle.working_directory is None, \ 71 | 'The working directory should have been cleared after deletion.' 72 | 73 | 74 | def test_bundle_file_factory(): 75 | bundle = Bundle() 76 | bundle.add_file(ldd) 77 | # Note that `ldd` is a shell script, and should bring in no dependencies. 78 | [file] = bundle.files 79 | new_file = bundle.file_factory(ldd) 80 | assert new_file is file, \ 81 | 'The same file should be returned instead of making a new one.' 82 | 83 | 84 | def test_bundle_hash(): 85 | bundle = Bundle(chroot=chroot) 86 | hashes = [bundle.hash] 87 | for filename in [fizz_buzz_glibc_32, fizz_buzz_glibc_64, fizz_buzz_musl_64]: 88 | bundle.add_file(filename) 89 | hashes.append(bundle.hash) 90 | assert len(hashes) == len(set(hashes)), 'All of the hashes should be unique.' 91 | assert all(len(hash) == 64 for hash in hashes), 'All of the hashes should have length 64.' 92 | 93 | 94 | def test_bundle_root(): 95 | try: 96 | bundle = Bundle(working_directory=True) 97 | assert bundle.hash in bundle.bundle_root, 'Bundle path should include the hash.' 98 | assert bundle.bundle_root.startswith(bundle.working_directory), \ 99 | 'The bundle root should be a subdirectory of the working directory.' 100 | except: # noqa: E722 101 | raise 102 | finally: 103 | bundle.delete_working_directory() 104 | 105 | 106 | @pytest.mark.parametrize('int,bytes,byteorder', [ 107 | (1234567890, b'\xd2\x02\x96I\x00\x00\x00\x00', 'little'), 108 | (1234567890, b'\x00\x00\x00\x00I\x96\x02\xd2', 'big'), 109 | (9876543210, b'\xea\x16\xb0L\x02\x00\x00\x00', 'little'), 110 | (9876543210, b'\x00\x00\x00\x02L\xb0\x16\xea', 'big'), 111 | ]) 112 | def test_bytes_to_int(int, bytes, byteorder): 113 | assert bytes_to_int(bytes, byteorder=byteorder) == int, 'Byte conversion should work.' 114 | 115 | 116 | @pytest.mark.parametrize('fizz_buzz,shell_launchers', [ 117 | (fizz_buzz_glibc_32, True), 118 | (fizz_buzz_glibc_32, False), 119 | (fizz_buzz_glibc_64, True), 120 | (fizz_buzz_glibc_64, False), 121 | (fizz_buzz_musl_64, True), 122 | (fizz_buzz_musl_64, False), 123 | ]) 124 | def test_create_unpackaged_bundle(fizz_buzz, shell_launchers): 125 | """This tests that the packaged executable runs as expected. At the very least, this 126 | tests that the symbolic links and launcher are functioning correctly. Unfortunately, 127 | it doesn't really test the linker overrides unless the required libraries are not 128 | present on the current system. FWIW, the CircleCI docker image being used is 129 | incompatible, so the continuous integration tests are more meaningful.""" 130 | root_directory = create_unpackaged_bundle( 131 | rename=[], executables=[fizz_buzz], chroot=chroot, shell_launchers=shell_launchers) 132 | try: 133 | binary_path = os.path.join(root_directory, 'bin', os.path.basename(fizz_buzz)) 134 | 135 | process = Popen([binary_path], stdout=PIPE, stderr=PIPE) 136 | stdout, stderr = process.communicate() 137 | assert 'FIZZBUZZ' in stdout.decode('utf-8') 138 | assert len(stderr.decode('utf-8')) == 0 139 | finally: 140 | assert root_directory.startswith('/tmp/') 141 | shutil.rmtree(root_directory) 142 | 143 | 144 | @pytest.mark.parametrize('detect', [False, True]) 145 | def test_create_unpackaged_bundle_detects_dependencies(detect): 146 | binary_name = 'ls' 147 | root_directory = create_unpackaged_bundle( 148 | rename=[], executables=[binary_name], detect=detect) 149 | try: 150 | # Determine the bundle root. 151 | binary_symlink = os.path.join(root_directory, 'bin', binary_name) 152 | binary_path = os.path.realpath(binary_symlink) 153 | dirname, basename = os.path.split(binary_path) 154 | while len(basename) != 64: 155 | dirname, basename = os.path.split(dirname) 156 | bundle_root = os.path.join(dirname, basename) 157 | 158 | man_directory = os.path.join(bundle_root, 'usr', 'share', 'man') 159 | assert os.path.exists(man_directory) == detect, \ 160 | 'The man directory should only exist when `detect=True`.' 161 | finally: 162 | assert root_directory.startswith('/tmp/') 163 | shutil.rmtree(root_directory) 164 | 165 | 166 | def test_create_unpackaged_bundle_has_correct_args(): 167 | root_directory = create_unpackaged_bundle( 168 | rename=[], executables=[echo_args_glibc_32], chroot=chroot) 169 | try: 170 | binary_path = os.path.join(root_directory, 'bin', os.path.basename(echo_args_glibc_32)) 171 | 172 | process = Popen([binary_path, 'arg1', 'arg2'], stdout=PIPE, stderr=PIPE) 173 | stdout, stderr = process.communicate() 174 | assert len(stderr.decode('utf-8')) == 0 175 | args = stdout.decode('utf-8').split('\n') 176 | assert os.path.basename(args[0]) == '%s-x' % os.path.basename(echo_args_glibc_32), \ 177 | 'The value of argv[0] should correspond to the local symlink.' 178 | assert args[1] == 'arg1' and args[2] == 'arg2', \ 179 | 'The other arguments should be passed through to the child process.' 180 | finally: 181 | assert root_directory.startswith('/tmp/') 182 | shutil.rmtree(root_directory) 183 | 184 | 185 | def test_create_unpackaged_bundle_has_correct_proc_self_exe(): 186 | root_directory = create_unpackaged_bundle( 187 | rename=[], executables=[echo_proc_self_exe_glibc_32], chroot=chroot) 188 | try: 189 | binary_path = os.path.join(root_directory, 'bin', 190 | os.path.basename(echo_proc_self_exe_glibc_32)) 191 | 192 | process = Popen([binary_path], stdout=PIPE, stderr=PIPE) 193 | stdout, stderr = process.communicate() 194 | assert len(stderr.decode('utf-8')) == 0 195 | proc_self_exe = stdout.decode('utf-8').strip() 196 | assert os.path.basename(proc_self_exe).startswith('linker-'), \ 197 | 'The linker should be the executing process.' 198 | relative_path = os.path.relpath(proc_self_exe, root_directory) 199 | assert relative_path.startswith('bundles/'), \ 200 | 'The process should be in the bundles directory.' 201 | finally: 202 | assert root_directory.startswith('/tmp/') 203 | shutil.rmtree(root_directory) 204 | 205 | 206 | def test_detect_elf_binary(): 207 | assert detect_elf_binary(fizz_buzz_glibc_32), 'The `fizz-buzz` file should be an ELF binary.' 208 | assert not detect_elf_binary(ldd), 'The `ldd` file should be a shell script.' 209 | 210 | 211 | @pytest.mark.parametrize('fizz_buzz,bits', [ 212 | (fizz_buzz_glibc_32, 32), 213 | (fizz_buzz_glibc_64, 64), 214 | (fizz_buzz_musl_64, 64), 215 | ]) 216 | def test_elf_bits(fizz_buzz, bits): 217 | fizz_buzz_elf = Elf(fizz_buzz, chroot=chroot) 218 | # Can be checked by running `file fizz-buzz`. 219 | assert fizz_buzz_elf.bits == bits, \ 220 | 'The fizz buzz executable should be %d-bit.' % bits 221 | 222 | 223 | @pytest.mark.parametrize('fizz_buzz', [ 224 | (fizz_buzz_glibc_32), 225 | (fizz_buzz_glibc_64), 226 | ]) 227 | def test_elf_dependencies(fizz_buzz): 228 | fizz_buzz_elf = Elf(fizz_buzz, chroot=chroot) 229 | direct_dependencies = fizz_buzz_elf.direct_dependencies 230 | all_dependencies = fizz_buzz_elf.dependencies 231 | assert set(direct_dependencies).issubset(all_dependencies), \ 232 | 'The direct dependencies should be a subset of all dependencies.' 233 | 234 | 235 | @pytest.mark.parametrize('fizz_buzz', [ 236 | (fizz_buzz_glibc_32), 237 | (fizz_buzz_glibc_64), 238 | (fizz_buzz_musl_64), 239 | ]) 240 | def test_elf_direct_dependencies(fizz_buzz): 241 | fizz_buzz_elf = Elf(fizz_buzz, chroot=chroot) 242 | dependencies = fizz_buzz_elf.direct_dependencies 243 | assert all(file.path.startswith(chroot) for file in dependencies), \ 244 | 'All dependencies should be located within the chroot.' 245 | assert len(dependencies), 'There should be at least one dependency.' 246 | 247 | # These don't apply to the musl binary. 248 | if 'glib' in fizz_buzz: 249 | assert len(dependencies) == 2, 'The linker and libc should be the only dependencies.' 250 | assert any('libc.so' in file.path for file in dependencies), \ 251 | '"libc" was not found as a direct dependency of the executable.' 252 | 253 | 254 | @pytest.mark.parametrize('fizz_buzz,expected_linker_path', [ 255 | (fizz_buzz_glibc_32, '/lib/ld-linux.so.2'), 256 | (fizz_buzz_glibc_64, '/lib64/ld-linux-x86-64.so.2'), 257 | (fizz_buzz_musl_64, '/lib/ld-musl-x86_64.so.1'), 258 | ]) 259 | def test_elf_linker(fizz_buzz, expected_linker_path): 260 | # Found by running `readelf -l fizz-buzz`. 261 | fizz_buzz_elf = Elf(fizz_buzz, chroot=chroot) 262 | expected_linker_path = os.path.join(chroot, os.path.relpath(expected_linker_path, '/')) 263 | assert fizz_buzz_elf.linker_file.path == expected_linker_path, \ 264 | 'The correct linker should be extracted from the ELF program header.' 265 | 266 | 267 | @pytest.mark.parametrize('fizz_buzz, expected_type', [ 268 | (fizz_buzz_glibc_32, 'shared'), 269 | (fizz_buzz_glibc_32_exe, 'executable'), 270 | (fizz_buzz_glibc_64, 'shared'), 271 | ]) 272 | def test_elf_type(fizz_buzz, expected_type): 273 | elf = Elf(fizz_buzz, chroot=chroot) 274 | assert elf.type == expected_type, 'Fizz buzz should match the expected ELF binary type.' 275 | 276 | 277 | def test_file_destination(): 278 | arch_file = File(os.path.join(ldd_output_directory, 'htop-arch.txt')) 279 | arch_directory = os.path.dirname(arch_file.destination) 280 | fizz_buzz_file = File(fizz_buzz_glibc_32, chroot=chroot) 281 | fizz_buzz_directory = os.path.dirname(fizz_buzz_file.destination) 282 | assert arch_directory == fizz_buzz_directory, \ 283 | 'Executable and non-executable files should be written to the same directory.' 284 | 285 | 286 | def test_file_executable(): 287 | fizz_buzz_file = File(fizz_buzz_glibc_32, chroot=chroot) 288 | arch_file = File(os.path.join(ldd_output_directory, 'htop-arch.txt')) 289 | assert fizz_buzz_file.executable, 'The fizz buzz executable should be executable.' 290 | assert not arch_file.executable, 'The arch text file should not be executable.' 291 | 292 | 293 | def test_file_elf(): 294 | fizz_buzz_file = File(fizz_buzz_glibc_32, chroot=chroot) 295 | arch_file = File(os.path.join(ldd_output_directory, 'htop-arch.txt')) 296 | assert fizz_buzz_file.elf, 'The fizz buzz executable should be an ELF binary.' 297 | assert not arch_file.elf, 'The arch text file should not be an ELF binary.' 298 | 299 | 300 | def test_file_hash(): 301 | amazon_file = File(os.path.join(ldd_output_directory, 'htop-amazon-linux.txt')) 302 | arch_file = File(os.path.join(ldd_output_directory, 'htop-arch.txt')) 303 | assert amazon_file.hash != arch_file.hash, 'The hashes should differ.' 304 | assert len(amazon_file.hash) == len(arch_file.hash) == 64, \ 305 | 'The hashes should have a consistent length of 64 characters.' 306 | 307 | # Found by executing `sha256sum fizz-buzz`. 308 | expected_hash = 'd54ab4714215d7822bf490df5cdf49bc3f32b4c85a439b109fc7581355f9d9c5' 309 | assert File(fizz_buzz_glibc_32, chroot=chroot).hash == expected_hash, 'Hashes should match.' 310 | 311 | 312 | @pytest.mark.parametrize('fizz_buzz', [ 313 | (fizz_buzz_glibc_32), 314 | (fizz_buzz_glibc_64), 315 | (fizz_buzz_musl_64), 316 | ]) 317 | def test_file_requires_launcher(fizz_buzz): 318 | file = File(fizz_buzz, chroot=chroot) 319 | assert file.requires_launcher, 'Fizz buzz should require a launcher.' 320 | assert all(not dependency.requires_launcher for dependency in file.elf.dependencies), \ 321 | 'All of the dependencies should not require launchers.' 322 | 323 | 324 | def test_file_symlink(): 325 | bundle = Bundle(chroot=chroot, working_directory=True) 326 | try: 327 | bundle.add_file(fizz_buzz_glibc_32) 328 | file = next(iter(bundle.files)) 329 | file.copy(bundle.working_directory) 330 | symlink = file.symlink(bundle.working_directory, bundle.bundle_root) 331 | assert os.path.islink(symlink), 'A symlink should have been created.' 332 | assert os.path.exists(symlink), 'The symlink should point to the actual file.' 333 | except: # noqa: E722 334 | raise 335 | finally: 336 | bundle.delete_working_directory() 337 | 338 | 339 | @pytest.mark.parametrize('filename_prefix', [ 340 | 'htop-amazon-linux', 341 | 'htop-arch', 342 | 'htop-ubuntu-14.04', 343 | ]) 344 | def test_parse_dependencies_from_ldd_output(filename_prefix): 345 | ldd_output_filename = filename_prefix + '.txt' 346 | with open(os.path.join(ldd_output_directory, ldd_output_filename)) as f: 347 | ldd_output = f.read() 348 | dependencies = parse_dependencies_from_ldd_output(ldd_output) 349 | 350 | ldd_results_filename = filename_prefix + '-dependencies.txt' 351 | with open(os.path.join(ldd_output_directory, ldd_results_filename)) as f: 352 | expected_dependencies = [line for line in f.read().split('\n') if len(line)] 353 | 354 | assert set(dependencies) == set(expected_dependencies), \ 355 | 'The dependencies were not parsed correctly from ldd output for "%s"' % filename_prefix 356 | 357 | 358 | def test_resolve_binary(): 359 | binary_directory = os.path.dirname(fizz_buzz_glibc_32) 360 | binary = os.path.basename(fizz_buzz_glibc_32) 361 | old_path = os.getenv('PATH', '') 362 | try: 363 | os.environ['PATH'] = '%s%s%s' % (binary_directory, os.pathsep, old_path) 364 | resolved_binary = resolve_binary(binary) 365 | assert resolved_binary == os.path.normpath(fizz_buzz_glibc_32), \ 366 | 'The full binary path was not resolved correctly.' 367 | finally: 368 | os.environ['PATH'] = old_path 369 | 370 | 371 | def test_resolve_file_path(): 372 | with pytest.raises(Exception): 373 | resolve_file_path(chroot) 374 | with pytest.raises(Exception): 375 | resolve_file_path(os.path.join(chroot, 'non-existent-file')) 376 | assert os.path.isabs(resolve_file_path(fizz_buzz_glibc_32)), \ 377 | 'The resolved path should be absolute.' 378 | 379 | 380 | def test_run_ldd(): 381 | assert any('libc.so' in line for line in run_ldd(ldd, fizz_buzz_glibc_32)), \ 382 | '"libc" was not found in the output of "ldd" for the executable.' 383 | 384 | 385 | def test_stored_property(): 386 | class Incrementer(object): 387 | def __init__(self): 388 | self.i = 0 389 | 390 | @stored_property 391 | def next(self): 392 | self.i += 1 393 | return self.i 394 | 395 | incrementer = Incrementer() 396 | for i in range(10): 397 | assert incrementer.next == 1, '`Incrementer.next` should not change.' 398 | -------------------------------------------------------------------------------- /tests/test_cli.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | import io 3 | import os 4 | import subprocess 5 | import tarfile 6 | import tempfile 7 | 8 | import pytest 9 | 10 | from exodus_bundler.bundling import logger 11 | from exodus_bundler.cli import configure_logging 12 | from exodus_bundler.cli import parse_args 13 | 14 | 15 | parent_directory = os.path.dirname(os.path.realpath(__file__)) 16 | chroot = os.path.join(parent_directory, 'data', 'binaries', 'chroot') 17 | fizz_buzz_glibc_32 = os.path.join(chroot, 'bin', 'fizz-buzz-glibc-32') 18 | fizz_buzz_glibc_32_exe = os.path.join(chroot, 'bin', 'fizz-buzz-glibc-32-exe') 19 | fizz_buzz_glibc_64 = os.path.join(chroot, 'bin', 'fizz-buzz-glibc-64') 20 | fizz_buzz_musl_64 = os.path.join(chroot, 'bin', 'fizz-buzz-musl-64') 21 | 22 | 23 | def run_exodus(args, **options): 24 | options['universal_newlines'] = options.get('universal_newlines', True) 25 | 26 | # Allow specifying content to pipe into stdin, with options['stdin'] 27 | if 'stdin' in options: 28 | input = options['stdin'].encode('utf-8') 29 | options['stdin'] = subprocess.PIPE 30 | else: 31 | input = None 32 | 33 | process = subprocess.Popen( 34 | ['exodus'] + args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, **options) 35 | stdout, stderr = process.communicate(input=input) 36 | return process.returncode, stdout, stderr 37 | 38 | 39 | def test_adding_additional_files(capsys): 40 | args = ['--chroot', chroot, '--output', '-', '--tarball', fizz_buzz_glibc_32] 41 | stdin = '\n'.join((fizz_buzz_glibc_32_exe, fizz_buzz_glibc_64)) 42 | returncode, stdout, stderr = run_exodus(args, universal_newlines=False, stdin=stdin) 43 | assert returncode == 0, "Exodus should have exited with a success status code, but didn't." 44 | stream = io.BytesIO(stdout) 45 | with tarfile.open(fileobj=stream, mode='r:gz') as f: 46 | names = f.getnames() 47 | assert 'exodus/bin/fizz-buzz-glibc-32' in names, stderr 48 | # These shouldn't be entrypoints, but should be included 49 | assert 'exodus/bin/fizz-buzz-glibc-32-exe' not in names, stderr 50 | assert 'exodus/bin/fizz-buzz-glibc-64' not in names, stderr 51 | assert any(fizz_buzz_glibc_32_exe in name for name in names), stderr 52 | assert any(fizz_buzz_glibc_64 in name for name in names), stderr 53 | 54 | 55 | def test_logging_outputs(capsys): 56 | # There should be no output before configuring the logger. 57 | logger.error('error') 58 | out, err = capsys.readouterr() 59 | print(out, err) 60 | assert len(out) == len(err) == 0 61 | 62 | # The different levels should be routed separately to stdout/stderr. 63 | configure_logging(verbose=True, quiet=False) 64 | logger.debug('debug') 65 | logger.warning('warn') 66 | logger.info('info') 67 | logger.error('error') 68 | out, err = capsys.readouterr() 69 | assert all(output in out for output in ('info')) 70 | assert all(output not in out for output in ('debug', 'warn', 'error')) 71 | assert all(output in err for output in ('warn', 'error')) 72 | assert all(output not in err for output in ('info', 'debug')) 73 | 74 | 75 | def test_missing_binary(capsys): 76 | # Without the --verbose flag. 77 | command = 'this-is-almost-definitely-not-going-to-be-a-command-anywhere' 78 | returncode, stdout, stderr = run_exodus([command]) 79 | assert returncode != 0, 'Running exodus should have failed.' 80 | assert 'Traceback' not in stderr, 'Traceback should not be included without the --verbose flag.' 81 | 82 | # With the --verbose flag. 83 | returncode, stdout, stderr = run_exodus(['--verbose', command]) 84 | assert returncode != 0, 'Running exodus should have failed.' 85 | assert 'Traceback' in stderr, 'Traceback should be included with the --verbose flag.' 86 | 87 | 88 | def test_required_argument(): 89 | with pytest.raises(SystemExit): 90 | parse_args([]) 91 | parse_args(['/bin/bash']) 92 | 93 | 94 | def test_return_type_is_dict(): 95 | assert type(parse_args(['/bin/bash'])) == dict 96 | 97 | 98 | def test_quiet_and_verbose_flags(): 99 | result = parse_args(['--quiet', '/bin/bash']) 100 | assert result['quiet'] and not result['verbose'] 101 | result = parse_args(['--verbose', '/bin/bash']) 102 | assert result['verbose'] and not result['quiet'] 103 | 104 | 105 | def test_writing_bundle_to_disk(): 106 | f, filename = tempfile.mkstemp(suffix='.sh') 107 | os.close(f) 108 | args = ['--chroot', chroot, '--output', filename, fizz_buzz_glibc_32] 109 | try: 110 | returncode, stdout, stderr = run_exodus(args) 111 | assert returncode == 0, "Exodus should have exited with a success status code, but didn't." 112 | with open(filename, 'rb') as f_in: 113 | first_line = f_in.readline().strip() 114 | assert first_line == b'#! /bin/bash', stderr 115 | finally: 116 | if os.path.exists(filename): 117 | os.unlink(filename) 118 | 119 | 120 | def test_writing_bundle_to_stdout(): 121 | args = ['--chroot', chroot, '--output', '-', fizz_buzz_glibc_32] 122 | returncode, stdout, stderr = run_exodus(args) 123 | assert returncode == 0, "Exodus should have exited with a success status code, but didn't." 124 | assert stdout.startswith('#! /bin/sh'), stderr 125 | 126 | 127 | def test_writing_tarball_to_disk(): 128 | f, filename = tempfile.mkstemp(suffix='.tgz') 129 | os.close(f) 130 | args = ['--chroot', chroot, '--output', filename, '--tarball', fizz_buzz_glibc_32] 131 | try: 132 | returncode, stdout, stderr = run_exodus(args) 133 | assert returncode == 0, "Exodus should have exited with a success status code, but didn't." 134 | assert tarfile.is_tarfile(filename), stderr 135 | with tarfile.open(filename, mode='r:gz') as f_in: 136 | assert 'exodus/bin/fizz-buzz-glibc-32' in f_in.getnames() 137 | finally: 138 | if os.path.exists(filename): 139 | os.unlink(filename) 140 | 141 | 142 | def test_writing_tarball_to_stdout(): 143 | args = ['--chroot', chroot, '--output', '-', '--tarball', fizz_buzz_glibc_32] 144 | returncode, stdout, stderr = run_exodus(args, universal_newlines=False) 145 | assert returncode == 0, "Exodus should have exited with a success status code, but didn't." 146 | stream = io.BytesIO(stdout) 147 | with tarfile.open(fileobj=stream, mode='r:gz') as f: 148 | assert 'exodus/bin/fizz-buzz-glibc-32' in f.getnames(), stderr 149 | -------------------------------------------------------------------------------- /tests/test_dependency_detection.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | import os 3 | 4 | from exodus_bundler.dependency_detection import detect_dependencies 5 | 6 | 7 | def test_detect_dependencies(): 8 | # This is a little janky, but the test suite won't run anywhere where it's not true. 9 | ls = '/usr/bin/ls' 10 | if not os.path.exists(ls): 11 | ls = '/bin/ls' 12 | assert os.path.exists(ls), 'This test assumes that `ls` is installed on the system.' 13 | 14 | dependencies = detect_dependencies(ls) 15 | assert any(ls in dependency for dependency in dependencies), \ 16 | '`%s` should have been detected as a dependency for `ls`.' % ls 17 | -------------------------------------------------------------------------------- /tests/test_input_parsing.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | import os 3 | 4 | from exodus_bundler.input_parsing import extract_exec_path 5 | from exodus_bundler.input_parsing import extract_open_path 6 | from exodus_bundler.input_parsing import extract_paths 7 | from exodus_bundler.input_parsing import extract_stat_path 8 | from exodus_bundler.input_parsing import strip_pid_prefix 9 | 10 | 11 | parent_directory = os.path.dirname(os.path.realpath(__file__)) 12 | strace_output_directory = os.path.join(parent_directory, 'data', 'strace-output') 13 | exodus_strace = os.path.join(strace_output_directory, 'exodus-output.txt') 14 | 15 | 16 | def test_extract_exec_path(): 17 | line = 'execve("/usr/bin/ls", ["ls"], 0x7ffea775ad70 /* 113 vars */) = 0' 18 | assert extract_exec_path(line) == '/usr/bin/ls', \ 19 | 'It should have extracted the path to the ls executable.' 20 | assert extract_exec_path('blah') is None, \ 21 | 'It should return `None` when there is no match.' 22 | 23 | 24 | def test_extract_no_paths(): 25 | input_paths = extract_paths('') 26 | assert input_paths == [], 'It should return an empty list.' 27 | 28 | 29 | def test_extract_open_path(): 30 | line = ( 31 | 'openat(AT_FDCWD, "/usr/lib/root/tls/x86_64/libcap.so.2", O_RDONLY|O_CLOEXEC) ' 32 | '= -1 ENOENT (No such file or directory)' 33 | ) 34 | assert extract_open_path(line) is None, 'Missing files should not return paths.' 35 | line = 'open(".", O_RDONLY|O_NONBLOCK|O_CLOEXEC|O_DIRECTORY) = 4' 36 | assert extract_open_path(line) is None, 'Opened directories should not return paths.' 37 | line = 'open("/usr/lib/locale/locale-archive", O_RDONLY|O_CLOEXEC) = 4' 38 | assert extract_open_path(line) == '/usr/lib/locale/locale-archive', \ 39 | 'An open() call should return a path.' 40 | line = 'openat(AT_FDCWD, "/usr/lib/libc.so.6", O_RDONLY|O_CLOEXEC) = 4' 41 | assert extract_open_path(line) == '/usr/lib/libc.so.6', \ 42 | 'An openat() call relative to the current directory should return a path.' 43 | 44 | 45 | def test_extract_raw_paths(): 46 | input_paths = [ 47 | '/absolute/path/to/file', 48 | './relative/path', 49 | '/another/absolute/path', 50 | ] 51 | input_paths_with_whitespace = \ 52 | [' ', ''] + [input_paths[0]] + [' '] + input_paths[1:] 53 | input_content = '\n'.join(input_paths_with_whitespace) 54 | extracted_paths = extract_paths(input_content) 55 | assert set(input_paths) == set(extracted_paths), \ 56 | 'The paths should have been extracted without the whitespace.' 57 | 58 | 59 | def test_extract_stat_path(): 60 | line = ( 61 | 'stat("/usr/local/lib/python3.6/encodings/__init__.py", ' 62 | '{st_mode=S_IFREG|0644, st_size=5642, ...}) = 0' 63 | ) 64 | expected_path = '/usr/local/lib/python3.6/encodings/__init__.py' 65 | assert extract_stat_path(line) == expected_path, \ 66 | 'The stat path should be extracted correctly.' 67 | line = ( 68 | 'stat("/usr/local/lib/python3.6/encodings/__init__.abi3.so", 0x7ffc9d6a0160) = -1 ' 69 | 'ENOENT (No such file or directory)' 70 | ) 71 | assert extract_stat_path(line) is None, \ 72 | 'Non-existent files should not be extracted.' 73 | 74 | 75 | def test_extract_strace_paths(): 76 | with open(exodus_strace, 'r') as f: 77 | content = f.read() 78 | extracted_paths = extract_paths(content, existing_only=False) 79 | expected_paths = [ 80 | # `execve()` call 81 | '/home/sangaline/projects/exodus/.env/bin/exodus', 82 | # `openat()` call 83 | '/usr/lib/libpthread.so.0', 84 | # `open()` call 85 | '/usr/lib/gconv/gconv-modules', 86 | ] 87 | 88 | for path in expected_paths: 89 | assert path in extracted_paths, \ 90 | '"%s" should be present in the extracted paths.' % path 91 | 92 | 93 | def test_strip_pid_prefix(): 94 | line = ( 95 | '[pid 655] execve("/usr/bin/musl-gcc", ["/usr/bin/musl-gcc", "-static", "-O3", ' 96 | '"/tmp/exodus-bundle-fqzw_lds.c", "-o", "/tmp/exodus-bundle-3p_c0osh"], [/* 45 vars */] ' 97 | '' 98 | ) 99 | assert strip_pid_prefix(line).startswith('execve('), 'The PID prefix should be stripped.' 100 | -------------------------------------------------------------------------------- /tests/test_launchers.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | import os 3 | import shutil 4 | import stat 5 | import tempfile 6 | from subprocess import PIPE 7 | from subprocess import Popen 8 | 9 | import pytest 10 | 11 | from exodus_bundler import launchers 12 | from exodus_bundler.bundling import create_unpackaged_bundle 13 | from exodus_bundler.launchers import CompilerNotFoundError 14 | from exodus_bundler.launchers import compile_diet 15 | from exodus_bundler.launchers import compile_musl 16 | from exodus_bundler.launchers import construct_bash_launcher 17 | from exodus_bundler.launchers import find_executable 18 | 19 | 20 | parent_directory = os.path.dirname(os.path.realpath(__file__)) 21 | chroot = os.path.join(parent_directory, 'data', 'binaries', 'chroot') 22 | echo_args_glibc_32 = os.path.join(chroot, 'bin', 'echo-args-glibc-32') 23 | fizz_buzz_source_file = os.path.join(parent_directory, 'data', 'binaries', 'fizz-buzz.c') 24 | 25 | 26 | def test_construct_bash_launcher(): 27 | linker, library_path, executable = '../lib/ld-linux.so.2', '../lib/', 'grep' 28 | script_content = construct_bash_launcher(linker=linker, library_path=library_path, 29 | executable=executable) 30 | assert script_content.startswith('#! /bin/bash\n') 31 | assert linker in script_content 32 | assert executable in script_content 33 | 34 | 35 | @pytest.mark.parametrize('compiler', ['diet', 'musl']) 36 | def test_compile(compiler): 37 | with open(fizz_buzz_source_file, 'r') as f: 38 | code = f.read() 39 | compile = compile_diet if compiler == 'diet' else None 40 | compile = compile or (compile_musl if compiler == 'musl' else None) 41 | try: 42 | content = compile(code) 43 | except CompilerNotFoundError: 44 | # We'll that's a bummer, but better to test these when the are available than 45 | # to not test them at all. 46 | return 47 | 48 | f, filename = tempfile.mkstemp() 49 | os.close(f) 50 | with open(filename, 'wb') as f: 51 | f.write(content) 52 | st = os.stat(filename) 53 | os.chmod(f.name, st.st_mode | stat.S_IXUSR) 54 | 55 | process = Popen(f.name, stdout=PIPE, stderr=PIPE) 56 | stdout, stderr = process.communicate() 57 | assert 'FIZZBUZZ' in stdout.decode('utf-8') 58 | assert len(stderr.decode('utf-8')) == 0 59 | 60 | 61 | def test_find_executable(): 62 | original_environment = os.environ.get('PATH') 63 | original_parent_directory = launchers.parent_directory 64 | 65 | root_directory = create_unpackaged_bundle( 66 | rename=[], executables=[echo_args_glibc_32], chroot=chroot) 67 | try: 68 | binary_name = os.path.basename(echo_args_glibc_32) 69 | binary_symlink = os.path.join(root_directory, 'bin', binary_name) 70 | binary_path = os.path.realpath(binary_symlink) 71 | # This is a pretend directory, but it doesn't check. 72 | launchers.parent_directory = os.path.join(os.path.dirname(binary_path), 'somewhere', 'else') 73 | os.environ['PATH'] = os.path.dirname(echo_args_glibc_32) 74 | assert find_executable(binary_name, skip_original_for_testing=True) == binary_path, \ 75 | 'It should have found the binary path "%s".' % binary_path 76 | finally: 77 | launchers.parent_directory = original_parent_directory 78 | os.environ['PATH'] = original_environment 79 | assert root_directory.startswith('/tmp/') 80 | shutil.rmtree(root_directory) 81 | -------------------------------------------------------------------------------- /tests/test_pytest.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | import pytest 3 | 4 | 5 | def test_catching_a_value_error(): 6 | """Temporary test to make sure that we can run tests.""" 7 | with pytest.raises(KeyError): 8 | {}['no-matching-key'] 9 | -------------------------------------------------------------------------------- /tests/test_templating.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | import os 3 | 4 | from exodus_bundler.templating import render_template 5 | from exodus_bundler.templating import render_template_file 6 | 7 | 8 | parent_directory = os.path.dirname(os.path.realpath(__file__)) 9 | data_directory = os.path.join(parent_directory, 'data') 10 | 11 | 12 | def test_render_template(): 13 | template = '{{greeting}}, my name is {{name}}.' 14 | expected = 'Hello, my name is Evan.' 15 | result = render_template(template, greeting='Hello', name='Evan') 16 | assert expected == result 17 | 18 | 19 | def test_render_template_file(): 20 | template_file = os.path.join(data_directory, 'template.txt') 21 | result = render_template_file(template_file, noun='word', location='here') 22 | with open(os.path.join(data_directory, 'template-result.txt'), 'r') as f: 23 | expected_result = f.read() 24 | assert result == expected_result 25 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | envlist = 3 | clean, 4 | check, 5 | {py27,py39}, 6 | report, 7 | 8 | [testenv] 9 | basepython = 10 | py27: {env:TOXPYTHON:python2.7} 11 | {clean,check,report,py39}: {env:TOXPYTHON:python3.9} 12 | setenv = 13 | PYTHONPATH={toxinidir}/tests 14 | PYTHONUNBUFFERED=yes 15 | passenv = 16 | * 17 | usedevelop = false 18 | deps = 19 | pytest 20 | pytest-sugar 21 | pytest-travis-fold 22 | pytest-cov 23 | commands = 24 | {posargs:py.test --cov --cov-report=term-missing -vv} 25 | 26 | [testenv:check] 27 | deps = 28 | docutils 29 | check-manifest 30 | flake8 31 | flake8-commas 32 | flake8-quotes 33 | readme-renderer 34 | pygments 35 | isort 36 | skip_install = true 37 | commands = 38 | flake8 src tests setup.py 39 | isort --verbose --check-only --diff --recursive src tests setup.py 40 | python setup.py check --strict --metadata --restructuredtext 41 | check-manifest {toxinidir} 42 | 43 | [testenv:report] 44 | deps = coverage 45 | skip_install = true 46 | commands = 47 | coverage report 48 | coverage html 49 | 50 | [testenv:clean] 51 | commands = coverage erase 52 | skip_install = true 53 | deps = coverage 54 | 55 | [flake8] 56 | ignore = E128,W504 57 | --------------------------------------------------------------------------------