├── .dockerignore ├── .editorconfig ├── .github └── workflows │ ├── debug.yml │ └── publish.yml ├── .gitignore ├── LICENSE ├── README.md ├── build ├── apt_disable_nvidia_repo.sh ├── apt_install_pkgs_for_dev.sh ├── configure_shell.sh ├── configure_sudo.sh ├── devel_cpu.sh ├── devel_cuda.sh ├── devel_shared.sh ├── fix_run_permission.sh ├── install_direnv.sh ├── install_fixuid.sh ├── install_openssh_server.sh ├── install_python.sh └── install_vscode_server.sh ├── dockerfile ├── devel_cpu ├── devel_cpu_debug └── devel_cuda ├── run ├── apt_set_mirror_aliyun.sh ├── apt_set_mirror_tencent.sh ├── devel_shared_init_post.sh ├── devel_shared_init_pre.sh ├── devel_shared_reentrant.sh ├── run.sh ├── run_devel_cpu_init_post.sh ├── run_devel_cpu_init_pre.sh ├── run_devel_cpu_reentrant.sh ├── run_devel_cuda_init_post.sh ├── run_devel_cuda_init_pre.sh └── run_devel_cuda_reentrant.sh └── tool ├── README.md ├── build_readme.py └── requirements.txt /.dockerignore: -------------------------------------------------------------------------------- 1 | **/.git 2 | **/.gitignore 3 | **/*.md 4 | -------------------------------------------------------------------------------- /.editorconfig: -------------------------------------------------------------------------------- 1 | # http://editorconfig.org 2 | 3 | root = true 4 | 5 | [*] 6 | indent_style = space 7 | indent_size = 4 8 | trim_trailing_whitespace = true 9 | insert_final_newline = true 10 | charset = utf-8 11 | end_of_line = lf 12 | 13 | [*.yml] 14 | indent_style = space 15 | indent_size = 2 16 | trim_trailing_whitespace = true 17 | insert_final_newline = true 18 | charset = utf-8 19 | end_of_line = lf 20 | 21 | [LICENSE] 22 | insert_final_newline = false 23 | 24 | [Makefile] 25 | indent_style = tab 26 | -------------------------------------------------------------------------------- /.github/workflows/debug.yml: -------------------------------------------------------------------------------- 1 | name: debug 2 | on: 3 | push: 4 | branches: 5 | - master 6 | paths-ignore: 7 | - "README.md" 8 | - ".*" 9 | - "tool/**" 10 | jobs: 11 | build-cpu: 12 | runs-on: ubuntu-latest 13 | strategy: 14 | matrix: 15 | ubuntu_version: ["20.04"] 16 | python_version: ["3.8"] 17 | steps: 18 | - uses: actions/checkout@v3 19 | 20 | - name: Docker build & push 21 | run: | 22 | export TAG=debug-devel-cpu-ubuntu${{ matrix.ubuntu_version }}-python${{ matrix.python_version }} 23 | 24 | docker build \ 25 | --no-cache \ 26 | --progress=plain \ 27 | --build-arg UBUNTU_VERSION=${{ matrix.ubuntu_version }} \ 28 | --build-arg PYTHON_VERSION=${{ matrix.python_version }} \ 29 | -t wden/wden:"$TAG" \ 30 | -f dockerfile/devel_cpu \ 31 | . 32 | 33 | docker login \ 34 | --username="${{ secrets.WDEN_HUAWEICLOUD_AREA }}@${{ secrets.WDEN_HUAWEICLOUD_AK }}" \ 35 | --password=${{ secrets.WDEN_HUAWEICLOUD_TOKEN }} \ 36 | "swr.${{ secrets.WDEN_HUAWEICLOUD_AREA }}.myhuaweicloud.com" 37 | docker tag wden/wden:"$TAG" swr."${{ secrets.WDEN_HUAWEICLOUD_AREA }}".myhuaweicloud.com/wden/wden:"$TAG" 38 | docker push swr."${{ secrets.WDEN_HUAWEICLOUD_AREA }}".myhuaweicloud.com/wden/wden:"$TAG" 39 | -------------------------------------------------------------------------------- /.github/workflows/publish.yml: -------------------------------------------------------------------------------- 1 | name: publish 2 | on: 3 | schedule: 4 | - cron: "0 0 * * *" 5 | workflow_dispatch: 6 | jobs: 7 | build-cuda: 8 | runs-on: ubuntu-latest 9 | strategy: 10 | matrix: 11 | # Follows the pytorch supported CUDA versions. 12 | cuda_version: 13 | - "11.6.2" 14 | - "11.7.1" 15 | - "11.8.0" 16 | cudnn_version: 17 | - "8" 18 | ubuntu_version: 19 | - "20.04" 20 | python_version: 21 | - "3.8" 22 | - "3.9" 23 | - "3.10" 24 | - "3.11" 25 | - "3.12" 26 | include: 27 | - cuda_version: "11.1.1" 28 | cudnn_version: "8" 29 | ubuntu_version: "20.04" 30 | python_version: "3.8" 31 | steps: 32 | - uses: actions/checkout@v3 33 | - name: Docker build & push 34 | run: | 35 | export TAG=devel-cuda${{ matrix.cuda_version }}-cudnn${{ matrix.cudnn_version }}-ubuntu${{ matrix.ubuntu_version }}-python${{ matrix.python_version }} 36 | 37 | docker build \ 38 | --no-cache \ 39 | --progress=plain \ 40 | --build-arg CUDA_VERSION=${{ matrix.cuda_version }} \ 41 | --build-arg CUDNN_VERSION=${{ matrix.cudnn_version }} \ 42 | --build-arg UBUNTU_VERSION=${{ matrix.ubuntu_version }} \ 43 | --build-arg PYTHON_VERSION=${{ matrix.python_version }} \ 44 | -t wden/wden:"$TAG" \ 45 | -f dockerfile/devel_cuda \ 46 | . 47 | 48 | docker login --username=wden --password=${{ secrets.WDEN_DOCKER_PASSWORD }} 49 | docker push wden/wden:"$TAG" 50 | 51 | docker login \ 52 | --username="${{ secrets.WDEN_HUAWEICLOUD_AREA }}@${{ secrets.WDEN_HUAWEICLOUD_AK }}" \ 53 | --password=${{ secrets.WDEN_HUAWEICLOUD_TOKEN }} \ 54 | "swr.${{ secrets.WDEN_HUAWEICLOUD_AREA }}.myhuaweicloud.com" 55 | docker tag wden/wden:"$TAG" swr."${{ secrets.WDEN_HUAWEICLOUD_AREA }}".myhuaweicloud.com/wden/wden:"$TAG" 56 | docker push swr."${{ secrets.WDEN_HUAWEICLOUD_AREA }}".myhuaweicloud.com/wden/wden:"$TAG" 57 | build-cpu: 58 | runs-on: ubuntu-latest 59 | strategy: 60 | matrix: 61 | ubuntu_version: 62 | - "20.04" 63 | python_version: 64 | - "3.8" 65 | - "3.9" 66 | - "3.10" 67 | - "3.11" 68 | steps: 69 | - uses: actions/checkout@v3 70 | 71 | - name: Docker build & push 72 | run: | 73 | export TAG=devel-cpu-ubuntu${{ matrix.ubuntu_version }}-python${{ matrix.python_version }} 74 | 75 | docker build \ 76 | --no-cache \ 77 | --progress=plain \ 78 | --build-arg UBUNTU_VERSION=${{ matrix.ubuntu_version }} \ 79 | --build-arg PYTHON_VERSION=${{ matrix.python_version }} \ 80 | -t wden/wden:"$TAG" \ 81 | -f dockerfile/devel_cpu \ 82 | . 83 | 84 | docker login --username=wden --password=${{ secrets.WDEN_DOCKER_PASSWORD }} 85 | docker push wden/wden:"$TAG" 86 | 87 | docker login \ 88 | --username="${{ secrets.WDEN_HUAWEICLOUD_AREA }}@${{ secrets.WDEN_HUAWEICLOUD_AK }}" \ 89 | --password=${{ secrets.WDEN_HUAWEICLOUD_TOKEN }} \ 90 | "swr.${{ secrets.WDEN_HUAWEICLOUD_AREA }}.myhuaweicloud.com" 91 | docker tag wden/wden:"$TAG" swr."${{ secrets.WDEN_HUAWEICLOUD_AREA }}".myhuaweicloud.com/wden/wden:"$TAG" 92 | docker push swr."${{ secrets.WDEN_HUAWEICLOUD_AREA }}".myhuaweicloud.com/wden/wden:"$TAG" 93 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | env/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | *.egg-info/ 24 | .installed.cfg 25 | *.egg 26 | 27 | # PyInstaller 28 | # Usually these files are written by a python script from a template 29 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 30 | *.manifest 31 | *.spec 32 | 33 | # Installer logs 34 | pip-log.txt 35 | pip-delete-this-directory.txt 36 | 37 | # Unit test / coverage reports 38 | htmlcov/ 39 | .tox/ 40 | .coverage 41 | .coverage.* 42 | .cache 43 | nosetests.xml 44 | coverage.xml 45 | *.cover 46 | .hypothesis/ 47 | .pytest_cache/ 48 | 49 | # Translations 50 | *.mo 51 | *.pot 52 | 53 | # Django stuff: 54 | *.log 55 | local_settings.py 56 | 57 | # Flask stuff: 58 | instance/ 59 | .webassets-cache 60 | 61 | # Scrapy stuff: 62 | .scrapy 63 | 64 | # Sphinx documentation 65 | docs/_build/ 66 | 67 | # PyBuilder 68 | target/ 69 | 70 | # Jupyter Notebook 71 | .ipynb_checkpoints 72 | 73 | # pyenv 74 | .python-version 75 | 76 | # celery beat schedule file 77 | celerybeat-schedule 78 | 79 | # SageMath parsed files 80 | *.sage.py 81 | 82 | # dotenv 83 | .env 84 | 85 | # virtualenv 86 | .venv 87 | venv/ 88 | ENV/ 89 | 90 | # Spyder project settings 91 | .spyderproject 92 | .spyproject 93 | 94 | # Rope project settings 95 | .ropeproject 96 | 97 | # mkdocs documentation 98 | /site 99 | 100 | # mypy 101 | .mypy_cache/ 102 | 103 | # direnv 104 | .envrc 105 | 106 | .vscode/ 107 | data/ 108 | .*DS_Store 109 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | 203 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | 2 | ## Table of Contents 3 | * [Overview](#Overview) 4 | * [Images](#Images) 5 | * [Hosted in Docker Hub](#Hosted-in-Docker-Hub) 6 | * [Hosted in Huawei Cloud](#Hosted-in-Huawei-Cloud) 7 | * [Usage](#Usage) 8 | * [UID and GID forwarding](#UID-and-GID-forwarding) 9 | * [Change the default cd folder](#Change-the-default-cd-folder) 10 | * [HTTP proxy setup](#HTTP-proxy-setup) 11 | * [SSH agent forwarding in macOS](#SSH-agent-forwarding-in-macOS) 12 | * [SSH agent forwarding in Linux](#SSH-agent-forwarding-in-Linux) 13 | * [SSH proxy](#SSH-proxy) 14 | * [SSH login to container](#SSH-login-to-container) 15 | * [Git config forwarding](#Git-config-forwarding) 16 | * [Bash history forwarding](#Bash-history-forwarding) 17 | * [Run as daemon](#Run-as-daemon) 18 | * [Use APT mirror sites](#Use-APT-mirror-sites) 19 | * [Upgrade `pip` to the latest version](#Upgrade-`pip`-to-the-latest-version) 20 | * [Use PyPI mirror sites](#Use-PyPI-mirror-sites) 21 | * [Run customized scirpts](#Run-customized-scirpts) 22 | 23 | 24 | ## Overview 25 | 26 | Goals 27 | 28 | - Attempt to provide an out-of-the-box experience to use docker container as the development environment, especially for Python developer, ML/DL engineer & researcher. 29 | - Setup a project boilerplate that is easy to understand. User with bash scripting and docker experience should be able to modify or extend the current codebase within 10 mins. While this project ships with the Python toolchain, most of the setup is not bound to Python, but instead is related to development experience (see the [Usage](#Usage) session). Hence this project could be extend to support variant scope development. 30 | 31 | Section [Images](#Images) documents the out-of-the-box prebuilt images. 32 | Section [Usage](#Usage) documents the typical use cases with examples. 33 | 34 | ## Images 35 | 36 | ### Hosted in Docker Hub 37 | 38 | Supported constructs are as followed: 39 | 40 | 41 | | Construct | `docker pull` command | 42 | | --------- | -------------------- | 43 | | CUDA=11.6.2, CUDNN=8, PYTHON=3.10, UBUNTU=20.04 | `docker pull wden/wden:devel-cuda11.6.2-cudnn8-ubuntu20.04-python3.10` | 44 | | CUDA=11.6.2, CUDNN=8, PYTHON=3.11, UBUNTU=20.04 | `docker pull wden/wden:devel-cuda11.6.2-cudnn8-ubuntu20.04-python3.11` | 45 | | CUDA=11.6.2, CUDNN=8, PYTHON=3.12, UBUNTU=20.04 | `docker pull wden/wden:devel-cuda11.6.2-cudnn8-ubuntu20.04-python3.12` | 46 | | CUDA=11.6.2, CUDNN=8, PYTHON=3.8, UBUNTU=20.04 | `docker pull wden/wden:devel-cuda11.6.2-cudnn8-ubuntu20.04-python3.8` | 47 | | CUDA=11.6.2, CUDNN=8, PYTHON=3.9, UBUNTU=20.04 | `docker pull wden/wden:devel-cuda11.6.2-cudnn8-ubuntu20.04-python3.9` | 48 | | CUDA=11.7.1, CUDNN=8, PYTHON=3.10, UBUNTU=20.04 | `docker pull wden/wden:devel-cuda11.7.1-cudnn8-ubuntu20.04-python3.10` | 49 | | CUDA=11.7.1, CUDNN=8, PYTHON=3.11, UBUNTU=20.04 | `docker pull wden/wden:devel-cuda11.7.1-cudnn8-ubuntu20.04-python3.11` | 50 | | CUDA=11.7.1, CUDNN=8, PYTHON=3.12, UBUNTU=20.04 | `docker pull wden/wden:devel-cuda11.7.1-cudnn8-ubuntu20.04-python3.12` | 51 | | CUDA=11.7.1, CUDNN=8, PYTHON=3.8, UBUNTU=20.04 | `docker pull wden/wden:devel-cuda11.7.1-cudnn8-ubuntu20.04-python3.8` | 52 | | CUDA=11.7.1, CUDNN=8, PYTHON=3.9, UBUNTU=20.04 | `docker pull wden/wden:devel-cuda11.7.1-cudnn8-ubuntu20.04-python3.9` | 53 | | CUDA=11.8.0, CUDNN=8, PYTHON=3.10, UBUNTU=20.04 | `docker pull wden/wden:devel-cuda11.8.0-cudnn8-ubuntu20.04-python3.10` | 54 | | CUDA=11.8.0, CUDNN=8, PYTHON=3.11, UBUNTU=20.04 | `docker pull wden/wden:devel-cuda11.8.0-cudnn8-ubuntu20.04-python3.11` | 55 | | CUDA=11.8.0, CUDNN=8, PYTHON=3.12, UBUNTU=20.04 | `docker pull wden/wden:devel-cuda11.8.0-cudnn8-ubuntu20.04-python3.12` | 56 | | CUDA=11.8.0, CUDNN=8, PYTHON=3.8, UBUNTU=20.04 | `docker pull wden/wden:devel-cuda11.8.0-cudnn8-ubuntu20.04-python3.8` | 57 | | CUDA=11.8.0, CUDNN=8, PYTHON=3.9, UBUNTU=20.04 | `docker pull wden/wden:devel-cuda11.8.0-cudnn8-ubuntu20.04-python3.9` | 58 | | PYTHON=3.10, UBUNTU=20.04 | `docker pull wden/wden:devel-cpu-ubuntu20.04-python3.10` | 59 | | PYTHON=3.11, UBUNTU=20.04 | `docker pull wden/wden:devel-cpu-ubuntu20.04-python3.11` | 60 | | PYTHON=3.8, UBUNTU=20.04 | `docker pull wden/wden:devel-cpu-ubuntu20.04-python3.8` | 61 | | PYTHON=3.9, UBUNTU=20.04 | `docker pull wden/wden:devel-cpu-ubuntu20.04-python3.9` | 62 | 63 | 64 | ### Hosted in Huawei Cloud 65 | 66 | As a known issue, pulling from docker hub is intolerably slow for users in China. To speed up, images are also hosted in Huawei Cloud: 67 | 68 | 69 | | Construct | `docker pull` command | 70 | | --------- | -------------------- | 71 | | CUDA=11.6.2, CUDNN=8, PYTHON=3.10, UBUNTU=20.04 | `docker pull swr.cn-east-3.myhuaweicloud.com/wden/wden:devel-cuda11.6.2-cudnn8-ubuntu20.04-python3.10 && docker tag swr.cn-east-3.myhuaweicloud.com/wden/wden:devel-cuda11.6.2-cudnn8-ubuntu20.04-python3.10 wden/wden:devel-cuda11.6.2-cudnn8-ubuntu20.04-python3.10 ` | 72 | | CUDA=11.6.2, CUDNN=8, PYTHON=3.11, UBUNTU=20.04 | `docker pull swr.cn-east-3.myhuaweicloud.com/wden/wden:devel-cuda11.6.2-cudnn8-ubuntu20.04-python3.11 && docker tag swr.cn-east-3.myhuaweicloud.com/wden/wden:devel-cuda11.6.2-cudnn8-ubuntu20.04-python3.11 wden/wden:devel-cuda11.6.2-cudnn8-ubuntu20.04-python3.11 ` | 73 | | CUDA=11.6.2, CUDNN=8, PYTHON=3.12, UBUNTU=20.04 | `docker pull swr.cn-east-3.myhuaweicloud.com/wden/wden:devel-cuda11.6.2-cudnn8-ubuntu20.04-python3.12 && docker tag swr.cn-east-3.myhuaweicloud.com/wden/wden:devel-cuda11.6.2-cudnn8-ubuntu20.04-python3.12 wden/wden:devel-cuda11.6.2-cudnn8-ubuntu20.04-python3.12 ` | 74 | | CUDA=11.6.2, CUDNN=8, PYTHON=3.8, UBUNTU=20.04 | `docker pull swr.cn-east-3.myhuaweicloud.com/wden/wden:devel-cuda11.6.2-cudnn8-ubuntu20.04-python3.8 && docker tag swr.cn-east-3.myhuaweicloud.com/wden/wden:devel-cuda11.6.2-cudnn8-ubuntu20.04-python3.8 wden/wden:devel-cuda11.6.2-cudnn8-ubuntu20.04-python3.8 ` | 75 | | CUDA=11.6.2, CUDNN=8, PYTHON=3.9, UBUNTU=20.04 | `docker pull swr.cn-east-3.myhuaweicloud.com/wden/wden:devel-cuda11.6.2-cudnn8-ubuntu20.04-python3.9 && docker tag swr.cn-east-3.myhuaweicloud.com/wden/wden:devel-cuda11.6.2-cudnn8-ubuntu20.04-python3.9 wden/wden:devel-cuda11.6.2-cudnn8-ubuntu20.04-python3.9 ` | 76 | | CUDA=11.7.1, CUDNN=8, PYTHON=3.10, UBUNTU=20.04 | `docker pull swr.cn-east-3.myhuaweicloud.com/wden/wden:devel-cuda11.7.1-cudnn8-ubuntu20.04-python3.10 && docker tag swr.cn-east-3.myhuaweicloud.com/wden/wden:devel-cuda11.7.1-cudnn8-ubuntu20.04-python3.10 wden/wden:devel-cuda11.7.1-cudnn8-ubuntu20.04-python3.10 ` | 77 | | CUDA=11.7.1, CUDNN=8, PYTHON=3.11, UBUNTU=20.04 | `docker pull swr.cn-east-3.myhuaweicloud.com/wden/wden:devel-cuda11.7.1-cudnn8-ubuntu20.04-python3.11 && docker tag swr.cn-east-3.myhuaweicloud.com/wden/wden:devel-cuda11.7.1-cudnn8-ubuntu20.04-python3.11 wden/wden:devel-cuda11.7.1-cudnn8-ubuntu20.04-python3.11 ` | 78 | | CUDA=11.7.1, CUDNN=8, PYTHON=3.12, UBUNTU=20.04 | `docker pull swr.cn-east-3.myhuaweicloud.com/wden/wden:devel-cuda11.7.1-cudnn8-ubuntu20.04-python3.12 && docker tag swr.cn-east-3.myhuaweicloud.com/wden/wden:devel-cuda11.7.1-cudnn8-ubuntu20.04-python3.12 wden/wden:devel-cuda11.7.1-cudnn8-ubuntu20.04-python3.12 ` | 79 | | CUDA=11.7.1, CUDNN=8, PYTHON=3.8, UBUNTU=20.04 | `docker pull swr.cn-east-3.myhuaweicloud.com/wden/wden:devel-cuda11.7.1-cudnn8-ubuntu20.04-python3.8 && docker tag swr.cn-east-3.myhuaweicloud.com/wden/wden:devel-cuda11.7.1-cudnn8-ubuntu20.04-python3.8 wden/wden:devel-cuda11.7.1-cudnn8-ubuntu20.04-python3.8 ` | 80 | | CUDA=11.7.1, CUDNN=8, PYTHON=3.9, UBUNTU=20.04 | `docker pull swr.cn-east-3.myhuaweicloud.com/wden/wden:devel-cuda11.7.1-cudnn8-ubuntu20.04-python3.9 && docker tag swr.cn-east-3.myhuaweicloud.com/wden/wden:devel-cuda11.7.1-cudnn8-ubuntu20.04-python3.9 wden/wden:devel-cuda11.7.1-cudnn8-ubuntu20.04-python3.9 ` | 81 | | CUDA=11.8.0, CUDNN=8, PYTHON=3.10, UBUNTU=20.04 | `docker pull swr.cn-east-3.myhuaweicloud.com/wden/wden:devel-cuda11.8.0-cudnn8-ubuntu20.04-python3.10 && docker tag swr.cn-east-3.myhuaweicloud.com/wden/wden:devel-cuda11.8.0-cudnn8-ubuntu20.04-python3.10 wden/wden:devel-cuda11.8.0-cudnn8-ubuntu20.04-python3.10 ` | 82 | | CUDA=11.8.0, CUDNN=8, PYTHON=3.11, UBUNTU=20.04 | `docker pull swr.cn-east-3.myhuaweicloud.com/wden/wden:devel-cuda11.8.0-cudnn8-ubuntu20.04-python3.11 && docker tag swr.cn-east-3.myhuaweicloud.com/wden/wden:devel-cuda11.8.0-cudnn8-ubuntu20.04-python3.11 wden/wden:devel-cuda11.8.0-cudnn8-ubuntu20.04-python3.11 ` | 83 | | CUDA=11.8.0, CUDNN=8, PYTHON=3.12, UBUNTU=20.04 | `docker pull swr.cn-east-3.myhuaweicloud.com/wden/wden:devel-cuda11.8.0-cudnn8-ubuntu20.04-python3.12 && docker tag swr.cn-east-3.myhuaweicloud.com/wden/wden:devel-cuda11.8.0-cudnn8-ubuntu20.04-python3.12 wden/wden:devel-cuda11.8.0-cudnn8-ubuntu20.04-python3.12 ` | 84 | | CUDA=11.8.0, CUDNN=8, PYTHON=3.8, UBUNTU=20.04 | `docker pull swr.cn-east-3.myhuaweicloud.com/wden/wden:devel-cuda11.8.0-cudnn8-ubuntu20.04-python3.8 && docker tag swr.cn-east-3.myhuaweicloud.com/wden/wden:devel-cuda11.8.0-cudnn8-ubuntu20.04-python3.8 wden/wden:devel-cuda11.8.0-cudnn8-ubuntu20.04-python3.8 ` | 85 | | CUDA=11.8.0, CUDNN=8, PYTHON=3.9, UBUNTU=20.04 | `docker pull swr.cn-east-3.myhuaweicloud.com/wden/wden:devel-cuda11.8.0-cudnn8-ubuntu20.04-python3.9 && docker tag swr.cn-east-3.myhuaweicloud.com/wden/wden:devel-cuda11.8.0-cudnn8-ubuntu20.04-python3.9 wden/wden:devel-cuda11.8.0-cudnn8-ubuntu20.04-python3.9 ` | 86 | | PYTHON=3.10, UBUNTU=20.04 | `docker pull swr.cn-east-3.myhuaweicloud.com/wden/wden:devel-cpu-ubuntu20.04-python3.10 && docker tag swr.cn-east-3.myhuaweicloud.com/wden/wden:devel-cpu-ubuntu20.04-python3.10 wden/wden:devel-cpu-ubuntu20.04-python3.10 ` | 87 | | PYTHON=3.11, UBUNTU=20.04 | `docker pull swr.cn-east-3.myhuaweicloud.com/wden/wden:devel-cpu-ubuntu20.04-python3.11 && docker tag swr.cn-east-3.myhuaweicloud.com/wden/wden:devel-cpu-ubuntu20.04-python3.11 wden/wden:devel-cpu-ubuntu20.04-python3.11 ` | 88 | | PYTHON=3.8, UBUNTU=20.04 | `docker pull swr.cn-east-3.myhuaweicloud.com/wden/wden:devel-cpu-ubuntu20.04-python3.8 && docker tag swr.cn-east-3.myhuaweicloud.com/wden/wden:devel-cpu-ubuntu20.04-python3.8 wden/wden:devel-cpu-ubuntu20.04-python3.8 ` | 89 | | PYTHON=3.9, UBUNTU=20.04 | `docker pull swr.cn-east-3.myhuaweicloud.com/wden/wden:devel-cpu-ubuntu20.04-python3.9 && docker tag swr.cn-east-3.myhuaweicloud.com/wden/wden:devel-cpu-ubuntu20.04-python3.9 wden/wden:devel-cpu-ubuntu20.04-python3.9 ` | 90 | 91 | 92 | ## Usage 93 | 94 | ### UID and GID forwarding 95 | 96 | ```bash 97 | ##################### 98 | # IN THE HOST SHELL # 99 | ##################### 100 | whoami 101 | << OUTPUT 102 | huntzhan 103 | OUTPUT 104 | 105 | echo "$(id -u):$(id -g)" 106 | << OUTPUT 107 | 501:20 108 | OUTPUT 109 | 110 | # Pass UID/GID to --user and mount a folder to container 111 | docker run \ 112 | --rm -it \ 113 | --user "$(id -u):$(id -g)" \ 114 | -v "$(pwd)":/data \ 115 | wden/wden:devel-cpu-ubuntu20.04-python3.8 116 | 117 | ########################## 118 | # IN THE CONTAINER SHELL # 119 | ########################## 120 | whoami 121 | << OUTPUT 122 | wden 123 | OUTPUT 124 | 125 | # Same UID/GID as host, thanks to https://github.com/boxboat/fixuid/ 126 | echo "$(id -u):$(id -g)" 127 | << OUTPUT 128 | 501:20 129 | OUTPUT 130 | 131 | cd /data 132 | touch foobar 133 | ls -l | grep foobar 134 | << OUTPUT 135 | -rw-r--r-- 1 wden dialout 0 Feb 5 14:24 foobar 136 | OUTPUT 137 | 138 | exit 139 | 140 | ##################### 141 | # IN THE HOST SHELL # 142 | ##################### 143 | ls -l | grep foobar 144 | << OUTPUT 145 | -rw-r--r-- 1 huntzhan staff 0 Feb 5 22:24 foobar 146 | OUTPUT 147 | ``` 148 | 149 | ### Change the default cd folder 150 | 151 | Env: 152 | 153 | * `CD_DEFAULT_FOLDER` : If set, initialize the shell and change the `cd` default folder to this path. 154 | 155 | ```bash 156 | ##################### 157 | # IN THE HOST SHELL # 158 | ##################### 159 | docker run \ 160 | --rm -it \ 161 | -e CD_DEFAULT_FOLDER=/data \ 162 | -v "$(pwd)":/data \ 163 | wden/wden:devel-cpu-ubuntu20.04-python3.8 164 | 165 | ########################## 166 | # IN THE CONTAINER SHELL # 167 | ########################## 168 | pwd 169 | << OUTPUT 170 | /data 171 | OUTPUT 172 | 173 | alias cd 174 | << OUTPUT 175 | alias cd='HOME='\''/data'\'' cd' 176 | OUTPUT 177 | ``` 178 | 179 | ### HTTP proxy setup 180 | 181 | Env: 182 | 183 | * `PROPAGATE_HTTPS_PROXY`: If set, will propagate `HTTPS_PROXY` to `HTTP_PROXY`, `https_proxy`, and `http_proxy` 184 | 185 | ```bash 186 | ##################### 187 | # IN THE HOST SHELL # 188 | ##################### 189 | docker run \ 190 | --rm -it \ 191 | -e PROPAGATE_HTTPS_PROXY=1 \ 192 | --add-host host.docker.internal:host-gateway \ 193 | -e HTTPS_PROXY='http://host.docker.internal:8889' \ 194 | wden/wden:devel-cpu-ubuntu20.04-python3.8 195 | 196 | ########################## 197 | # IN THE CONTAINER SHELL # 198 | ########################## 199 | cat << EOF 200 | ${HTTPS_PROXY} 201 | ${HTTP_PROXY} 202 | ${https_proxy} 203 | ${http_proxy} 204 | EOF 205 | 206 | << OUTPUT 207 | http://host.docker.internal:8889 208 | http://host.docker.internal:8889 209 | http://host.docker.internal:8889 210 | http://host.docker.internal:8889 211 | OUTPUT 212 | ``` 213 | 214 | ### SSH agent forwarding in macOS 215 | 216 | ```bash 217 | ##################### 218 | # IN THE HOST SHELL # 219 | ##################### 220 | # Store passphrases in your keychain. 221 | ssh-add -K /path/to/private-key-file 222 | 223 | # Add all identities stored in your keychain. 224 | # Make sure key identity is added before launching container. 225 | ssh-add -A 226 | 227 | # 1. The option "shared" in volume change the file permission of 228 | # /run/ssh-auth.sock to 777 to make the file accessible by non-root user. 229 | # 2. If SSH_AUTH_SOCK is set, the setup for SSH agent forwarding will be triggered. 230 | docker run \ 231 | --rm -it \ 232 | -v /run/host-services/ssh-auth.sock:/run/ssh-auth.sock:shared \ 233 | -e SSH_AUTH_SOCK="/run/ssh-auth.sock" \ 234 | wden/wden:devel-cpu-ubuntu20.04-python3.8 235 | 236 | 237 | ########################## 238 | # IN THE CONTAINER SHELL # 239 | ########################## 240 | # Test connection (suppose you have added the github private key). 241 | ssh -T git@github.com 242 | << OUTPUT 243 | Warning: Permanently added 'github.com,13.250.177.223' (RSA) to the list of known hosts. 244 | Hi huntzhan! You've successfully authenticated, but GitHub does not provide shell access. 245 | OUTPUT 246 | 247 | # ssh alias is set to bypass host checking. 248 | alias ssh 249 | << OUTPUT 250 | alias ssh='ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no' 251 | OUTPUT 252 | 253 | # Same here, to faciliate git operations. 254 | echo $GIT_SSH_COMMAND 255 | << OUTPUT 256 | ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no 257 | OUTPUT 258 | ``` 259 | 260 | ### SSH agent forwarding in Linux 261 | 262 | ```bash 263 | ##################### 264 | # IN THE HOST SHELL # 265 | ##################### 266 | # Make sure ssh-agent is up. 267 | echo $SSH_AUTH_SOCK 268 | << OUTPUT 269 | /tmp/ssh-adpxy7vPDaj7/agent.15833 270 | OUTPUT 271 | 272 | # Mount the socket file to container. 273 | docker run \ 274 | --rm -it \ 275 | -v "$SSH_AUTH_SOCK":/run/ssh-auth.sock:shared \ 276 | -e SSH_AUTH_SOCK="/run/ssh-auth.sock" \ 277 | wden/wden:devel-cpu-ubuntu20.04-python3.8 278 | 279 | 280 | ########################## 281 | # IN THE CONTAINER SHELL # 282 | ########################## 283 | # Same as 'SSH agent forwarding in macOS' 284 | ``` 285 | 286 | ### SSH proxy 287 | 288 | Env: 289 | 290 | * `SSH_SOCKS5_PROXY`: Should be formatted as `:`. If set, use such socks5 proxy in ssh connection. 291 | 292 | ```bash 293 | ##################### 294 | # IN THE HOST SHELL # 295 | ##################### 296 | # NOTE: 297 | # 1. SSH_AUTH_SOCK is not necessary. 298 | # 2. `host.docker.internal` refers to the hostname of the macOS host. 299 | docker run \ 300 | --rm -it \ 301 | -v /run/host-services/ssh-auth.sock:/run/ssh-auth.sock:shared \ 302 | -e SSH_AUTH_SOCK="/run/ssh-auth.sock" \ 303 | --add-host host.docker.internal:host-gateway \ 304 | -e SSH_SOCKS5_PROXY="host.docker.internal:1089" \ 305 | wden/wden:devel-cpu-ubuntu20.04-python3.8 306 | 307 | ########################## 308 | # IN THE CONTAINER SHELL # 309 | ########################## 310 | alias ssh 311 | << OUTPUT 312 | alias ssh='ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o ProxyCommand='\''ncat --proxy-type socks5 --proxy host.docker.internal:1089 %h %p'\''' 313 | OUTPUT 314 | 315 | echo $GIT_SSH_COMMAND 316 | << OUTPUT 317 | ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o ProxyCommand='ncat --proxy-type socks5 --proxy host.docker.internal:1089 %h %p' 318 | OUTPUT 319 | 320 | ssh -T git@github.com 321 | << OUTPUT 322 | Warning: Permanently added 'github.com' (RSA) to the list of known hosts. 323 | Hi huntzhan! You've successfully authenticated, but GitHub does not provide shell access. 324 | OUTPUT 325 | ``` 326 | 327 | ### SSH login to container 328 | 329 | Env: 330 | 331 | * `SSHD_AUTHORIZED_KEYS`: By default, [ssh_wden_rsa_key](https://github.com/vkit-x/wden-ssh-key/blob/master/ssh_wden_rsa_key.pub) has been setup. If you are conserned about the security, set this env to overwrite the `authorized_keys` file. 332 | * `SSHD_PORT`: If set, start the sshd service and bind to this port. 333 | * `DISABLE_SCREEN_DAEMON`: Disable the screen session for remote login. 334 | 335 | ```bash 336 | ##################### 337 | # IN THE HOST SHELL # 338 | ##################### 339 | docker run \ 340 | --rm -it \ 341 | -e SSHD_PORT=3333 \ 342 | wden/wden:devel-cpu-ubuntu20.04-python3.8 343 | 344 | ######################### 345 | # IN ANOTHER HOST SHELL # 346 | ######################### 347 | ssh wden@localhost \ 348 | -p 3333 \ 349 | -o IdentitiesOnly=yes \ 350 | -o IdentityFile=/path/to/wden-ssh-key/ssh_wden_rsa_key 351 | ``` 352 | 353 | Vscode `Remote - SSH` is well supported! 354 | 355 | ### Git config forwarding 356 | 357 | ```bash 358 | ##################### 359 | # IN THE HOST SHELL # 360 | ##################### 361 | docker run \ 362 | --rm -it \ 363 | -v "$HOME"/.gitconfig:/etc/gitconfig:ro \ 364 | wden/wden:devel-cpu-ubuntu20.04-python3.8 365 | 366 | ########################## 367 | # IN THE CONTAINER SHELL # 368 | ########################## 369 | git config --list | cat 370 | << OUTPUT 371 | user.name=Hunt Zhan 372 | user.email=huntzhan.dev@gmail.com 373 | credential.helper=osxkeychain 374 | OUTPUT 375 | ``` 376 | 377 | ### Bash history forwarding 378 | 379 | ```bash 380 | ##################### 381 | # IN THE HOST SHELL # 382 | ##################### 383 | # "$HOME"/.bash_history will be forwarded to and changed by the container. 384 | # https://ss64.com/bash/history.html 385 | # NOTE: backup ~/.bash_history if you want to forward it. 386 | docker run \ 387 | --rm -it \ 388 | -v /path/to/your/bash_history:/run/.bash_history \ 389 | wden/wden:devel-cpu-ubuntu20.04-python3.8 390 | 391 | ########################## 392 | # IN THE CONTAINER SHELL # 393 | ########################## 394 | history 395 | << OUTPUT 396 | 397 | OUTPUT 398 | ``` 399 | 400 | ### Run as daemon 401 | 402 | ```bash 403 | ##################### 404 | # IN THE HOST SHELL # 405 | ##################### 406 | # NOTE: `-it` must be added. 407 | docker run \ 408 | -d -it \ 409 | wden/wden:devel-cpu-ubuntu20.04-python3.8 410 | ``` 411 | 412 | ### Use APT mirror sites 413 | 414 | Env: 415 | 416 | * `APT_SET_MIRROR_TENCENT` : If set, switch to use [Tencent's mirror](https://mirrors.cloud.tencent.com/help/ubuntu.html). 417 | * `APT_SET_MIRROR_ALIYUN` : If set, switch to use [Aliyun's mirror](https://developer.aliyun.com/mirror/ubuntu). 418 | 419 | ```bash 420 | ##################### 421 | # IN THE HOST SHELL # 422 | ##################### 423 | docker run \ 424 | --rm -it \ 425 | -e APT_SET_MIRROR_TENCENT=1 \ 426 | wden/wden:devel-cpu-ubuntu20.04-python3.8 427 | 428 | ########################## 429 | # IN THE CONTAINER SHELL # 430 | ########################## 431 | << OUTPUT 432 | Run apt_set_mirror_tencent.sh 433 | Get:1 http://mirrors.cloud.tencent.com/ubuntu bionic InRelease [242 kB] 434 | Get:2 http://mirrors.cloud.tencent.com/ubuntu bionic-security InRelease [88.7 kB] 435 | Hit:3 http://ppa.launchpad.net/deadsnakes/ppa/ubuntu bionic InRelease 436 | Get:4 http://mirrors.cloud.tencent.com/ubuntu bionic-updates InRelease [88.7 kB] 437 | Get:5 http://mirrors.cloud.tencent.com/ubuntu bionic/universe Sources [11.5 MB] 438 | ... 439 | OUTPUT 440 | ``` 441 | 442 | ### Upgrade `pip` to the latest version 443 | 444 | Env: 445 | 446 | * `PIP_UPGRADE_TO_LATEST`: If set, `pip` will be upgraded to the latest version. 447 | 448 | ### Use PyPI mirror sites 449 | 450 | Env: 451 | 452 | * `PIP_SET_INDEX_TENCENT`: Use Tencent's PyPI index. 453 | * `PIP_SET_INDEX_ALIYUN` : Use Aliyun's PyPI index. 454 | 455 | ```bash 456 | ##################### 457 | # IN THE HOST SHELL # 458 | ##################### 459 | docker run \ 460 | --rm -it \ 461 | -e PIP_SET_INDEX_TENCENT=1 \ 462 | wden/wden:devel-cpu-ubuntu20.04-python3.8 463 | 464 | ########################## 465 | # IN THE CONTAINER SHELL # 466 | ########################## 467 | echo $PIP_INDEX_URL 468 | << OUTPUT 469 | https://mirrors.cloud.tencent.com/pypi/simple/ 470 | OUTPUT 471 | ``` 472 | 473 | ### Run customized scirpts 474 | 475 | Env: 476 | 477 | * `CUSTOMIZED_INIT_SH`: If set, this var should be the path to your script that will be executed only once during container setup. 478 | * `CUSTOMIZED_REENTRANT_SH`: If set, this var should be the path to your script that will be executed in every login session. 479 | 480 | ```bash 481 | ##################### 482 | # IN THE HOST SHELL # 483 | ##################### 484 | docker run \ 485 | --rm -it \ 486 | -v /path/to/customized_init.sh:/run/customized_init.sh \ 487 | -v /path/to/customized_reentrant.sh:/run/customized_reentrant.sh \ 488 | -e CUSTOMIZED_INIT_SH='/run/customized_init.sh'\ 489 | -e CUSTOMIZED_REENTRANT_SH='/run/customized_reentrant.sh'\ 490 | wden/wden:devel-cpu-ubuntu20.04-python3.8 491 | ``` 492 | -------------------------------------------------------------------------------- /build/apt_disable_nvidia_repo.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -euo pipefail 3 | trap "echo 'error: Script failed: see failed command above'" ERR 4 | 5 | cd /etc/apt/sources.list.d/ 6 | mv cuda.list cuda.list.disabled || true 7 | mv nvidia-ml.list nvidia-ml.list.disabled || true 8 | -------------------------------------------------------------------------------- /build/apt_install_pkgs_for_dev.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -euo pipefail 3 | trap "echo 'error: Script failed: see failed command above'" ERR 4 | 5 | apt-get update 6 | 7 | # Suppress tzdata interactive prompt required by with Python >= 3.9. 8 | # This one cannot be moved to install_python.sh. 9 | export DEBIAN_FRONTEND="noninteractive" 10 | apt-get install -y tzdata 11 | 12 | declare -a pkgs=( 13 | sudo 14 | 15 | # Network. 16 | net-tools 17 | iputils-ping 18 | curl 19 | wget 20 | 21 | # ncat for ssh proxy. 22 | # Ubuntu 18.04 23 | nmap 24 | # Ubuntu 20.04 25 | ncat 26 | 27 | # Editor 28 | vim 29 | neovim 30 | 31 | # APT. 32 | apt-transport-https 33 | ca-certificates 34 | gnupg 35 | software-properties-common 36 | 37 | # GCC, libraries and utilities for build. 38 | build-essential 39 | 40 | # Utility. 41 | rsync 42 | zip 43 | unzip 44 | p7zip-full 45 | git 46 | ) 47 | 48 | apt-get install -y "${pkgs[@]}" 49 | 50 | # Install the latest version of cmake. 51 | # https://apt.kitware.com/ 52 | wget -O - https://apt.kitware.com/keys/kitware-archive-latest.asc 2>/dev/null \ 53 | | gpg --dearmor - \ 54 | | tee /etc/apt/trusted.gpg.d/kitware.gpg >/dev/null 55 | source /etc/os-release 56 | apt-add-repository -y "deb https://apt.kitware.com/ubuntu/ ${UBUNTU_CODENAME} main" 57 | apt-get update 58 | apt-get install -y cmake 59 | 60 | # Install watchexec. 61 | wget 'https://github.com/watchexec/watchexec/releases/download/v1.22.2/watchexec-1.22.2-x86_64-unknown-linux-musl.deb' \ 62 | -O /tmp/watchexec-1.22.2-x86_64-unknown-linux-musl.deb 63 | apt-get install -y /tmp/watchexec-1.22.2-x86_64-unknown-linux-musl.deb 64 | rm -f /tmp/watchexec-1.22.2-x86_64-unknown-linux-musl.deb 65 | 66 | # Install pyright. 67 | # https://github.com/nodesource/distributions 68 | mkdir -p /etc/apt/keyrings 69 | curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg 70 | NODE_MAJOR=18 71 | echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_$NODE_MAJOR.x nodistro main" | tee /etc/apt/sources.list.d/nodesource.list 72 | apt-get update 73 | apt-get install -y nodejs 74 | node -v 75 | npm -v 76 | npm install -g pyright 77 | pyright --version 78 | -------------------------------------------------------------------------------- /build/configure_shell.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -euo pipefail 3 | trap "echo 'error: Script failed: see failed command above'" ERR 4 | 5 | echo "Install oh-my-bash." 6 | # After installation, /root/.bashrc is configured with oh-my-bash setup. 7 | bash -c "$(curl -fsSL https://raw.githubusercontent.com/ohmybash/oh-my-bash/master/tools/install.sh)" 8 | 9 | # Disable auto update and notification. 10 | mv /root/.bashrc /root/.oh-my-bash.bashrc 11 | 12 | BASHRC_CONFIG=$( 13 | cat << 'EOF' 14 | 15 | # oh-my.zsh 16 | if [ -z "$DISABLE_OH_MY_BASH" ] ; then 17 | export DISABLE_UPDATE_PROMPT=true 18 | export DISABLE_AUTO_UPDATE=true 19 | source /root/.oh-my-bash.bashrc 20 | fi 21 | 22 | EOF 23 | ) 24 | echo "$BASHRC_CONFIG" | tee -a /root/.bashrc > /dev/null 25 | 26 | # Grant the wden user read & execute permissions. 27 | chmod -R 755 /root/ 28 | 29 | echo "Install screen." 30 | # Required by daemon session. 31 | apt-get install -y screen 32 | 33 | echo "Setup .bashrc for wden user." 34 | # Backup the original ~/.bashrc. 35 | # NOTE: The `-` is short for `--login`, meaning a login shell will be started. 36 | # Since login shell loads .bash_profile, executing this `su - ...` statement before changing 37 | # .bash_profile can mitigate confusing error message. 38 | su - $FIXUID_USER \ 39 | -c "cp ~/.bashrc ~/.bashrc.bak" 40 | # This script is invoked as an interactive non-login shell. 41 | # https://www.gnu.org/software/bash/manual/html_node/Bash-Startup-Files.html 42 | # 43 | # Hence, the following script will be executed for shell attached to screen session. 44 | BASHRC=$( 45 | cat << 'EOF' 46 | 47 | export BASHRC_EXECUTED=1 48 | 49 | # Load oh-my-zsh. 50 | source /root/.bashrc 51 | 52 | # Load Python. 53 | source ~/.bash_python 54 | 55 | # Phrase reentrant. 56 | source ~/.bash-session-env 57 | source "$WDEN_RUN_FOLDER"/run_${WDEN_RUN_TAG}_reentrant.sh 58 | 59 | EOF 60 | ) 61 | echo "$BASHRC" | tee "/home/${FIXUID_USER}/.bashrc" > /dev/null 62 | 63 | echo "Setup the .bash_profile for wden user." 64 | # This script is invoked as an interactive login shell, or with --login. 65 | # https://www.gnu.org/software/bash/manual/html_node/Bash-Startup-Files.html 66 | # 67 | # Hence, we set CMD instruction to execute the run.sh script, in which bash is explicitly executed 68 | # with --login option. In other words, the following script will be executed when user 69 | # - runs `docker run -it ...`, entering the shell that is in docker run session. 70 | # - remote login through ssh. 71 | BASH_PROFILE=$( 72 | cat << 'EOF' 73 | 74 | # Load oh-my-zsh. 75 | source /root/.bashrc 76 | 77 | # Load Python. 78 | source ~/.bash_python 79 | 80 | # Phrase init_pre. 81 | if [ ! -f ~/.bash-session-env ]; then 82 | # Shell is in docker run session. In other words, shell IS NOT in screen daemon session. 83 | # Hence the following script will be execute only once. During the setup, 84 | # - `~/.bash-session-env` is created. 85 | # - screen daemon session is initialized. 86 | # - ... 87 | source "${WDEN_RUN_FOLDER}/run_${WDEN_RUN_TAG}_init_pre.sh" 88 | 89 | # NOTE: If the following statement is place before init_pre.sh, 90 | # the screen daemon session will inherits this var. 91 | export IN_DOCKER_RUN_SESSION=1 92 | 93 | else 94 | # Shell IS in screen daemon session. 95 | # The init_pre script has be executed. At this moment, simply load the env. 96 | source ~/.bash-session-env 97 | 98 | fi 99 | 100 | # Phrase reentrant. 101 | # This scirpt contains statements could be executed multiple times. 102 | # NOTE: Remote ssh login will be attached to screen session at the end of this script. 103 | # 104 | # TODO: The scirpt sets some environment variables, which are also dumped to ~/.bash-session-env. 105 | # Those environment variables can be removed from ~/.bash-session-env. 106 | source "$WDEN_RUN_FOLDER"/run_${WDEN_RUN_TAG}_reentrant.sh 107 | 108 | # Phrase init_post. 109 | if [ -n "$IN_DOCKER_RUN_SESSION" ] ; then 110 | # Again, script will be execute only once. 111 | source "${WDEN_RUN_FOLDER}/run_${WDEN_RUN_TAG}_init_post.sh" 112 | fi 113 | 114 | EOF 115 | ) 116 | echo "$BASH_PROFILE" | tee "/home/${FIXUID_USER}/.bash_profile" > /dev/null 117 | -------------------------------------------------------------------------------- /build/configure_sudo.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -euo pipefail 3 | trap "echo 'error: Script failed: see failed command above'" ERR 4 | 5 | # Reset root password. 6 | echo "root:asdlkj" | chpasswd 7 | # Allow all user to sudo without password. 8 | echo 'ALL ALL=(ALL) NOPASSWD:ALL' >> /etc/sudoers 9 | -------------------------------------------------------------------------------- /build/devel_cpu.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -euo pipefail 3 | trap "echo 'error: Script failed: see failed command above'" ERR 4 | 5 | "$WDEN_BUILD_FOLDER"/devel_shared.sh 6 | -------------------------------------------------------------------------------- /build/devel_cuda.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -euo pipefail 3 | trap "echo 'error: Script failed: see failed command above'" ERR 4 | 5 | # Don't waste time to query nvidia repo. 6 | "$WDEN_BUILD_FOLDER"/apt_disable_nvidia_repo.sh 7 | "$WDEN_BUILD_FOLDER"/devel_shared.sh 8 | -------------------------------------------------------------------------------- /build/devel_shared.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -euo pipefail 3 | trap "echo 'error: Script failed: see failed command above'" ERR 4 | 5 | apt-get update 6 | 7 | "$WDEN_BUILD_FOLDER"/apt_install_pkgs_for_dev.sh 8 | "$WDEN_BUILD_FOLDER"/install_fixuid.sh 9 | "$WDEN_BUILD_FOLDER"/install_python.sh 10 | "$WDEN_BUILD_FOLDER"/configure_sudo.sh 11 | "$WDEN_BUILD_FOLDER"/install_openssh_server.sh 12 | "$WDEN_BUILD_FOLDER"/install_vscode_server.sh 13 | "$WDEN_BUILD_FOLDER"/install_direnv.sh 14 | "$WDEN_BUILD_FOLDER"/fix_run_permission.sh 15 | "$WDEN_BUILD_FOLDER"/configure_shell.sh 16 | -------------------------------------------------------------------------------- /build/fix_run_permission.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -euo pipefail 3 | trap "echo 'error: Script failed: see failed command above'" ERR 4 | 5 | chmod -R +rx "$WDEN_RUN_FOLDER" 6 | -------------------------------------------------------------------------------- /build/install_direnv.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -euo pipefail 3 | trap "echo 'error: Script failed: see failed command above'" ERR 4 | 5 | su - $FIXUID_USER \ 6 | -c "curl -sfL https://direnv.net/install.sh | bash" 7 | -------------------------------------------------------------------------------- /build/install_fixuid.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -euo pipefail 3 | trap "echo 'error: Script failed: see failed command above'" ERR 4 | 5 | #https://github.com/boxboat/fixuid 6 | addgroup \ 7 | --gid 1000 \ 8 | "$FIXUID_GROUP" 9 | 10 | adduser \ 11 | --uid 1000 \ 12 | --ingroup "$FIXUID_GROUP" \ 13 | --home /home/"$FIXUID_USER" \ 14 | --shell /bin/bash \ 15 | --disabled-password \ 16 | --gecos "" \ 17 | "$FIXUID_USER" 18 | 19 | export URL='https://github.com/boxboat/fixuid/releases/download/v0.6.0/fixuid-0.6.0-linux-amd64.tar.gz' 20 | curl -SsL $URL | tar -C /usr/local/bin -xzf - 21 | chown root:root /usr/local/bin/fixuid 22 | chmod 4755 /usr/local/bin/fixuid 23 | 24 | mkdir -p /etc/fixuid 25 | printf 'user: %s\ngroup: %s\n' "$FIXUID_USER" "$FIXUID_GROUP" > /etc/fixuid/config.yml 26 | -------------------------------------------------------------------------------- /build/install_openssh_server.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -euo pipefail 3 | trap "echo 'error: Script failed: see failed command above'" ERR 4 | 5 | apt-get install -y openssh-server 6 | 7 | git clone https://github.com/vkit-x/wden-ssh-key.git /tmp/wden-ssh-key 8 | 9 | # Host key. 10 | cp /tmp/wden-ssh-key/ssh_host_* /etc/ssh 11 | 12 | # Login key. 13 | mkdir "/home/${FIXUID_USER}/.ssh" 14 | SSHD_AUTHORIZED_KEYS="/home/${FIXUID_USER}/.ssh/authorized_keys" 15 | cat /tmp/wden-ssh-key/ssh_wden_rsa_key.pub > "$SSHD_AUTHORIZED_KEYS" 16 | chown -R "${FIXUID_USER}:${FIXUID_GROUP}" "/home/${FIXUID_USER}/.ssh" 17 | chmod 600 "$SSHD_AUTHORIZED_KEYS" 18 | 19 | rm -rf /tmp/wden-ssh-key 20 | -------------------------------------------------------------------------------- /build/install_python.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -euo pipefail 3 | trap "echo 'error: Script failed: see failed command above'" ERR 4 | 5 | add-apt-repository ppa:deadsnakes/ppa -y 6 | apt-get update 7 | 8 | # Install Python. 9 | apt-get install -y \ 10 | python"$PYTHON_VERSION" \ 11 | python"$PYTHON_VERSION"-dev \ 12 | python"$PYTHON_VERSION"-venv 13 | 14 | su - $FIXUID_USER \ 15 | -c 'mkdir -p ~/.local/bin' 16 | su - $FIXUID_USER \ 17 | -c 'ln -s /usr/bin/python3 ~/.local/bin/python' 18 | su - $FIXUID_USER \ 19 | -c 'ln -s /usr/bin/python3 ~/.local/bin/python3' 20 | 21 | # Configs for FIXUID_USER. 22 | FIXUID_USER_BASH_PYTHON_CONFIG=$( 23 | cat << EOF 24 | 25 | if [ -z "\$BASH_PYTHON_FLAG" ] ; then 26 | export PATH=/home/${FIXUID_USER}/.local/bin:\$PATH 27 | export BASH_PYTHON_FLAG=1 28 | fi 29 | 30 | EOF 31 | ) 32 | echo "$FIXUID_USER_BASH_PYTHON_CONFIG" > "/home/${FIXUID_USER}/.bash_python" 33 | 34 | su - $FIXUID_USER \ 35 | -c 'touch ~/.bash_profile' 36 | echo '. ~/.bash_python' >> "/home/${FIXUID_USER}/.bash_profile" 37 | 38 | # Install the latest version of pip to FIXUID_USER. 39 | su - $FIXUID_USER \ 40 | -c "curl --silent --show-error --retry 5 https://bootstrap.pypa.io/get-pip.py | python${PYTHON_VERSION}" 41 | # Make pip --user as default mode. 42 | su - $FIXUID_USER \ 43 | -c "mkdir ~/.pip && touch ~/.pip/pip.conf" 44 | 45 | PIP_CONFIG=$( 46 | cat << EOF 47 | 48 | [global] 49 | user = yes 50 | 51 | EOF 52 | ) 53 | echo "$PIP_CONFIG" | tee -a "/home/${FIXUID_USER}/.pip/pip.conf" > /dev/null 54 | 55 | # Show. 56 | su - $FIXUID_USER \ 57 | -c "python --version" 58 | su - $FIXUID_USER \ 59 | -c "python3 --version" 60 | su - $FIXUID_USER \ 61 | -c "pip --version" 62 | su - $FIXUID_USER \ 63 | -c "pip3 --version" 64 | -------------------------------------------------------------------------------- /build/install_vscode_server.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -euo pipefail 3 | trap "echo 'error: Script failed: see failed command above'" ERR 4 | 5 | # Install vscode remote server. 6 | wget https://update.code.visualstudio.com/latest/server-linux-x64/stable \ 7 | -O /tmp/vscode-server-linux-x64.tar.gz 8 | tar xvzf /tmp/vscode-server-linux-x64.tar.gz -C /tmp/ 9 | rm -f /tmp/vscode-server-linux-x64.tar.gz 10 | 11 | apt-get install -y jq 12 | 13 | VSCODE_SERVER_COMMIT=$(cat /tmp/vscode-server-linux-x64/product.json | jq -r '.commit') 14 | mkdir -p "/home/${FIXUID_USER}/.vscode-server/bin" 15 | mv /tmp/vscode-server-linux-x64 "/home/${FIXUID_USER}/.vscode-server/bin/${VSCODE_SERVER_COMMIT}" 16 | chown -R "${FIXUID_USER}:${FIXUID_GROUP}" "/home/${FIXUID_USER}/.vscode-server" 17 | 18 | # Install vscode plugin. (must run as target user) 19 | CODE_SERVER="/home/${FIXUID_USER}/.vscode-server/bin/${VSCODE_SERVER_COMMIT}/bin/code-server" 20 | su - $FIXUID_USER \ 21 | -c "${CODE_SERVER} --install-extension ms-python.python" 22 | su - $FIXUID_USER \ 23 | -c "${CODE_SERVER} --install-extension editorconfig.editorconfig" 24 | su - $FIXUID_USER \ 25 | -c "${CODE_SERVER} --install-extension canna.box-comment" 26 | # Remove jupyter plugin. 27 | su - $FIXUID_USER \ 28 | -c "${CODE_SERVER} --uninstall-extension ms-toolsai.jupyter" || true 29 | -------------------------------------------------------------------------------- /dockerfile/devel_cpu: -------------------------------------------------------------------------------- 1 | ARG UBUNTU_VERSION 2 | 3 | FROM ubuntu:${UBUNTU_VERSION} 4 | 5 | ARG PYTHON_VERSION 6 | 7 | ENV PYTHONIOENCODING=UTF-8 \ 8 | PYTHONUTF8=1 \ 9 | LANG=C.UTF-8 \ 10 | LC_CTYPE=C.UTF-8 \ 11 | LC_ALL=C.UTF-8 \ 12 | FIXUID_USER=wden \ 13 | FIXUID_GROUP=wden \ 14 | WDEN_BUILD_FOLDER=/.wden-build \ 15 | WDEN_RUN_FOLDER=/.wden-run \ 16 | WDEN_RUN_TAG='devel_cpu' 17 | 18 | COPY build "$WDEN_BUILD_FOLDER" 19 | COPY run "$WDEN_RUN_FOLDER" 20 | 21 | RUN "$WDEN_BUILD_FOLDER"/devel_cpu.sh 22 | 23 | USER "$FIXUID_USER":"$FIXUID_GROUP" 24 | ENTRYPOINT ["fixuid", "-q"] 25 | CMD "$WDEN_RUN_FOLDER"/run.sh 26 | -------------------------------------------------------------------------------- /dockerfile/devel_cpu_debug: -------------------------------------------------------------------------------- 1 | FROM wden/wden:devel-cpu-ubuntu20.04-python3.8 2 | COPY run "$WDEN_RUN_FOLDER" 3 | -------------------------------------------------------------------------------- /dockerfile/devel_cuda: -------------------------------------------------------------------------------- 1 | ARG CUDA_VERSION 2 | ARG CUDNN_VERSION 3 | ARG UBUNTU_VERSION 4 | 5 | # https://hub.docker.com/r/nvidia/cuda/tags 6 | FROM nvidia/cuda:${CUDA_VERSION}-cudnn${CUDNN_VERSION}-devel-ubuntu${UBUNTU_VERSION} 7 | 8 | ARG PYTHON_VERSION 9 | 10 | ENV PYTHONIOENCODING=UTF-8 \ 11 | PYTHONUTF8=1 \ 12 | LANG=C.UTF-8 \ 13 | LC_CTYPE=C.UTF-8 \ 14 | LC_ALL=C.UTF-8 \ 15 | FIXUID_USER=wden \ 16 | FIXUID_GROUP=wden \ 17 | WDEN_BUILD_FOLDER=/.wden-build \ 18 | WDEN_RUN_FOLDER=/.wden-run \ 19 | WDEN_RUN_TAG='devel_cuda' 20 | 21 | COPY build "$WDEN_BUILD_FOLDER" 22 | COPY run "$WDEN_RUN_FOLDER" 23 | 24 | RUN "$WDEN_BUILD_FOLDER"/devel_cuda.sh 25 | 26 | USER "$FIXUID_USER":"$FIXUID_GROUP" 27 | ENTRYPOINT ["fixuid", "-q"] 28 | CMD "$WDEN_RUN_FOLDER"/run.sh 29 | -------------------------------------------------------------------------------- /run/apt_set_mirror_aliyun.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -euo pipefail 3 | trap "echo 'error: Script failed: see failed command above'" ERR 4 | 5 | echo "Run apt_set_mirror_aliyun.sh" 6 | 7 | source /etc/os-release 8 | 9 | SOURCES_LIST=$( 10 | cat << EOF 11 | 12 | deb http://mirrors.aliyun.com/ubuntu/ ${UBUNTU_CODENAME} main restricted universe multiverse 13 | deb-src http://mirrors.aliyun.com/ubuntu/ ${UBUNTU_CODENAME} main restricted universe multiverse 14 | 15 | deb http://mirrors.aliyun.com/ubuntu/ ${UBUNTU_CODENAME}-security main restricted universe multiverse 16 | deb-src http://mirrors.aliyun.com/ubuntu/ ${UBUNTU_CODENAME}-security main restricted universe multiverse 17 | 18 | deb http://mirrors.aliyun.com/ubuntu/ ${UBUNTU_CODENAME}-updates main restricted universe multiverse 19 | deb-src http://mirrors.aliyun.com/ubuntu/ ${UBUNTU_CODENAME}-updates main restricted universe multiverse 20 | 21 | deb http://mirrors.aliyun.com/ubuntu/ ${UBUNTU_CODENAME}-proposed main restricted universe multiverse 22 | deb-src http://mirrors.aliyun.com/ubuntu/ ${UBUNTU_CODENAME}-proposed main restricted universe multiverse 23 | 24 | deb http://mirrors.aliyun.com/ubuntu/ ${UBUNTU_CODENAME}-backports main restricted universe multiverse 25 | deb-src http://mirrors.aliyun.com/ubuntu/ ${UBUNTU_CODENAME}-backports main restricted universe multiverse 26 | 27 | EOF 28 | ) 29 | echo "$SOURCES_LIST" | tee /etc/apt/sources.list > /dev/null 30 | 31 | apt-get update 32 | -------------------------------------------------------------------------------- /run/apt_set_mirror_tencent.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -euo pipefail 3 | trap "echo 'error: Script failed: see failed command above'" ERR 4 | 5 | echo "Run apt_set_mirror_tencent.sh" 6 | 7 | source /etc/os-release 8 | 9 | SOURCES_LIST=$( 10 | cat << EOF 11 | 12 | deb http://mirrors.cloud.tencent.com/ubuntu/ ${UBUNTU_CODENAME} main restricted universe multiverse 13 | deb http://mirrors.cloud.tencent.com/ubuntu/ ${UBUNTU_CODENAME}-security main restricted universe multiverse 14 | deb http://mirrors.cloud.tencent.com/ubuntu/ ${UBUNTU_CODENAME}-updates main restricted universe multiverse 15 | #deb http://mirrors.cloud.tencent.com/ubuntu/ ${UBUNTU_CODENAME}-proposed main restricted universe multiverse 16 | #deb http://mirrors.cloud.tencent.com/ubuntu/ ${UBUNTU_CODENAME}-backports main restricted universe multiverse 17 | deb-src http://mirrors.cloud.tencent.com/ubuntu/ ${UBUNTU_CODENAME} main restricted universe multiverse 18 | deb-src http://mirrors.cloud.tencent.com/ubuntu/ ${UBUNTU_CODENAME}-security main restricted universe multiverse 19 | deb-src http://mirrors.cloud.tencent.com/ubuntu/ ${UBUNTU_CODENAME}-updates main restricted universe multiverse 20 | #deb-src http://mirrors.cloud.tencent.com/ubuntu/ ${UBUNTU_CODENAME}-proposed main restricted universe multiverse 21 | #deb-src http://mirrors.cloud.tencent.com/ubuntu/ ${UBUNTU_CODENAME}-backports main restricted universe multiverse 22 | 23 | EOF 24 | ) 25 | echo "$SOURCES_LIST" | tee /etc/apt/sources.list > /dev/null 26 | 27 | apt-get update 28 | -------------------------------------------------------------------------------- /run/devel_shared_init_post.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | if [ -n "$CUSTOMIZED_INIT_SH" ] ; then 4 | source "$CUSTOMIZED_INIT_SH" 5 | fi 6 | -------------------------------------------------------------------------------- /run/devel_shared_init_pre.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | echo "Setting up container..." 4 | 5 | function patch_file_permission { 6 | FILE=$1 7 | OCTAL_PERMISSIONS=$2 8 | CUR_OCTAL_PERMISSIONS=$(stat -c '%a' "$FILE") 9 | if [ "$CUR_OCTAL_PERMISSIONS" != "$OCTAL_PERMISSIONS" ] ; then 10 | echo "WARNING: Patching the permissions of FILE=${FILE}" \ 11 | "${CUR_OCTAL_PERMISSIONS} => ${OCTAL_PERMISSIONS}" 12 | sudo chmod "$OCTAL_PERMISSIONS" "$FILE" 13 | echo "Done." 14 | fi 15 | } 16 | 17 | # Patch device permissions in privileged mode. 18 | patch_file_permission /dev/console '620' 19 | patch_file_permission /dev/full '666' 20 | patch_file_permission /dev/null '666' 21 | patch_file_permission /dev/random '666' 22 | patch_file_permission /dev/tty '666' 23 | patch_file_permission /dev/urandom '666' 24 | patch_file_permission /dev/zero '666' 25 | 26 | # APT mirror. 27 | if [ -n "$APT_SET_MIRROR_TENCENT" ] ; then 28 | sudo "$WDEN_RUN_FOLDER"/apt_set_mirror_tencent.sh 29 | elif [ -n "$APT_SET_MIRROR_ALIYUN" ] ; then 30 | sudo "$WDEN_RUN_FOLDER"/apt_set_mirror_aliyun.sh 31 | fi 32 | 33 | # SSH agent. 34 | if [ -n "$SSH_AUTH_SOCK" ] ; then 35 | patch_file_permission "$SSH_AUTH_SOCK" '777' 36 | fi 37 | 38 | # SSH login. 39 | if [ -n "$SSHD_AUTHORIZED_KEYS" ] ; then 40 | echo "$SSHD_AUTHORIZED_KEYS" > "/home/${FIXUID_USER}/.ssh/authorized_keys" 41 | fi 42 | 43 | if [ -n "$SSHD_PORT" ] ; then 44 | sudo sed -i "s|#Port 22|Port ${SSHD_PORT}|g" /etc/ssh/sshd_config 45 | sudo service ssh start 46 | fi 47 | 48 | # Pip index. 49 | if [ -n "$PIP_SET_INDEX_TENCENT" ] ; then 50 | export PIP_INDEX_URL='https://mirrors.cloud.tencent.com/pypi/simple/' 51 | elif [ -n "$PIP_SET_INDEX_ALIYUN" ] ; then 52 | export PIP_INDEX_URL='https://mirrors.aliyun.com/pypi/simple/' 53 | fi 54 | 55 | # Pip. 56 | if [ -n "$PIP_UPGRADE_TO_LATEST" ] ; then 57 | python -m pip install -U pip 58 | fi 59 | 60 | # Screen session. 61 | if [ -z "$DISABLE_SCREEN_DAEMON" ] ; then 62 | # screen config. 63 | # https://www.gnu.org/software/screen/manual/html_node/Bind.html#Bind 64 | # https://www.gnu.org/software/screen/manual/html_node/Default-Key-Bindings.html#Default-Key-Bindings 65 | # https://www.gnu.org/software/screen/manual/html_node/Bind-Examples.html#Bind-Examples 66 | SCREENRC=$( 67 | cat << 'EOF' 68 | 69 | # As login shell 70 | defshell -bash 71 | 72 | # https://vim.fandom.com/wiki/GNU_Screen_integration 73 | term screen-256color 74 | maptimeout 5 75 | 76 | # Reset some envs since screen session inherits envs. 77 | unsetenv BASH_PYTHON_FLAG 78 | unsetenv PATH 79 | 80 | # alternate screen. 81 | altscreen on 82 | 83 | # Unbind all the bindings. 84 | unbindall 85 | 86 | # Then enable the following bindings. 87 | bind -c demo d detach 88 | 89 | # C-n 90 | bindkey "^N" command -c demo 91 | 92 | EOF 93 | ) 94 | echo "$SCREENRC" | tee -a ~/.screenrc > /dev/null 95 | # Start up screen as daemon. 96 | args=() 97 | if [ -n "$SCREEN_DAEMON_LOG" ] ; then 98 | args=(-L -Logfile "$SCREEN_DAEMON_LOG") 99 | fi 100 | # Setting `-e` to avoid conflicts with C-a binding. 101 | screen -dmS daemon -e '^Nn' "${args[@]}" 102 | fi 103 | 104 | BASH_SESSION_ENV=$( 105 | cat << EOF 106 | 107 | # Customized. 108 | export DISABLE_OH_MY_BASH='$DISABLE_OH_MY_BASH' 109 | export PIP_SET_INDEX_TENCENT='$PIP_SET_INDEX_TENCENT' 110 | export PIP_SET_INDEX_ALIYUN='$PIP_SET_INDEX_ALIYUN' 111 | export CD_DEFAULT_FOLDER='$CD_DEFAULT_FOLDER' 112 | export SSH_AUTH_SOCK='$SSH_AUTH_SOCK' 113 | export SSH_SOCKS5_PROXY='$SSH_SOCKS5_PROXY' 114 | export PROPAGATE_HTTPS_PROXY='$PROPAGATE_HTTPS_PROXY' 115 | export CUSTOMIZED_REENTRANT_SH='$CUSTOMIZED_REENTRANT_SH' 116 | export DISABLE_SCREEN_DAEMON='$DISABLE_SCREEN_DAEMON' 117 | 118 | # Dockerfile. 119 | export PYTHONIOENCODING='$PYTHONIOENCODING' 120 | export PYTHONUTF8='$PYTHONUTF8' 121 | export LANG='$LANG' 122 | export LC_CTYPE='$LC_CTYPE' 123 | export LC_ALL='$LC_ALL' 124 | export FIXUID_USER='$FIXUID_USER' 125 | export FIXUID_GROUP='$FIXUID_GROUP' 126 | export WDEN_BUILD_FOLDER='$WDEN_BUILD_FOLDER' 127 | export WDEN_RUN_FOLDER='$WDEN_RUN_FOLDER' 128 | export WDEN_RUN_TAG='$WDEN_RUN_TAG' 129 | 130 | # Pip. 131 | export PIP_INDEX_URL='$PIP_INDEX_URL' 132 | 133 | # HTTP proxy. 134 | export HTTPS_PROXY='$HTTPS_PROXY' 135 | export HTTP_PROXY='$HTTP_PROXY' 136 | export https_proxy='$https_proxy' 137 | export http_proxy='$http_proxy' 138 | 139 | EOF 140 | ) 141 | echo "$BASH_SESSION_ENV" | tee -a ~/.bash-session-env > /dev/null 142 | 143 | BASH_SESSION_ENV=$( 144 | cat << 'EOF' 145 | 146 | # Disable logout & exit command if is in screen session. 147 | function _nop { 148 | echo 'Use (Ctrl+n d) to to leave the shell.' 149 | } 150 | 151 | if [ -n "$STY" ] ; then 152 | set -o ignoreeof 153 | alias logout=_nop 154 | alias exit=_nop 155 | fi 156 | 157 | EOF 158 | ) 159 | echo "$BASH_SESSION_ENV" | tee -a ~/.bash-session-env > /dev/null 160 | 161 | echo "Finished container setup..." 162 | -------------------------------------------------------------------------------- /run/devel_shared_reentrant.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # Home folder. 4 | if [ -n "$CD_DEFAULT_FOLDER" ] ; then 5 | if [ -d "$CD_DEFAULT_FOLDER" ] ; then 6 | alias cd="HOME='${CD_DEFAULT_FOLDER}' cd" 7 | cd "$CD_DEFAULT_FOLDER" 8 | else 9 | echo "WARNING: CD_DEFAULT_FOLDER=${CD_DEFAULT_FOLDER} not exists." 10 | fi 11 | fi 12 | 13 | # SSH agent & proxy. 14 | export SSH_OPTIONS='' 15 | 16 | if [ -n "$SSH_AUTH_SOCK" ] ; then 17 | export SSH_OPTIONS='-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no' 18 | fi 19 | 20 | if [ -n "$SSH_SOCKS5_PROXY" ] ; then 21 | export SSH_OPTIONS="${SSH_OPTIONS} -o ProxyCommand='ncat --proxy-type socks5 --proxy ${SSH_SOCKS5_PROXY} %h %p'" 22 | export GIT_SSH_COMMAND="ssh ${SSH_OPTIONS}" 23 | alias ssh="ssh ${SSH_OPTIONS}" 24 | fi 25 | 26 | # HTTP proxy. 27 | if [ -n "$PROPAGATE_HTTPS_PROXY" ] && [ -n "$HTTPS_PROXY" ] ; then 28 | export HTTP_PROXY="$HTTPS_PROXY" 29 | export https_proxy="$HTTPS_PROXY" 30 | export http_proxy="$HTTPS_PROXY" 31 | fi 32 | 33 | # Customized script. 34 | if [ -n "$CUSTOMIZED_REENTRANT_SH" ] ; then 35 | source "$CUSTOMIZED_REENTRANT_SH" 36 | fi 37 | 38 | # Direnv. 39 | eval "$(direnv hook bash)" 40 | 41 | # Attach screen session. 42 | if [ -z "$DISABLE_SCREEN_DAEMON" ] \ 43 | && [ -z "$IN_DOCKER_RUN_SESSION" ] \ 44 | && [ -z "$BASHRC_EXECUTED" ] \ 45 | && [ -z "$STY" ] ; then 46 | SCREEN_LIST=$(screen -list) 47 | if [[ "$SCREEN_LIST" =~ "Attached" ]] ; then 48 | echo "screen session was attached, stop attaching..." 49 | else 50 | screen -a -A -U -r 51 | exit 52 | fi 53 | fi 54 | 55 | function reattach-screen-session { 56 | screen -d -r 57 | exit 58 | } 59 | -------------------------------------------------------------------------------- /run/run.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | exec bash --login 4 | -------------------------------------------------------------------------------- /run/run_devel_cpu_init_post.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | source "$WDEN_RUN_FOLDER"/devel_shared_init_post.sh 4 | -------------------------------------------------------------------------------- /run/run_devel_cpu_init_pre.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | source "$WDEN_RUN_FOLDER"/devel_shared_init_pre.sh 4 | -------------------------------------------------------------------------------- /run/run_devel_cpu_reentrant.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | source "$WDEN_RUN_FOLDER"/devel_shared_reentrant.sh 4 | -------------------------------------------------------------------------------- /run/run_devel_cuda_init_post.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | source "$WDEN_RUN_FOLDER"/devel_shared_init_post.sh 4 | -------------------------------------------------------------------------------- /run/run_devel_cuda_init_pre.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | source "$WDEN_RUN_FOLDER"/devel_shared_init_pre.sh 4 | -------------------------------------------------------------------------------- /run/run_devel_cuda_reentrant.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | source "$WDEN_RUN_FOLDER"/devel_shared_reentrant.sh 4 | -------------------------------------------------------------------------------- /tool/README.md: -------------------------------------------------------------------------------- 1 | [TOC] 2 | 3 | ## Overview 4 | 5 | Goals 6 | 7 | - Attempt to provide an out-of-the-box experience to use docker container as the development environment, especially for Python developer, ML/DL engineer & researcher. 8 | - Setup a project boilerplate that is easy to understand. User with bash scripting and docker experience should be able to modify or extend the current codebase within 10 mins. While this project ships with the Python toolchain, most of the setup is not bound to Python, but instead is related to development experience (see the [Usage](#Usage) session). Hence this project could be extend to support variant scope development. 9 | 10 | Section [Images](#Images) documents the out-of-the-box prebuilt images. 11 | Section [Usage](#Usage) documents the typical use cases with examples. 12 | 13 | ## Images 14 | 15 | ### Hosted in Docker Hub 16 | 17 | Supported constructs are as followed: 18 | 19 | [TABLE_DOCKER_HUB] 20 | 21 | ### Hosted in Huawei Cloud 22 | 23 | As a known issue, pulling from docker hub is intolerably slow for users in China. To speed up, images are also hosted in Huawei Cloud: 24 | 25 | [TABLE_HUAWEICLOUD] 26 | 27 | ## Usage 28 | 29 | ### UID and GID forwarding 30 | 31 | ```bash 32 | ##################### 33 | # IN THE HOST SHELL # 34 | ##################### 35 | whoami 36 | << OUTPUT 37 | huntzhan 38 | OUTPUT 39 | 40 | echo "$(id -u):$(id -g)" 41 | << OUTPUT 42 | 501:20 43 | OUTPUT 44 | 45 | # Pass UID/GID to --user and mount a folder to container 46 | docker run \ 47 | --rm -it \ 48 | --user "$(id -u):$(id -g)" \ 49 | -v "$(pwd)":/data \ 50 | wden/wden:devel-cpu-ubuntu20.04-python3.8 51 | 52 | ########################## 53 | # IN THE CONTAINER SHELL # 54 | ########################## 55 | whoami 56 | << OUTPUT 57 | wden 58 | OUTPUT 59 | 60 | # Same UID/GID as host, thanks to https://github.com/boxboat/fixuid/ 61 | echo "$(id -u):$(id -g)" 62 | << OUTPUT 63 | 501:20 64 | OUTPUT 65 | 66 | cd /data 67 | touch foobar 68 | ls -l | grep foobar 69 | << OUTPUT 70 | -rw-r--r-- 1 wden dialout 0 Feb 5 14:24 foobar 71 | OUTPUT 72 | 73 | exit 74 | 75 | ##################### 76 | # IN THE HOST SHELL # 77 | ##################### 78 | ls -l | grep foobar 79 | << OUTPUT 80 | -rw-r--r-- 1 huntzhan staff 0 Feb 5 22:24 foobar 81 | OUTPUT 82 | ``` 83 | 84 | ### Change the default cd folder 85 | 86 | Env: 87 | 88 | * `CD_DEFAULT_FOLDER` : If set, initialize the shell and change the `cd` default folder to this path. 89 | 90 | ```bash 91 | ##################### 92 | # IN THE HOST SHELL # 93 | ##################### 94 | docker run \ 95 | --rm -it \ 96 | -e CD_DEFAULT_FOLDER=/data \ 97 | -v "$(pwd)":/data \ 98 | wden/wden:devel-cpu-ubuntu20.04-python3.8 99 | 100 | ########################## 101 | # IN THE CONTAINER SHELL # 102 | ########################## 103 | pwd 104 | << OUTPUT 105 | /data 106 | OUTPUT 107 | 108 | alias cd 109 | << OUTPUT 110 | alias cd='HOME='\''/data'\'' cd' 111 | OUTPUT 112 | ``` 113 | 114 | ### HTTP proxy setup 115 | 116 | Env: 117 | 118 | * `PROPAGATE_HTTPS_PROXY`: If set, will propagate `HTTPS_PROXY` to `HTTP_PROXY`, `https_proxy`, and `http_proxy` 119 | 120 | ```bash 121 | ##################### 122 | # IN THE HOST SHELL # 123 | ##################### 124 | docker run \ 125 | --rm -it \ 126 | -e PROPAGATE_HTTPS_PROXY=1 \ 127 | --add-host host.docker.internal:host-gateway \ 128 | -e HTTPS_PROXY='http://host.docker.internal:8889' \ 129 | wden/wden:devel-cpu-ubuntu20.04-python3.8 130 | 131 | ########################## 132 | # IN THE CONTAINER SHELL # 133 | ########################## 134 | cat << EOF 135 | ${HTTPS_PROXY} 136 | ${HTTP_PROXY} 137 | ${https_proxy} 138 | ${http_proxy} 139 | EOF 140 | 141 | << OUTPUT 142 | http://host.docker.internal:8889 143 | http://host.docker.internal:8889 144 | http://host.docker.internal:8889 145 | http://host.docker.internal:8889 146 | OUTPUT 147 | ``` 148 | 149 | ### SSH agent forwarding in macOS 150 | 151 | ```bash 152 | ##################### 153 | # IN THE HOST SHELL # 154 | ##################### 155 | # Store passphrases in your keychain. 156 | ssh-add -K /path/to/private-key-file 157 | 158 | # Add all identities stored in your keychain. 159 | # Make sure key identity is added before launching container. 160 | ssh-add -A 161 | 162 | # 1. The option "shared" in volume change the file permission of 163 | # /run/ssh-auth.sock to 777 to make the file accessible by non-root user. 164 | # 2. If SSH_AUTH_SOCK is set, the setup for SSH agent forwarding will be triggered. 165 | docker run \ 166 | --rm -it \ 167 | -v /run/host-services/ssh-auth.sock:/run/ssh-auth.sock:shared \ 168 | -e SSH_AUTH_SOCK="/run/ssh-auth.sock" \ 169 | wden/wden:devel-cpu-ubuntu20.04-python3.8 170 | 171 | 172 | ########################## 173 | # IN THE CONTAINER SHELL # 174 | ########################## 175 | # Test connection (suppose you have added the github private key). 176 | ssh -T git@github.com 177 | << OUTPUT 178 | Warning: Permanently added 'github.com,13.250.177.223' (RSA) to the list of known hosts. 179 | Hi huntzhan! You've successfully authenticated, but GitHub does not provide shell access. 180 | OUTPUT 181 | 182 | # ssh alias is set to bypass host checking. 183 | alias ssh 184 | << OUTPUT 185 | alias ssh='ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no' 186 | OUTPUT 187 | 188 | # Same here, to faciliate git operations. 189 | echo $GIT_SSH_COMMAND 190 | << OUTPUT 191 | ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no 192 | OUTPUT 193 | ``` 194 | 195 | ### SSH agent forwarding in Linux 196 | 197 | ```bash 198 | ##################### 199 | # IN THE HOST SHELL # 200 | ##################### 201 | # Make sure ssh-agent is up. 202 | echo $SSH_AUTH_SOCK 203 | << OUTPUT 204 | /tmp/ssh-adpxy7vPDaj7/agent.15833 205 | OUTPUT 206 | 207 | # Mount the socket file to container. 208 | docker run \ 209 | --rm -it \ 210 | -v "$SSH_AUTH_SOCK":/run/ssh-auth.sock:shared \ 211 | -e SSH_AUTH_SOCK="/run/ssh-auth.sock" \ 212 | wden/wden:devel-cpu-ubuntu20.04-python3.8 213 | 214 | 215 | ########################## 216 | # IN THE CONTAINER SHELL # 217 | ########################## 218 | # Same as 'SSH agent forwarding in macOS' 219 | ``` 220 | 221 | ### SSH proxy 222 | 223 | Env: 224 | 225 | * `SSH_SOCKS5_PROXY`: Should be formatted as `:`. If set, use such socks5 proxy in ssh connection. 226 | 227 | ```bash 228 | ##################### 229 | # IN THE HOST SHELL # 230 | ##################### 231 | # NOTE: 232 | # 1. SSH_AUTH_SOCK is not necessary. 233 | # 2. `host.docker.internal` refers to the hostname of the macOS host. 234 | docker run \ 235 | --rm -it \ 236 | -v /run/host-services/ssh-auth.sock:/run/ssh-auth.sock:shared \ 237 | -e SSH_AUTH_SOCK="/run/ssh-auth.sock" \ 238 | --add-host host.docker.internal:host-gateway \ 239 | -e SSH_SOCKS5_PROXY="host.docker.internal:1089" \ 240 | wden/wden:devel-cpu-ubuntu20.04-python3.8 241 | 242 | ########################## 243 | # IN THE CONTAINER SHELL # 244 | ########################## 245 | alias ssh 246 | << OUTPUT 247 | alias ssh='ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o ProxyCommand='\''ncat --proxy-type socks5 --proxy host.docker.internal:1089 %h %p'\''' 248 | OUTPUT 249 | 250 | echo $GIT_SSH_COMMAND 251 | << OUTPUT 252 | ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o ProxyCommand='ncat --proxy-type socks5 --proxy host.docker.internal:1089 %h %p' 253 | OUTPUT 254 | 255 | ssh -T git@github.com 256 | << OUTPUT 257 | Warning: Permanently added 'github.com' (RSA) to the list of known hosts. 258 | Hi huntzhan! You've successfully authenticated, but GitHub does not provide shell access. 259 | OUTPUT 260 | ``` 261 | 262 | ### SSH login to container 263 | 264 | Env: 265 | 266 | * `SSHD_AUTHORIZED_KEYS`: By default, [ssh_wden_rsa_key](https://github.com/vkit-x/wden-ssh-key/blob/master/ssh_wden_rsa_key.pub) has been setup. If you are conserned about the security, set this env to overwrite the `authorized_keys` file. 267 | * `SSHD_PORT`: If set, start the sshd service and bind to this port. 268 | * `DISABLE_SCREEN_DAEMON`: Disable the screen session for remote login. 269 | 270 | ```bash 271 | ##################### 272 | # IN THE HOST SHELL # 273 | ##################### 274 | docker run \ 275 | --rm -it \ 276 | -e SSHD_PORT=3333 \ 277 | wden/wden:devel-cpu-ubuntu20.04-python3.8 278 | 279 | ######################### 280 | # IN ANOTHER HOST SHELL # 281 | ######################### 282 | ssh wden@localhost \ 283 | -p 3333 \ 284 | -o IdentitiesOnly=yes \ 285 | -o IdentityFile=/path/to/wden-ssh-key/ssh_wden_rsa_key 286 | ``` 287 | 288 | Vscode `Remote - SSH` is well supported! 289 | 290 | ### Git config forwarding 291 | 292 | ```bash 293 | ##################### 294 | # IN THE HOST SHELL # 295 | ##################### 296 | docker run \ 297 | --rm -it \ 298 | -v "$HOME"/.gitconfig:/etc/gitconfig:ro \ 299 | wden/wden:devel-cpu-ubuntu20.04-python3.8 300 | 301 | ########################## 302 | # IN THE CONTAINER SHELL # 303 | ########################## 304 | git config --list | cat 305 | << OUTPUT 306 | user.name=Hunt Zhan 307 | user.email=huntzhan.dev@gmail.com 308 | credential.helper=osxkeychain 309 | OUTPUT 310 | ``` 311 | 312 | ### Bash history forwarding 313 | 314 | ```bash 315 | ##################### 316 | # IN THE HOST SHELL # 317 | ##################### 318 | # "$HOME"/.bash_history will be forwarded to and changed by the container. 319 | # https://ss64.com/bash/history.html 320 | # NOTE: backup ~/.bash_history if you want to forward it. 321 | docker run \ 322 | --rm -it \ 323 | -v /path/to/your/bash_history:/run/.bash_history \ 324 | wden/wden:devel-cpu-ubuntu20.04-python3.8 325 | 326 | ########################## 327 | # IN THE CONTAINER SHELL # 328 | ########################## 329 | history 330 | << OUTPUT 331 | 332 | OUTPUT 333 | ``` 334 | 335 | ### Run as daemon 336 | 337 | ```bash 338 | ##################### 339 | # IN THE HOST SHELL # 340 | ##################### 341 | # NOTE: `-it` must be added. 342 | docker run \ 343 | -d -it \ 344 | wden/wden:devel-cpu-ubuntu20.04-python3.8 345 | ``` 346 | 347 | ### Use APT mirror sites 348 | 349 | Env: 350 | 351 | * `APT_SET_MIRROR_TENCENT` : If set, switch to use [Tencent's mirror](https://mirrors.cloud.tencent.com/help/ubuntu.html). 352 | * `APT_SET_MIRROR_ALIYUN` : If set, switch to use [Aliyun's mirror](https://developer.aliyun.com/mirror/ubuntu). 353 | 354 | ```bash 355 | ##################### 356 | # IN THE HOST SHELL # 357 | ##################### 358 | docker run \ 359 | --rm -it \ 360 | -e APT_SET_MIRROR_TENCENT=1 \ 361 | wden/wden:devel-cpu-ubuntu20.04-python3.8 362 | 363 | ########################## 364 | # IN THE CONTAINER SHELL # 365 | ########################## 366 | << OUTPUT 367 | Run apt_set_mirror_tencent.sh 368 | Get:1 http://mirrors.cloud.tencent.com/ubuntu bionic InRelease [242 kB] 369 | Get:2 http://mirrors.cloud.tencent.com/ubuntu bionic-security InRelease [88.7 kB] 370 | Hit:3 http://ppa.launchpad.net/deadsnakes/ppa/ubuntu bionic InRelease 371 | Get:4 http://mirrors.cloud.tencent.com/ubuntu bionic-updates InRelease [88.7 kB] 372 | Get:5 http://mirrors.cloud.tencent.com/ubuntu bionic/universe Sources [11.5 MB] 373 | ... 374 | OUTPUT 375 | ``` 376 | 377 | ### Upgrade `pip` to the latest version 378 | 379 | Env: 380 | 381 | * `PIP_UPGRADE_TO_LATEST`: If set, `pip` will be upgraded to the latest version. 382 | 383 | ### Use PyPI mirror sites 384 | 385 | Env: 386 | 387 | * `PIP_SET_INDEX_TENCENT`: Use Tencent's PyPI index. 388 | * `PIP_SET_INDEX_ALIYUN` : Use Aliyun's PyPI index. 389 | 390 | ```bash 391 | ##################### 392 | # IN THE HOST SHELL # 393 | ##################### 394 | docker run \ 395 | --rm -it \ 396 | -e PIP_SET_INDEX_TENCENT=1 \ 397 | wden/wden:devel-cpu-ubuntu20.04-python3.8 398 | 399 | ########################## 400 | # IN THE CONTAINER SHELL # 401 | ########################## 402 | echo $PIP_INDEX_URL 403 | << OUTPUT 404 | https://mirrors.cloud.tencent.com/pypi/simple/ 405 | OUTPUT 406 | ``` 407 | 408 | ### Run customized scirpts 409 | 410 | Env: 411 | 412 | * `CUSTOMIZED_INIT_SH`: If set, this var should be the path to your script that will be executed only once during container setup. 413 | * `CUSTOMIZED_REENTRANT_SH`: If set, this var should be the path to your script that will be executed in every login session. 414 | 415 | ```bash 416 | ##################### 417 | # IN THE HOST SHELL # 418 | ##################### 419 | docker run \ 420 | --rm -it \ 421 | -v /path/to/customized_init.sh:/run/customized_init.sh \ 422 | -v /path/to/customized_reentrant.sh:/run/customized_reentrant.sh \ 423 | -e CUSTOMIZED_INIT_SH='/run/customized_init.sh'\ 424 | -e CUSTOMIZED_REENTRANT_SH='/run/customized_reentrant.sh'\ 425 | wden/wden:devel-cpu-ubuntu20.04-python3.8 426 | ``` 427 | -------------------------------------------------------------------------------- /tool/build_readme.py: -------------------------------------------------------------------------------- 1 | ''' 2 | pip install -r tool/requirements.txt 3 | 4 | fib tool/build_readme.py:build \ 5 | --readme_md="./tool/README.md" \ 6 | --main_yml="./.github/workflows/publish.yml" \ 7 | --output="./README.md" 8 | ''' 9 | 10 | import subprocess 11 | import iolite as io 12 | import yaml 13 | import itertools 14 | 15 | 16 | def combination(matrix): 17 | matrix.pop('include', None) 18 | excludes = matrix.pop('exclude', []) 19 | 20 | comb = [] 21 | 22 | def dfs(keys, cur): 23 | skip = False 24 | for exclude in excludes: 25 | all_hit = True 26 | for ex_key, ex_val in exclude.items(): 27 | if ex_key not in cur: 28 | all_hit = False 29 | break 30 | if cur[ex_key] != ex_val: 31 | all_hit = False 32 | break 33 | if all_hit: 34 | skip = True 35 | break 36 | 37 | if skip: 38 | print(f'skip {cur}') 39 | return 40 | 41 | if not keys: 42 | comb.append(cur.copy()) 43 | return 44 | 45 | key = keys.pop() 46 | for val in matrix[key]: 47 | cur[key] = val 48 | dfs(keys, cur) 49 | cur.pop(key) 50 | keys.append(key) 51 | 52 | dfs(list(matrix.keys()), {}) 53 | 54 | return comb 55 | 56 | 57 | def build(readme_md, main_yml, output): 58 | assert readme_md != output 59 | template = io.file(readme_md, exists=True).read_text() 60 | 61 | # TOC. 62 | result = subprocess.run( 63 | [ 64 | 'markdown-toc', 65 | '-t', 66 | 'github', 67 | '--no-write', 68 | readme_md, 69 | ], 70 | capture_output=True, 71 | ) 72 | result.check_returncode() 73 | toc = result.stdout.decode().strip() 74 | 75 | template = template.replace('[TOC]', f'\n#{toc}\n') 76 | 77 | # Tables. 78 | config = yaml.safe_load(io.file(main_yml, exists=True).read_text()) 79 | 80 | # CUDA. 81 | gpu_matrix = config['jobs']['build-cuda']['strategy']['matrix'] 82 | gpu_comb = combination(gpu_matrix.copy()) 83 | gpu_tag_cb = [] 84 | for cb in gpu_comb: 85 | cuda_version = cb['cuda_version'] 86 | cudnn_version = cb['cudnn_version'] 87 | ubuntu_version = cb['ubuntu_version'] 88 | python_version = cb['python_version'] 89 | tag = f'devel-cuda{cuda_version}-cudnn{cudnn_version}-ubuntu{ubuntu_version}-python{python_version}' 90 | gpu_tag_cb.append((tag, cb)) 91 | gpu_tag_cb = sorted(gpu_tag_cb, key=lambda p: p[0]) 92 | 93 | # CPU. 94 | cpu_matrix = config['jobs']['build-cpu']['strategy']['matrix'] 95 | cpu_comb = combination(cpu_matrix.copy()) 96 | cpu_tag_cb = [] 97 | for cb in cpu_comb: 98 | ubuntu_version = cb['ubuntu_version'] 99 | python_version = cb['python_version'] 100 | tag = f'devel-cpu-ubuntu{ubuntu_version}-python{python_version}' 101 | cpu_tag_cb.append((tag, cb)) 102 | cpu_tag_cb = sorted(cpu_tag_cb, key=lambda p: p[0]) 103 | 104 | def build_table(registry_prefix): 105 | table = [ 106 | '| Construct | `docker pull` command |', 107 | '| --------- | -------------------- |', 108 | ] 109 | for tag, cb in itertools.chain(gpu_tag_cb, cpu_tag_cb): 110 | bases = [] 111 | for key in sorted(cb): 112 | val = cb[key] 113 | if key.endswith('_version'): 114 | key = key[:-len('_version')] 115 | key = key.upper() 116 | bases.append(f'{key}={val}') 117 | bases = ', '.join(bases) 118 | if not registry_prefix: 119 | table.append(f'| {bases} | `docker pull wden/wden:{tag}` |') 120 | else: 121 | table.append(f'| {bases} | `docker pull {registry_prefix}wden/wden:{tag} && docker tag {registry_prefix}wden/wden:{tag} wden/wden:{tag} ` |') 122 | return '\n'.join(table) 123 | 124 | table_docker_hub = build_table('') 125 | table_huaweicloud = build_table('swr.cn-east-3.myhuaweicloud.com/') 126 | 127 | template = template.replace('[TABLE_DOCKER_HUB]', f'\n{table_docker_hub}\n') 128 | template = template.replace('[TABLE_HUAWEICLOUD]', f'\n{table_huaweicloud}\n') 129 | 130 | io.file(output).write_text(template) 131 | -------------------------------------------------------------------------------- /tool/requirements.txt: -------------------------------------------------------------------------------- 1 | markdown-toc 2 | pyyaml 3 | fireball 4 | iolite 5 | yapf 6 | flake8 7 | ipython 8 | --------------------------------------------------------------------------------