├── .github ├── dependabot.yml └── workflows │ └── release.yml ├── .gitignore ├── LICENSE ├── README.md ├── embed_util ├── embedded_files.go ├── file_list.go └── packer.go ├── example └── main.go ├── go.mod ├── go.sum ├── hack ├── build-tag.sh └── next-build-num.sh ├── internal ├── cleanup_python.go ├── fs.go └── tar.go ├── pip ├── embed_pip_packages.go ├── generate │ └── main.go ├── internal │ ├── data │ │ ├── .gitignore │ │ └── embed.go │ └── requirements.txt └── pip_lib.go └── python ├── embedded_python.go ├── embedded_python_test.go ├── generate └── main.go ├── internal └── data │ ├── .gitignore │ └── dummy.go ├── python.go └── python_test.go /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | 3 | updates: 4 | - package-ecosystem: "github-actions" 5 | directory: "/" 6 | schedule: 7 | interval: "weekly" 8 | - package-ecosystem: "gomod" 9 | directory: "/" 10 | schedule: 11 | interval: "daily" 12 | -------------------------------------------------------------------------------- /.github/workflows/release.yml: -------------------------------------------------------------------------------- 1 | name: release 2 | 3 | on: 4 | push: 5 | branches: 6 | - 'main' 7 | pull_request: 8 | branches: 9 | - 'main' 10 | 11 | env: 12 | PYTHON_STANDALONE_VERSIONS: | 13 | [ 14 | "20241219" 15 | ] 16 | PYTHON_VERSIONS: | 17 | [ 18 | "3.10.16", 19 | "3.11.11", 20 | "3.12.8", 21 | "3.13.1", 22 | ] 23 | 24 | jobs: 25 | build-matrix: 26 | runs-on: ubuntu-latest 27 | outputs: 28 | PYTHON_STANDALONE_VERSIONS: ${{ env.PYTHON_STANDALONE_VERSIONS }} 29 | PYTHON_VERSIONS: ${{ env.PYTHON_VERSIONS }} 30 | steps: 31 | - name: noop 32 | run: | 33 | echo noop 34 | 35 | build-tag: 36 | needs: 37 | - build-matrix 38 | strategy: 39 | matrix: 40 | pythonStandaloneVersion: ${{ fromJSON(needs.build-matrix.outputs.PYTHON_STANDALONE_VERSIONS) }} 41 | pythonVersion: ${{ fromJSON(needs.build-matrix.outputs.PYTHON_VERSIONS) }} 42 | fail-fast: false 43 | runs-on: ubuntu-22.04 44 | steps: 45 | - name: clone 46 | run: | 47 | # can't use actions/checkout here as transferring the shallow clone fails when using upload-/download-artifact 48 | git clone https://github.com/$GITHUB_REPOSITORY . --depth=1 49 | - name: checkout PR 50 | if: ${{ github.event_name == 'pull_request' }} 51 | run: | 52 | echo fetching pull/${{ github.ref_name }} 53 | git fetch origin pull/${{ github.ref_name }}:pr --depth=1 54 | git checkout pr 55 | - name: checkout branch 56 | if: ${{ github.event_name == 'push' }} 57 | run: | 58 | echo fetching ${{ github.ref_name }} 59 | git fetch origin ${{ github.ref_name }} --depth=1 60 | git checkout ${{ github.ref_name }} 61 | - name: Set up Go 62 | uses: actions/setup-go@v5 63 | with: 64 | go-version: 1.19 65 | - name: build-tag 66 | run: | 67 | git config --global user.email "no@mail.exists" 68 | git config --global user.name "go-embed-python releaser" 69 | BUILD_NUM=$(./hack/next-build-num.sh ${{ matrix.pythonStandaloneVersion }} ${{ matrix.pythonVersion }}) 70 | ./hack/build-tag.sh ${{ matrix.pythonStandaloneVersion }} ${{ matrix.pythonVersion }} $BUILD_NUM 71 | echo $BUILD_NUM > build-num 72 | - name: git gc 73 | run: | 74 | git gc 75 | - name: rename .git 76 | run: | 77 | mv .git git-dir 78 | - uses: actions/upload-artifact@v4 79 | with: 80 | name: workdir-${{ matrix.pythonStandaloneVersion }} ${{ matrix.pythonVersion }} 81 | path: | 82 | git-dir 83 | build-num 84 | 85 | tests: 86 | needs: 87 | - build-matrix 88 | - build-tag 89 | strategy: 90 | matrix: 91 | os: 92 | - ubuntu-22.04 93 | - macos-13 94 | - windows-2022 95 | pythonStandaloneVersion: ${{ fromJSON(needs.build-matrix.outputs.PYTHON_STANDALONE_VERSIONS) }} 96 | pythonVersion: ${{ fromJSON(needs.build-matrix.outputs.PYTHON_VERSIONS) }} 97 | fail-fast: false 98 | runs-on: ${{ matrix.os }} 99 | steps: 100 | - uses: actions/download-artifact@v4 101 | with: 102 | name: workdir-${{ matrix.pythonStandaloneVersion }} ${{ matrix.pythonVersion }} 103 | - name: rename .git back 104 | run: | 105 | mv git-dir .git 106 | - name: checkout tag 107 | shell: bash 108 | run: | 109 | git reset --hard 110 | git checkout v0.0.0-${{ matrix.pythonVersion }}-${{ matrix.pythonStandaloneVersion }}-$(cat build-num) 111 | - name: Set up Go 112 | uses: actions/setup-go@v5 113 | with: 114 | go-version: 1.19 115 | - name: run tests 116 | shell: bash 117 | run: | 118 | go test ./... -v 119 | 120 | release: 121 | needs: 122 | - build-matrix 123 | - tests 124 | strategy: 125 | matrix: 126 | pythonStandaloneVersion: ${{ fromJSON(needs.build-matrix.outputs.PYTHON_STANDALONE_VERSIONS) }} 127 | pythonVersion: ${{ fromJSON(needs.build-matrix.outputs.PYTHON_VERSIONS) }} 128 | fail-fast: false 129 | runs-on: ubuntu-22.04 130 | if: ${{ github.event_name == 'push' && github.ref_name == 'main' }} 131 | permissions: 132 | contents: write 133 | actions: write 134 | steps: 135 | - uses: actions/download-artifact@v4 136 | with: 137 | name: workdir-${{ matrix.pythonStandaloneVersion }} ${{ matrix.pythonVersion }} 138 | - name: rename .git back 139 | run: | 140 | mv git-dir .git 141 | - name: update remote url 142 | run: | 143 | git remote set-url origin https://token:$GITHUB_TOKEN@github.com/$GITHUB_REPOSITORY 144 | env: 145 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 146 | - name: push tag 147 | run: | 148 | git push origin v0.0.0-${{ matrix.pythonVersion }}-${{ matrix.pythonStandaloneVersion }}-$(cat build-num) 149 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /.idea 2 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | APPENDIX: How to apply the Apache License to your work. 180 | 181 | To apply the Apache License to your work, attach the following 182 | boilerplate notice, with the fields enclosed by brackets "[]" 183 | replaced with your own identifying information. (Don't include 184 | the brackets!) The text should be enclosed in the appropriate 185 | comment syntax for the file format. We also recommend that a 186 | file or class name and description of purpose be included on the 187 | same "printed page" as the copyright notice for easier 188 | identification within third-party archives. 189 | 190 | Copyright [yyyy] [name of copyright owner] 191 | 192 | Licensed under the Apache License, Version 2.0 (the "License"); 193 | you may not use this file except in compliance with the License. 194 | You may obtain a copy of the License at 195 | 196 | http://www.apache.org/licenses/LICENSE-2.0 197 | 198 | Unless required by applicable law or agreed to in writing, software 199 | distributed under the License is distributed on an "AS IS" BASIS, 200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 201 | See the License for the specific language governing permissions and 202 | limitations under the License. 203 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Embedded Python Interpreter for Go 2 | 3 | This library provides an embedded distribution of Python, which should work out-of-the box on a selected set of 4 | architectures and operating systems. 5 | 6 | This library does not require CGO and solely relies on executing Python inside another process. It does not rely 7 | on CPython binding to work. There is also no need to have Python pre-installed on the target host. 8 | 9 | You really only have to depend on this library and invoke it as follows: 10 | 11 | ```go 12 | import ( 13 | "github.com/kluctl/go-embed-python/python" 14 | "os" 15 | ) 16 | 17 | func main() { 18 | ep, err := python.NewEmbeddedPython("example") 19 | if err != nil { 20 | panic(err) 21 | } 22 | 23 | cmd, err := ep.PythonCmd("-c", "print('hello')") 24 | if err != nil { 25 | panic(err) 26 | } 27 | cmd.Stdout = os.Stdout 28 | cmd.Stderr = os.Stderr 29 | err = cmd.Run() 30 | if err != nil { 31 | panic(err) 32 | } 33 | } 34 | ``` 35 | 36 | ## Supported architectures 37 | The following operating systems and architectures are supported: 38 | * darwin-amd64 39 | * darwin-arm64 40 | * linux-amd64 41 | * linux-arm64 42 | * windows-amd64 43 | 44 | ## Releases 45 | Releases in this library are handled a bit different from what one might be used to. This library does currently not 46 | follow a versioning schema comparable to sematic versioning. This might however change in the future. 47 | 48 | Right now, every tagged release is compromised of the Python interpreter version, the [python-standalone](https://github.com/astral-sh/python-build-standalone) 49 | and a build number. For example, the release version `v0.0.0-3.11.6-20241219-2` belongs to Python version 3.11.6, 50 | the [20241219](https://github.com/astral-sh/python-build-standalone/releases/tag/20241219) version of python-standalone 51 | and build number 2. The release version currently always has v0.0.0 as its own version. 52 | 53 | The way versioning is handled might result in popular dependency management tools (e.g. dependabot) to not work as you 54 | might require it. Please watch out to not accidentally upgrade your Python version! 55 | 56 | ## How it works 57 | This library uses the standalone Python distributions found at https://github.com/astral-sh/python-build-standalone as 58 | the base. 59 | 60 | The `./hack/build-tag.sh` script is used to invoke `python/generate` and `pip/generate`, which then downloads, extracts 61 | and packages all supported Python distributions. The script then also creates a tag which then can be used as a dependency 62 | in your project. 63 | 64 | The tagged release internally embed all Python sources and binaries via `//go:embed`. The `EmbeddedPython` object 65 | is then used as a helper utility to access the embedded distribution. 66 | 67 | `EmbeddedPython` is created via `NewEmbeddedPython`, which will extract the embedded distribution into a temporary folder. 68 | Extraction is optimized in a way that it is only executed when needed (by verifying integrity of previously extracted 69 | distributions). 70 | 71 | ## Upgrading python 72 | The Python version and downloaded distributions are controlled via the `.github/workflows/release.yml` workflow. It 73 | contains a matrix of supported distributions. To upgrade Python, edit this workflow and create a pull request. 74 | 75 | ## Embedding Python libraries into your applications 76 | This library provides utilities/helpers to allow embedding of external libraries into your own application. 77 | 78 | To do this, create a simple generator application inside your application/library, for example in `internal/my-python-libs/generate/main.go`: 79 | 80 | ```go 81 | package main 82 | 83 | import ( 84 | "github.com/kluctl/go-embed-python/pip" 85 | ) 86 | 87 | func main() { 88 | err := pip.CreateEmbeddedPipPackagesForKnownPlatforms("requirements.txt", "./data/") 89 | if err != nil { 90 | panic(err) 91 | } 92 | } 93 | ``` 94 | 95 | Then create add the `//go:generate go run ./generate` statement to a .go file above the generator source, e.g. in `internal/my-python-libs/dummy.go`: 96 | ``` 97 | package internal 98 | 99 | //go:generate go run ./generate 100 | ``` 101 | 102 | And the requirements.txt in `internal/my-python-libs/requirements.txt`: 103 | ``` 104 | jinja2==3.1.2 105 | ``` 106 | 107 | When running `go generate ./...` inside your application/library, you'll get the referenced Python libraries installed 108 | to `internal/my-python-libs/data`. The embedded data is then available via `data.Data` and can be passed to 109 | `embed_util.NewEmbeddedFiles()` for extraction. 110 | 111 | The path returned by `EmbeddedFiles.GetExtractedPath()` can then be added to the `EmbeddedPython` by calling 112 | `AddPythonPath` on it. 113 | 114 | An example of all this can be found in https://github.com/kluctl/go-jinja2 115 | 116 | # Why another go+python solution? 117 | There are already multiple implementations of go-bindings for Python, which however all rely on CGO and/or dynamic 118 | linking. I experimented a lot with these and was not able to make it stable enough so that I could use it without fear 119 | of the process crashing after some time. I even got to the point where I implemented my own dynamic library loader that 120 | was not depending on CGO, but ultimately gave up when I realized that it would not work on all platforms. 121 | 122 | The only solution that was left was to spawn a Python process and use some kind of inter-process communication. For this 123 | to work reliably, without any dependencies on the host system, it was required to embed a fully working Python 124 | distribution into my Go binaries. I managed to make this flexible enough to put into a library so that others might 125 | benefit as well. 126 | 127 | Initially, this approach/code was part of https://github.com/kluctl/kluctl to allow Jinja2 templates in Go. The Jinja2 128 | part can now be found in https://github.com/kluctl/go-jinja2. 129 | -------------------------------------------------------------------------------- /embed_util/embedded_files.go: -------------------------------------------------------------------------------- 1 | package embed_util 2 | 3 | import ( 4 | "bytes" 5 | "compress/gzip" 6 | "fmt" 7 | "github.com/gofrs/flock" 8 | "io" 9 | "io/fs" 10 | "os" 11 | "path/filepath" 12 | ) 13 | 14 | type EmbeddedFiles struct { 15 | tmpDir string 16 | extractedPath string 17 | } 18 | 19 | func NewEmbeddedFiles(embedFs fs.FS, name string) (*EmbeddedFiles, error) { 20 | tmpDir := filepath.Join(os.TempDir(), fmt.Sprintf("go-embedded-%s", name)) 21 | return NewEmbeddedFilesWithTmpDir(embedFs, tmpDir, true) 22 | } 23 | 24 | func NewEmbeddedFilesWithTmpDir(embedFs fs.FS, tmpDir string, withHashInDir bool) (*EmbeddedFiles, error) { 25 | e := &EmbeddedFiles{ 26 | tmpDir: tmpDir, 27 | } 28 | err := e.extract(embedFs, withHashInDir) 29 | if err != nil { 30 | return nil, err 31 | } 32 | return e, nil 33 | } 34 | 35 | func (e *EmbeddedFiles) Cleanup() error { 36 | if e.extractedPath == "" { 37 | return nil 38 | } 39 | err := os.RemoveAll(e.extractedPath) 40 | e.extractedPath = "" 41 | return err 42 | } 43 | 44 | func (e *EmbeddedFiles) GetExtractedPath() string { 45 | return e.extractedPath 46 | } 47 | 48 | func (e *EmbeddedFiles) extract(embedFs fs.FS, withHashInDir bool) error { 49 | fl, err := e.readOrBuildFileList(embedFs) 50 | if err != nil { 51 | return err 52 | } 53 | 54 | flHash := fl.Hash() 55 | 56 | if withHashInDir { 57 | e.extractedPath = fmt.Sprintf("%s-%s", e.tmpDir, flHash[:16]) 58 | } else { 59 | e.extractedPath = e.tmpDir 60 | } 61 | err = os.MkdirAll(filepath.Dir(e.extractedPath), 0o755) 62 | if err != nil { 63 | return err 64 | } 65 | 66 | lock := flock.New(e.extractedPath + ".lock") 67 | err = lock.Lock() 68 | if err != nil { 69 | return err 70 | } 71 | defer lock.Close() 72 | 73 | err = os.MkdirAll(e.extractedPath, 0o755) 74 | if err != nil { 75 | return err 76 | } 77 | 78 | err = e.copyEmbeddedFilesToTmp(embedFs, fl) 79 | if err != nil { 80 | return err 81 | } 82 | 83 | return nil 84 | } 85 | 86 | func (e *EmbeddedFiles) readOrBuildFileList(embedFs fs.FS) (*fileList, error) { 87 | flStr, err := fs.ReadFile(embedFs, "files.json") 88 | if err != nil { 89 | if os.IsNotExist(err) { 90 | return buildFileListFromFs(embedFs) 91 | } 92 | return nil, err 93 | } 94 | 95 | fl, err := readFileList(string(flStr)) 96 | if err != nil { 97 | return nil, err 98 | } 99 | return fl, nil 100 | } 101 | 102 | func (e *EmbeddedFiles) copyEmbeddedFilesToTmp(embedFs fs.FS, fl *fileList) error { 103 | m := make(map[string]fileListEntry) 104 | 105 | for _, fle := range fl.Files { 106 | m[fle.Name] = fle 107 | } 108 | 109 | for _, fle := range fl.Files { 110 | resolvedFle := fle 111 | for resolvedFle.Mode.Type() == fs.ModeSymlink { 112 | if filepath.IsAbs(resolvedFle.Symlink) { 113 | return fmt.Errorf("abs path not allowed: %s", resolvedFle.Symlink) 114 | } 115 | sl := filepath.Clean(filepath.Join(filepath.Dir(resolvedFle.Name), resolvedFle.Symlink)) 116 | fle2, ok := m[sl] 117 | if !ok { 118 | return fmt.Errorf("symlink %s at %s could not be resolved", resolvedFle.Symlink, resolvedFle.Name) 119 | } 120 | resolvedFle = fle2 121 | if resolvedFle.Mode.IsDir() { 122 | return fmt.Errorf("symlinked dirs not supported at the moment: %s -> %s", fle.Name, resolvedFle.Name) 123 | } 124 | } 125 | 126 | path := filepath.Join(e.extractedPath, fle.Name) 127 | existingSt, err := os.Lstat(path) 128 | if err == nil { 129 | if resolvedFle.Mode.Type() == existingSt.Mode().Type() { 130 | if resolvedFle.Mode.IsDir() { 131 | continue 132 | } else if existingSt.Size() == resolvedFle.Size { 133 | // unchanged 134 | continue 135 | } 136 | } 137 | err = os.RemoveAll(path) 138 | if err != nil { 139 | return err 140 | } 141 | } 142 | 143 | if fle.Mode.IsDir() { 144 | err := os.MkdirAll(path, resolvedFle.Mode.Perm()) 145 | if err != nil { 146 | return err 147 | } 148 | continue 149 | } else if !resolvedFle.Mode.IsRegular() { 150 | continue 151 | } 152 | 153 | var data []byte 154 | 155 | if resolvedFle.Compressed { 156 | data, err = fs.ReadFile(embedFs, resolvedFle.Name+".gz") 157 | if err != nil { 158 | return err 159 | } 160 | gz, err := gzip.NewReader(bytes.NewReader(data)) 161 | if err != nil { 162 | return err 163 | } 164 | data, err = io.ReadAll(gz) 165 | _ = gz.Close() 166 | if err != nil { 167 | return err 168 | } 169 | } else { 170 | data, err = fs.ReadFile(embedFs, resolvedFle.Name) 171 | if err != nil { 172 | return err 173 | } 174 | } 175 | 176 | err = os.WriteFile(path, data, resolvedFle.Mode.Perm()) 177 | if err != nil { 178 | return err 179 | } 180 | } 181 | 182 | return nil 183 | } 184 | -------------------------------------------------------------------------------- /embed_util/file_list.go: -------------------------------------------------------------------------------- 1 | package embed_util 2 | 3 | import ( 4 | "crypto/sha256" 5 | "encoding/hex" 6 | "encoding/json" 7 | "fmt" 8 | "io/fs" 9 | "net/http" 10 | "os" 11 | "path/filepath" 12 | "sort" 13 | ) 14 | 15 | type fileList struct { 16 | ContentHash string `json:"contentHash"` 17 | Files []fileListEntry `json:"files"` 18 | } 19 | 20 | type fileListEntry struct { 21 | Name string `json:"name"` 22 | Size int64 `json:"size"` 23 | Mode fs.FileMode `json:"perm"` 24 | Symlink string `json:"symlink,omitempty"` 25 | Compressed bool `json:"compressed,omitempty"` 26 | } 27 | 28 | func readFileList(fileListStr string) (*fileList, error) { 29 | var fl fileList 30 | err := json.Unmarshal([]byte(fileListStr), &fl) 31 | if err != nil { 32 | return nil, err 33 | } 34 | return &fl, err 35 | } 36 | 37 | func buildFileListFromDir(dir string) (*fileList, error) { 38 | var fl fileList 39 | 40 | err := filepath.Walk(dir, func(path string, info fs.FileInfo, err error) error { 41 | relPath, err := filepath.Rel(dir, path) 42 | if err != nil { 43 | return err 44 | } 45 | 46 | if relPath == "." || relPath == "files.json" { 47 | return nil 48 | } 49 | 50 | fle := fileListEntry{ 51 | Name: relPath, 52 | Size: info.Size(), 53 | Mode: info.Mode(), 54 | } 55 | 56 | if info.Mode().Type() == fs.ModeSymlink { 57 | sl, err := os.Readlink(path) 58 | if err != nil { 59 | return err 60 | } 61 | fle.Symlink = sl 62 | fle.Mode &= ^fs.ModePerm 63 | } else if info.Mode().IsDir() { 64 | fle.Size = 0 65 | } else if info.Mode().IsRegular() { 66 | fle.Compressed = shouldCompress(path) 67 | } 68 | 69 | fl.Files = append(fl.Files, fle) 70 | return nil 71 | }) 72 | if err != nil { 73 | return nil, err 74 | } 75 | sort.Slice(fl.Files, func(i, j int) bool { 76 | return fl.Files[i].Name < fl.Files[j].Name 77 | }) 78 | return &fl, nil 79 | } 80 | 81 | func buildFileListFromFs(embedFs fs.FS) (*fileList, error) { 82 | var fl fileList 83 | 84 | err := fs.WalkDir(embedFs, ".", func(path string, d fs.DirEntry, err error) error { 85 | if path == "." || path == "files.json" { 86 | return nil 87 | } 88 | 89 | info, err := d.Info() 90 | if err != nil { 91 | return err 92 | } 93 | 94 | fle := fileListEntry{ 95 | Name: path, 96 | Size: info.Size(), 97 | Mode: info.Mode() | 0o600, 98 | } 99 | 100 | if info.Mode().Type() == fs.ModeSymlink { 101 | return fmt.Errorf("symlink not supported in buildFileListFromFs") 102 | } else if info.Mode().IsDir() { 103 | fle.Size = 0 104 | } 105 | 106 | fl.Files = append(fl.Files, fle) 107 | return nil 108 | }) 109 | if err != nil { 110 | return nil, err 111 | } 112 | sort.Slice(fl.Files, func(i, j int) bool { 113 | return fl.Files[i].Name < fl.Files[j].Name 114 | }) 115 | return &fl, nil 116 | } 117 | 118 | func shouldCompress(path string) bool { 119 | f, err := os.Open(path) 120 | if err != nil { 121 | return false 122 | } 123 | defer f.Close() 124 | data := make([]byte, 512) 125 | n, err := f.Read(data) 126 | if err != nil { 127 | return false 128 | } 129 | if http.DetectContentType(data[:n]) == "application/octet-stream" { 130 | return true 131 | } 132 | return false 133 | } 134 | 135 | func (fl *fileList) toMap() map[string]fileListEntry { 136 | m := make(map[string]fileListEntry) 137 | for _, e := range fl.Files { 138 | m[e.Name] = e 139 | } 140 | return m 141 | } 142 | 143 | func (fl *fileList) Hash() string { 144 | h := sha256.New() 145 | e := json.NewEncoder(h) 146 | err := e.Encode(fl) 147 | if err != nil { 148 | panic(err) 149 | } 150 | return hex.EncodeToString(h.Sum(nil)) 151 | } 152 | -------------------------------------------------------------------------------- /embed_util/packer.go: -------------------------------------------------------------------------------- 1 | package embed_util 2 | 3 | import ( 4 | "bytes" 5 | "compress/gzip" 6 | "crypto/sha256" 7 | "encoding/binary" 8 | "encoding/hex" 9 | "encoding/json" 10 | "fmt" 11 | log "github.com/sirupsen/logrus" 12 | "golang.org/x/sync/errgroup" 13 | "os" 14 | "path/filepath" 15 | "strings" 16 | ) 17 | 18 | func CopyForEmbed(out string, dir string) error { 19 | fl, err := buildFileListFromDir(dir) 20 | if err != nil { 21 | return err 22 | } 23 | 24 | log.Infof("copying to %s with %d files", out, len(fl.Files)) 25 | err = copyFiles(out, dir, fl) 26 | if err != nil { 27 | return err 28 | } 29 | 30 | return doWriteFilesList(dir, out, fl) 31 | } 32 | 33 | func BuildAndWriteFilesList(dir string) error { 34 | fl, err := buildFileListFromDir(dir) 35 | if err != nil { 36 | return err 37 | } 38 | return doWriteFilesList(dir, dir, fl) 39 | } 40 | 41 | func doWriteFilesList(srcDir string, outDir string, fl *fileList) error { 42 | var err error 43 | fl.ContentHash, err = calcContentHash(srcDir, fl) 44 | if err != nil { 45 | return err 46 | } 47 | b, err := json.MarshalIndent(fl, "", " ") 48 | if err != nil { 49 | return err 50 | } 51 | 52 | err = os.WriteFile(filepath.Join(outDir, "files.json"), b, 0o644) 53 | if err != nil { 54 | return err 55 | } 56 | return nil 57 | } 58 | 59 | func WriteEmbedGoFile(targetDir string, goOs string, goArch string) error { 60 | var embedSrc, fname string 61 | if goOs == "" { 62 | embedSrc = ` 63 | package data 64 | 65 | import "embed" 66 | 67 | //go:embed all:* 68 | var Data embed.FS 69 | ` 70 | fname = "embed.go" 71 | } else { 72 | embedSrc = fmt.Sprintf(` 73 | package data 74 | 75 | import ( 76 | "embed" 77 | "io/fs" 78 | ) 79 | 80 | //go:embed all:%s-%s 81 | var _data embed.FS 82 | var Data, _ = fs.Sub(_data, "%s-%s") 83 | `, goOs, goArch, goOs, goArch) 84 | fname = strings.ReplaceAll(fmt.Sprintf("embed_%s_%s.go", goOs, goArch), "-", "_") 85 | } 86 | 87 | return os.WriteFile(filepath.Join(targetDir, fname), []byte(embedSrc), 0o644) 88 | } 89 | 90 | func copyFiles(out string, dir string, fl *fileList) error { 91 | var g errgroup.Group 92 | g.SetLimit(8) 93 | 94 | for _, fle := range fl.Files { 95 | fle := fle 96 | path := filepath.Join(dir, fle.Name) 97 | 98 | st, err := os.Lstat(path) 99 | if err != nil { 100 | return err 101 | } 102 | 103 | outPath := filepath.Join(out, fle.Name) 104 | err = os.MkdirAll(filepath.Dir(outPath), 0o755) 105 | if err != nil { 106 | return err 107 | } 108 | 109 | if !st.Mode().IsRegular() { 110 | continue 111 | } 112 | 113 | g.Go(func() error { 114 | data, err := os.ReadFile(path) 115 | if err != nil { 116 | return err 117 | } 118 | 119 | if fle.Compressed { 120 | b := bytes.NewBuffer(make([]byte, 0, len(data))) 121 | gz, err := gzip.NewWriterLevel(b, gzip.BestCompression) 122 | if err != nil { 123 | return err 124 | } 125 | _, err = gz.Write(data) 126 | if err != nil { 127 | _ = gz.Close() 128 | return err 129 | } 130 | err = gz.Flush() 131 | _ = gz.Close() 132 | if err != nil { 133 | return err 134 | } 135 | data = b.Bytes() 136 | outPath += ".gz" 137 | } 138 | 139 | err = os.WriteFile(outPath, data, 0o644) 140 | if err != nil { 141 | return err 142 | } 143 | return nil 144 | }) 145 | } 146 | err := g.Wait() 147 | if err != nil { 148 | return err 149 | } 150 | 151 | return nil 152 | } 153 | 154 | func calcContentHash(dir string, fl *fileList) (string, error) { 155 | hash := sha256.New() 156 | for _, fle := range fl.Files { 157 | path := filepath.Join(dir, fle.Name) 158 | st, err := os.Lstat(path) 159 | if err != nil { 160 | return "", err 161 | } 162 | if st.Mode().Type() == os.ModeSymlink { 163 | sl, err := os.Readlink(path) 164 | if err != nil { 165 | return "", err 166 | } 167 | _ = binary.Write(hash, binary.LittleEndian, "symlink") 168 | _ = binary.Write(hash, binary.LittleEndian, sl) 169 | } else if st.Mode().IsDir() { 170 | err = os.MkdirAll(path, fle.Mode.Perm()) 171 | if err != nil { 172 | return "", err 173 | } 174 | _ = binary.Write(hash, binary.LittleEndian, "dir") 175 | _ = binary.Write(hash, binary.LittleEndian, fle.Name) 176 | } else if st.Mode().IsRegular() { 177 | outPath := filepath.Join(dir, fle.Name) 178 | data, err := os.ReadFile(outPath) 179 | if err != nil { 180 | return "", err 181 | } 182 | 183 | _ = binary.Write(hash, binary.LittleEndian, "regular") 184 | _ = binary.Write(hash, binary.LittleEndian, fle.Name) 185 | _ = binary.Write(hash, binary.LittleEndian, data) 186 | } 187 | } 188 | return hex.EncodeToString(hash.Sum(nil)), nil 189 | } 190 | -------------------------------------------------------------------------------- /example/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "github.com/kluctl/go-embed-python/python" 5 | "os" 6 | ) 7 | 8 | func main() { 9 | ep, err := python.NewEmbeddedPython("example") 10 | if err != nil { 11 | panic(err) 12 | } 13 | 14 | cmd, err := ep.PythonCmd("-c", "print('hello')") 15 | if err != nil { 16 | panic(err) 17 | } 18 | cmd.Stdout = os.Stdout 19 | cmd.Stderr = os.Stderr 20 | err = cmd.Run() 21 | if err != nil { 22 | panic(err) 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/kluctl/go-embed-python 2 | 3 | go 1.19 4 | 5 | require ( 6 | github.com/gobwas/glob v0.2.3 7 | github.com/gofrs/flock v0.12.1 8 | github.com/klauspost/compress v1.17.11 9 | github.com/sirupsen/logrus v1.9.3 10 | github.com/stretchr/testify v1.10.0 11 | golang.org/x/sync v0.10.0 12 | ) 13 | 14 | require ( 15 | github.com/davecgh/go-spew v1.1.1 // indirect 16 | github.com/kr/text v0.2.0 // indirect 17 | github.com/pmezard/go-difflib v1.0.0 // indirect 18 | github.com/rogpeppe/go-internal v1.13.1 // indirect 19 | golang.org/x/sys v0.28.0 // indirect 20 | gopkg.in/yaml.v3 v3.0.1 // indirect 21 | ) 22 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= 2 | github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 3 | github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= 4 | github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= 5 | github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y= 6 | github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8= 7 | github.com/gofrs/flock v0.12.1 h1:MTLVXXHf8ekldpJk3AKicLij9MdwOWkZ+a/jHHZby9E= 8 | github.com/gofrs/flock v0.12.1/go.mod h1:9zxTsyu5xtJ9DK+1tFZyibEV7y3uwDxPPfbxeeHCoD0= 9 | github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc= 10 | github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0= 11 | github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= 12 | github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= 13 | github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= 14 | github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= 15 | github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= 16 | github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= 17 | github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= 18 | github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= 19 | github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= 20 | github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= 21 | github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= 22 | github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= 23 | github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= 24 | github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= 25 | golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ= 26 | golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= 27 | golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 28 | golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= 29 | golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= 30 | gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= 31 | gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= 32 | gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= 33 | gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 34 | gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= 35 | gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= 36 | -------------------------------------------------------------------------------- /hack/build-tag.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | set -e 4 | 5 | DIR=$(cd $(dirname $0) && pwd) 6 | cd $DIR/.. 7 | 8 | PYTHON_STANDALONE_VERSION=$1 9 | PYTHON_VERSION=$2 10 | BUILD_NUM=$3 11 | 12 | if [ "$PYTHON_STANDALONE_VERSION" = "" ]; then 13 | echo "missing python-standalone version" 14 | exit 1 15 | fi 16 | 17 | if [ "$PYTHON_VERSION" = "" ]; then 18 | echo "missing python version" 19 | exit 1 20 | fi 21 | 22 | if [ "$BUILD_NUM" = "" ]; then 23 | echo "missing build num" 24 | exit 1 25 | fi 26 | 27 | if [ ! -z "$(git status --porcelain)" ]; then 28 | echo "working directory is dirty!" 29 | exit 1 30 | fi 31 | 32 | go run ./python/generate --python-standalone-version=$PYTHON_STANDALONE_VERSION --python-version $PYTHON_VERSION 33 | go run ./pip/generate 34 | 35 | TAG=v0.0.0-$PYTHON_VERSION-$PYTHON_STANDALONE_VERSION-$BUILD_NUM 36 | 37 | echo "checking out temporary branch" 38 | git checkout --detach 39 | git add -f python/internal/data 40 | git add -f pip/internal/data 41 | git commit -m "added python $PYTHON_VERSION from python-standalone $PYTHON_STANDALONE_VERSION" 42 | git tag -f $TAG 43 | git checkout - 44 | -------------------------------------------------------------------------------- /hack/next-build-num.sh: -------------------------------------------------------------------------------- 1 | set -e 2 | 3 | DIR=$(cd $(dirname $0) && pwd) 4 | cd $DIR/.. 5 | 6 | PYTHON_STANDALONE_VERSION=$1 7 | PYTHON_VERSION=$2 8 | 9 | if [ "$PYTHON_STANDALONE_VERSION" = "" ]; then 10 | echo "missing python-standalone version" 11 | exit 1 12 | fi 13 | 14 | if [ "$PYTHON_VERSION" = "" ]; then 15 | echo "missing python version" 16 | exit 1 17 | fi 18 | 19 | REMOTE_TAGS=$(git ls-remote) 20 | LOCAL_TAGS=$(git tag) 21 | #echo REMOTE_TAGS=$REMOTE_TAGS 22 | #echo LOCAL_TAGS=$LOCAL_TAGS 23 | 24 | BUILD_NUM=1 25 | 26 | while true; do 27 | TAG=v0.0.0-$PYTHON_VERSION-$PYTHON_STANDALONE_VERSION-$BUILD_NUM 28 | if [ "$(echo $REMOTE_TAGS | grep "refs/tags/$TAG")" != "" -o "$(echo $LOCAL_TAGS | grep "$TAG")" != "" ] ; then 29 | BUILD_NUM=$(($BUILD_NUM+1)) 30 | else 31 | break 32 | fi 33 | done 34 | 35 | echo $BUILD_NUM 36 | -------------------------------------------------------------------------------- /internal/cleanup_python.go: -------------------------------------------------------------------------------- 1 | package internal 2 | 3 | import ( 4 | "github.com/gobwas/glob" 5 | "io/fs" 6 | "os" 7 | "path/filepath" 8 | ) 9 | 10 | var DefaultPythonRemovePatterns = []glob.Glob{ 11 | glob.MustCompile("__pycache__"), 12 | glob.MustCompile("**/__pycache__"), 13 | glob.MustCompile("**.a"), 14 | glob.MustCompile("**.pdb"), 15 | glob.MustCompile("**.pyc"), 16 | glob.MustCompile("**/test_*.py"), 17 | glob.MustCompile("**/*.dist-info"), 18 | } 19 | 20 | func CleanupPythonDir(dir string, keepPatterns []glob.Glob) error { 21 | var removes []string 22 | err := filepath.Walk(dir, func(path string, info fs.FileInfo, err error) error { 23 | relPath, err := filepath.Rel(dir, path) 24 | if err != nil { 25 | return err 26 | } 27 | for _, p := range DefaultPythonRemovePatterns { 28 | if p.Match(relPath) { 29 | removes = append(removes, path) 30 | } 31 | } 32 | if len(keepPatterns) != 0 && !info.Mode().IsDir() { 33 | keep := false 34 | for _, p := range keepPatterns { 35 | if p.Match(relPath) { 36 | keep = true 37 | break 38 | } 39 | } 40 | if !keep { 41 | removes = append(removes, path) 42 | } 43 | } 44 | return nil 45 | }) 46 | 47 | for _, r := range removes { 48 | err = os.RemoveAll(r) 49 | if err != nil && !os.IsNotExist(err) { 50 | return err 51 | } 52 | } 53 | 54 | err = removeEmptyDirs(dir) 55 | if err != nil { 56 | return err 57 | } 58 | 59 | return err 60 | } 61 | 62 | func removeEmptyDirs(dir string) error { 63 | for true { 64 | didRemove, err := removeEmptyDirs2(dir) 65 | if err != nil { 66 | return err 67 | } 68 | if !didRemove { 69 | break 70 | } 71 | } 72 | return nil 73 | } 74 | 75 | func removeEmptyDirs2(dir string) (bool, error) { 76 | var removes []string 77 | err := filepath.Walk(dir, func(path string, info fs.FileInfo, err error) error { 78 | if info.IsDir() { 79 | des, err := os.ReadDir(path) 80 | if err != nil { 81 | return err 82 | } 83 | if len(des) == 0 { 84 | removes = append(removes, path) 85 | } 86 | } 87 | return nil 88 | }) 89 | if err != nil { 90 | return false, err 91 | } 92 | 93 | for _, r := range removes { 94 | err = os.Remove(r) 95 | if err != nil { 96 | return false, err 97 | } 98 | } 99 | return len(removes) != 0, nil 100 | } 101 | -------------------------------------------------------------------------------- /internal/fs.go: -------------------------------------------------------------------------------- 1 | package internal 2 | 3 | import ( 4 | "os" 5 | ) 6 | 7 | func Exists(path string) bool { 8 | _, err := os.Stat(path) 9 | if err != nil { 10 | return false 11 | } 12 | return true 13 | } 14 | -------------------------------------------------------------------------------- /internal/tar.go: -------------------------------------------------------------------------------- 1 | package internal 2 | 3 | import ( 4 | "archive/tar" 5 | "fmt" 6 | "io" 7 | "os" 8 | "path/filepath" 9 | "strings" 10 | ) 11 | 12 | func ExtractTarStream(r io.Reader, targetPath string) error { 13 | tarReader := tar.NewReader(r) 14 | for true { 15 | header, err := tarReader.Next() 16 | if err == io.EOF { 17 | break 18 | } 19 | 20 | if err != nil { 21 | return fmt.Errorf("ExtractTarStream: Next() failed: %w", err) 22 | } 23 | 24 | if !validRelPath(header.Name) { 25 | return fmt.Errorf("tar contained invalid name error %q", header.Name) 26 | } 27 | 28 | p := filepath.FromSlash(header.Name) 29 | p = filepath.Join(targetPath, p) 30 | 31 | err = os.MkdirAll(filepath.Dir(p), 0755) 32 | if err != nil { 33 | return err 34 | } 35 | 36 | switch header.Typeflag { 37 | case tar.TypeDir: 38 | if err := os.MkdirAll(p, 0755); err != nil { 39 | return fmt.Errorf("ExtractTarStream: Mkdir() failed: %w", err) 40 | } 41 | case tar.TypeReg: 42 | _ = os.Remove(p) // we allow overwriting, which easily happens on case insensitive filesystems 43 | outFile, err := os.Create(p) 44 | if err != nil { 45 | return fmt.Errorf("ExtractTarStream: Create() failed: %w", err) 46 | } 47 | _, err = io.Copy(outFile, tarReader) 48 | _ = outFile.Close() 49 | if err != nil { 50 | return fmt.Errorf("ExtractTarStream: Copy() failed: %w", err) 51 | } 52 | err = os.Chmod(p, header.FileInfo().Mode()) 53 | if err != nil { 54 | return fmt.Errorf("ExtractTarStream: Chmod() failed: %w", err) 55 | } 56 | err = os.Chtimes(p, header.AccessTime, header.ModTime) 57 | if err != nil { 58 | return err 59 | } 60 | case tar.TypeSymlink: 61 | _ = os.Remove(p) // we allow overwriting, which easily happens on case insensitive filesystems 62 | if err := os.Symlink(header.Linkname, p); err != nil { 63 | return fmt.Errorf("ExtractTarStream: Symlink() failed: %w", err) 64 | } 65 | default: 66 | return fmt.Errorf("ExtractTarStream: uknown type %v in %v", header.Typeflag, header.Name) 67 | } 68 | } 69 | return nil 70 | } 71 | 72 | func validRelPath(p string) bool { 73 | if p == "" || strings.Contains(p, `\`) || strings.HasPrefix(p, "/") || strings.Contains(p, "../") { 74 | return false 75 | } 76 | return true 77 | } 78 | -------------------------------------------------------------------------------- /pip/embed_pip_packages.go: -------------------------------------------------------------------------------- 1 | package pip 2 | 3 | import ( 4 | "fmt" 5 | "github.com/kluctl/go-embed-python/embed_util" 6 | "github.com/kluctl/go-embed-python/internal" 7 | "github.com/kluctl/go-embed-python/python" 8 | "math/rand" 9 | "os" 10 | "path/filepath" 11 | "strings" 12 | ) 13 | 14 | func CreateEmbeddedPipPackagesForKnownPlatforms(requirementsFile string, targetDir string) error { 15 | platforms := map[string][]string{ 16 | "darwin-amd64": {"macosx_11_0_x86_64", "macosx_12_0_x86_64"}, 17 | "darwin-arm64": {"macosx_11_0_arm64", "macosx_12_0_arm64"}, 18 | "linux-amd64": {"manylinux_2_17_x86_64", "manylinux_2_28_x86_64", "manylinux2014_x86_64"}, 19 | "linux-arm64": {"manylinux_2_17_aarch64", "manylinux_2_28_aarch64", "manylinux2014_aarch64"}, 20 | "windows-amd64": {"win_amd64"}, 21 | } 22 | 23 | for goPlatform, pipPlatforms := range platforms { 24 | s := strings.Split(goPlatform, "-") 25 | goOs, goArch := s[0], s[1] 26 | err := CreateEmbeddedPipPackages(requirementsFile, goOs, goArch, pipPlatforms, targetDir) 27 | if err != nil { 28 | return err 29 | } 30 | } 31 | return nil 32 | } 33 | 34 | func CreateEmbeddedPipPackages(requirementsFile string, goOs string, goArch string, pipPlatforms []string, targetDir string) error { 35 | name := fmt.Sprintf("pip-%d", rand.Uint32()) 36 | 37 | // ensure we have a stable extract path for the python distribution (otherwise shebangs won't be stable) 38 | tmpDir := filepath.Join("/tmp", fmt.Sprintf("python-pip-%s-%s", goOs, goArch)) 39 | ep, err := python.NewEmbeddedPythonWithTmpDir(tmpDir, false) 40 | if err != nil { 41 | return err 42 | } 43 | defer ep.Cleanup() 44 | 45 | pipLib, err := NewPipLib(name) 46 | if err != nil { 47 | return err 48 | } 49 | defer pipLib.Cleanup() 50 | 51 | ep.AddPythonPath(pipLib.GetExtractedPath()) 52 | 53 | return CreateEmbeddedPipPackages2(ep, requirementsFile, goOs, goArch, pipPlatforms, targetDir) 54 | } 55 | 56 | func CreateEmbeddedPipPackages2(ep *python.EmbeddedPython, requirementsFile string, goOs string, goArch string, pipPlatforms []string, targetDir string) error { 57 | tmpDir, err := os.MkdirTemp("", "pip-") 58 | if err != nil { 59 | return err 60 | } 61 | defer os.RemoveAll(tmpDir) 62 | 63 | err = pipInstall(ep, requirementsFile, pipPlatforms, tmpDir) 64 | if err != nil { 65 | return err 66 | } 67 | 68 | err = os.MkdirAll(targetDir, 0o755) 69 | if err != nil { 70 | return err 71 | } 72 | 73 | platformTargetDir := targetDir 74 | if goOs != "" { 75 | platformTargetDir = filepath.Join(platformTargetDir, fmt.Sprintf("%s-%s", goOs, goArch)) 76 | } 77 | 78 | if internal.Exists(platformTargetDir) { 79 | err = os.RemoveAll(platformTargetDir) 80 | if err != nil { 81 | return err 82 | } 83 | } 84 | 85 | err = os.Mkdir(platformTargetDir, 0o755) 86 | if err != nil { 87 | return err 88 | } 89 | 90 | err = embed_util.CopyForEmbed(platformTargetDir, tmpDir) 91 | if err != nil { 92 | return err 93 | } 94 | 95 | err = embed_util.WriteEmbedGoFile(targetDir, goOs, goArch) 96 | if err != nil { 97 | return err 98 | } 99 | 100 | return nil 101 | } 102 | 103 | func pipInstall(ep *python.EmbeddedPython, requirementsFile string, platforms []string, targetDir string) error { 104 | args := []string{"-m", "pip", "install", "-r", requirementsFile, "-t", targetDir} 105 | if len(platforms) != 0 { 106 | for _, p := range platforms { 107 | args = append(args, "--platform", p) 108 | } 109 | args = append(args, "--only-binary=:all:") 110 | } 111 | 112 | cmd, err := ep.PythonCmd(args...) 113 | if err != nil { 114 | return err 115 | } 116 | cmd.Stdout = os.Stdout 117 | cmd.Stderr = os.Stderr 118 | err = cmd.Run() 119 | if err != nil { 120 | return err 121 | } 122 | err = internal.CleanupPythonDir(targetDir, nil) 123 | if err != nil { 124 | return err 125 | } 126 | return nil 127 | } 128 | -------------------------------------------------------------------------------- /pip/generate/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "fmt" 5 | "github.com/kluctl/go-embed-python/pip" 6 | "github.com/kluctl/go-embed-python/python" 7 | "io" 8 | "net/http" 9 | "os" 10 | "path/filepath" 11 | ) 12 | 13 | func main() { 14 | targetDir := "./pip/internal/data" 15 | 16 | // ensure we have a stable extract path for the python distribution (otherwise shebangs won't be stable) 17 | tmpDir := filepath.Join("/tmp", fmt.Sprintf("python-pip-bootstrap")) 18 | ep, err := python.NewEmbeddedPythonWithTmpDir(tmpDir, false) 19 | if err != nil { 20 | panic(err) 21 | } 22 | defer ep.Cleanup() 23 | 24 | bootstrapPip(ep) 25 | 26 | err = pip.CreateEmbeddedPipPackages2(ep, "./pip/internal/requirements.txt", "", "", nil, targetDir) 27 | if err != nil { 28 | panic(err) 29 | } 30 | } 31 | 32 | func bootstrapPip(ep *python.EmbeddedPython) { 33 | getPip := downloadGetPip() 34 | defer os.Remove(getPip) 35 | 36 | cmd, err := ep.PythonCmd(getPip) 37 | if err != nil { 38 | panic(err) 39 | } 40 | cmd.Stdout = os.Stdout 41 | cmd.Stderr = os.Stderr 42 | err = cmd.Run() 43 | if err != nil { 44 | panic(err) 45 | } 46 | } 47 | 48 | func downloadGetPip() string { 49 | resp, err := http.Get("https://bootstrap.pypa.io/get-pip.py") 50 | if err != nil { 51 | panic(err) 52 | } 53 | if resp.StatusCode < 200 || resp.StatusCode > 299 { 54 | panic("failed to download get-pip.py: " + resp.Status) 55 | } 56 | 57 | tmpFile, err := os.CreateTemp("", "get-pip.py") 58 | if err != nil { 59 | panic(err) 60 | } 61 | defer tmpFile.Close() 62 | 63 | _, err = io.Copy(tmpFile, resp.Body) 64 | if err != nil { 65 | os.Remove(tmpFile.Name()) 66 | panic(err) 67 | } 68 | 69 | return tmpFile.Name() 70 | } 71 | -------------------------------------------------------------------------------- /pip/internal/data/.gitignore: -------------------------------------------------------------------------------- 1 | * 2 | !embed.go 3 | !.gitignore 4 | -------------------------------------------------------------------------------- /pip/internal/data/embed.go: -------------------------------------------------------------------------------- 1 | 2 | package data 3 | 4 | import "embed" 5 | 6 | //go:embed all:* 7 | var Data embed.FS 8 | -------------------------------------------------------------------------------- /pip/internal/requirements.txt: -------------------------------------------------------------------------------- 1 | pip==24.3.1 2 | -------------------------------------------------------------------------------- /pip/pip_lib.go: -------------------------------------------------------------------------------- 1 | package pip 2 | 3 | import ( 4 | "fmt" 5 | "github.com/kluctl/go-embed-python/embed_util" 6 | "github.com/kluctl/go-embed-python/pip/internal/data" 7 | ) 8 | 9 | func NewPipLib(name string) (*embed_util.EmbeddedFiles, error) { 10 | return embed_util.NewEmbeddedFiles(data.Data, fmt.Sprintf("pip-%s", name)) 11 | } 12 | -------------------------------------------------------------------------------- /python/embedded_python.go: -------------------------------------------------------------------------------- 1 | package python 2 | 3 | import ( 4 | "fmt" 5 | "github.com/kluctl/go-embed-python/embed_util" 6 | "github.com/kluctl/go-embed-python/python/internal/data" 7 | ) 8 | 9 | type EmbeddedPython struct { 10 | e *embed_util.EmbeddedFiles 11 | Python 12 | } 13 | 14 | // NewEmbeddedPython creates a new EmbeddedPython instance. The embedded source code and python binaries are 15 | // extracted on demand using the given name as the base for the temporary directory. You should ensure that the chosen 16 | // name does collide with other consumers of this library. 17 | func NewEmbeddedPython(name string) (*EmbeddedPython, error) { 18 | e, err := embed_util.NewEmbeddedFiles(data.Data, fmt.Sprintf("python-%s", name)) 19 | if err != nil { 20 | return nil, err 21 | } 22 | return &EmbeddedPython{ 23 | e: e, 24 | Python: NewPython(WithPythonHome(e.GetExtractedPath())), 25 | }, nil 26 | } 27 | 28 | func NewEmbeddedPythonWithTmpDir(tmpDir string, withHashInDir bool) (*EmbeddedPython, error) { 29 | e, err := embed_util.NewEmbeddedFilesWithTmpDir(data.Data, tmpDir, withHashInDir) 30 | if err != nil { 31 | return nil, err 32 | } 33 | return &EmbeddedPython{ 34 | e: e, 35 | Python: NewPython(WithPythonHome(e.GetExtractedPath())), 36 | }, nil 37 | } 38 | 39 | func (ep *EmbeddedPython) Cleanup() error { 40 | return ep.e.Cleanup() 41 | } 42 | 43 | func (ep *EmbeddedPython) GetExtractedPath() string { 44 | return ep.e.GetExtractedPath() 45 | } 46 | -------------------------------------------------------------------------------- /python/embedded_python_test.go: -------------------------------------------------------------------------------- 1 | package python 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "github.com/kluctl/go-embed-python/internal" 7 | "github.com/stretchr/testify/assert" 8 | "io" 9 | "math/rand" 10 | "testing" 11 | ) 12 | 13 | func TestEmbeddedPython(t *testing.T) { 14 | rndName := fmt.Sprintf("test-%d", rand.Uint32()) 15 | ep, err := NewEmbeddedPython(rndName) 16 | assert.NoError(t, err) 17 | defer ep.Cleanup() 18 | path := ep.GetExtractedPath() 19 | assert.NotEqual(t, path, "") 20 | pexe, _ := ep.GetExePath() 21 | assert.True(t, internal.Exists(pexe)) 22 | 23 | cmd, _ := ep.PythonCmd("-c", "print('test test')") 24 | stdout, err := cmd.StdoutPipe() 25 | assert.NoError(t, err) 26 | defer stdout.Close() 27 | 28 | err = cmd.Start() 29 | assert.NoError(t, err) 30 | 31 | stdoutStr, err := io.ReadAll(stdout) 32 | assert.NoError(t, err) 33 | 34 | err = cmd.Wait() 35 | assert.NoError(t, err) 36 | 37 | stdoutStr = bytes.TrimSpace(stdoutStr) 38 | assert.Equal(t, "test test", string(stdoutStr)) 39 | } 40 | 41 | func TestPrintSystemInfo(t *testing.T) { 42 | getSystemInfo := ` 43 | import platform, sys 44 | 45 | print("system info:") 46 | print("sys.version=" + sys.version) 47 | 48 | print("platform.python_version=" + platform.python_version()) 49 | print("platform.machine=" + platform.machine()) 50 | print("platform.version=" + platform.version()) 51 | print("platform.platform=" + platform.platform()) 52 | print("platform.release=" + platform.release()) 53 | print("platform.uname=" + str(platform.uname())) 54 | print("platform.system=" + platform.system()) 55 | print("platform.processor=" + platform.processor()) 56 | ` 57 | 58 | rndName := fmt.Sprintf("test-%d", rand.Uint32()) 59 | ep, err := NewEmbeddedPython(rndName) 60 | assert.NoError(t, err) 61 | defer ep.Cleanup() 62 | path := ep.GetExtractedPath() 63 | assert.NotEqual(t, path, "") 64 | pexe, _ := ep.GetExePath() 65 | assert.True(t, internal.Exists(pexe)) 66 | 67 | cmd, _ := ep.PythonCmd("-c", getSystemInfo) 68 | stdout, _ := cmd.StdoutPipe() 69 | stderr, _ := cmd.StderrPipe() 70 | defer stdout.Close() 71 | defer stderr.Close() 72 | 73 | err = cmd.Start() 74 | assert.NoError(t, err) 75 | 76 | stdoutStr, _ := io.ReadAll(stdout) 77 | stderrStr, _ := io.ReadAll(stderr) 78 | t.Log("stdout=" + string(stdoutStr)) 79 | t.Log("stderr=" + string(stderrStr)) 80 | 81 | err = cmd.Wait() 82 | assert.NoError(t, err) 83 | } 84 | -------------------------------------------------------------------------------- /python/generate/main.go: -------------------------------------------------------------------------------- 1 | package main 2 | 3 | import ( 4 | "flag" 5 | "fmt" 6 | "github.com/gobwas/glob" 7 | "github.com/klauspost/compress/zstd" 8 | "github.com/kluctl/go-embed-python/embed_util" 9 | "github.com/kluctl/go-embed-python/internal" 10 | log "github.com/sirupsen/logrus" 11 | "io" 12 | "net/http" 13 | "os" 14 | "path/filepath" 15 | "strings" 16 | "sync" 17 | ) 18 | 19 | var ( 20 | pythonStandaloneVersion = flag.String("python-standalone-version", "", "specify the python-standalone version. Check https://github.com/astral-sh/python-build-standalone/releases/ for available options.") 21 | pythonVersion = flag.String("python-version", "", "specify the python version.") 22 | preparePath = flag.String("prepare-path", filepath.Join(os.TempDir(), "python-download"), "specify the path where the python executables are downloaded and prepared. automatically creates a temporary directory if unset") 23 | runPrepare = flag.Bool("prepare", true, "if set, python executables will be downloaded and prepared for packing at the configured path") 24 | runPack = flag.Bool("pack", true, "if set, previously prepared python executables will be packed into their redistributable form") 25 | pythonVersionBase string 26 | ) 27 | 28 | var archMapping = map[string]string{ 29 | "amd64": "x86_64", 30 | "386": "i686", 31 | "arm64": "aarch64", 32 | } 33 | 34 | var removeLibs = []string{ 35 | "ensurepip", 36 | "idlelib", 37 | "lib2to3", 38 | "pydoc_data", 39 | "site-packages", 40 | "test", 41 | "turtledemo", 42 | "bin", // not really a library, but erroneously installed by jsonpath_ng 43 | } 44 | 45 | var keepNixPatterns = []glob.Glob{ 46 | glob.MustCompile("bin/**"), 47 | glob.MustCompile("lib/*.so*"), 48 | glob.MustCompile("lib/*.dylib"), 49 | glob.MustCompile("lib/python3.*/**"), 50 | } 51 | var keepWinPatterns = []glob.Glob{ 52 | glob.MustCompile("Lib/**"), 53 | glob.MustCompile("DLLs/**"), 54 | glob.MustCompile("*.dll"), 55 | glob.MustCompile("*.exe"), 56 | } 57 | 58 | var downloadLock sync.Mutex 59 | 60 | func main() { 61 | flag.Parse() 62 | 63 | if *pythonVersion == "" || *pythonStandaloneVersion == "" { 64 | log.Fatal("missing flags") 65 | } 66 | 67 | log.Infof("python-standalone-version=%s", *pythonStandaloneVersion) 68 | log.Infof("python-version=%s", *pythonVersion) 69 | 70 | pythonVersionBase = strings.Join(strings.Split(*pythonVersion, ".")[0:2], ".") 71 | 72 | targetPath := "./python/internal/data" 73 | 74 | var wg sync.WaitGroup 75 | 76 | type job struct { 77 | os string 78 | arch string 79 | dist string 80 | keepPatterns []glob.Glob 81 | } 82 | 83 | jobs := []job{ 84 | {"linux", "amd64", "unknown-linux-gnu-pgo+lto-full", keepNixPatterns}, 85 | {"linux", "arm64", "unknown-linux-gnu-lto-full", keepNixPatterns}, 86 | {"darwin", "amd64", "apple-darwin-pgo+lto-full", keepNixPatterns}, 87 | {"darwin", "arm64", "apple-darwin-pgo+lto-full", keepNixPatterns}, 88 | {"windows", "amd64", "pc-windows-msvc-shared-pgo-full", keepWinPatterns}, 89 | } 90 | for _, j := range jobs { 91 | j := j 92 | wg.Add(1) 93 | go func() { 94 | if *runPrepare { 95 | downloadAndPrepare(j.os, j.arch, j.dist, j.keepPatterns) 96 | } 97 | if *runPack { 98 | packPrepared(j.os, j.arch, j.dist, targetPath) 99 | } 100 | wg.Done() 101 | }() 102 | } 103 | wg.Wait() 104 | } 105 | 106 | func downloadAndPrepare(osName string, arch string, dist string, keepPatterns []glob.Glob) { 107 | downloadPath := download(osName, arch, dist) 108 | 109 | extractPath := downloadPath + ".extracted" 110 | err := os.RemoveAll(extractPath) 111 | if err != nil { 112 | log.Panic(err) 113 | } 114 | 115 | extract(downloadPath, extractPath) 116 | 117 | installPath := filepath.Join(extractPath, "python", "install") 118 | 119 | var libPath string 120 | if osName == "windows" { 121 | libPath = filepath.Join(installPath, "Lib") 122 | } else { 123 | libPath = filepath.Join(installPath, "lib", fmt.Sprintf("python%s", pythonVersionBase)) 124 | } 125 | 126 | for _, lib := range removeLibs { 127 | _ = os.RemoveAll(filepath.Join(libPath, lib)) 128 | } 129 | 130 | err = internal.CleanupPythonDir(installPath, keepPatterns) 131 | if err != nil { 132 | panic(err) 133 | } 134 | } 135 | 136 | func packPrepared(osName string, arch string, dist string, targetPath string) { 137 | extractPath := generateDownloadPath(arch, dist) + ".extracted" 138 | installPath := filepath.Join(extractPath, "python", "install") 139 | err := embed_util.CopyForEmbed(filepath.Join(targetPath, fmt.Sprintf("%s-%s", osName, arch)), installPath) 140 | if err != nil { 141 | panic(err) 142 | } 143 | 144 | err = embed_util.WriteEmbedGoFile(targetPath, osName, arch) 145 | if err != nil { 146 | panic(err) 147 | } 148 | 149 | f, err := os.Create(filepath.Join(targetPath, "PYTHON_VERSION")) 150 | if err != nil { 151 | panic(err) 152 | } 153 | defer f.Close() 154 | _, err = fmt.Fprintf(f, "PYTHON_VERSION=%q\nPYTHON_STANDALONE_VERSION=%q\n", *pythonVersion, *pythonStandaloneVersion) 155 | if err != nil { 156 | panic(err) 157 | } 158 | } 159 | 160 | func generateDownloadPath(arch string, dist string) string { 161 | pythonArch, ok := archMapping[arch] 162 | if !ok { 163 | log.Errorf("arch %s not supported", arch) 164 | os.Exit(1) 165 | } 166 | fname := fmt.Sprintf("cpython-%s+%s-%s-%s.tar.zst", *pythonVersion, *pythonStandaloneVersion, pythonArch, dist) 167 | return filepath.Join(*preparePath, fname) 168 | } 169 | 170 | func download(osName string, arch string, dist string) string { 171 | downloadLock.Lock() 172 | defer downloadLock.Unlock() 173 | 174 | downloadPath := generateDownloadPath(arch, dist) 175 | fname := filepath.Base(downloadPath) 176 | downloadUrl := fmt.Sprintf("https://github.com/astral-sh/python-build-standalone/releases/download/%s/%s", *pythonStandaloneVersion, fname) 177 | 178 | if _, err := os.Stat(downloadPath); err == nil { 179 | log.Infof("skipping download of %s", downloadUrl) 180 | return downloadPath 181 | } 182 | 183 | err := os.MkdirAll(filepath.Dir(downloadPath), 0o755) 184 | if err != nil { 185 | log.Errorf("mkdirs failed: %v", err) 186 | os.Exit(1) 187 | } 188 | log.Infof("downloading %s", downloadUrl) 189 | 190 | r, err := http.Get(downloadUrl) 191 | if err != nil { 192 | log.Errorf("download failed: %v", err) 193 | os.Exit(1) 194 | } 195 | if r.StatusCode == http.StatusNotFound { 196 | log.Errorf("404 not found") 197 | os.Exit(1) 198 | } 199 | defer r.Body.Close() 200 | 201 | fileData, err := io.ReadAll(r.Body) 202 | 203 | err = os.WriteFile(downloadPath, fileData, 0o640) 204 | if err != nil { 205 | log.Errorf("writing file failed: %v", err) 206 | os.Remove(downloadPath) 207 | os.Exit(1) 208 | } 209 | 210 | return downloadPath 211 | } 212 | 213 | func extract(archivePath string, targetPath string) string { 214 | f, err := os.Open(archivePath) 215 | if err != nil { 216 | log.Errorf("opening file failed: %v", err) 217 | os.Exit(1) 218 | } 219 | defer f.Close() 220 | 221 | z, err := zstd.NewReader(f) 222 | if err != nil { 223 | log.Errorf("decompression failed: %v", err) 224 | os.Exit(1) 225 | } 226 | defer z.Close() 227 | 228 | log.Infof("decompressing %s", archivePath) 229 | err = internal.ExtractTarStream(z, targetPath) 230 | if err != nil { 231 | log.Errorf("decompression failed: %v", err) 232 | os.Exit(1) 233 | } 234 | 235 | return targetPath 236 | } 237 | -------------------------------------------------------------------------------- /python/internal/data/.gitignore: -------------------------------------------------------------------------------- 1 | # we ignore these here, but the build-tag.sh script will force-add these anyway 2 | darwin-amd64 3 | darwin-arm64 4 | linux-amd64 5 | linux-arm64 6 | windows-amd64 7 | embed_*.go -------------------------------------------------------------------------------- /python/internal/data/dummy.go: -------------------------------------------------------------------------------- 1 | package data 2 | 3 | // PLEASE READ THIS!!!! 4 | // This file is really just a dummy. The release process will remove this file and generate some read embedded files 5 | // and commit these into a temporary branch and then tag it. This is to avoid clogging up the main branch with too many 6 | // binary files, which would be a very bad experience when pulling in go-embed-python as a dependency. 7 | -------------------------------------------------------------------------------- /python/python.go: -------------------------------------------------------------------------------- 1 | package python 2 | 3 | import ( 4 | "fmt" 5 | "os" 6 | "os/exec" 7 | "path/filepath" 8 | "runtime" 9 | "strings" 10 | ) 11 | 12 | type Python interface { 13 | GetExeName() string 14 | GetExePath() (string, error) 15 | AddPythonPath(p string) 16 | PythonCmd(args ...string) (*exec.Cmd, error) 17 | PythonCmd2(args []string) (*exec.Cmd, error) 18 | } 19 | 20 | type python struct { 21 | pythonHome string 22 | pythonPath []string 23 | } 24 | 25 | type PythonOpt func(o *python) 26 | 27 | func WithPythonHome(home string) PythonOpt { 28 | return func(o *python) { 29 | o.pythonHome = home 30 | } 31 | } 32 | 33 | func NewPython(opts ...PythonOpt) Python { 34 | ep := &python{} 35 | 36 | for _, o := range opts { 37 | o(ep) 38 | } 39 | 40 | return ep 41 | } 42 | 43 | func (ep *python) GetExeName() string { 44 | suffix := "" 45 | if runtime.GOOS == "windows" { 46 | suffix = ".exe" 47 | } else { 48 | suffix = "3" 49 | } 50 | return "python" + suffix 51 | } 52 | 53 | func (ep *python) GetExePath() (string, error) { 54 | if ep.pythonHome == "" { 55 | p, err := exec.LookPath(ep.GetExeName()) 56 | if err != nil { 57 | return "", fmt.Errorf("failed to determine %s path: %w", ep.GetExeName(), err) 58 | } 59 | return p, nil 60 | } else { 61 | var p string 62 | if runtime.GOOS == "windows" { 63 | p = filepath.Join(ep.pythonHome, ep.GetExeName()) 64 | } else { 65 | p = filepath.Join(ep.pythonHome, "bin", ep.GetExeName()) 66 | } 67 | if _, err := os.Stat(p); err != nil { 68 | return "", fmt.Errorf("failed to determine %s path: %w", ep.GetExeName(), err) 69 | } 70 | return p, nil 71 | } 72 | } 73 | 74 | func (ep *python) AddPythonPath(p string) { 75 | ep.pythonPath = append(ep.pythonPath, p) 76 | } 77 | 78 | func (ep *python) PythonCmd(args ...string) (*exec.Cmd, error) { 79 | return ep.PythonCmd2(args) 80 | } 81 | 82 | func (ep *python) PythonCmd2(args []string) (*exec.Cmd, error) { 83 | exePath, err := ep.GetExePath() 84 | if err != nil { 85 | return nil, err 86 | } 87 | 88 | cmd := exec.Command(exePath, args...) 89 | cmd.Env = os.Environ() 90 | 91 | if ep.pythonHome != "" { 92 | cmd.Env = append(cmd.Env, fmt.Sprintf("PYTHONHOME=%s", ep.pythonHome)) 93 | } 94 | 95 | if len(ep.pythonPath) != 0 { 96 | pythonPathEnv := fmt.Sprintf("PYTHONPATH=%s", strings.Join(ep.pythonPath, string(os.PathListSeparator))) 97 | cmd.Env = append(cmd.Env, pythonPathEnv) 98 | } 99 | 100 | return cmd, nil 101 | } 102 | -------------------------------------------------------------------------------- /python/python_test.go: -------------------------------------------------------------------------------- 1 | package python 2 | 3 | import ( 4 | "bytes" 5 | "github.com/kluctl/go-embed-python/internal" 6 | "github.com/stretchr/testify/assert" 7 | "io" 8 | "testing" 9 | ) 10 | 11 | func TestExternalPython(t *testing.T) { 12 | ep := NewPython() 13 | 14 | pexe, _ := ep.GetExePath() 15 | assert.True(t, internal.Exists(pexe)) 16 | 17 | cmd, _ := ep.PythonCmd("-c", "print('test test')") 18 | stdout, err := cmd.StdoutPipe() 19 | assert.NoError(t, err) 20 | defer stdout.Close() 21 | 22 | err = cmd.Start() 23 | assert.NoError(t, err) 24 | 25 | stdoutStr, err := io.ReadAll(stdout) 26 | assert.NoError(t, err) 27 | 28 | err = cmd.Wait() 29 | assert.NoError(t, err) 30 | 31 | stdoutStr = bytes.TrimSpace(stdoutStr) 32 | assert.Equal(t, "test test", string(stdoutStr)) 33 | } 34 | --------------------------------------------------------------------------------