├── .github └── workflows │ ├── bionic.yml │ └── focal.yml ├── .gitignore ├── LICENSE ├── README.md ├── ci ├── build-swiftlint-static.sh ├── build-swiftlint.sh └── install-macos-toolchain.sh ├── setup-swiftpm-toolchain └── swift-autolink-extract.swift /.github/workflows/bionic.yml: -------------------------------------------------------------------------------- 1 | name: bionic 2 | 3 | concurrency: 4 | group: ${{ github.workflow }}-${{ github.ref }} 5 | cancel-in-progress: true 6 | 7 | on: 8 | push: 9 | branches: [ "main" ] 10 | pull_request: 11 | branches: [ "main" ] 12 | schedule: 13 | # Every Wednesday at 11:03 UTC. 14 | - cron: '3 11 * * 3' 15 | 16 | jobs: 17 | bionic-build-swiftlint: 18 | runs-on: macos-latest 19 | steps: 20 | - uses: actions/checkout@v3 21 | - name: build swiftlint 22 | run: ./ci/build-swiftlint.sh bionic x86_64 23 | - uses: actions/upload-artifact@v2 24 | with: 25 | name: swiftlint-bionic 26 | path: swiftlint 27 | bionic-run-swiftlint: 28 | needs: bionic-build-swiftlint 29 | runs-on: ubuntu-18.04 30 | container: swift:5.6.2-bionic 31 | steps: 32 | - uses: actions/checkout@v3 33 | - uses: actions/download-artifact@v3 34 | with: 35 | name: swiftlint-bionic 36 | - run: chmod +x ./swiftlint 37 | - run: ./swiftlint 38 | bionic-build-swiftlint-static: 39 | runs-on: macos-latest 40 | steps: 41 | - uses: actions/checkout@v3 42 | - name: build swiftlint 43 | run: ./ci/build-swiftlint-static.sh bionic x86_64 44 | - uses: actions/upload-artifact@v2 45 | with: 46 | name: swiftlint-bionic-static 47 | path: swiftlint 48 | - uses: actions/upload-artifact@v2 49 | with: 50 | name: lib_InternalSwiftSyntaxParser-bionic 51 | path: ./toolchain-bionic-x86_64-5.6.2/swift-5.6.2-RELEASE-ubuntu18.04/usr/lib/swift_static/linux/lib_InternalSwiftSyntaxParser.so 52 | - uses: actions/upload-artifact@v2 53 | with: 54 | name: libBlocksRuntime-bionic 55 | path: ./toolchain-bionic-x86_64-5.6.2/swift-5.6.2-RELEASE-ubuntu18.04/usr/lib/swift/linux/libBlocksRuntime.so 56 | - uses: actions/upload-artifact@v2 57 | with: 58 | name: libdispatch-bionic 59 | path: ./toolchain-bionic-x86_64-5.6.2/swift-5.6.2-RELEASE-ubuntu18.04/usr/lib/swift/linux/libdispatch.so 60 | - uses: actions/upload-artifact@v2 61 | with: 62 | name: libsourcekitdInProc-bionic 63 | path: ./toolchain-bionic-x86_64-5.6.2/swift-5.6.2-RELEASE-ubuntu18.04/usr/lib/libsourcekitdInProc.so 64 | # TODO: Fix this job 65 | # bionic-run-swiftlint-static: 66 | # needs: bionic-build-swiftlint-static 67 | # runs-on: ubuntu-18.04 68 | # steps: 69 | # - uses: actions/checkout@v3 70 | # - uses: actions/download-artifact@v3 71 | # with: 72 | # name: swiftlint-bionic-static 73 | # - uses: actions/download-artifact@v3 74 | # with: 75 | # name: lib_InternalSwiftSyntaxParser-bionic 76 | # path: /tmp 77 | # - uses: actions/download-artifact@v3 78 | # with: 79 | # name: libdispatch-bionic 80 | # path: /tmp 81 | # - uses: actions/download-artifact@v3 82 | # with: 83 | # name: libBlocksRuntime-bionic 84 | # path: /tmp 85 | # - uses: actions/download-artifact@v3 86 | # with: 87 | # name: libsourcekitdInProc-bionic 88 | # path: /tmp 89 | # - run: chmod +x ./swiftlint 90 | # - run: LD_LIBRARY_PATH=/tmp ./swiftlint 91 | -------------------------------------------------------------------------------- /.github/workflows/focal.yml: -------------------------------------------------------------------------------- 1 | name: focal 2 | 3 | concurrency: 4 | group: ${{ github.workflow }}-${{ github.ref }} 5 | cancel-in-progress: true 6 | 7 | on: 8 | push: 9 | branches: [ "main" ] 10 | pull_request: 11 | branches: [ "main" ] 12 | schedule: 13 | # Every Wednesday at 11:03 UTC. 14 | - cron: '3 11 * * 3' 15 | 16 | jobs: 17 | focal-build-swiftlint: 18 | runs-on: macos-latest 19 | steps: 20 | - uses: actions/checkout@v3 21 | - name: build swiftlint 22 | run: ./ci/build-swiftlint.sh focal x86_64 23 | - uses: actions/upload-artifact@v2 24 | with: 25 | name: swiftlint-focal 26 | path: swiftlint 27 | focal-run-swiftlint: 28 | needs: focal-build-swiftlint 29 | runs-on: ubuntu-20.04 30 | container: swift:5.6.2-focal 31 | steps: 32 | - uses: actions/checkout@v3 33 | - uses: actions/download-artifact@v3 34 | with: 35 | name: swiftlint-focal 36 | - run: chmod +x ./swiftlint 37 | - run: ./swiftlint 38 | focal-build-swiftlint-static: 39 | runs-on: macos-latest 40 | steps: 41 | - uses: actions/checkout@v3 42 | - name: build swiftlint 43 | run: ./ci/build-swiftlint-static.sh focal x86_64 44 | - uses: actions/upload-artifact@v2 45 | with: 46 | name: swiftlint-focal-static 47 | path: swiftlint 48 | - uses: actions/upload-artifact@v2 49 | with: 50 | name: lib_InternalSwiftSyntaxParser-focal 51 | path: ./toolchain-focal-x86_64-5.6.2/swift-5.6.2-RELEASE-ubuntu20.04/usr/lib/swift_static/linux/lib_InternalSwiftSyntaxParser.so 52 | - uses: actions/upload-artifact@v2 53 | with: 54 | name: libBlocksRuntime-focal 55 | path: ./toolchain-focal-x86_64-5.6.2/swift-5.6.2-RELEASE-ubuntu20.04/usr/lib/swift/linux/libBlocksRuntime.so 56 | - uses: actions/upload-artifact@v2 57 | with: 58 | name: libdispatch-focal 59 | path: ./toolchain-focal-x86_64-5.6.2/swift-5.6.2-RELEASE-ubuntu20.04/usr/lib/swift/linux/libdispatch.so 60 | - uses: actions/upload-artifact@v2 61 | with: 62 | name: libsourcekitdInProc-focal 63 | path: ./toolchain-focal-x86_64-5.6.2/swift-5.6.2-RELEASE-ubuntu20.04/usr/lib/libsourcekitdInProc.so 64 | focal-run-swiftlint-static: 65 | needs: focal-build-swiftlint-static 66 | runs-on: ubuntu-20.04 67 | steps: 68 | - uses: actions/checkout@v3 69 | - uses: actions/download-artifact@v3 70 | with: 71 | name: swiftlint-focal-static 72 | - uses: actions/download-artifact@v3 73 | with: 74 | name: lib_InternalSwiftSyntaxParser-focal 75 | path: /tmp 76 | - uses: actions/download-artifact@v3 77 | with: 78 | name: libdispatch-focal 79 | path: /tmp 80 | - uses: actions/download-artifact@v3 81 | with: 82 | name: libBlocksRuntime-focal 83 | path: /tmp 84 | - uses: actions/download-artifact@v3 85 | with: 86 | name: libsourcekitdInProc-focal 87 | path: /tmp 88 | - run: chmod +x ./swiftlint 89 | - run: LD_LIBRARY_PATH=/tmp ./swiftlint 90 | # arm binaries aren't tested because of x86_64 CI machines on GitHub actions. 91 | # Hopefully building is good enough 92 | focal-build-swiftlint-arm: 93 | runs-on: macos-latest 94 | steps: 95 | - uses: actions/checkout@v3 96 | - name: build swiftlint 97 | run: ./ci/build-swiftlint.sh focal arm64 98 | focal-build-swiftlint-static-arm: 99 | runs-on: macos-latest 100 | steps: 101 | - uses: actions/checkout@v3 102 | - name: build swiftlint 103 | run: ./ci/build-swiftlint-static.sh focal arm64 104 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | /toolchain-*/ 2 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2022 Keith Smiley (http://keith.so) 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy of 4 | this software and associated documentation files (the 'Software'), to deal in 5 | the Software without restriction, including without limitation the rights to 6 | use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of 7 | the Software, and to permit persons to whom the Software is furnished to do so, 8 | subject to the following conditions: 9 | 10 | The above copyright notice and this permission notice shall be included in all 11 | copies or substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS 15 | FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR 16 | COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER 17 | IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 18 | CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 19 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # swiftpm-linux-cross 2 | 3 | This repo provides easy scripts for setting up a toolchain that allows 4 | you to cross compile Linux binaries from a macOS machine using Swift 5 | Package Manager. This can be useful for building your release binaries 6 | on your host machine instead of in a potentially slower VM or docker 7 | container. 8 | 9 | ## Replaced by [swift-sdk-generator](https://github.com/apple/swift-sdk-generator) 10 | 11 | This repo might still be a useful example if you need to manage this 12 | yourself, but moving forward you should use the new 13 | [swift-sdk-generator](https://github.com/apple/swift-sdk-generator) 14 | project instead. 15 | 16 | ## Usage 17 | 18 | ```sh 19 | $ ./setup-swiftpm-toolchain 20 | Done! To use run: 21 | 22 | swift build --destination /Users/ksmiley/dev/swiftpm-linux-cross/toolchain-focal-x86_64-5.6.2/destination.json 23 | 24 | Or for static stdlib builds run: 25 | 26 | swift build --static-swift-stdlib --destination /Users/ksmiley/dev/swiftpm-linux-cross/toolchain-focal-x86_64-5.6.2/destination_static.json 27 | ``` 28 | 29 | See the help to customize the OS, architecture, Swift version, or 30 | provide your own system package dependencies. 31 | 32 | ```sh 33 | $ ./setup-swiftpm-toolchain --help 34 | usage: setup-swiftpm-toolchain [-h] [--ubuntu-release {bionic,focal}] [--arch {x86_64,arm64}] [--swift-version SWIFT_VERSION] [packages ...] 35 | 36 | positional arguments: 37 | packages Extra OS packages your build requires 38 | 39 | options: 40 | -h, --help show this help message and exit 41 | --ubuntu-release {bionic,focal} 42 | The target Ubuntu release you will run on 43 | --arch {x86_64,arm64} 44 | The target architecture to build for 45 | --swift-version SWIFT_VERSION 46 | The version of Swift you're building with and for 47 | ``` 48 | 49 | Currently this only supports targeting Ubuntu, but it should be extended 50 | to support more operating systems. 51 | 52 | ## Installation 53 | 54 | ### With [homebrew](https://brew.sh) 55 | 56 | ``` 57 | brew install keith/formulae/swiftpm-linux-cross 58 | ``` 59 | 60 | ### Manually 61 | 62 | 1. Clone this repo 63 | 2. Run any `setup-swiftpm-toolchains` inside the repo 64 | 65 | ## Matching Swift Versions 66 | 67 | To use this your local version of Swift much match the version of Swift 68 | the toolchain was created for. This might mean that you have to 69 | [download a Swift toolchain](https://www.swift.org/download) instead of 70 | using the one bundled with Xcode. If this is required you will see an 71 | error when building like: 72 | 73 | ``` 74 | :0: error: compiled module was created by a different version of the compiler; rebuild 'Swift' and try again 75 | ``` 76 | 77 | Once you install the matching toolchain version you can then build using 78 | the `TOOLCHAINS` environment variable like this: 79 | 80 | ``` 81 | TOOLCHAINS=org.swift.562202206021a swift build --destination /Users/ksmiley/dev/swiftpm-linux-cross/toolchain-focal-x86_64-5.6.2/destination.json 82 | ``` 83 | 84 | You can fetch the identifier used here from the `Info.plist` bundled 85 | with the toolchain: 86 | 87 | ``` 88 | /usr/libexec/PlistBuddy -c "print :CFBundleIdentifier" /Library/Developer/Toolchains/swift-latest.xctoolchain/Info.plist 89 | ``` 90 | 91 | This path varies either in `/Library/Developer/Toolchains` or 92 | `~/Library/Developer/Toolchains` depending on how you install it. 93 | 94 | ## References 95 | 96 | - [This script][script] is outdated but is a useful reference 97 | 98 | [script]: https://github.com/apple/swift-package-manager/blob/401640a3d1553ba1a340f9b53e40b3c7442b5452/Utilities/build_ubuntu_cross_compilation_toolchain 99 | -------------------------------------------------------------------------------- /ci/build-swiftlint-static.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -euo pipefail 4 | set -x 5 | 6 | readonly os=$1 7 | readonly arch=$2 8 | 9 | toolchain_id=$(./ci/install-macos-toolchain.sh) 10 | 11 | ./setup-swiftpm-toolchain --ubuntu-release "$os" --arch "$arch" 12 | readonly destination=$PWD/toolchain-$os-$arch-5.6.2/destination_static.json 13 | readonly binary_path=$PWD/swiftlint 14 | 15 | cd "$(mktemp -d)" 16 | git clone --branch ks/switch-to-swiftpm-conditional-dependencies-api --recursive --depth 1 https://github.com/realm/SwiftLint.git 17 | cd SwiftLint 18 | 19 | TOOLCHAINS="$toolchain_id" swift build --destination "$destination" --static-swift-stdlib -Xlinker -lCFURLSessionInterface -Xlinker -lCFXMLInterface -Xlinker -lcurl -Xlinker -lxml2 -Xlinker -lz -Xlinker -llzma -Xlinker -licuuc -Xlinker -licudata 20 | cp ./.build/debug/swiftlint "$binary_path" 21 | -------------------------------------------------------------------------------- /ci/build-swiftlint.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -euo pipefail 4 | set -x 5 | 6 | readonly os=$1 7 | readonly arch=$2 8 | 9 | toolchain_id=$(./ci/install-macos-toolchain.sh) 10 | 11 | ./setup-swiftpm-toolchain --ubuntu-release "$os" --arch "$arch" 12 | readonly destination=$PWD/toolchain-$os-$arch-5.6.2/destination.json 13 | readonly binary_path=$PWD/swiftlint 14 | 15 | cd "$(mktemp -d)" 16 | git clone --branch ks/switch-to-swiftpm-conditional-dependencies-api --recursive --depth 1 https://github.com/realm/SwiftLint.git 17 | cd SwiftLint 18 | 19 | TOOLCHAINS="$toolchain_id" swift build --destination "$destination" 20 | cp ./.build/debug/swiftlint "$binary_path" 21 | -------------------------------------------------------------------------------- /ci/install-macos-toolchain.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -euo pipefail 4 | set -x 5 | 6 | wget https://download.swift.org/swift-5.6.2-release/xcode/swift-5.6.2-RELEASE/swift-5.6.2-RELEASE-osx.pkg >&2 7 | installer -target CurrentUserHomeDirectory -pkg swift-5.6.2-RELEASE-osx.pkg >&2 8 | 9 | /usr/libexec/PlistBuddy -c "print :CFBundleIdentifier" ~/Library/Developer/Toolchains/swift-latest.xctoolchain/Info.plist 10 | -------------------------------------------------------------------------------- /setup-swiftpm-toolchain: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | from pathlib import Path 4 | from typing import List, Tuple 5 | import argparse 6 | import contextlib 7 | import gzip 8 | import json 9 | import multiprocessing.pool 10 | import os 11 | import shutil 12 | import subprocess 13 | import sys 14 | import tarfile 15 | import tempfile 16 | import urllib.error 17 | import urllib.request 18 | 19 | _FOCAL_PACKAGES = [ 20 | "libbrotli-dev", 21 | "libc6", 22 | "libc6-dev", 23 | "libcurl4", 24 | "libcurl4-openssl-dev", 25 | "libgcc-9-dev", 26 | "libgcc-s1", 27 | "libicu-dev", 28 | "liblzma-dev", 29 | "libssl-dev", 30 | "libstdc++-9-dev", 31 | "libxml2-dev", 32 | "linux-libc-dev", 33 | "zlib1g-dev", 34 | ] 35 | _BIONIC_PACKAGES = [ 36 | "libc6", 37 | "libc6-dev", 38 | "libcurl4", 39 | "libcurl4-openssl-dev", 40 | "libgcc-7-dev", 41 | "libgcc1", 42 | "libicu-dev", 43 | "liblzma-dev", 44 | "libssl-dev", 45 | "libstdc++-7-dev", 46 | "libunwind-dev", 47 | "libxml2-dev", 48 | "linux-libc-dev", 49 | "zlib1g-dev", 50 | ] 51 | 52 | 53 | class Arch: 54 | def __init__( 55 | self, id: str, swift_id: str, ubuntu_id: str, ubuntu_mirror: str 56 | ): 57 | self.id = id 58 | self.swift_id = swift_id 59 | self.ubuntu_id = ubuntu_id 60 | self.ubuntu_mirror = ubuntu_mirror 61 | 62 | 63 | ARM64 = Arch( 64 | "arm64", "aarch64", "arm64", "http://ports.ubuntu.com/ubuntu-ports" 65 | ) 66 | X86_64 = Arch( 67 | "x86_64", "x86_64", "amd64", "http://gb.archive.ubuntu.com/ubuntu" 68 | ) 69 | 70 | 71 | def _build_parser() -> argparse.ArgumentParser: 72 | parser = argparse.ArgumentParser() 73 | parser.add_argument( 74 | "--ubuntu-release", 75 | choices=("bionic", "focal"), 76 | default="focal", 77 | help="The target Ubuntu release you will run on", 78 | ) 79 | parser.add_argument( 80 | "--arch", 81 | choices=("x86_64", "arm64"), 82 | default="x86_64", 83 | help="The target architecture to build for", 84 | ) 85 | parser.add_argument( 86 | "--swift-version", 87 | default="5.6.2", 88 | help="The version of Swift you're building with and for", 89 | ) 90 | parser.add_argument( 91 | "packages", nargs="*", help="Extra OS packages your build requires" 92 | ) 93 | return parser 94 | 95 | 96 | def _install_llvm() -> None: 97 | try: 98 | subprocess.check_call( 99 | ["brew", "--prefix", "llvm"], 100 | stdout=subprocess.DEVNULL, 101 | stderr=subprocess.DEVNULL, 102 | ) 103 | except subprocess.CalledProcessError: 104 | print("Installing LLVM...") 105 | subprocess.check_call(["brew", "install", "llvm"]) 106 | 107 | 108 | def _setup_toolchain_root(os: str, arch: Arch, swift_version: str) -> Path: 109 | toolchain_dir = Path(f"toolchain-{os}-{arch.id}-{swift_version}") 110 | if toolchain_dir.is_dir(): 111 | shutil.rmtree(toolchain_dir) 112 | elif toolchain_dir.exists(): 113 | toolchain_dir.unlink() 114 | if toolchain_dir.exists(): 115 | raise SystemExit( 116 | f"error: failed to remove '{toolchain_dir}', please delete it and re-run" 117 | ) 118 | toolchain_dir.mkdir() 119 | return toolchain_dir 120 | 121 | 122 | def _download_extract_swift( 123 | os: str, arch: Arch, swift_version: str, toolchain_dir: Path 124 | ) -> Path: 125 | if os == "focal": 126 | v1 = "ubuntu2004" 127 | v2 = "ubuntu20.04" 128 | elif os == "bionic": 129 | v1 = "ubuntu1804" 130 | v2 = "ubuntu18.04" 131 | else: 132 | raise SystemExit(f"error: unsupported os: {os}") 133 | 134 | arch_suffix = "" 135 | if arch.id == "arm64": 136 | arch_suffix = "-aarch64" 137 | 138 | swift_url = f"https://download.swift.org/swift-{swift_version}-release/{v1}{arch_suffix}/swift-{swift_version}-RELEASE/swift-{swift_version}-RELEASE-{v2}{arch_suffix}.tar.gz" 139 | swift_archive = Path(f"/tmp/swift-{os}-{arch.id}-{swift_version}.tar.gz") 140 | if not swift_archive.exists(): 141 | print(f"Downloading Swift {swift_version} for {os}-{arch.id}...") 142 | # TODO: progress reporting 143 | urllib.request.urlretrieve(swift_url, swift_archive) 144 | 145 | print("Extracting Swift...") 146 | shutil.unpack_archive(swift_archive, toolchain_dir) 147 | return next(toolchain_dir.glob("swift-*")) 148 | 149 | 150 | def _setup_swift_bin(swift_root: Path) -> Path: 151 | swift_bin = swift_root / "usr" / "cross-bin" 152 | swift_bin.mkdir() 153 | os.symlink("/usr/bin/ar", swift_bin.absolute() / "ar") 154 | llvm_root = Path( 155 | subprocess.check_output( 156 | ["brew", "--prefix", "llvm"], 157 | ) 158 | .decode() 159 | .strip() 160 | ) 161 | os.symlink(llvm_root / "bin" / "ld.lld", swift_bin.absolute() / "ld.lld") 162 | subprocess.check_call( 163 | [ 164 | "swiftc", 165 | Path(__file__).resolve().parent / "swift-autolink-extract.swift", 166 | "-o", 167 | swift_bin / "swift-autolink-extract", 168 | ] 169 | ) 170 | return swift_bin 171 | 172 | 173 | def _write_destination_file( 174 | arch: Arch, 175 | toolchain_dir: Path, 176 | swift_root: Path, 177 | swift_bin: Path, 178 | sdk_root: Path, 179 | ) -> Tuple[Path, Path]: 180 | contents = { 181 | "version": 1, 182 | "sdk": str(sdk_root.absolute()), 183 | "toolchain-bin-dir": str(swift_bin.absolute()), 184 | "target": f"{arch.swift_id}-unknown-linux-gnu", 185 | "extra-cc-flags": ["-fPIC"], 186 | "extra-cpp-flags": ["-lstdc++"], 187 | "extra-swiftc-flags": [ 188 | "-use-ld=lld", 189 | "-Xlinker", 190 | "-rpath", 191 | "-Xlinker", 192 | "/usr/lib/swift/linux", 193 | "-tools-directory", 194 | str(swift_bin.absolute()), 195 | "-resource-dir", 196 | str((swift_root / "usr" / "lib" / "swift").absolute()), 197 | "-sdk", 198 | str(sdk_root.absolute()), 199 | ], 200 | } 201 | 202 | destination = toolchain_dir / "destination.json" 203 | with open(destination, "w") as f: 204 | json.dump(contents, f, indent=4) 205 | 206 | contents = { 207 | "version": 1, 208 | "sdk": str(sdk_root.absolute()), 209 | "toolchain-bin-dir": str(swift_bin.absolute()), 210 | "target": f"{arch.swift_id}-unknown-linux-gnu", 211 | "extra-cc-flags": ["-fPIC"], 212 | "extra-cpp-flags": ["-lstdc++"], 213 | "extra-swiftc-flags": [ 214 | "-use-ld=lld", 215 | "-Xlinker", 216 | "-rpath", 217 | "-Xlinker", 218 | "/usr/lib/swift/linux", 219 | "-tools-directory", 220 | str(swift_bin.absolute()), 221 | "-resource-dir", 222 | str((swift_root / "usr" / "lib" / "swift_static").absolute()), 223 | "-sdk", 224 | str(sdk_root.absolute()), 225 | ], 226 | } 227 | 228 | destination_static = toolchain_dir / "destination_static.json" 229 | with open(destination_static, "w") as f: 230 | json.dump(contents, f, indent=4) 231 | 232 | return destination, destination_static 233 | 234 | 235 | @contextlib.contextmanager 236 | def restore_pwd(): 237 | pwd = os.getcwd() 238 | try: 239 | yield 240 | finally: 241 | os.chdir(pwd) 242 | 243 | 244 | def _download_and_extract_package( 245 | name: str, url: str, package_dir: Path 246 | ) -> None: 247 | name = url.split("/")[-1] 248 | output_path = Path(f"/tmp/{name}.deb") 249 | try: 250 | urllib.request.urlretrieve(url, output_path) 251 | except urllib.error.HTTPError as e: 252 | print( 253 | f"error: failed to download {name} from {url}: {e}", 254 | file=sys.stderr, 255 | ) 256 | raise 257 | with tempfile.TemporaryDirectory() as dirname: 258 | with restore_pwd(): 259 | os.chdir(dirname) 260 | subprocess.check_call(["ar", "x", output_path]) 261 | with tarfile.open("data.tar.xz") as f: 262 | f.extractall(package_dir) 263 | 264 | 265 | def _download_packages( 266 | os: str, arch: Arch, packages: List[str], package_dir: Path 267 | ) -> None: 268 | package_archive = Path(f"/tmp/packages-{os}-{arch.id}.gz") 269 | if not package_archive.exists(): 270 | print(f"Downloading package list for {os}-{arch.id}...") 271 | package_url = f"{arch.ubuntu_mirror}/dists/{os}/main/binary-{arch.ubuntu_id}/Packages.gz" 272 | # TODO: progress reporting 273 | urllib.request.urlretrieve(package_url, package_archive) 274 | 275 | with gzip.open(package_archive, "rb") as f: 276 | files = [ 277 | x 278 | for x in f.read().decode().splitlines() 279 | if x.startswith("Filename: ") 280 | ] 281 | 282 | if os == "focal": 283 | os_packages = _FOCAL_PACKAGES 284 | elif os == "bionic": 285 | os_packages = _BIONIC_PACKAGES 286 | else: 287 | raise SystemExit(f"error: unsupported os: {os}") 288 | 289 | os_packages += packages 290 | 291 | package_paths = {} 292 | needed_packages = set(os_packages) 293 | for filename in files: 294 | if not needed_packages: 295 | break 296 | for package in needed_packages: 297 | # Filename: pool/main/c/curl/libcurl4-openssl-dev_7.68.0-1ubuntu2_amd64.deb 298 | last_component = filename.split("/")[-1] 299 | if last_component.startswith(f"{package}_"): 300 | package_paths[package] = filename.split(" ")[-1] 301 | needed_packages.remove(package) 302 | break 303 | 304 | missing_packages = set(os_packages) - set(package_paths.keys()) 305 | if missing_packages: 306 | raise SystemExit( 307 | "Failed to find some packages, please report this issue: {}".format( 308 | " ".join(sorted(missing_packages)) 309 | ) 310 | ) 311 | 312 | pool = multiprocessing.pool.Pool() 313 | results = [] 314 | for name, path in package_paths.items(): 315 | results.append( 316 | pool.apply_async( 317 | _download_and_extract_package, 318 | (name, f"{arch.ubuntu_mirror}/{path}", package_dir.absolute()), 319 | ) 320 | ) 321 | 322 | pool.close() 323 | pool.join() 324 | 325 | for result in results: 326 | if not result.successful(): 327 | raise SystemExit(f"error: {result.get()}") 328 | 329 | broken_libraries_dir = ( 330 | package_dir / "usr" / "lib" / f"{arch.swift_id}-linux-gnu" 331 | ) 332 | destination_dir = package_dir / "lib" / f"{arch.swift_id}-linux-gnu" 333 | _fix_package_symlinks(broken_libraries_dir, destination_dir) 334 | broken_libraries_dir = next( 335 | ( 336 | package_dir / "usr" / "lib" / "gcc" / f"{arch.swift_id}-linux-gnu" 337 | ).glob("*") 338 | ) 339 | _fix_package_symlinks(broken_libraries_dir, broken_libraries_dir) 340 | 341 | 342 | def _fix_package_symlinks( 343 | broken_libraries_dir: Path, destination_dir: Path 344 | ) -> None: 345 | assert broken_libraries_dir.exists() 346 | assert destination_dir.exists() 347 | for lib in broken_libraries_dir.glob("*.so*"): 348 | # Ignore valid symlinks or normal files 349 | if not lib.is_symlink() or lib.exists(): 350 | continue 351 | 352 | # TODO: Ideally this would be relative to be more portable 353 | dest = (destination_dir / lib.readlink().name).absolute() 354 | if not dest.exists(): 355 | # TODO: Ideally no invalid symlinks would remain, but some do 356 | continue 357 | 358 | lib.unlink() 359 | lib.symlink_to(dest) 360 | 361 | 362 | def _apply_static_fixups(swift_root: Path) -> None: 363 | dynamic_lib_dir = swift_root / "usr" / "lib" / "swift" 364 | static_lib_dir = swift_root / "usr" / "lib" / "swift_static" 365 | shutil.copytree( 366 | dynamic_lib_dir / "_InternalSwiftSyntaxParser", 367 | static_lib_dir / "_InternalSwiftSyntaxParser", 368 | ) 369 | shutil.copy( 370 | dynamic_lib_dir / "linux" / "lib_InternalSwiftSyntaxParser.so", 371 | static_lib_dir / "linux", 372 | ) 373 | shutil.copytree( 374 | dynamic_lib_dir / "_InternalSwiftScan", 375 | static_lib_dir / "_InternalSwiftScan", 376 | ) 377 | shutil.copy( 378 | dynamic_lib_dir / "linux" / "lib_InternalSwiftScan.so", 379 | static_lib_dir / "linux", 380 | ) 381 | 382 | 383 | def _main( 384 | os: str, arch: Arch, swift_version: str, packages: List[str] 385 | ) -> None: 386 | _install_llvm() 387 | toolchain_dir = _setup_toolchain_root(os, arch, swift_version) 388 | swift_root = _download_extract_swift( 389 | os, arch, swift_version, toolchain_dir 390 | ) 391 | swift_bin = _setup_swift_bin(swift_root) 392 | _apply_static_fixups(swift_root) 393 | sdk_root = toolchain_dir / "packages" 394 | _download_packages(os, arch, packages, sdk_root) 395 | destination, static_destination = _write_destination_file( 396 | arch, toolchain_dir, swift_root, swift_bin, sdk_root 397 | ) 398 | 399 | print( 400 | f"""\ 401 | Done! To use run: 402 | 403 | swift build --destination {destination.absolute()} 404 | 405 | Or for static stdlib builds run: 406 | 407 | swift build --static-swift-stdlib --destination {static_destination.absolute()}\ 408 | """ 409 | ) 410 | 411 | 412 | if __name__ == "__main__": 413 | args = _build_parser().parse_args() 414 | if args.arch == "x86_64": 415 | arch = X86_64 416 | elif args.arch == "arm64": 417 | arch = ARM64 418 | else: 419 | raise SystemExit(f"error: invalid arch: {args.arch}") 420 | 421 | if args.arch == "arm64" and args.ubuntu_release == "bionic": 422 | raise SystemExit( 423 | f"error: Swift didn't vendor arm64 toolchains until Ubuntu Focal, either use Focal or target x86_64" 424 | ) 425 | 426 | _main(args.ubuntu_release, arch, args.swift_version, args.packages) 427 | -------------------------------------------------------------------------------- /swift-autolink-extract.swift: -------------------------------------------------------------------------------- 1 | import Foundation 2 | 3 | let task = Process() 4 | task.launchPath = "/usr/bin/xcrun" 5 | task.arguments = ["swift-frontend", "--driver-mode=swift-autolink-extract"] + Array(ProcessInfo.processInfo.arguments.dropFirst()) 6 | task.launch() 7 | task.waitUntilExit() 8 | exit(task.terminationStatus) 9 | --------------------------------------------------------------------------------