├── .editorconfig ├── .github ├── CODEOWNERS └── workflows │ └── pull_request.yml ├── .gitignore ├── .license_header_template ├── .licenseignore ├── .swift-format ├── .swiftformat ├── .swiftformatignore ├── .unacceptablelanguageignore ├── Brewfile ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── Docker ├── Dockerfile ├── docker-compose.2204.59.yaml ├── docker-compose.2204.main.yaml └── docker-compose.yaml ├── LICENSE.txt ├── Package.resolved ├── Package.swift ├── README.md ├── Sources ├── AsyncProcess │ ├── ChunkSequence.swift │ ├── EOFSequence.swift │ ├── FileContentStream.swift │ ├── NIOAsyncPipeWriter.swift │ ├── ProcessExecutor+Convenience.swift │ ├── ProcessExecutor.swift │ └── ProcessExit.swift ├── GeneratorCLI │ └── GeneratorCLI.swift ├── Helpers │ ├── ThrowingDefer.swift │ └── Vendor │ │ ├── QueryEngine │ │ ├── CacheKey.swift │ │ ├── FileCacheRecord.swift │ │ ├── FileLock.swift │ │ ├── Query.swift │ │ ├── QueryEngine.swift │ │ ├── SQLite.swift │ │ └── SQLiteBackedCache.swift │ │ ├── README.md │ │ ├── Triple.swift │ │ └── _AsyncFileSystem │ │ ├── AsyncFileSystem.swift │ │ ├── ConcurrencySupport.swift │ │ ├── MockFileSystem.swift │ │ ├── OSFileSystem.swift │ │ ├── OpenReadableFile.swift │ │ ├── OpenWritableFile.swift │ │ ├── ReadableFileStream.swift │ │ └── WritableStream.swift ├── SwiftSDKGenerator │ ├── Artifacts │ │ └── DownloadableArtifacts.swift │ ├── ArtifactsArchiveMetadata.swift │ ├── Dockerfiles │ │ └── Ubuntu │ │ │ └── 22.04 │ │ │ └── Dockerfile │ ├── Extensions │ │ └── String+hasAnyPrefix.swift │ ├── Generator │ │ ├── SwiftSDKGenerator+Copy.swift │ │ ├── SwiftSDKGenerator+Download.swift │ │ ├── SwiftSDKGenerator+Entrypoint.swift │ │ ├── SwiftSDKGenerator+Fixup.swift │ │ ├── SwiftSDKGenerator+Metadata.swift │ │ ├── SwiftSDKGenerator+Unpack.swift │ │ └── SwiftSDKGenerator.swift │ ├── PathsConfiguration.swift │ ├── PlatformModels │ │ ├── LinuxDistribution.swift │ │ ├── Triple.swift │ │ └── VersionsConfiguration.swift │ ├── Queries │ │ ├── CMakeBuildQuery.swift │ │ ├── DownloadArtifactQuery.swift │ │ ├── DownloadFileQuery.swift │ │ └── TarExtractQuery.swift │ ├── Serialization │ │ ├── SwiftSDKMetadata.swift │ │ └── Toolset.swift │ ├── SwiftSDKRecipes │ │ ├── LinuxRecipe.swift │ │ ├── SwiftSDKRecipe.swift │ │ └── WebAssemblyRecipe.swift │ └── SystemUtils │ │ ├── ByteBuffer+Utils.swift │ │ ├── FileOperationError.swift │ │ ├── GeneratorError.swift │ │ ├── HTTPClient+Download.swift │ │ ├── Shell.swift │ │ ├── UnixName.swift │ │ └── which.swift └── SystemSQLite │ ├── module.modulemap │ └── sqlite.h ├── Tests ├── AsyncProcessTests │ ├── AsyncByteBufferLineSequenceTests.swift │ ├── Helpers+LogRecorderHandler.swift │ └── IntegrationTests.swift ├── GeneratorEngineTests │ └── EngineTests.swift ├── HelpersTests │ └── ThrowingDeferTests.swift ├── MacrosTests │ └── MacrosTests.swift └── SwiftSDKGeneratorTests │ ├── ArchitectureMappingTests.swift │ ├── EndToEndTests.swift │ ├── Generator │ └── SwiftSDKGenerator+MetadataTests.swift │ └── SwiftSDKRecipes │ ├── LinuxRecipeTests.swift │ └── WebAssemblyRecipe.swift └── Utilities └── git.commit.template /.editorconfig: -------------------------------------------------------------------------------- 1 | [*] 2 | indent_style = space 3 | indent_size = 2 4 | -------------------------------------------------------------------------------- /.github/CODEOWNERS: -------------------------------------------------------------------------------- 1 | # This file is a list of the people responsible for ensuring that patches for a 2 | # particular part of Swift are reviewed, either by themselves or by someone else. 3 | # They are also the gatekeepers for their part of Swift, with the final word on 4 | # what goes in or not. 5 | # 6 | # The list is sorted by surname and formatted to allow easy grepping and 7 | # beautification by scripts. The fields are: name (N), email (E), web-address 8 | # (W), PGP key ID and fingerprint (P), description (D), and snail-mail address 9 | # (S). 10 | 11 | # N: Max Desiatov 12 | # E: m_desiatov@apple.com 13 | 14 | ### 15 | 16 | # The following lines are used by GitHub to automatically recommend reviewers. 17 | 18 | * @MaxDesiatov 19 | -------------------------------------------------------------------------------- /.github/workflows/pull_request.yml: -------------------------------------------------------------------------------- 1 | name: Pull request 2 | 3 | on: 4 | pull_request: 5 | types: [opened, reopened, synchronize] 6 | 7 | jobs: 8 | tests: 9 | name: Test 10 | uses: swiftlang/github-workflows/.github/workflows/swift_package_test.yml@main 11 | with: 12 | linux_pre_build_command: apt-get update && apt-get install -y locales locales-all libsqlite3-dev 13 | enable_windows_checks: false 14 | soundness: 15 | name: Soundness 16 | uses: swiftlang/github-workflows/.github/workflows/soundness.yml@main 17 | with: 18 | api_breakage_check_enabled: false 19 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | /.build 3 | /.index-build 4 | /Packages 5 | /*.xcodeproj 6 | xcuserdata/ 7 | DerivedData/ 8 | .netrc 9 | .swiftpm 10 | tmp 11 | /Artifacts 12 | /Bundles 13 | .vscode 14 | -------------------------------------------------------------------------------- /.license_header_template: -------------------------------------------------------------------------------- 1 | @@===----------------------------------------------------------------------===@@ 2 | @@ 3 | @@ This source file is part of the Swift open source project 4 | @@ 5 | @@ Copyright (c) YEARS Apple Inc. and the Swift project authors 6 | @@ Licensed under Apache License v2.0 with Runtime Library Exception 7 | @@ 8 | @@ See https://swift.org/LICENSE.txt for license information 9 | @@ See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 10 | @@ 11 | @@===----------------------------------------------------------------------===@@ 12 | 13 | -------------------------------------------------------------------------------- /.licenseignore: -------------------------------------------------------------------------------- 1 | .gitignore 2 | .swiftformat 3 | .swiftformatignore 4 | .editorconfig 5 | .unacceptablelanguageignore 6 | Brewfile 7 | Package.swift 8 | Package.resolved 9 | *.md 10 | *.txt 11 | *.yml 12 | **/.editorconfig 13 | **/*.docc/** 14 | **/*.entitlements 15 | **/*.input 16 | **/*.modulemap 17 | **/*.plist 18 | **/*.xcodeproj/** 19 | **/CODEOWNERS 20 | **/Dockerfile 21 | **/Package.swift 22 | Utilities/git.commit.template 23 | -------------------------------------------------------------------------------- /.swift-format: -------------------------------------------------------------------------------- 1 | { 2 | "fileScopedDeclarationPrivacy" : { 3 | "accessLevel" : "private" 4 | }, 5 | "indentConditionalCompilationBlocks" : true, 6 | "indentSwitchCaseLabels" : false, 7 | "indentation" : { 8 | "spaces" : 2 9 | }, 10 | "lineBreakAroundMultilineExpressionChainComponents" : false, 11 | "lineBreakBeforeControlFlowKeywords" : false, 12 | "lineBreakBeforeEachArgument" : true, 13 | "lineBreakBeforeEachGenericRequirement" : false, 14 | "lineLength" : 120, 15 | "maximumBlankLines" : 1, 16 | "multiElementCollectionTrailingCommas" : true, 17 | "noAssignmentInExpressions" : { 18 | "allowedFunctions" : [ 19 | "XCTAssertNoThrow" 20 | ] 21 | }, 22 | "prioritizeKeepingFunctionOutputTogether" : false, 23 | "respectsExistingLineBreaks" : true, 24 | "rules" : { 25 | "AllPublicDeclarationsHaveDocumentation" : false, 26 | "AlwaysUseLiteralForEmptyCollectionInit" : false, 27 | "AlwaysUseLowerCamelCase" : false, 28 | "AmbiguousTrailingClosureOverload" : true, 29 | "BeginDocumentationCommentWithOneLineSummary" : false, 30 | "DoNotUseSemicolons" : true, 31 | "DontRepeatTypeInStaticProperties" : true, 32 | "FileScopedDeclarationPrivacy" : true, 33 | "FullyIndirectEnum" : true, 34 | "GroupNumericLiterals" : true, 35 | "IdentifiersMustBeASCII" : true, 36 | "NeverForceUnwrap" : false, 37 | "NeverUseForceTry" : false, 38 | "NeverUseImplicitlyUnwrappedOptionals" : false, 39 | "NoAccessLevelOnExtensionDeclaration" : true, 40 | "NoAssignmentInExpressions" : true, 41 | "NoBlockComments" : true, 42 | "NoCasesWithOnlyFallthrough" : true, 43 | "NoEmptyTrailingClosureParentheses" : true, 44 | "NoLabelsInCasePatterns" : true, 45 | "NoLeadingUnderscores" : false, 46 | "NoParensAroundConditions" : true, 47 | "NoPlaygroundLiterals" : true, 48 | "NoVoidReturnOnFunctionSignature" : true, 49 | "OmitExplicitReturns" : false, 50 | "OneCasePerLine" : true, 51 | "OneVariableDeclarationPerLine" : true, 52 | "OnlyOneTrailingClosureArgument" : true, 53 | "OrderedImports" : true, 54 | "ReplaceForEachWithForLoop" : true, 55 | "ReturnVoidInsteadOfEmptyTuple" : true, 56 | "TypeNamesShouldBeCapitalized" : true, 57 | "UseEarlyExits" : false, 58 | "UseExplicitNilCheckInConditions" : true, 59 | "UseLetInEveryBoundCaseVariable" : false, 60 | "UseShorthandTypeNames" : true, 61 | "UseSingleLinePropertyGetter" : true, 62 | "UseSynthesizedInitializer" : true, 63 | "UseTripleSlashForDocumentationComments" : true, 64 | "UseWhereClausesInForLoops" : false, 65 | "ValidateDocumentationComments" : false 66 | }, 67 | "spacesAroundRangeFormationOperators" : false, 68 | "tabWidth" : 8, 69 | "version" : 1 70 | } 71 | -------------------------------------------------------------------------------- /.swiftformat: -------------------------------------------------------------------------------- 1 | --indent 2 2 | --indentcase false 3 | --trimwhitespace always 4 | --voidtype tuple 5 | --nospaceoperators ..<,... 6 | --ifdef noindent 7 | --stripunusedargs closure-only 8 | --maxwidth 120 9 | --wraparguments before-first 10 | --funcattributes prev-line 11 | --varattributes prev-line 12 | --disable andOperator 13 | 14 | # Insert explicit `self` where applicable. 15 | --self insert 16 | 17 | --exclude Sources/Helpers/Vendor 18 | 19 | --swiftversion 5.8 20 | -------------------------------------------------------------------------------- /.swiftformatignore: -------------------------------------------------------------------------------- 1 | Sources/Helpers/Vendor/* 2 | Sources/AsyncProcess/ProcessExecutor.swift 3 | -------------------------------------------------------------------------------- /.unacceptablelanguageignore: -------------------------------------------------------------------------------- 1 | Sources/AsyncProcess/ProcessExecutor.swift 2 | Tests/AsyncProcessTests/IntegrationTests.swift 3 | -------------------------------------------------------------------------------- /Brewfile: -------------------------------------------------------------------------------- 1 | brew 'xz' 2 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Code of Conduct 2 | 3 | The code of conduct for this project can be found at https://swift.org/code-of-conduct. 4 | 5 | 6 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | ## Legal 2 | 3 | By submitting a pull request, you represent that you have the right to license 4 | your contribution to Apple and the community, and agree by submitting the patch 5 | that your contributions are licensed under the Apache 2.0 license (see 6 | `LICENSE.txt`). 7 | 8 | ## How to submit a bug report 9 | 10 | Please ensure to specify the following: 11 | 12 | * Commit hash or a tagged version with which the bug is reproducible 13 | * Contextual information (e.g. what you were trying to achieve with the project) 14 | * Simplest possible steps to reproduce 15 | * More complex the steps are, lower the priority will be. 16 | * A pull request with failing test case is preferred, but it's just fine to paste the test case into the issue description. 17 | * Anything that might be relevant in your opinion, such as: 18 | * Swift version or the output of `swift --version` 19 | * OS version and the output of `uname -a` 20 | * Network configuration 21 | 22 | ### Example 23 | 24 | ``` 25 | Commit hash: b17a8a9f0f814c01a56977680cb68d8a779c951f 26 | 27 | Context: 28 | While running the generator, I noticed that ... 29 | 30 | Steps to reproduce: 31 | 1. ... 32 | 2. ... 33 | 3. ... 34 | 4. ... 35 | 36 | $ swift --version 37 | Swift version 4.0.2 (swift-4.0.2-RELEASE) 38 | Target: x86_64-unknown-linux-gnu 39 | 40 | Operating system: Ubuntu Linux 16.04 64-bit 41 | 42 | $ uname -a 43 | Linux beefy.machine 4.4.0-101-generic #124-Ubuntu SMP Fri Nov 10 18:29:59 UTC 2017 x86_64 x86_64 x86_64 GNU/Linux 44 | ``` 45 | 46 | ## Writing a Patch 47 | 48 | A good patch is: 49 | 50 | 1. Concise, and contains as few changes as needed to achieve the end result. 51 | 2. Tested, ensuring that any tests provided failed before the patch and pass after it. 52 | 3. Documented, adding API documentation as needed to cover new functions and properties. 53 | 4. Accompanied by a great commit message, using our commit message template. 54 | 55 | ### Commit Message Template 56 | 57 | We require that your commit messages match our template. The easiest way to do that is to get git to help you by explicitly using the template. To do that, `cd` to the root of our repository and run: 58 | 59 | git config commit.template Utilities/git.commit.template 60 | 61 | ## How to contribute your work 62 | 63 | Please open a pull request at https://github.com/apple/swift-sdk-generator. Make sure the CI passes, and then wait for code review. 64 | -------------------------------------------------------------------------------- /Docker/Dockerfile: -------------------------------------------------------------------------------- 1 | ARG swift_version=5.9 2 | ARG ubuntu_version=jammy 3 | ARG base_image=swift:$swift_version-$ubuntu_version 4 | FROM $base_image 5 | # needed to do again after FROM due to docker limitation 6 | ARG swift_version 7 | ARG ubuntu_version 8 | 9 | # set as UTF-8 10 | RUN apt-get update && apt-get install -y locales locales-all libsqlite3-dev 11 | ENV LC_ALL en_US.UTF-8 12 | ENV LANG en_US.UTF-8 13 | ENV LANGUAGE en_US.UTF-8 14 | 15 | # tools 16 | RUN mkdir -p $HOME/.tools 17 | RUN echo 'export PATH="$HOME/.tools:$PATH"' >> $HOME/.profile 18 | -------------------------------------------------------------------------------- /Docker/docker-compose.2204.59.yaml: -------------------------------------------------------------------------------- 1 | version: "3" 2 | 3 | services: 4 | 5 | runtime-setup: 6 | image: swift-sdk-generator:22.04-5.9 7 | build: 8 | args: 9 | base_image: "swiftlang/swift:nightly-5.9-jammy" 10 | 11 | test: 12 | image: swift-sdk-generator:22.04-5.9 13 | environment: 14 | # - WARN_AS_ERROR_ARG=-Xswiftc -warnings-as-errors # need to fix S-CL-F sendability first 15 | - IMPORT_CHECK_ARG=--explicit-target-dependency-import-check error 16 | # - SANITIZER_ARG=--sanitize=thread # TSan broken still 17 | 18 | shell: 19 | image: swift-sdk-generator:22.04-5.9 20 | -------------------------------------------------------------------------------- /Docker/docker-compose.2204.main.yaml: -------------------------------------------------------------------------------- 1 | version: "3" 2 | 3 | services: 4 | 5 | runtime-setup: 6 | image: swift-sdk-generator:22.04-main 7 | build: 8 | args: 9 | base_image: "swiftlang/swift:nightly-main-jammy" 10 | 11 | test: 12 | image: swift-sdk-generator:22.04-main 13 | environment: 14 | # - WARN_AS_ERROR_ARG=-Xswiftc -warnings-as-errors # need to fix S-CL-F sendability first 15 | - IMPORT_CHECK_ARG=--explicit-target-dependency-import-check error 16 | # - SANITIZER_ARG=--sanitize=thread # TSan broken still 17 | 18 | shell: 19 | image: swift-sdk-generator:22.04-main 20 | -------------------------------------------------------------------------------- /Docker/docker-compose.yaml: -------------------------------------------------------------------------------- 1 | # this file is not designed to be run directly 2 | # instead, use the docker-compose.. files 3 | # eg docker-compose -f docker/docker-compose.yaml -f docker/docker-compose.2204.59.yaml run test 4 | version: "3" 5 | 6 | services: 7 | 8 | runtime-setup: 9 | image: swift-sdk-generator:default 10 | build: 11 | context: . 12 | dockerfile: Dockerfile 13 | 14 | common: &common 15 | image: swift-sdk-generator:default 16 | depends_on: [runtime-setup] 17 | volumes: 18 | - ~/.ssh:/root/.ssh 19 | - ..:/code:z 20 | working_dir: /code 21 | 22 | test: 23 | <<: *common 24 | environment: 25 | - JENKINS_URL 26 | command: /bin/bash -xcl "swift test $${WARN_AS_ERROR_ARG-} $${SANITIZER_ARG-} $${IMPORT_CHECK_ARG-}" 27 | 28 | # util 29 | 30 | shell: 31 | <<: *common 32 | entrypoint: /bin/bash 33 | -------------------------------------------------------------------------------- /Package.resolved: -------------------------------------------------------------------------------- 1 | { 2 | "pins" : [ 3 | { 4 | "identity" : "async-http-client", 5 | "kind" : "remoteSourceControl", 6 | "location" : "https://github.com/swift-server/async-http-client.git", 7 | "state" : { 8 | "revision" : "16f7e62c08c6969899ce6cc277041e868364e5cf", 9 | "version" : "1.19.0" 10 | } 11 | }, 12 | { 13 | "identity" : "swift-argument-parser", 14 | "kind" : "remoteSourceControl", 15 | "location" : "https://github.com/apple/swift-argument-parser", 16 | "state" : { 17 | "revision" : "41982a3656a71c768319979febd796c6fd111d5c", 18 | "version" : "1.5.0" 19 | } 20 | }, 21 | { 22 | "identity" : "swift-async-algorithms", 23 | "kind" : "remoteSourceControl", 24 | "location" : "https://github.com/apple/swift-async-algorithms.git", 25 | "state" : { 26 | "revision" : "6ae9a051f76b81cc668305ceed5b0e0a7fd93d20", 27 | "version" : "1.0.1" 28 | } 29 | }, 30 | { 31 | "identity" : "swift-atomics", 32 | "kind" : "remoteSourceControl", 33 | "location" : "https://github.com/apple/swift-atomics.git", 34 | "state" : { 35 | "revision" : "cd142fd2f64be2100422d658e7411e39489da985", 36 | "version" : "1.2.0" 37 | } 38 | }, 39 | { 40 | "identity" : "swift-collections", 41 | "kind" : "remoteSourceControl", 42 | "location" : "https://github.com/apple/swift-collections.git", 43 | "state" : { 44 | "revision" : "9bf03ff58ce34478e66aaee630e491823326fd06", 45 | "version" : "1.1.3" 46 | } 47 | }, 48 | { 49 | "identity" : "swift-crypto", 50 | "kind" : "remoteSourceControl", 51 | "location" : "https://github.com/apple/swift-crypto.git", 52 | "state" : { 53 | "revision" : "b51f1d6845b353a2121de1c6a670738ec33561a6", 54 | "version" : "3.1.0" 55 | } 56 | }, 57 | { 58 | "identity" : "swift-http-types", 59 | "kind" : "remoteSourceControl", 60 | "location" : "https://github.com/apple/swift-http-types", 61 | "state" : { 62 | "revision" : "12358d55a3824bd5fed310b999ea8cf83a9a1a65", 63 | "version" : "1.0.3" 64 | } 65 | }, 66 | { 67 | "identity" : "swift-log", 68 | "kind" : "remoteSourceControl", 69 | "location" : "https://github.com/apple/swift-log.git", 70 | "state" : { 71 | "revision" : "9cb486020ebf03bfa5b5df985387a14a98744537", 72 | "version" : "1.6.1" 73 | } 74 | }, 75 | { 76 | "identity" : "swift-nio", 77 | "kind" : "remoteSourceControl", 78 | "location" : "https://github.com/apple/swift-nio.git", 79 | "state" : { 80 | "revision" : "4c4453b489cf76e6b3b0f300aba663eb78182fad", 81 | "version" : "2.70.0" 82 | } 83 | }, 84 | { 85 | "identity" : "swift-nio-extras", 86 | "kind" : "remoteSourceControl", 87 | "location" : "https://github.com/apple/swift-nio-extras.git", 88 | "state" : { 89 | "revision" : "363da63c1966405764f380c627409b2f9d9e710b", 90 | "version" : "1.21.0" 91 | } 92 | }, 93 | { 94 | "identity" : "swift-nio-http2", 95 | "kind" : "remoteSourceControl", 96 | "location" : "https://github.com/apple/swift-nio-http2.git", 97 | "state" : { 98 | "revision" : "0904bf0feb5122b7e5c3f15db7df0eabe623dd87", 99 | "version" : "1.30.0" 100 | } 101 | }, 102 | { 103 | "identity" : "swift-nio-ssl", 104 | "kind" : "remoteSourceControl", 105 | "location" : "https://github.com/apple/swift-nio-ssl.git", 106 | "state" : { 107 | "revision" : "4fb7ead803e38949eb1d6fabb849206a72c580f3", 108 | "version" : "2.23.0" 109 | } 110 | }, 111 | { 112 | "identity" : "swift-nio-transport-services", 113 | "kind" : "remoteSourceControl", 114 | "location" : "https://github.com/apple/swift-nio-transport-services.git", 115 | "state" : { 116 | "revision" : "e7403c35ca6bb539a7ca353b91cc2d8ec0362d58", 117 | "version" : "1.19.0" 118 | } 119 | }, 120 | { 121 | "identity" : "swift-system", 122 | "kind" : "remoteSourceControl", 123 | "location" : "https://github.com/apple/swift-system", 124 | "state" : { 125 | "revision" : "d2ba781702a1d8285419c15ee62fd734a9437ff5", 126 | "version" : "1.3.2" 127 | } 128 | } 129 | ], 130 | "version" : 2 131 | } 132 | -------------------------------------------------------------------------------- /Package.swift: -------------------------------------------------------------------------------- 1 | // swift-tools-version: 5.9 2 | // The swift-tools-version declares the minimum version of Swift required to build this package. 3 | 4 | import PackageDescription 5 | 6 | let package = Package( 7 | name: "swift-sdk-generator", 8 | platforms: [.macOS(.v13)], 9 | products: [ 10 | // Products define the executables and libraries a package produces, and make them visible to other packages. 11 | .executable( 12 | name: "swift-sdk-generator", 13 | targets: ["GeneratorCLI"] 14 | ) 15 | ], 16 | targets: [ 17 | // Targets are the basic building blocks of a package. A target can define a module or a test suite. 18 | // Targets can depend on other targets in this package, and on products in packages this package depends on. 19 | .executableTarget( 20 | name: "GeneratorCLI", 21 | dependencies: [ 22 | "SwiftSDKGenerator", 23 | .product(name: "ArgumentParser", package: "swift-argument-parser"), 24 | ], 25 | swiftSettings: [ 26 | .enableExperimentalFeature("StrictConcurrency=complete") 27 | ] 28 | ), 29 | .target( 30 | name: "SwiftSDKGenerator", 31 | dependencies: [ 32 | .target(name: "AsyncProcess"), 33 | .product(name: "AsyncAlgorithms", package: "swift-async-algorithms"), 34 | .product(name: "NIOHTTP1", package: "swift-nio"), 35 | .product(name: "Logging", package: "swift-log"), 36 | .product(name: "SystemPackage", package: "swift-system"), 37 | "Helpers", 38 | ], 39 | exclude: ["Dockerfiles"], 40 | swiftSettings: [ 41 | .enableExperimentalFeature("StrictConcurrency=complete") 42 | ] 43 | ), 44 | .testTarget( 45 | name: "SwiftSDKGeneratorTests", 46 | dependencies: [ 47 | "SwiftSDKGenerator" 48 | ], 49 | swiftSettings: [ 50 | .enableExperimentalFeature("StrictConcurrency=complete") 51 | ] 52 | ), 53 | .testTarget( 54 | name: "GeneratorEngineTests", 55 | dependencies: [ 56 | "Helpers" 57 | ] 58 | ), 59 | .target( 60 | name: "Helpers", 61 | dependencies: [ 62 | "SystemSQLite", 63 | .product(name: "Crypto", package: "swift-crypto"), 64 | .product(name: "Logging", package: "swift-log"), 65 | .product(name: "SystemPackage", package: "swift-system"), 66 | ], 67 | exclude: ["Vendor/README.md"], 68 | swiftSettings: [ 69 | .enableExperimentalFeature("StrictConcurrency=complete") 70 | ] 71 | ), 72 | .testTarget( 73 | name: "HelpersTests", 74 | dependencies: [ 75 | "Helpers" 76 | ] 77 | ), 78 | .systemLibrary(name: "SystemSQLite", pkgConfig: "sqlite3"), 79 | .target( 80 | name: "AsyncProcess", 81 | dependencies: [ 82 | .product(name: "Atomics", package: "swift-atomics"), 83 | .product(name: "AsyncAlgorithms", package: "swift-async-algorithms"), 84 | .product(name: "Logging", package: "swift-log"), 85 | .product(name: "NIO", package: "swift-nio"), 86 | .product(name: "DequeModule", package: "swift-collections"), 87 | .product(name: "SystemPackage", package: "swift-system"), 88 | ] 89 | ), 90 | .testTarget( 91 | name: "AsyncProcessTests", 92 | dependencies: [ 93 | "AsyncProcess", 94 | .product(name: "Atomics", package: "swift-atomics"), 95 | .product(name: "AsyncAlgorithms", package: "swift-async-algorithms"), 96 | .product(name: "NIO", package: "swift-nio"), 97 | .product(name: "NIOConcurrencyHelpers", package: "swift-nio"), 98 | .product(name: "Logging", package: "swift-log"), 99 | ] 100 | ), 101 | ], 102 | swiftLanguageVersions: [.v5, .version("6")] 103 | ) 104 | 105 | struct Configuration { 106 | let useAsyncHttpClient: Bool 107 | let useLocalDependencies: Bool 108 | init(SWIFT_SDK_GENERATOR_DISABLE_AHC: Bool, SWIFTCI_USE_LOCAL_DEPS: Bool) { 109 | self.useAsyncHttpClient = !SWIFT_SDK_GENERATOR_DISABLE_AHC && !SWIFTCI_USE_LOCAL_DEPS 110 | self.useLocalDependencies = SWIFTCI_USE_LOCAL_DEPS 111 | } 112 | } 113 | 114 | let configuration = Configuration( 115 | SWIFT_SDK_GENERATOR_DISABLE_AHC: Context.environment["SWIFT_SDK_GENERATOR_DISABLE_AHC"] != nil, 116 | SWIFTCI_USE_LOCAL_DEPS: Context.environment["SWIFTCI_USE_LOCAL_DEPS"] != nil 117 | ) 118 | 119 | if configuration.useAsyncHttpClient { 120 | package.dependencies.append( 121 | .package(url: "https://github.com/swift-server/async-http-client.git", from: "1.19.0") 122 | ) 123 | let targetsToAppend: Set = ["SwiftSDKGenerator", "Helpers"] 124 | for target in package.targets.filter({ targetsToAppend.contains($0.name) }) { 125 | target.dependencies.append( 126 | .product(name: "AsyncHTTPClient", package: "async-http-client") 127 | ) 128 | } 129 | } 130 | 131 | if configuration.useLocalDependencies { 132 | package.dependencies += [ 133 | .package(path: "../swift-system"), 134 | .package(path: "../swift-argument-parser"), 135 | .package(path: "../swift-async-algorithms"), 136 | .package(path: "../swift-atomics"), 137 | .package(path: "../swift-collections"), 138 | .package(path: "../swift-crypto"), 139 | .package(path: "../swift-nio"), 140 | .package(path: "../swift-log"), 141 | ] 142 | } else { 143 | package.dependencies += [ 144 | .package(url: "https://github.com/apple/swift-system", from: "1.3.0"), 145 | .package(url: "https://github.com/apple/swift-argument-parser", from: "1.4.0"), 146 | .package(url: "https://github.com/apple/swift-async-algorithms.git", exact: "1.0.1"), 147 | .package(url: "https://github.com/apple/swift-atomics.git", from: "1.2.0"), 148 | .package(url: "https://github.com/apple/swift-collections.git", from: "1.1.2"), 149 | .package(url: "https://github.com/apple/swift-crypto.git", from: "3.1.0"), 150 | .package(url: "https://github.com/apple/swift-nio.git", from: "2.65.0"), 151 | .package(url: "https://github.com/apple/swift-log.git", from: "1.5.4"), 152 | ] 153 | } 154 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Swift SDK Generator 2 | 3 | ## Overview 4 | 5 | With Swift supporting many different platforms, cross-compilation can boost developer productivity. In certain cases it's 6 | the only way to build a Swift package. 7 | 8 | [SE-0387](https://github.com/swiftlang/swift-evolution/blob/main/proposals/0387-cross-compilation-destinations.md) proposal 9 | introduces Swift SDKs, which bundle together all components required for cross-compilation in a single archive, and 10 | make cross builds as easy as running a single command. 11 | 12 | Swift SDK authors can assemble such archive manually, but the goal of Swift SDK Generator developed in this repository 13 | is to automate this task as much as possible. If you're a platform maintainer, or someone who would like to make 14 | cross-compiling easy to your favorite platform, you can tailor the generator source code to your needs and publish 15 | a newly generated Swift SDK for users to install. 16 | 17 | ## Requirements 18 | 19 | Usage of Swift SDKs requires Swift 5.9, follow [installation instructions on swift.org](https://www.swift.org/install/) to install it first. 20 | 21 | After that, verify that the `experimental-sdk` command is available: 22 | 23 | ``` 24 | swift experimental-sdk list 25 | ``` 26 | 27 | The output will either state that no Swift SDKs are available, or produce a list of those you previously had 28 | installed, in case you've used the `swift experimental-sdk install` command before. 29 | 30 | ### macOS Requirements 31 | 32 | The generator depends on the `xz` utility for more efficient downloading of package lists for Ubuntu. This is optional, but can be installed via the included `Brewfile`: 33 | 34 | ```bash 35 | brew bundle install 36 | ``` 37 | 38 | If `xz` is not found, the generator will fallback on `gzip`. 39 | 40 | ## Supported platforms and minimum versions 41 | 42 | macOS as a host platform and Linux as both host and target platforms are supported by the generator. 43 | The generator also allows cross-compiling between any Linux distributions officially supported by the Swift project. 44 | 45 | | Platform | Supported Version as Host | Supported Version as Target | 46 | | -: | :- | :- | 47 | | macOS (arm64) | ✅ macOS 13.0+ | ❌ | 48 | | macOS (x86_64) | ✅ macOS 13.0+[^1] | ❌ | 49 | | Ubuntu | ✅ 20.04+ | ✅ 20.04+ | 50 | | Debian | ✅ 11, 12[^2] | ✅ 11, 12[^2] | 51 | | RHEL | ✅ Fedora 39, UBI 9 | ✅ Fedora 39, UBI 9[^3] | 52 | | Amazon Linux 2 | ✅ Supported | ✅ Supported[^3] | 53 | 54 | [^1]: Since LLVM project doesn't provide pre-built binaries of `lld` for macOS on x86_64, it will be automatically built 55 | from sources by the generator, which will increase its run by at least 15 minutes on recent hardware. You will also 56 | need CMake and Ninja preinstalled (e.g. via `brew install cmake ninja`). 57 | [^2]: Swift does not officially support Debian 11 or Debian 12 with Swift versions before 5.10.1. However, the Ubuntu 20.04/22.04 toolchains can be used with Debian 11 and 12 (respectively) since they are binary compatible. 58 | [^3]: These versions are technically supported but require custom commands and a Docker container to build the Swift SDK, as the generator will not download dependencies for these distributions automatically. See [issue #138](https://github.com/swiftlang/swift-sdk-generator/issues/138). 59 | 60 | ## How to use it 61 | 62 | Clone this repository into a directory of your choice and make it the current directory. Build and run it with this command: 63 | 64 | ``` 65 | swift run swift-sdk-generator make-linux-sdk 66 | ``` 67 | 68 | This will download required components and produce a Swift SDK for Ubuntu Jammy in the `Bundles` subdirectory. Follow the steps 69 | printed at the end of generator's output for installing the newly generated Swift SDK. 70 | 71 | Additional command-line options are available for specifying target platform features, such as Linux distribution name, 72 | version, and target CPU architecture. Pass `--help` flag to see all of the available options: 73 | 74 | ``` 75 | swift run swift-sdk-generator make-linux-sdk --help 76 | ``` 77 | 78 | After installing a Swift SDK, verify that it's available to SwiftPM: 79 | 80 | ``` 81 | swift experimental-sdk list 82 | ``` 83 | 84 | The output of the last command should contain `ubuntu22.04`. Note the full Swift SDK ID in the output, we'll refer to it 85 | subsequently as ``. 86 | 87 | Create a new project to verify that the SDK works: 88 | 89 | ``` 90 | mkdir cross-compilation-test 91 | cd cross-compilation-test 92 | swift package init --type executable 93 | ``` 94 | 95 | Build this project with the SDK: 96 | 97 | ``` 98 | swift build --experimental-swift-sdk 99 | ``` 100 | 101 | Verify that the produced binary is compatible with Linux: 102 | 103 | ``` 104 | file .build/debug/cross-compilation-test 105 | ``` 106 | 107 | That should produce output similar to this: 108 | 109 | ``` 110 | .build/debug/cross-compilation-test: ELF 64-bit LSB pie executable, ARM aarch64, version 1 (SYSV), 111 | dynamically linked, interpreter /lib/ld-linux-aarch64.so.1, for GNU/Linux 3.7.0, with debug_info, not stripped 112 | ``` 113 | 114 | You can then copy this binary to a Docker image that has Swift runtime libraries installed. For example, 115 | for Ubuntu Jammy and Swift 5.9 this would be `swift:5.9-jammy-slim`. If you'd like to copy the binary to 116 | an arbitrary Ubuntu Jammy system, make sure you pass `--static-swift-stdlib` flag to `swift build`, in addition 117 | to the `--experimental-swift-sdk` option. 118 | 119 | ## Building an SDK from a container image 120 | 121 | You can base your SDK on a container image, such as one of the 122 | [official Swift images](https://hub.docker.com/_/swift). By 123 | default, the command below will build an SDK based on the Ubuntu 124 | Jammy image: 125 | ``` 126 | swift run swift-sdk-generator make-linux-sdk --with-docker 127 | ``` 128 | To build a RHEL images, use the `--linux-distribution-name` option. 129 | The following command will build a `ubi9`-based image: 130 | ``` 131 | swift run swift-sdk-generator make-linux-sdk --with-docker --linux-distribution-name rhel 132 | ``` 133 | 134 | You can also specify the base container image by name: 135 | 136 | ``` 137 | swift run swift-sdk-generator make-linux-sdk --from-container-image swift:5.9-jammy 138 | ``` 139 | 140 | ``` 141 | swift run swift-sdk-generator make-linux-sdk --with-docker --linux-distribution-name rhel --from-container-image swift:5.9-rhel-ubi9 142 | ``` 143 | 144 | ### Including extra Linux libraries 145 | 146 | If your project depends on Linux libraries which are not part of a 147 | standard base image, you can build your SDK from a custom container 148 | image which includes them. 149 | 150 | Prepare a `Dockerfile` which derives from one of the standard images 151 | and installs the packages you need. This example installs SQLite 152 | and its dependencies on top of the Swift project's Ubuntu Jammy image: 153 | 154 | ``` 155 | FROM swift:5.9-jammy 156 | RUN apt update && apt -y install libsqlite3-dev && apt -y clean 157 | ``` 158 | 159 | Build the new container image: 160 | ``` 161 | docker build -t my-custom-image -f Dockerfile . 162 | ``` 163 | 164 | Finally, build your custom SDK: 165 | ``` 166 | swift run swift-sdk-generator make-linux-sdk --with-docker --from-container-image my-custom-image:latest --sdk-name 5.9-ubuntu-with-sqlite 167 | ``` 168 | 169 | ## Swift SDK distribution 170 | 171 | The `.artifactbundle` directory produced in the previous section can be packaged as a `.tar.gz` archive and redistributed 172 | in this form. Users of such Swift SDK bundle archive can easily install it with `swift experimental-sdk install` 173 | command, which supports both local file system paths and public `http://` and `https://` URLs as an argument. 174 | 175 | 176 | ## Contributing 177 | 178 | There are several ways to contribute to Swift SDK Generator. To learn about the policies, best practices that govern contributions to the Swift project, and instructions for setting up the development environment please read the [Contributor Guide](CONTRIBUTING.md). 179 | 180 | If you're interested in adding support for a new platform, please open an issue on this repository first so that the best implementation strategy can be discussed before proceeding with an implementation. 181 | 182 | ## Reporting issues 183 | 184 | If you have any trouble with the Swift SDK Generator, help is available. We recommend: 185 | 186 | * Generator's [bug tracker](https://github.com/swiftlang/swift-sdk-generator/issues); 187 | * The [Swift Forums](https://forums.swift.org/c/development/swiftpm/). 188 | 189 | When reporting an issue please follow the bug reporting guidelines, they can be found in [contribution guide](./CONTRIBUTING.md#how-to-submit-a-bug-report). 190 | 191 | If you’re not comfortable sharing your question with the list, contact details for the code owners can be found in [CODEOWNERS](.github/CODEOWNERS); however, Swift Forums is usually the best place to go for help. 192 | 193 | ## License 194 | 195 | Copyright 2022 - 2024 Apple Inc. and the Swift project authors. Licensed under Apache License v2.0 with Runtime Library Exception. 196 | 197 | See [https://swift.org/LICENSE.txt](https://swift.org/LICENSE.txt) for license information. 198 | 199 | See [https://swift.org/CONTRIBUTORS.txt](https://swift.org/CONTRIBUTORS.txt) for Swift project authors. 200 | -------------------------------------------------------------------------------- /Sources/AsyncProcess/ChunkSequence.swift: -------------------------------------------------------------------------------- 1 | //===----------------------------------------------------------------------===// 2 | // 3 | // This source file is part of the Swift open source project 4 | // 5 | // Copyright (c) 2022-2023 Apple Inc. and the Swift project authors 6 | // Licensed under Apache License v2.0 with Runtime Library Exception 7 | // 8 | // See https://swift.org/LICENSE.txt for license information 9 | // See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 10 | // 11 | //===----------------------------------------------------------------------===// 12 | 13 | import NIO 14 | 15 | #if os(Linux) || os(Android) || os(Windows) 16 | @preconcurrency import Foundation 17 | #else 18 | import Foundation 19 | #endif 20 | 21 | public struct IllegalStreamConsumptionError: Error { 22 | var description: String 23 | } 24 | 25 | public struct ChunkSequence: AsyncSequence & Sendable { 26 | private let fileHandle: FileHandle? 27 | private let group: EventLoopGroup 28 | 29 | public init(takingOwnershipOfFileHandle fileHandle: FileHandle?, group: EventLoopGroup) { 30 | self.group = group 31 | self.fileHandle = fileHandle 32 | } 33 | 34 | public func makeAsyncIterator() -> AsyncIterator { 35 | // This will close the file handle. 36 | AsyncIterator(try! self.fileHandle?.fileContentStream(eventLoop: self.group.any())) 37 | } 38 | 39 | public typealias Element = ByteBuffer 40 | public struct AsyncIterator: AsyncIteratorProtocol { 41 | public typealias Element = ByteBuffer 42 | typealias UnderlyingSequence = FileContentStream 43 | 44 | private var underlyingIterator: UnderlyingSequence.AsyncIterator? 45 | 46 | init(_ underlyingSequence: UnderlyingSequence?) { 47 | self.underlyingIterator = underlyingSequence?.makeAsyncIterator() 48 | } 49 | 50 | public mutating func next() async throws -> Element? { 51 | if self.underlyingIterator != nil { 52 | return try await self.underlyingIterator!.next() 53 | } else { 54 | throw IllegalStreamConsumptionError( 55 | description: """ 56 | Either `.discard`ed, `.inherit`ed or redirected this stream to a `.fileHandle`, 57 | cannot also consume it. To consume, please `.stream` it. 58 | """ 59 | ) 60 | } 61 | } 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /Sources/AsyncProcess/EOFSequence.swift: -------------------------------------------------------------------------------- 1 | //===----------------------------------------------------------------------===// 2 | // 3 | // This source file is part of the Swift open source project 4 | // 5 | // Copyright (c) 2022-2023 Apple Inc. and the Swift project authors 6 | // Licensed under Apache License v2.0 with Runtime Library Exception 7 | // 8 | // See https://swift.org/LICENSE.txt for license information 9 | // See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 10 | // 11 | //===----------------------------------------------------------------------===// 12 | 13 | public struct EOFSequence: AsyncSequence & Sendable { 14 | public typealias Element = Element 15 | 16 | public struct AsyncIterator: AsyncIteratorProtocol { 17 | public mutating func next() async throws -> Element? { 18 | nil 19 | } 20 | } 21 | 22 | public init(of type: Element.Type = Element.self) {} 23 | 24 | public func makeAsyncIterator() -> AsyncIterator { 25 | AsyncIterator() 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /Sources/AsyncProcess/NIOAsyncPipeWriter.swift: -------------------------------------------------------------------------------- 1 | //===----------------------------------------------------------------------===// 2 | // 3 | // This source file is part of the Swift open source project 4 | // 5 | // Copyright (c) 2022-2023 Apple Inc. and the Swift project authors 6 | // Licensed under Apache License v2.0 with Runtime Library Exception 7 | // 8 | // See https://swift.org/LICENSE.txt for license information 9 | // See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 10 | // 11 | //===----------------------------------------------------------------------===// 12 | 13 | import Foundation 14 | import NIO 15 | 16 | struct NIOAsyncPipeWriter where Chunks.Element == ByteBuffer { 17 | static func sinkSequenceInto( 18 | _ chunks: Chunks, 19 | takingOwnershipOfFD fd: CInt, 20 | eventLoop: EventLoop 21 | ) async throws { 22 | let channel = try await NIOPipeBootstrap(group: eventLoop) 23 | .channelOption(ChannelOptions.allowRemoteHalfClosure, value: true) 24 | .channelOption(ChannelOptions.autoRead, value: false) 25 | .takingOwnershipOfDescriptor( 26 | output: fd 27 | ).get() 28 | channel.close(mode: .input, promise: nil) 29 | defer { 30 | channel.close(promise: nil) 31 | } 32 | for try await chunk in chunks { 33 | try await channel.writeAndFlush(chunk).get() 34 | } 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /Sources/AsyncProcess/ProcessExit.swift: -------------------------------------------------------------------------------- 1 | //===----------------------------------------------------------------------===// 2 | // 3 | // This source file is part of the Swift open source project 4 | // 5 | // Copyright (c) 2022-2023 Apple Inc. and the Swift project authors 6 | // Licensed under Apache License v2.0 with Runtime Library Exception 7 | // 8 | // See https://swift.org/LICENSE.txt for license information 9 | // See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 10 | // 11 | //===----------------------------------------------------------------------===// 12 | 13 | public enum ProcessExitReason: Hashable & Sendable { 14 | case exit(CInt) 15 | case signal(CInt) 16 | 17 | public func throwIfNonZero() throws { 18 | switch self { 19 | case .exit(0): 20 | return 21 | default: 22 | throw ProcessExecutionError(self) 23 | } 24 | } 25 | } 26 | 27 | public struct ProcessExecutionError: Error & Hashable & Sendable { 28 | public var exitReason: ProcessExitReason 29 | 30 | public init(_ exitResult: ProcessExitReason) { 31 | self.exitReason = exitResult 32 | } 33 | } 34 | 35 | extension ProcessExecutionError: CustomStringConvertible { 36 | public var description: String { 37 | "process exited non-zero: \(self.exitReason)" 38 | } 39 | } 40 | -------------------------------------------------------------------------------- /Sources/Helpers/ThrowingDefer.swift: -------------------------------------------------------------------------------- 1 | //===----------------------------------------------------------------------===// 2 | // 3 | // This source file is part of the Swift open source project 4 | // 5 | // Copyright (c) 2023 Apple Inc. and the Swift project authors 6 | // Licensed under Apache License v2.0 with Runtime Library Exception 7 | // 8 | // See https://swift.org/LICENSE.txt for license information 9 | // See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 10 | // 11 | //===----------------------------------------------------------------------===// 12 | 13 | /// Runs a cleanup closure (`deferred`) after a given `work` closure, 14 | /// making sure `deferred` is run also when `work` throws an error. 15 | /// - Parameters: 16 | /// - work: The work that should be performed. Will always be executed. 17 | /// - deferred: The cleanup that needs to be done in any case. 18 | /// - Throws: Any error thrown by `deferred` or `work` (in that order). 19 | /// - Returns: The result of `work`. 20 | /// - Note: If `work` **and** `deferred` throw an error, 21 | /// the one thrown by `deferred` is thrown from this function. 22 | /// - SeeAlso: ``withAsyncThrowing(do:defer:)`` 23 | public func withThrowing( 24 | do work: () throws -> T, 25 | defer deferred: () throws -> Void 26 | ) throws -> T { 27 | do { 28 | let result = try work() 29 | try deferred() 30 | return result 31 | } catch { 32 | try deferred() 33 | throw error 34 | } 35 | } 36 | 37 | /// Runs an async cleanup closure (`deferred`) after a given async `work` closure, 38 | /// making sure `deferred` is run also when `work` throws an error. 39 | /// - Parameters: 40 | /// - work: The work that should be performed. Will always be executed. 41 | /// - deferred: The cleanup that needs to be done in any case. 42 | /// - Throws: Any error thrown by `deferred` or `work` (in that order). 43 | /// - Returns: The result of `work`. 44 | /// - Note: If `work` **and** `deferred` throw an error, 45 | /// the one thrown by `deferred` is thrown from this function. 46 | /// - SeeAlso: ``withThrowing(do:defer:)`` 47 | public func withAsyncThrowing( 48 | do work: @Sendable () async throws -> T, 49 | defer deferred: @Sendable () async throws -> Void 50 | ) async throws -> T { 51 | do { 52 | let result = try await work() 53 | try await deferred() 54 | return result 55 | } catch { 56 | try await deferred() 57 | throw error 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /Sources/Helpers/Vendor/QueryEngine/CacheKey.swift: -------------------------------------------------------------------------------- 1 | //===----------------------------------------------------------------------===// 2 | // 3 | // This source file is part of the Swift open source project 4 | // 5 | // Copyright (c) 2023-2024 Apple Inc. and the Swift project authors 6 | // Licensed under Apache License v2.0 with Runtime Library Exception 7 | // 8 | // See https://swift.org/LICENSE.txt for license information 9 | // See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 10 | // 11 | //===----------------------------------------------------------------------===// 12 | 13 | @_exported import protocol Crypto.HashFunction 14 | import struct Foundation.URL 15 | import struct SystemPackage.FilePath 16 | 17 | /// Indicates that values of a conforming type can be hashed with an arbitrary hashing function. Unlike `Hashable`, 18 | /// this protocol doesn't utilize random seed values and produces consistent hash values across process launches. 19 | public protocol CacheKey: Encodable {} 20 | 21 | /// Types that cannot be decomposed more to be hashed 22 | protocol LeafCacheKey: CacheKey { 23 | func hash(with hashFunction: inout some HashFunction) 24 | } 25 | 26 | extension Bool: LeafCacheKey { 27 | func hash(with hashFunction: inout some HashFunction) { 28 | String(reflecting: Self.self).hash(with: &hashFunction) 29 | hashFunction.update(data: self ? [1] : [0]) 30 | } 31 | } 32 | 33 | extension Int: LeafCacheKey { 34 | func hash(with hashFunction: inout some HashFunction) { 35 | String(reflecting: Self.self).hash(with: &hashFunction) 36 | withUnsafeBytes(of: self) { 37 | hashFunction.update(data: $0) 38 | } 39 | } 40 | } 41 | 42 | extension Int8: LeafCacheKey { 43 | func hash(with hashFunction: inout some HashFunction) { 44 | String(reflecting: Self.self).hash(with: &hashFunction) 45 | withUnsafeBytes(of: self) { 46 | hashFunction.update(data: $0) 47 | } 48 | } 49 | } 50 | 51 | extension Int16: LeafCacheKey { 52 | func hash(with hashFunction: inout some HashFunction) { 53 | String(reflecting: Self.self).hash(with: &hashFunction) 54 | withUnsafeBytes(of: self) { 55 | hashFunction.update(data: $0) 56 | } 57 | } 58 | } 59 | 60 | extension Int32: LeafCacheKey { 61 | func hash(with hashFunction: inout some HashFunction) { 62 | String(reflecting: Self.self).hash(with: &hashFunction) 63 | withUnsafeBytes(of: self) { 64 | hashFunction.update(data: $0) 65 | } 66 | } 67 | } 68 | 69 | extension Int64: LeafCacheKey { 70 | func hash(with hashFunction: inout some HashFunction) { 71 | String(reflecting: Self.self).hash(with: &hashFunction) 72 | withUnsafeBytes(of: self) { 73 | hashFunction.update(data: $0) 74 | } 75 | } 76 | } 77 | 78 | extension UInt: LeafCacheKey { 79 | func hash(with hashFunction: inout some HashFunction) { 80 | String(reflecting: Self.self).hash(with: &hashFunction) 81 | withUnsafeBytes(of: self) { 82 | hashFunction.update(data: $0) 83 | } 84 | } 85 | } 86 | 87 | extension UInt8: LeafCacheKey { 88 | func hash(with hashFunction: inout some HashFunction) { 89 | String(reflecting: Self.self).hash(with: &hashFunction) 90 | withUnsafeBytes(of: self) { 91 | hashFunction.update(data: $0) 92 | } 93 | } 94 | } 95 | 96 | extension UInt16: LeafCacheKey { 97 | func hash(with hashFunction: inout some HashFunction) { 98 | String(reflecting: Self.self).hash(with: &hashFunction) 99 | withUnsafeBytes(of: self) { 100 | hashFunction.update(data: $0) 101 | } 102 | } 103 | } 104 | 105 | extension UInt32: LeafCacheKey { 106 | func hash(with hashFunction: inout some HashFunction) { 107 | String(reflecting: Self.self).hash(with: &hashFunction) 108 | withUnsafeBytes(of: self) { 109 | hashFunction.update(data: $0) 110 | } 111 | } 112 | } 113 | 114 | extension UInt64: LeafCacheKey { 115 | func hash(with hashFunction: inout some HashFunction) { 116 | String(reflecting: Self.self).hash(with: &hashFunction) 117 | withUnsafeBytes(of: self) { 118 | hashFunction.update(data: $0) 119 | } 120 | } 121 | } 122 | 123 | extension Float: LeafCacheKey { 124 | func hash(with hashFunction: inout some HashFunction) { 125 | String(reflecting: Self.self).hash(with: &hashFunction) 126 | withUnsafeBytes(of: self) { 127 | hashFunction.update(data: $0) 128 | } 129 | } 130 | } 131 | 132 | extension Double: LeafCacheKey { 133 | func hash(with hashFunction: inout some HashFunction) { 134 | String(reflecting: Self.self).hash(with: &hashFunction) 135 | withUnsafeBytes(of: self) { 136 | hashFunction.update(data: $0) 137 | } 138 | } 139 | } 140 | 141 | extension String: LeafCacheKey { 142 | func hash(with hashFunction: inout some HashFunction) { 143 | var t = String(reflecting: Self.self) 144 | t.withUTF8 { 145 | hashFunction.update(data: $0) 146 | } 147 | var x = self 148 | x.withUTF8 { 149 | hashFunction.update(data: $0) 150 | } 151 | } 152 | } 153 | 154 | extension FilePath: LeafCacheKey { 155 | func hash(with hashFunction: inout some HashFunction) { 156 | String(reflecting: Self.self).hash(with: &hashFunction) 157 | self.string.hash(with: &hashFunction) 158 | } 159 | } 160 | 161 | extension FilePath.Component: LeafCacheKey { 162 | func hash(with hashFunction: inout some HashFunction) { 163 | String(reflecting: Self.self).hash(with: &hashFunction) 164 | self.string.hash(with: &hashFunction) 165 | } 166 | } 167 | 168 | extension URL: LeafCacheKey { 169 | func hash(with hashFunction: inout some HashFunction) { 170 | // Until Swift 6.0, the fully-qualified name of the URL type was `Foundation.URL`. 171 | // After the adoption of FoundationEssentials, the name changed to `FoundationEssentials.URL`. 172 | // This difference causes the hashes to change, so for backwards compatibility we pin the 173 | // type name to `Foundation.URL`. 174 | "Foundation.URL".hash(with: &hashFunction) 175 | self.description.hash(with: &hashFunction) 176 | } 177 | } 178 | 179 | extension Array: CacheKey where Element == FilePath.Component { 180 | func hash(with hashFunction: inout some HashFunction) { 181 | String(reflecting: Self.self).hash(with: &hashFunction) 182 | map(\.string).joined(separator: "\n").hash(with: &hashFunction) 183 | } 184 | } 185 | -------------------------------------------------------------------------------- /Sources/Helpers/Vendor/QueryEngine/FileCacheRecord.swift: -------------------------------------------------------------------------------- 1 | //===----------------------------------------------------------------------===// 2 | // 3 | // This source file is part of the Swift open source project 4 | // 5 | // Copyright (c) 2023-2024 Apple Inc. and the Swift project authors 6 | // Licensed under Apache License v2.0 with Runtime Library Exception 7 | // 8 | // See https://swift.org/LICENSE.txt for license information 9 | // See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 10 | // 11 | //===----------------------------------------------------------------------===// 12 | 13 | // FIXME: need a new swift-system tag to remove `@preconcurrency` 14 | @preconcurrency import struct SystemPackage.FilePath 15 | 16 | public struct FileCacheRecord: Sendable { 17 | public let path: FilePath 18 | public let hash: String 19 | } 20 | 21 | extension FileCacheRecord: Codable { 22 | enum CodingKeys: CodingKey { 23 | case path 24 | case hash 25 | } 26 | 27 | // FIXME: `Codable` on `FilePath` is broken, thus all `Codable` types with `FilePath` properties need a custom impl. 28 | public init(from decoder: any Decoder) throws { 29 | let container = try decoder.container(keyedBy: CodingKeys.self) 30 | self.path = try FilePath(container.decode(String.self, forKey: .path)) 31 | self.hash = try container.decode(String.self, forKey: .hash) 32 | } 33 | 34 | public func encode(to encoder: any Encoder) throws { 35 | var container = encoder.container(keyedBy: CodingKeys.self) 36 | try container.encode(self.path.string, forKey: .path) 37 | try container.encode(self.hash, forKey: .hash) 38 | } 39 | } 40 | 41 | extension OpenReadableFile { 42 | func hash(with hashFunction: inout some HashFunction) async throws { 43 | let stream = try await self.read() 44 | 45 | for try await bytes in stream { 46 | hashFunction.update(data: bytes) 47 | } 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /Sources/Helpers/Vendor/QueryEngine/FileLock.swift: -------------------------------------------------------------------------------- 1 | //===----------------------------------------------------------------------===// 2 | // 3 | // This source file is part of the Swift open source project 4 | // 5 | // Copyright (c) 2014-2024 Apple Inc. and the Swift project authors 6 | // Licensed under Apache License v2.0 with Runtime Library Exception 7 | // 8 | // See https://swift.org/LICENSE.txt for license information 9 | // See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 10 | // 11 | //===----------------------------------------------------------------------===// 12 | 13 | import Foundation 14 | import SystemPackage 15 | 16 | enum ProcessLockError: Error { 17 | case unableToAquireLock(errno: Int32) 18 | } 19 | 20 | extension ProcessLockError: CustomNSError { 21 | public var errorUserInfo: [String: Any] { 22 | [NSLocalizedDescriptionKey: "\(self)"] 23 | } 24 | } 25 | 26 | /// Provides functionality to acquire a lock on a file via POSIX's flock() method. 27 | /// It can be used for things like serializing concurrent mutations on a shared resource 28 | /// by multiple instances of a process. The `FileLock` is not thread-safe. 29 | final class FileLock { 30 | enum LockType { 31 | case exclusive 32 | case shared 33 | } 34 | 35 | enum Error: Swift.Error { 36 | case noEntry(FilePath) 37 | case notDirectory(FilePath) 38 | } 39 | 40 | /// File descriptor to the lock file. 41 | #if os(Windows) 42 | private var handle: HANDLE? 43 | #else 44 | private var fileDescriptor: CInt? 45 | #endif 46 | 47 | /// Path to the lock file. 48 | private let lockFile: FilePath 49 | 50 | /// Create an instance of FileLock at the path specified 51 | /// 52 | /// Note: The parent directory path should be a valid directory. 53 | init(at lockFile: FilePath) { 54 | self.lockFile = lockFile 55 | } 56 | 57 | /// Try to acquire a lock. This method will block until lock the already aquired by other process. 58 | /// 59 | /// Note: This method can throw if underlying POSIX methods fail. 60 | func lock(type: LockType = .exclusive) throws { 61 | #if os(Windows) 62 | if self.handle == nil { 63 | let h: HANDLE = self.lockFile.pathString.withCString(encodedAs: UTF16.self) { 64 | CreateFileW( 65 | $0, 66 | UInt32(GENERIC_READ) | UInt32(GENERIC_WRITE), 67 | UInt32(FILE_SHARE_READ) | UInt32(FILE_SHARE_WRITE), 68 | nil, 69 | DWORD(OPEN_ALWAYS), 70 | DWORD(FILE_ATTRIBUTE_NORMAL), 71 | nil 72 | ) 73 | } 74 | if h == INVALID_HANDLE_VALUE { 75 | throw FileSystemError(errno: Int32(GetLastError()), self.lockFile) 76 | } 77 | self.handle = h 78 | } 79 | var overlapped = OVERLAPPED() 80 | overlapped.Offset = 0 81 | overlapped.OffsetHigh = 0 82 | overlapped.hEvent = nil 83 | switch type { 84 | case .exclusive: 85 | if !LockFileEx( 86 | self.handle, 87 | DWORD(LOCKFILE_EXCLUSIVE_LOCK), 88 | 0, 89 | UInt32.max, 90 | UInt32.max, 91 | &overlapped 92 | ) { 93 | throw ProcessLockError.unableToAquireLock(errno: Int32(GetLastError())) 94 | } 95 | case .shared: 96 | if !LockFileEx( 97 | self.handle, 98 | 0, 99 | 0, 100 | UInt32.max, 101 | UInt32.max, 102 | &overlapped 103 | ) { 104 | throw ProcessLockError.unableToAquireLock(errno: Int32(GetLastError())) 105 | } 106 | } 107 | #else 108 | // Open the lock file. 109 | if self.fileDescriptor == nil { 110 | do { 111 | self.fileDescriptor = try FileDescriptor.open( 112 | self.lockFile, 113 | .writeOnly, 114 | options: [.create, .closeOnExec], 115 | permissions: [.groupReadWrite, .ownerReadWrite, .otherReadWrite] 116 | ).rawValue 117 | } catch { 118 | throw error.attach(self.lockFile) 119 | } 120 | } 121 | // Acquire lock on the file. 122 | while true { 123 | if type == .exclusive && flock(self.fileDescriptor!, LOCK_EX) == 0 { 124 | break 125 | } else if type == .shared && flock(self.fileDescriptor!, LOCK_SH) == 0 { 126 | break 127 | } 128 | // Retry if interrupted. 129 | if errno == EINTR { continue } 130 | throw ProcessLockError.unableToAquireLock(errno: errno) 131 | } 132 | #endif 133 | } 134 | 135 | /// Unlock the held lock. 136 | public func unlock() { 137 | #if os(Windows) 138 | var overlapped = OVERLAPPED() 139 | overlapped.Offset = 0 140 | overlapped.OffsetHigh = 0 141 | overlapped.hEvent = nil 142 | UnlockFileEx(self.handle, 0, UInt32.max, UInt32.max, &overlapped) 143 | #else 144 | guard let fd = fileDescriptor else { return } 145 | flock(fd, LOCK_UN) 146 | #endif 147 | } 148 | 149 | deinit { 150 | #if os(Windows) 151 | guard let handle else { return } 152 | CloseHandle(handle) 153 | #else 154 | guard let fd = fileDescriptor else { return } 155 | close(fd) 156 | #endif 157 | } 158 | 159 | /// Execute the given block while holding the lock. 160 | private func withLock(type: LockType = .exclusive, _ body: () throws -> T) throws -> T { 161 | try self.lock(type: type) 162 | defer { unlock() } 163 | return try body() 164 | } 165 | 166 | /// Execute the given block while holding the lock. 167 | private func withLock(type: LockType = .exclusive, _ body: () async throws -> T) async throws -> T { 168 | try self.lock(type: type) 169 | defer { unlock() } 170 | return try await body() 171 | } 172 | 173 | private static func prepareLock( 174 | fileToLock: FilePath, 175 | at lockFilesDirectory: FilePath? = nil, 176 | _ type: LockType = .exclusive 177 | ) throws -> FileLock { 178 | let fm = FileManager.default 179 | 180 | // unless specified, we use the tempDirectory to store lock files 181 | let lockFilesDirectory = lockFilesDirectory ?? FilePath(fm.temporaryDirectory.path) 182 | var isDirectory: ObjCBool = false 183 | if !fm.fileExists(atPath: lockFilesDirectory.string, isDirectory: &isDirectory) { 184 | throw Error.noEntry(lockFilesDirectory) 185 | } 186 | if !isDirectory.boolValue { 187 | throw Error.notDirectory(lockFilesDirectory) 188 | } 189 | // use the parent path to generate unique filename in temp 190 | var lockFileName = 191 | ( 192 | FilePath(URL(string: fileToLock.removingLastComponent().string)!.resolvingSymlinksInPath().path) 193 | .appending(fileToLock.lastComponent!) 194 | ) 195 | .components.map(\.string).joined(separator: "_") 196 | .replacingOccurrences(of: ":", with: "_") + ".lock" 197 | #if os(Windows) 198 | // NTFS has an ARC limit of 255 codepoints 199 | var lockFileUTF16 = lockFileName.utf16.suffix(255) 200 | while String(lockFileUTF16) == nil { 201 | lockFileUTF16 = lockFileUTF16.dropFirst() 202 | } 203 | lockFileName = String(lockFileUTF16) ?? lockFileName 204 | #else 205 | // back off until it occupies at most `NAME_MAX` UTF-8 bytes but without splitting scalars 206 | // (we might split clusters but it's not worth the effort to keep them together as long as we get a valid file name) 207 | var lockFileUTF8 = lockFileName.utf8.suffix(Int(NAME_MAX)) 208 | while String(lockFileUTF8) == nil { 209 | // in practice this will only be a few iterations 210 | lockFileUTF8 = lockFileUTF8.dropFirst() 211 | } 212 | // we will never end up with nil since we have ASCII characters at the end 213 | lockFileName = String(lockFileUTF8) ?? lockFileName 214 | #endif 215 | let lockFilePath = lockFilesDirectory.appending(lockFileName) 216 | 217 | return FileLock(at: lockFilePath) 218 | } 219 | 220 | static func withLock( 221 | fileToLock: FilePath, 222 | lockFilesDirectory: FilePath? = nil, 223 | type: LockType = .exclusive, 224 | body: () throws -> T 225 | ) throws -> T { 226 | let lock = try Self.prepareLock(fileToLock: fileToLock, at: lockFilesDirectory, type) 227 | return try lock.withLock(type: type, body) 228 | } 229 | 230 | static func withLock( 231 | fileToLock: FilePath, 232 | lockFilesDirectory: FilePath? = nil, 233 | type: LockType = .exclusive, 234 | body: () async throws -> T 235 | ) async throws -> T { 236 | let lock = try Self.prepareLock(fileToLock: fileToLock, at: lockFilesDirectory, type) 237 | return try await lock.withLock(type: type, body) 238 | } 239 | } 240 | -------------------------------------------------------------------------------- /Sources/Helpers/Vendor/QueryEngine/QueryEngine.swift: -------------------------------------------------------------------------------- 1 | //===----------------------------------------------------------------------===// 2 | // 3 | // This source file is part of the Swift open source project 4 | // 5 | // Copyright (c) 2023-2024 Apple Inc. and the Swift project authors 6 | // Licensed under Apache License v2.0 with Runtime Library Exception 7 | // 8 | // See https://swift.org/LICENSE.txt for license information 9 | // See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 10 | // 11 | //===----------------------------------------------------------------------===// 12 | 13 | import Crypto 14 | import Logging 15 | @preconcurrency import SystemPackage 16 | 17 | #if canImport(AsyncHTTPClient) 18 | import AsyncHTTPClient 19 | #endif 20 | 21 | public func withQueryEngine( 22 | _ fileSystem: some AsyncFileSystem, 23 | _ logger: Logger, 24 | cacheLocation: SQLite.Location, 25 | _ body: @Sendable (QueryEngine) async throws -> Void 26 | ) async throws { 27 | let engine = QueryEngine( 28 | fileSystem, 29 | logger, 30 | cacheLocation: cacheLocation 31 | ) 32 | 33 | try await withAsyncThrowing { 34 | try await body(engine) 35 | } defer: { 36 | try await engine.shutDown() 37 | } 38 | } 39 | 40 | /// Cacheable computations engine. Currently the engine makes an assumption that computations produce same results for 41 | /// the same query values and write results to a single file path. 42 | public actor QueryEngine { 43 | private(set) var cacheHits = 0 44 | private(set) var cacheMisses = 0 45 | 46 | public let fileSystem: any AsyncFileSystem 47 | 48 | #if canImport(AsyncHTTPClient) 49 | public let httpClient = HTTPClient() 50 | #endif 51 | 52 | public let observabilityScope: Logger 53 | private let resultsCache: SQLiteBackedCache 54 | private var isShutDown = false 55 | 56 | /// Creates a new instance of the ``QueryEngine`` actor. Requires an explicit call 57 | /// to ``QueryEngine//shutdown`` before the instance is deinitialized. The recommended approach to resource 58 | /// management is to place `engine.shutDown()` when the engine is no longer used, but is not deinitialized yet. 59 | /// - Parameter fileSystem: Implementation of a file system this engine should use. 60 | /// - Parameter cacheLocation: Location of cache storage used by the engine. 61 | /// - Parameter logger: Logger to use during queries execution. 62 | init( 63 | _ fileSystem: any AsyncFileSystem, 64 | _ observabilityScope: Logger, 65 | cacheLocation: SQLite.Location = .memory 66 | ) { 67 | self.fileSystem = fileSystem 68 | self.observabilityScope = observabilityScope 69 | self.resultsCache = SQLiteBackedCache(tableName: "cache_table", location: cacheLocation, observabilityScope) 70 | } 71 | 72 | public func shutDown() async throws { 73 | precondition(!self.isShutDown, "`QueryEngine/shutDown` should be called only once") 74 | try self.resultsCache.close() 75 | 76 | #if canImport(AsyncHTTPClient) 77 | try await httpClient.shutdown() 78 | #endif 79 | 80 | self.isShutDown = true 81 | } 82 | 83 | deinit { 84 | let isShutDown = self.isShutDown 85 | precondition( 86 | isShutDown, 87 | "`QueryEngine/shutDown` should be called explicitly on instances of `Engine` before deinitialization" 88 | ) 89 | } 90 | 91 | /// Executes a given query if no cached result of it is available. Otherwise fetches the result from engine's cache. 92 | /// - Parameter query: A query value to execute. 93 | /// - Returns: A file path to query's result recorded in a file. 94 | public subscript(_ query: some Query) -> FileCacheRecord { 95 | get async throws { 96 | let hashEncoder = HashEncoder() 97 | try hashEncoder.encode(query.cacheKey) 98 | let key = hashEncoder.finalize() 99 | 100 | if let fileRecord = try resultsCache.get(key, as: FileCacheRecord.self) { 101 | let fileHash = try await self.fileSystem.withOpenReadableFile(fileRecord.path) { 102 | var hashFunction = SHA512() 103 | try await $0.hash(with: &hashFunction) 104 | return hashFunction.finalize().description 105 | } 106 | 107 | if fileHash == fileRecord.hash { 108 | self.cacheHits += 1 109 | return fileRecord 110 | } 111 | } 112 | 113 | self.cacheMisses += 1 114 | let resultPath = try await query.run(engine: self) 115 | 116 | let resultHash = try await self.fileSystem.withOpenReadableFile(resultPath) { 117 | var hashFunction = SHA512() 118 | try await $0.hash(with: &hashFunction) 119 | return hashFunction.finalize().description 120 | } 121 | let result = FileCacheRecord(path: resultPath, hash: resultHash) 122 | 123 | // FIXME: update `SQLiteBackedCache` to store `resultHash` directly instead of relying on string conversions 124 | try self.resultsCache.set(key, to: result) 125 | 126 | return result 127 | } 128 | } 129 | } 130 | -------------------------------------------------------------------------------- /Sources/Helpers/Vendor/QueryEngine/SQLite.swift: -------------------------------------------------------------------------------- 1 | //===----------------------------------------------------------------------===// 2 | // 3 | // This source file is part of the Swift open source project 4 | // 5 | // Copyright (c) 2014-2024 Apple Inc. and the Swift project authors 6 | // Licensed under Apache License v2.0 with Runtime Library Exception 7 | // 8 | // See https://swift.org/LICENSE.txt for license information 9 | // See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 10 | // 11 | //===----------------------------------------------------------------------===// 12 | 13 | import struct Foundation.Data 14 | import SystemPackage 15 | import SystemSQLite 16 | 17 | extension FilePath: @unchecked Sendable {} 18 | extension FilePath.Component: @unchecked Sendable {} 19 | 20 | /// A minimal SQLite wrapper. 21 | public final class SQLite { 22 | enum Error: Swift.Error, Equatable { 23 | case databaseFull 24 | case message(String) 25 | } 26 | 27 | /// The location of the database. 28 | let location: Location 29 | 30 | /// The configuration for the database. 31 | let configuration: Configuration 32 | 33 | /// Pointer to the database. 34 | let db: OpaquePointer 35 | 36 | /// Create or open the database at the given path. 37 | /// 38 | /// The database is opened in serialized mode. 39 | init(location: Location, configuration: Configuration = Configuration()) throws { 40 | self.location = location 41 | self.configuration = configuration 42 | 43 | var handle: OpaquePointer? 44 | try Self.checkError( 45 | { 46 | sqlite3_open_v2( 47 | location.pathString, 48 | &handle, 49 | SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE | SQLITE_OPEN_FULLMUTEX, 50 | nil 51 | ) 52 | }, 53 | description: "Unable to open database at \(self.location)" 54 | ) 55 | 56 | guard let db = handle else { 57 | throw Error.message("Unable to open database at \(self.location)") 58 | } 59 | self.db = db 60 | try Self.checkError({ sqlite3_extended_result_codes(db, 1) }, description: "Unable to configure database") 61 | try Self.checkError( 62 | { sqlite3_busy_timeout(db, self.configuration.busyTimeoutMilliseconds) }, 63 | description: "Unable to configure database busy timeout" 64 | ) 65 | if let maxPageCount = self.configuration.maxPageCount { 66 | try self.exec(query: "PRAGMA max_page_count=\(maxPageCount);") 67 | } 68 | } 69 | 70 | /// Prepare the given query. 71 | func prepare(query: String) throws -> PreparedStatement { 72 | try PreparedStatement(db: self.db, query: query) 73 | } 74 | 75 | /// Directly execute the given query. 76 | /// 77 | /// Note: Use withCString for string arguments. 78 | func exec(query queryString: String, args: [CVarArg] = [], _ callback: SQLiteExecCallback? = nil) throws { 79 | let query = withVaList(args) { ptr in 80 | sqlite3_vmprintf(queryString, ptr) 81 | } 82 | 83 | let wcb = callback.map { CallbackWrapper($0) } 84 | let callbackCtx = wcb.map { Unmanaged.passUnretained($0).toOpaque() } 85 | 86 | var err: UnsafeMutablePointer? 87 | try Self.checkError { sqlite3_exec(self.db, query, sqlite_callback, callbackCtx, &err) } 88 | 89 | sqlite3_free(query) 90 | 91 | if let err { 92 | let errorString = String(cString: err) 93 | sqlite3_free(err) 94 | throw Error.message(errorString) 95 | } 96 | } 97 | 98 | func close() throws { 99 | try Self.checkError { sqlite3_close(self.db) } 100 | } 101 | 102 | typealias SQLiteExecCallback = ([Column]) -> () 103 | 104 | struct Configuration { 105 | var busyTimeoutMilliseconds: Int32 106 | var maxSizeInBytes: Int? 107 | 108 | // https://www.sqlite.org/pgszchng2016.html 109 | private let defaultPageSizeInBytes = 1024 110 | 111 | init() { 112 | self.busyTimeoutMilliseconds = 5000 113 | self.maxSizeInBytes = .none 114 | } 115 | 116 | public var maxSizeInMegabytes: Int? { 117 | get { 118 | self.maxSizeInBytes.map { $0 / (1024 * 1024) } 119 | } 120 | set { 121 | self.maxSizeInBytes = newValue.map { $0 * 1024 * 1024 } 122 | } 123 | } 124 | 125 | public var maxPageCount: Int? { 126 | self.maxSizeInBytes.map { $0 / self.defaultPageSizeInBytes } 127 | } 128 | } 129 | 130 | public enum Location: Sendable { 131 | case path(FilePath) 132 | case memory 133 | case temporary 134 | 135 | var pathString: String { 136 | switch self { 137 | case let .path(path): 138 | return path.string 139 | case .memory: 140 | return ":memory:" 141 | case .temporary: 142 | return "" 143 | } 144 | } 145 | } 146 | 147 | /// Represents an sqlite value. 148 | enum SQLiteValue { 149 | case null 150 | case string(String) 151 | case int(Int) 152 | case blob(Data) 153 | } 154 | 155 | /// Represents a row returned by called step() on a prepared statement. 156 | struct Row { 157 | /// The pointer to the prepared statement. 158 | let stmt: OpaquePointer 159 | 160 | /// Get integer at the given column index. 161 | func int(at index: Int32) -> Int { 162 | Int(sqlite3_column_int64(self.stmt, index)) 163 | } 164 | 165 | /// Get blob data at the given column index. 166 | func blob(at index: Int32) -> Data { 167 | let bytes = sqlite3_column_blob(stmt, index)! 168 | let count = sqlite3_column_bytes(stmt, index) 169 | return Data(bytes: bytes, count: Int(count)) 170 | } 171 | 172 | /// Get string at the given column index. 173 | func string(at index: Int32) -> String { 174 | String(cString: sqlite3_column_text(self.stmt, index)) 175 | } 176 | } 177 | 178 | struct Column { 179 | var name: String 180 | var value: String 181 | } 182 | 183 | /// Represents a prepared statement. 184 | struct PreparedStatement { 185 | typealias sqlite3_destructor_type = @convention(c) (UnsafeMutableRawPointer?) -> () 186 | static let SQLITE_STATIC = unsafeBitCast(0, to: sqlite3_destructor_type.self) 187 | static let SQLITE_TRANSIENT = unsafeBitCast(-1, to: sqlite3_destructor_type.self) 188 | 189 | /// The pointer to the prepared statement. 190 | let stmt: OpaquePointer 191 | 192 | init(db: OpaquePointer, query: String) throws { 193 | var stmt: OpaquePointer? 194 | try SQLite.checkError { sqlite3_prepare_v2(db, query, -1, &stmt, nil) } 195 | self.stmt = stmt! 196 | } 197 | 198 | /// Evaluate the prepared statement. 199 | @discardableResult 200 | func step() throws -> Row? { 201 | let result = sqlite3_step(stmt) 202 | 203 | switch result { 204 | case SQLITE_DONE: 205 | return nil 206 | case SQLITE_ROW: 207 | return Row(stmt: self.stmt) 208 | default: 209 | throw Error.message(String(cString: sqlite3_errstr(result))) 210 | } 211 | } 212 | 213 | /// Bind the given arguments to the statement. 214 | func bind(_ arguments: [SQLiteValue]) throws { 215 | for (idx, argument) in arguments.enumerated() { 216 | let idx = Int32(idx) + 1 217 | switch argument { 218 | case .null: 219 | try checkError { sqlite3_bind_null(self.stmt, idx) } 220 | case let .int(int): 221 | try checkError { sqlite3_bind_int64(self.stmt, idx, Int64(int)) } 222 | case let .string(str): 223 | try checkError { sqlite3_bind_text(self.stmt, idx, str, -1, Self.SQLITE_TRANSIENT) } 224 | case let .blob(blob): 225 | try checkError { 226 | blob.withUnsafeBytes { ptr in 227 | sqlite3_bind_blob( 228 | self.stmt, 229 | idx, 230 | ptr.baseAddress, 231 | Int32(blob.count), 232 | Self.SQLITE_TRANSIENT 233 | ) 234 | } 235 | } 236 | } 237 | } 238 | } 239 | 240 | /// Reset the prepared statement. 241 | func reset() throws { 242 | try SQLite.checkError { sqlite3_reset(self.stmt) } 243 | } 244 | 245 | /// Clear bindings from the prepared statement. 246 | func clearBindings() throws { 247 | try SQLite.checkError { sqlite3_clear_bindings(self.stmt) } 248 | } 249 | 250 | /// Finalize the statement and free up resources. 251 | func finalize() throws { 252 | try SQLite.checkError { sqlite3_finalize(self.stmt) } 253 | } 254 | } 255 | 256 | fileprivate class CallbackWrapper { 257 | var callback: SQLiteExecCallback 258 | init(_ callback: @escaping SQLiteExecCallback) { 259 | self.callback = callback 260 | } 261 | } 262 | 263 | private static func checkError(_ fn: () -> Int32, description prefix: String? = .none) throws { 264 | let result = fn() 265 | if result != SQLITE_OK { 266 | var description = String(cString: sqlite3_errstr(result)) 267 | switch description.lowercased() { 268 | case "database or disk is full": 269 | throw Error.databaseFull 270 | default: 271 | if let prefix { 272 | description = "\(prefix): \(description)" 273 | } 274 | throw Error.message(description) 275 | } 276 | } 277 | } 278 | } 279 | 280 | // Explicitly mark this class as non-Sendable 281 | @available(*, unavailable) 282 | extension SQLite: Sendable {} 283 | 284 | private func sqlite_callback( 285 | _ ctx: UnsafeMutableRawPointer?, 286 | _ numColumns: Int32, 287 | _ columns: UnsafeMutablePointer?>?, 288 | _ columnNames: UnsafeMutablePointer?>? 289 | ) -> Int32 { 290 | guard let ctx else { return 0 } 291 | guard let columnNames, let columns else { return 0 } 292 | let numColumns = Int(numColumns) 293 | var result: [SQLite.Column] = [] 294 | 295 | for idx in 0...fromOpaque(ctx).takeUnretainedValue() 308 | wcb.callback(result) 309 | 310 | return 0 311 | } 312 | -------------------------------------------------------------------------------- /Sources/Helpers/Vendor/QueryEngine/SQLiteBackedCache.swift: -------------------------------------------------------------------------------- 1 | //===----------------------------------------------------------------------===// 2 | // 3 | // This source file is part of the Swift open source project 4 | // 5 | // Copyright (c) 2021-2024 Apple Inc. and the Swift project authors 6 | // Licensed under Apache License v2.0 with Runtime Library Exception 7 | // 8 | // See https://swift.org/LICENSE.txt for license information 9 | // See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 10 | // 11 | //===----------------------------------------------------------------------===// 12 | 13 | import Foundation 14 | import Logging 15 | import SystemPackage 16 | 17 | /// SQLite backed persistent cache. 18 | final class SQLiteBackedCache { 19 | typealias Key = String 20 | 21 | let tableName: String 22 | let location: SQLite.Location 23 | let configuration: Configuration 24 | private let logger: Logger 25 | 26 | private var state = State.idle 27 | 28 | private let jsonEncoder: JSONEncoder 29 | private let jsonDecoder: JSONDecoder 30 | 31 | /// Creates a SQLite-backed cache. 32 | /// 33 | /// - Parameters: 34 | /// - tableName: The SQLite table name. Must follow SQLite naming rules (e.g., no spaces). 35 | /// - location: SQLite.Location 36 | /// - configuration: Optional. Configuration for the cache. 37 | init( 38 | tableName: String, 39 | location: SQLite.Location, 40 | configuration: Configuration = .init(), 41 | _ logger: Logger 42 | ) { 43 | self.tableName = tableName 44 | self.location = location 45 | self.logger = logger 46 | self.configuration = configuration 47 | self.jsonEncoder = JSONEncoder() 48 | self.jsonDecoder = JSONDecoder() 49 | } 50 | 51 | deinit { 52 | try? self.withStateLock { 53 | if case let .connected(db) = self.state { 54 | // TODO: we could wrap the failure here with diagnostics if it was available 55 | assertionFailure("db should be closed") 56 | try db.close() 57 | } 58 | } 59 | } 60 | 61 | public func close() throws { 62 | try self.withStateLock { 63 | if case let .connected(db) = self.state { 64 | try db.close() 65 | } 66 | self.state = .disconnected 67 | } 68 | } 69 | 70 | private func put( 71 | key: some Sequence, 72 | value: some Codable, 73 | replace: Bool = false 74 | ) throws { 75 | do { 76 | let query = "INSERT OR \(replace ? "REPLACE" : "IGNORE") INTO \(self.tableName) VALUES (?, ?);" 77 | try self.executeStatement(query) { statement in 78 | let data = try self.jsonEncoder.encode(value) 79 | let bindings: [SQLite.SQLiteValue] = [ 80 | .blob(Data(key)), 81 | .blob(data), 82 | ] 83 | try statement.bind(bindings) 84 | try statement.step() 85 | } 86 | } catch let error as SQLite.Error where error == .databaseFull { 87 | if !self.configuration.truncateWhenFull { 88 | throw error 89 | } 90 | self.logger.warning( 91 | "truncating \(self.tableName) cache database since it reached max size of \(self.configuration.maxSizeInBytes ?? 0) bytes" 92 | ) 93 | try self.executeStatement("DELETE FROM \(self.tableName);") { statement in 94 | try statement.step() 95 | } 96 | try self.put(key: key, value: value, replace: replace) 97 | } catch { 98 | throw error 99 | } 100 | } 101 | 102 | func get(_ key: some Sequence, as: Value.Type) throws -> Value? { 103 | let query = "SELECT value FROM \(self.tableName) WHERE key = ? LIMIT 1;" 104 | return try self.executeStatement(query) { statement -> Value? in 105 | try statement.bind([.blob(Data(key))]) 106 | let data = try statement.step()?.blob(at: 0) 107 | return try data.flatMap { 108 | try self.jsonDecoder.decode(Value.self, from: $0) 109 | } 110 | } 111 | } 112 | 113 | func set(_ key: some Sequence, to value: some Codable) throws { 114 | try self.put(key: key, value: value, replace: true) 115 | } 116 | 117 | func remove(key: Key) throws { 118 | let query = "DELETE FROM \(self.tableName) WHERE key = ?;" 119 | try self.executeStatement(query) { statement in 120 | try statement.bind([.string(key)]) 121 | try statement.step() 122 | } 123 | } 124 | 125 | @discardableResult 126 | private func executeStatement(_ query: String, _ body: (SQLite.PreparedStatement) throws -> T) throws -> T { 127 | try self.withDB { db in 128 | let result: Result 129 | let statement = try db.prepare(query: query) 130 | do { 131 | result = try .success(body(statement)) 132 | } catch { 133 | result = .failure(error) 134 | } 135 | try statement.finalize() 136 | switch result { 137 | case let .failure(error): 138 | throw error 139 | case let .success(value): 140 | return value 141 | } 142 | } 143 | } 144 | 145 | private func withDB(_ body: (SQLite) throws -> T) throws -> T { 146 | let createDB = { () throws -> SQLite in 147 | let db = try SQLite(location: self.location, configuration: self.configuration.underlying) 148 | try self.createSchemaIfNecessary(db: db) 149 | return db 150 | } 151 | 152 | let db: SQLite 153 | let fm = FileManager.default 154 | switch (self.location, self.state) { 155 | case let (.path(path), .connected(database)): 156 | if fm.fileExists(atPath: path.string) { 157 | db = database 158 | } else { 159 | try database.close() 160 | try fm.createDirectory(atPath: path.removingLastComponent().string, withIntermediateDirectories: true) 161 | db = try createDB() 162 | } 163 | case let (.path(path), _): 164 | if !fm.fileExists(atPath: path.string) { 165 | try fm.createDirectory(atPath: path.removingLastComponent().string, withIntermediateDirectories: true) 166 | } 167 | db = try createDB() 168 | case let (_, .connected(database)): 169 | db = database 170 | case (_, _): 171 | db = try createDB() 172 | } 173 | self.state = .connected(db) 174 | return try body(db) 175 | } 176 | 177 | private func createSchemaIfNecessary(db: SQLite) throws { 178 | let table = """ 179 | CREATE TABLE IF NOT EXISTS \(self.tableName) ( 180 | key STRING PRIMARY KEY NOT NULL, 181 | value BLOB NOT NULL 182 | ); 183 | """ 184 | 185 | try db.exec(query: table) 186 | try db.exec(query: "PRAGMA journal_mode=WAL;") 187 | } 188 | 189 | private func withStateLock(_ body: () throws -> T) throws -> T { 190 | switch self.location { 191 | case let .path(path): 192 | let fm = FileManager.default 193 | if !fm.fileExists(atPath: path.string) { 194 | try fm.createDirectory(atPath: path.removingLastComponent().string, withIntermediateDirectories: true) 195 | } 196 | 197 | return try FileLock.withLock(fileToLock: path, body: body) 198 | case .memory, .temporary: 199 | return try body() 200 | } 201 | } 202 | 203 | private enum State { 204 | case idle 205 | case connected(SQLite) 206 | case disconnected 207 | } 208 | 209 | public struct Configuration { 210 | var truncateWhenFull: Bool 211 | 212 | fileprivate var underlying: SQLite.Configuration 213 | 214 | init() { 215 | self.underlying = .init() 216 | self.truncateWhenFull = true 217 | self.maxSizeInMegabytes = 100 218 | // see https://www.sqlite.org/c3ref/busy_timeout.html 219 | self.busyTimeoutMilliseconds = 1000 220 | } 221 | 222 | var maxSizeInMegabytes: Int? { 223 | get { 224 | self.underlying.maxSizeInMegabytes 225 | } 226 | set { 227 | self.underlying.maxSizeInMegabytes = newValue 228 | } 229 | } 230 | 231 | var maxSizeInBytes: Int? { 232 | get { 233 | self.underlying.maxSizeInBytes 234 | } 235 | set { 236 | self.underlying.maxSizeInBytes = newValue 237 | } 238 | } 239 | 240 | var busyTimeoutMilliseconds: Int32 { 241 | get { 242 | self.underlying.busyTimeoutMilliseconds 243 | } 244 | set { 245 | self.underlying.busyTimeoutMilliseconds = newValue 246 | } 247 | } 248 | } 249 | } 250 | 251 | // Explicitly mark this class as non-Sendable 252 | @available(*, unavailable) 253 | extension SQLiteBackedCache: Sendable {} 254 | -------------------------------------------------------------------------------- /Sources/Helpers/Vendor/README.md: -------------------------------------------------------------------------------- 1 | The `Triple.swift` file in this directory has been vendored from the 2 | [swift-driver](https://github.com/swiftlang/swift-driver) project. 3 | 4 | The `_AsyncFileSystem` and `QueryEngine` code in this directory has been vendored from the 5 | [swift-package-manager](https://github.com/swiftlang/swift-package-manager) project. 6 | 7 | The purpose of this vendoring is not to bring these entire packages as dependencies. 8 | -------------------------------------------------------------------------------- /Sources/Helpers/Vendor/_AsyncFileSystem/AsyncFileSystem.swift: -------------------------------------------------------------------------------- 1 | //===----------------------------------------------------------------------===// 2 | // 3 | // This source file is part of the Swift open source project 4 | // 5 | // Copyright (c) 2023-2024 Apple Inc. and the Swift project authors 6 | // Licensed under Apache License v2.0 with Runtime Library Exception 7 | // 8 | // See https://swift.org/LICENSE.txt for license information 9 | // See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 10 | // 11 | //===----------------------------------------------------------------------===// 12 | 13 | import _Concurrency 14 | @preconcurrency import struct SystemPackage.Errno 15 | @preconcurrency import struct SystemPackage.FilePath 16 | 17 | /// An abstract file system protocol with first-class support for Swift Concurrency. 18 | public protocol AsyncFileSystem: Actor { 19 | /// Whether a file exists on the file system. 20 | /// - Parameter path: Absolute path to the file to check. 21 | /// - Returns: `true` if the file exists, `false` otherwise. 22 | func exists(_ path: FilePath) async -> Bool 23 | 24 | /// Temporarily opens a read-only file within a scope defined by a given closure. 25 | /// - Parameters: 26 | /// - path: Absolute path to a readable file on this file system. 27 | /// - body: Closure that has temporary read-only access to the open file. The underlying file handle is closed 28 | /// when this closure returns, thus in the current absence of `~Escapable` types in Swift 6.0, users should take 29 | /// care not to allow this handle to escape the closure. 30 | /// - Returns: Result of the `body` closure. 31 | func withOpenReadableFile( 32 | _ path: FilePath, 33 | _ body: @Sendable (_ fileHandle: OpenReadableFile) async throws -> T 34 | ) async throws -> T 35 | 36 | /// Temporarily opens a write-only file within a scope defined by a given closure. 37 | /// - Parameters: 38 | /// - path: Absolute path to a writable file on this file system. 39 | /// - body: Closure that has temporary write-only access to the open file. The underlying file handle is closed 40 | /// when this closure returns, thus in the current absence of `~Escapable` types in Swift 6.0, users should take 41 | /// care not to allow this handle to escape the closure. 42 | /// - Returns: Result of the `body` closure. 43 | func withOpenWritableFile( 44 | _ path: FilePath, 45 | _ body: @Sendable (_ fileHandle: OpenWritableFile) async throws -> T 46 | ) async throws -> T 47 | } 48 | 49 | /// Errors that can be thrown by the ``AsyncFileSystem`` type. 50 | public enum AsyncFileSystemError: Error { 51 | /// A file with the associated ``FilePath`` value does not exist. 52 | case fileDoesNotExist(FilePath) 53 | 54 | /// A wrapper for the underlying `SystemPackage` error types that attaches a ``FilePath`` value to it. 55 | case systemError(FilePath, Errno) 56 | } 57 | 58 | extension Error { 59 | /// Makes a system error value more actionable and readable by the end user. 60 | /// - Parameter path: absolute path to the file, operations on which caused this error. 61 | /// - Returns: An ``AsyncFileSystemError`` value augmented by the given file path. 62 | func attach(_ path: FilePath) -> any Error { 63 | if let error = self as? Errno { 64 | return AsyncFileSystemError.systemError(path, error) 65 | } else { 66 | return self 67 | } 68 | } 69 | } 70 | 71 | extension AsyncFileSystem { 72 | public func write(_ path: FilePath, bytes: some Collection & Sendable) async throws { 73 | try await self.withOpenWritableFile(path) { fileHandle in 74 | try await fileHandle.write(bytes) 75 | } 76 | } 77 | } 78 | -------------------------------------------------------------------------------- /Sources/Helpers/Vendor/_AsyncFileSystem/ConcurrencySupport.swift: -------------------------------------------------------------------------------- 1 | //===----------------------------------------------------------------------===// 2 | // 3 | // This source file is part of the Swift open source project 4 | // 5 | // Copyright (c) 2020-2024 Apple Inc. and the Swift project authors 6 | // Licensed under Apache License v2.0 with Runtime Library Exception 7 | // 8 | // See https://swift.org/LICENSE.txt for license information 9 | // See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 10 | // 11 | //===----------------------------------------------------------------------===// 12 | 13 | import _Concurrency 14 | import class Dispatch.DispatchQueue 15 | 16 | extension DispatchQueue { 17 | /// Schedules blocking synchronous work item on this ``DispatchQueue`` instance. 18 | /// - Parameter work: Blocking synchronous closure that should be scheduled on this queue. 19 | /// - Returns: Result of the `work` closure. 20 | func scheduleOnQueue(work: @escaping @Sendable () throws -> T) async throws -> T { 21 | try await withCheckedThrowingContinuation { continuation in 22 | self.async { 23 | do { 24 | continuation.resume(returning: try work()) 25 | } catch { 26 | continuation.resume(throwing: error) 27 | } 28 | } 29 | } 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /Sources/Helpers/Vendor/_AsyncFileSystem/MockFileSystem.swift: -------------------------------------------------------------------------------- 1 | //===----------------------------------------------------------------------===// 2 | // 3 | // This source file is part of the Swift open source project 4 | // 5 | // Copyright (c) 2023-2024 Apple Inc. and the Swift project authors 6 | // Licensed under Apache License v2.0 with Runtime Library Exception 7 | // 8 | // See https://swift.org/LICENSE.txt for license information 9 | // See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 10 | // 11 | //===----------------------------------------------------------------------===// 12 | 13 | @preconcurrency import struct SystemPackage.FilePath 14 | 15 | /// In-memory implementation of `AsyncFileSystem` for mocking and testing purposes. 16 | public actor MockFileSystem: AsyncFileSystem { 17 | /// The default size of chunks in bytes read by this file system. 18 | public static let defaultChunkSize = 512 * 1024 19 | 20 | /// Maximum size of chunks in bytes read by this instance of file system. 21 | let readChunkSize: Int 22 | 23 | /// Underlying in-memory dictionary-based storage for this mock file system. 24 | final class Storage { 25 | init(_ content: [FilePath: [UInt8]]) { 26 | self.content = content 27 | } 28 | 29 | var content: [FilePath: [UInt8]] 30 | } 31 | 32 | /// Concrete instance of the underlying storage used by this file system. 33 | private let storage: Storage 34 | 35 | /// Creates a new instance of the mock file system. 36 | /// - Parameters: 37 | /// - content: Dictionary of paths to their in-memory contents to use for seeding the file system. 38 | /// - readChunkSize: Size of chunks in bytes produced by this file system when reading files. 39 | public init(content: [FilePath: [UInt8]] = [:], readChunkSize: Int = defaultChunkSize) { 40 | self.storage = .init(content) 41 | self.readChunkSize = readChunkSize 42 | } 43 | 44 | public func exists(_ path: FilePath) -> Bool { 45 | self.storage.content.keys.contains(path) 46 | } 47 | 48 | /// Appends a sequence of bytes to a file. 49 | /// - Parameters: 50 | /// - path: absolute path of the file to append bytes to. 51 | /// - bytes: sequence of bytes to append to file's contents. 52 | func append(path: FilePath, bytes: some Sequence) { 53 | storage.content[path, default: []].append(contentsOf: bytes) 54 | } 55 | 56 | public func withOpenReadableFile( 57 | _ path: FilePath, 58 | _ body: (OpenReadableFile) async throws -> T 59 | ) async throws -> T { 60 | guard let bytes = storage.content[path] else { 61 | throw AsyncFileSystemError.fileDoesNotExist(path) 62 | } 63 | return try await body(.init(chunkSize: self.readChunkSize, fileHandle: .mock(bytes))) 64 | } 65 | 66 | public func withOpenWritableFile( 67 | _ path: FilePath, 68 | _ body: (OpenWritableFile) async throws -> T 69 | ) async throws -> T { 70 | try await body(.init(storage: .mock(self), path: path)) 71 | } 72 | } 73 | -------------------------------------------------------------------------------- /Sources/Helpers/Vendor/_AsyncFileSystem/OSFileSystem.swift: -------------------------------------------------------------------------------- 1 | //===----------------------------------------------------------------------===// 2 | // 3 | // This source file is part of the Swift open source project 4 | // 5 | // Copyright (c) 2023-2024 Apple Inc. and the Swift project authors 6 | // Licensed under Apache License v2.0 with Runtime Library Exception 7 | // 8 | // See https://swift.org/LICENSE.txt for license information 9 | // See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 10 | // 11 | //===----------------------------------------------------------------------===// 12 | 13 | import Foundation 14 | import SystemPackage 15 | 16 | public actor OSFileSystem: AsyncFileSystem { 17 | public static let defaultChunkSize = 512 * 1024 18 | 19 | let readChunkSize: Int 20 | private let ioQueue = DispatchQueue(label: "org.swift.sdk-generator-io") 21 | 22 | public init(readChunkSize: Int = defaultChunkSize) { 23 | self.readChunkSize = readChunkSize 24 | } 25 | 26 | public func withOpenReadableFile( 27 | _ path: FilePath, 28 | _ body: (OpenReadableFile) async throws -> T 29 | ) async throws -> T { 30 | let fd = try FileDescriptor.open(path, .readOnly) 31 | // Can't use ``FileDescriptor//closeAfter` here, as that doesn't support async closures. 32 | do { 33 | let result = try await body(.init(chunkSize: readChunkSize, fileHandle: .real(fd, self.ioQueue))) 34 | try fd.close() 35 | return result 36 | } catch { 37 | try fd.close() 38 | throw error.attach(path) 39 | } 40 | } 41 | 42 | public func withOpenWritableFile( 43 | _ path: FilePath, 44 | _ body: (OpenWritableFile) async throws -> T 45 | ) async throws -> T { 46 | let fd = try FileDescriptor.open(path, .writeOnly) 47 | do { 48 | let result = try await body(.init(storage: .real(fd, self.ioQueue), path: path)) 49 | try fd.close() 50 | return result 51 | } catch { 52 | try fd.close() 53 | throw error.attach(path) 54 | } 55 | } 56 | 57 | public func exists(_ path: SystemPackage.FilePath) async -> Bool { 58 | FileManager.default.fileExists(atPath: path.string) 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /Sources/Helpers/Vendor/_AsyncFileSystem/OpenReadableFile.swift: -------------------------------------------------------------------------------- 1 | //===----------------------------------------------------------------------===// 2 | // 3 | // This source file is part of the Swift open source project 4 | // 5 | // Copyright (c) 2023-2024 Apple Inc. and the Swift project authors 6 | // Licensed under Apache License v2.0 with Runtime Library Exception 7 | // 8 | // See https://swift.org/LICENSE.txt for license information 9 | // See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 10 | // 11 | //===----------------------------------------------------------------------===// 12 | 13 | import class Dispatch.DispatchQueue 14 | import struct SystemPackage.FileDescriptor 15 | 16 | /// A read-only thread-safe handle to an open file. 17 | public struct OpenReadableFile: Sendable { 18 | /// Maximum size of chunks in bytes produced when reading this file handle. 19 | let chunkSize: Int 20 | 21 | /// Underlying storage for this file handle, dependent on the file system type that produced it. 22 | enum Storage { 23 | /// Operating system file descriptor and a queue used for reading from that file descriptor without blocking 24 | /// the Swift Concurrency thread pool. 25 | case real(FileDescriptor, DispatchQueue) 26 | 27 | /// Complete contents of the file represented by this handle stored in memory as an array of bytes. 28 | case mock([UInt8]) 29 | } 30 | 31 | /// Concrete instance of underlying file storage. 32 | let fileHandle: Storage 33 | 34 | /// Creates a readable ``AsyncSequence`` that can be iterated on to read from this file handle. 35 | /// - Returns: `ReadableFileStream` value conforming to ``AsyncSequence``, ready for asynchronous iteration. 36 | public func read() async throws -> ReadableFileStream { 37 | switch self.fileHandle { 38 | case let .real(fileDescriptor, ioQueue): 39 | return ReadableFileStream.real( 40 | .init( 41 | fileDescriptor: fileDescriptor, 42 | ioQueue: ioQueue, 43 | readChunkSize: self.chunkSize 44 | ) 45 | ) 46 | 47 | case .mock(let array): 48 | return ReadableFileStream.mock(.init(bytes: array, chunkSize: self.chunkSize)) 49 | } 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /Sources/Helpers/Vendor/_AsyncFileSystem/OpenWritableFile.swift: -------------------------------------------------------------------------------- 1 | //===----------------------------------------------------------------------===// 2 | // 3 | // This source file is part of the Swift open source project 4 | // 5 | // Copyright (c) 2023-2024 Apple Inc. and the Swift project authors 6 | // Licensed under Apache License v2.0 with Runtime Library Exception 7 | // 8 | // See https://swift.org/LICENSE.txt for license information 9 | // See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 10 | // 11 | //===----------------------------------------------------------------------===// 12 | 13 | import class Dispatch.DispatchQueue 14 | @preconcurrency import struct SystemPackage.FileDescriptor 15 | import struct SystemPackage.FilePath 16 | 17 | /// A write-only thread-safe handle to an open file. 18 | public actor OpenWritableFile: WritableStream { 19 | /// Underlying storage for this file handle, dependent on the file system type that produced it. 20 | enum Storage { 21 | /// Operating system file descriptor and a queue used for reading from that file descriptor without blocking 22 | /// the Swift Concurrency thread pool. 23 | case real(FileDescriptor, DispatchQueue) 24 | 25 | /// Reference to the ``MockFileSystem`` actor that provides file storage. 26 | case mock(MockFileSystem) 27 | } 28 | 29 | /// Concrete instance of underlying storage. 30 | let storage: Storage 31 | 32 | /// Absolute path to the file represented by this file handle. 33 | let path: FilePath 34 | 35 | /// Whether the underlying file descriptor has been closed. 36 | private var isClosed = false 37 | 38 | /// Creates a new write-only file handle. 39 | /// - Parameters: 40 | /// - storage: Underlying storage for the file. 41 | /// - path: Absolute path to the file on the file system that provides `storage`. 42 | init(storage: OpenWritableFile.Storage, path: FilePath) { 43 | self.storage = storage 44 | self.path = path 45 | } 46 | 47 | /// Writes a sequence of bytes to the buffer. 48 | public func write(_ bytes: some Collection & Sendable) async throws { 49 | assert(!isClosed) 50 | switch self.storage { 51 | case let .real(fileDescriptor, queue): 52 | let path = self.path 53 | try await queue.scheduleOnQueue { 54 | do { 55 | let writtenBytesCount = try fileDescriptor.writeAll(bytes) 56 | assert(bytes.count == writtenBytesCount) 57 | } catch { 58 | throw error.attach(path) 59 | } 60 | } 61 | case let .mock(storage): 62 | await storage.append(path: self.path, bytes: bytes) 63 | } 64 | } 65 | 66 | /// Closes the underlying stream handle. It is a programmer error to write to a stream after it's closed. 67 | public func close() async throws { 68 | isClosed = true 69 | 70 | guard case let .real(fileDescriptor, queue) = self.storage else { 71 | return 72 | } 73 | 74 | try await queue.scheduleOnQueue { 75 | try fileDescriptor.close() 76 | } 77 | } 78 | } 79 | 80 | -------------------------------------------------------------------------------- /Sources/Helpers/Vendor/_AsyncFileSystem/ReadableFileStream.swift: -------------------------------------------------------------------------------- 1 | //===----------------------------------------------------------------------===// 2 | // 3 | // This source file is part of the Swift open source project 4 | // 5 | // Copyright (c) 2023-2024 Apple Inc. and the Swift project authors 6 | // Licensed under Apache License v2.0 with Runtime Library Exception 7 | // 8 | // See https://swift.org/LICENSE.txt for license information 9 | // See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 10 | // 11 | //===----------------------------------------------------------------------===// 12 | 13 | import _Concurrency 14 | import SystemPackage 15 | import class Dispatch.DispatchQueue 16 | 17 | /// Type-erasure wrapper over underlying file system readable streams. 18 | public enum ReadableFileStream: AsyncSequence { 19 | public typealias Element = ArraySlice 20 | 21 | case real(RealReadableFileStream) 22 | case mock(MockReadableFileStream) 23 | 24 | public enum Iterator: AsyncIteratorProtocol { 25 | case real(RealReadableFileStream.Iterator) 26 | case mock(MockReadableFileStream.Iterator) 27 | 28 | public func next() async throws -> ArraySlice? { 29 | switch self { 30 | case .real(let local): 31 | return try await local.next() 32 | case .mock(let virtual): 33 | return try await virtual.next() 34 | } 35 | } 36 | } 37 | 38 | public func makeAsyncIterator() -> Iterator { 39 | switch self { 40 | case .real(let real): 41 | return .real(real.makeAsyncIterator()) 42 | case .mock(let mock): 43 | return .mock(mock.makeAsyncIterator()) 44 | } 45 | } 46 | } 47 | 48 | /// A stream of file contents from the real file system provided by the OS. 49 | public struct RealReadableFileStream: AsyncSequence { 50 | public typealias Element = ArraySlice 51 | 52 | let fileDescriptor: FileDescriptor 53 | let ioQueue: DispatchQueue 54 | let readChunkSize: Int 55 | 56 | public final class Iterator: AsyncIteratorProtocol { 57 | init(_ fileDescriptor: FileDescriptor, ioQueue: DispatchQueue, readChunkSize: Int) { 58 | self.fileDescriptor = fileDescriptor 59 | self.ioQueue = ioQueue 60 | self.chunkSize = readChunkSize 61 | } 62 | 63 | private let fileDescriptor: FileDescriptor 64 | private let ioQueue: DispatchQueue 65 | private let chunkSize: Int 66 | 67 | public func next() async throws -> ArraySlice? { 68 | let chunkSize = self.chunkSize 69 | let fileDescriptor = self.fileDescriptor 70 | 71 | return try await ioQueue.scheduleOnQueue { 72 | var buffer = [UInt8](repeating: 0, count: chunkSize) 73 | 74 | let bytesRead = try buffer.withUnsafeMutableBytes { 75 | try fileDescriptor.read(into: $0) 76 | } 77 | 78 | guard bytesRead > 0 else { 79 | return nil 80 | } 81 | 82 | buffer.removeLast(chunkSize - bytesRead) 83 | return buffer[...] 84 | } 85 | } 86 | } 87 | 88 | public func makeAsyncIterator() -> Iterator { 89 | Iterator(self.fileDescriptor, ioQueue: ioQueue, readChunkSize: self.readChunkSize) 90 | } 91 | } 92 | 93 | 94 | /// A stream of file contents backed by an in-memory array of bytes. 95 | public struct MockReadableFileStream: AsyncSequence { 96 | public typealias Element = ArraySlice 97 | 98 | public final class Iterator: AsyncIteratorProtocol { 99 | init(bytes: [UInt8], chunkSize: Int) { 100 | self.bytes = bytes 101 | self.chunkSize = chunkSize 102 | } 103 | 104 | private let chunkSize: Int 105 | var bytes: [UInt8] 106 | private var position = 0 107 | 108 | public func next() async throws -> ArraySlice? { 109 | let nextPosition = Swift.min(bytes.count, position + chunkSize) 110 | 111 | guard nextPosition > position else { 112 | return nil 113 | } 114 | 115 | defer { self.position = nextPosition } 116 | return self.bytes[position.. Iterator { 124 | Iterator(bytes: self.bytes, chunkSize: self.chunkSize) 125 | } 126 | } 127 | -------------------------------------------------------------------------------- /Sources/Helpers/Vendor/_AsyncFileSystem/WritableStream.swift: -------------------------------------------------------------------------------- 1 | //===----------------------------------------------------------------------===// 2 | // 3 | // This source file is part of the Swift open source project 4 | // 5 | // Copyright (c) 2014-2025 Apple Inc. and the Swift project authors 6 | // Licensed under Apache License v2.0 with Runtime Library Exception 7 | // 8 | // See https://swift.org/LICENSE.txt for license information 9 | // See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 10 | // 11 | //===----------------------------------------------------------------------===// 12 | 13 | import _Concurrency 14 | 15 | /// An asynchronous output byte stream. 16 | /// 17 | /// This protocol is designed to be able to support efficient streaming to 18 | /// different output destinations, e.g., a file or an in memory buffer. 19 | /// 20 | /// The stream is generally used in conjunction with the ``WritableStream/write(_:)`` function. 21 | /// For example: 22 | /// ```swift 23 | /// let stream = fileSystem.withOpenWritableFile { stream in 24 | /// stream.write("Hello, world!".utf8) 25 | /// } 26 | /// ``` 27 | /// would write the UTF8 encoding of "Hello, world!" to the stream. 28 | public protocol WritableStream: Actor { 29 | /// Writes a sequence of bytes to the buffer. 30 | func write(_ bytes: some Collection & Sendable) async throws 31 | 32 | /// Closes the underlying stream handle. It is a programmer error to write to a stream after it's closed. 33 | func close() async throws 34 | } 35 | -------------------------------------------------------------------------------- /Sources/SwiftSDKGenerator/Artifacts/DownloadableArtifacts.swift: -------------------------------------------------------------------------------- 1 | //===----------------------------------------------------------------------===// 2 | // 3 | // This source file is part of the Swift open source project 4 | // 5 | // Copyright (c) 2022-2023 Apple Inc. and the Swift project authors 6 | // Licensed under Apache License v2.0 with Runtime Library Exception 7 | // 8 | // See https://swift.org/LICENSE.txt for license information 9 | // See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 10 | // 11 | //===----------------------------------------------------------------------===// 12 | 13 | import Helpers 14 | 15 | import struct Foundation.URL 16 | import struct SystemPackage.FilePath 17 | 18 | extension Triple { 19 | fileprivate var llvmBinaryURLSuffix: String { 20 | switch (self.os, self.arch) { 21 | case (.linux, .aarch64): return "aarch64-linux-gnu" 22 | case (.linux, .x86_64): return "x86_64-linux-gnu-ubuntu-22.04" 23 | case (.macosx, .aarch64): return "arm64-apple-darwin22.0" 24 | case (.macosx, .x86_64): return "x86_64-apple-darwin22.0" 25 | default: fatalError("\(self) is not supported as LLVM host platform yet") 26 | } 27 | } 28 | } 29 | 30 | typealias CPUMapping = [Triple.Arch: String] 31 | 32 | struct DownloadableArtifacts: Sendable { 33 | struct Item: Sendable, CacheKey { 34 | let remoteURL: URL 35 | var localPath: FilePath 36 | let isPrebuilt: Bool 37 | } 38 | 39 | let hostSwift: Item 40 | private(set) var hostLLVM: Item 41 | let targetSwift: Item 42 | 43 | private let versions: VersionsConfiguration 44 | private let paths: PathsConfiguration 45 | 46 | init( 47 | hostTriple: Triple, 48 | targetTriple: Triple, 49 | _ versions: VersionsConfiguration, 50 | _ paths: PathsConfiguration 51 | ) throws { 52 | self.versions = versions 53 | self.paths = paths 54 | 55 | if hostTriple.os == .linux { 56 | // Amazon Linux 2 is chosen for its best compatibility with all Swift-supported Linux hosts 57 | let hostArchSuffix = 58 | hostTriple.arch == .aarch64 ? "-\(Triple.Arch.aarch64.linuxConventionName)" : "" 59 | self.hostSwift = .init( 60 | remoteURL: versions.swiftDownloadURL( 61 | subdirectory: "amazonlinux2\(hostArchSuffix)", 62 | platform: "amazonlinux2\(hostArchSuffix)", 63 | fileExtension: "tar.gz" 64 | ), 65 | localPath: paths.artifactsCachePath 66 | .appending("host_swift_\(versions.swiftVersion)_\(hostTriple.triple).tar.gz"), 67 | isPrebuilt: true 68 | ) 69 | } else { 70 | self.hostSwift = .init( 71 | remoteURL: versions.swiftDownloadURL( 72 | subdirectory: "xcode", 73 | platform: "osx", 74 | fileExtension: "pkg" 75 | ), 76 | localPath: paths.artifactsCachePath 77 | .appending("host_swift_\(versions.swiftVersion)_\(hostTriple.triple).pkg"), 78 | isPrebuilt: true 79 | ) 80 | } 81 | 82 | self.hostLLVM = .init( 83 | remoteURL: URL( 84 | string: """ 85 | https://github.com/llvm/llvm-project/releases/download/llvmorg-\( 86 | versions.lldVersion 87 | )/clang+llvm-\( 88 | versions.lldVersion 89 | )-\(hostTriple.llvmBinaryURLSuffix).tar.xz 90 | """ 91 | )!, 92 | localPath: paths.artifactsCachePath 93 | .appending("host_llvm_\(versions.lldVersion)_\(hostTriple.triple).tar.xz"), 94 | isPrebuilt: true 95 | ) 96 | 97 | self.targetSwift = .init( 98 | remoteURL: versions.swiftDownloadURL(), 99 | localPath: paths.artifactsCachePath 100 | .appending( 101 | "target_swift_\(versions.swiftVersion)_\(versions.swiftPlatform)_\(targetTriple.archName).tar.gz" 102 | ), 103 | isPrebuilt: true 104 | ) 105 | } 106 | 107 | mutating func useLLVMSources() { 108 | self.hostLLVM = .init( 109 | remoteURL: URL( 110 | string: """ 111 | https://github.com/llvm/llvm-project/releases/download/llvmorg-\( 112 | self.versions.lldVersion 113 | )/llvm-project-\( 114 | self.versions.lldVersion 115 | ).src.tar.xz 116 | """ 117 | )!, 118 | localPath: self.paths.artifactsCachePath 119 | .appending("llvm_\(self.versions.lldVersion).src.tar.xz"), 120 | isPrebuilt: false 121 | ) 122 | } 123 | } 124 | -------------------------------------------------------------------------------- /Sources/SwiftSDKGenerator/ArtifactsArchiveMetadata.swift: -------------------------------------------------------------------------------- 1 | //===----------------------------------------------------------------------===// 2 | // 3 | // This source file is part of the Swift open source project 4 | // 5 | // Copyright (c) 2022 Apple Inc. and the Swift project authors 6 | // Licensed under Apache License v2.0 with Runtime Library Exception 7 | // 8 | // See https://swift.org/LICENSE.txt for license information 9 | // See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 10 | // 11 | //===----------------------------------------------------------------------===// 12 | 13 | import Foundation 14 | 15 | public struct ArtifactsArchiveMetadata: Equatable, Codable { 16 | public let schemaVersion: String 17 | public let artifacts: [String: Artifact] 18 | 19 | public init(schemaVersion: String, artifacts: [String: Artifact]) { 20 | self.schemaVersion = schemaVersion 21 | self.artifacts = artifacts 22 | } 23 | 24 | public struct Artifact: Equatable, Codable { 25 | let type: ArtifactType 26 | let version: String 27 | let variants: [Variant] 28 | 29 | public init( 30 | type: ArtifactsArchiveMetadata.ArtifactType, 31 | version: String, 32 | variants: [Variant] 33 | ) { 34 | self.type = type 35 | self.version = version 36 | self.variants = variants 37 | } 38 | } 39 | 40 | public enum ArtifactType: String, RawRepresentable, Codable { 41 | case swiftSDK 42 | } 43 | 44 | public struct Variant: Equatable, Codable { 45 | let path: String 46 | let supportedTriples: [String]? 47 | 48 | public init(path: String, supportedTriples: [String]?) { 49 | self.path = path 50 | self.supportedTriples = supportedTriples 51 | } 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /Sources/SwiftSDKGenerator/Dockerfiles/Ubuntu/22.04/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM swift:5.8-jammy 2 | 3 | # Uncomment and refine the list of packages you'd like to be installed 4 | # in the target environment. 5 | # RUN apt-get update && \ 6 | # apt-get install -y librdkafka-dev 7 | -------------------------------------------------------------------------------- /Sources/SwiftSDKGenerator/Extensions/String+hasAnyPrefix.swift: -------------------------------------------------------------------------------- 1 | //===----------------------------------------------------------------------===// 2 | // 3 | // This source file is part of the Swift open source project 4 | // 5 | // Copyright (c) 2025 Apple Inc. and the Swift project authors 6 | // Licensed under Apache License v2.0 with Runtime Library Exception 7 | // 8 | // See https://swift.org/LICENSE.txt for license information 9 | // See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 10 | // 11 | //===----------------------------------------------------------------------===// 12 | 13 | extension String { 14 | func hasAnyPrefix(from array: [String]) -> Bool { 15 | for item in array { 16 | if self.hasPrefix(item) { 17 | return true 18 | } 19 | } 20 | return false 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Copy.swift: -------------------------------------------------------------------------------- 1 | //===----------------------------------------------------------------------===// 2 | // 3 | // This source file is part of the Swift open source project 4 | // 5 | // Copyright (c) 2022-2023 Apple Inc. and the Swift project authors 6 | // Licensed under Apache License v2.0 with Runtime Library Exception 7 | // 8 | // See https://swift.org/LICENSE.txt for license information 9 | // See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 10 | // 11 | //===----------------------------------------------------------------------===// 12 | 13 | import Foundation 14 | import SystemPackage 15 | 16 | extension SwiftSDKGenerator { 17 | func copyTargetSwiftFromDocker( 18 | targetDistribution: LinuxDistribution, 19 | baseDockerImage: String, 20 | sdkDirPath: FilePath 21 | ) async throws { 22 | logger.info("Launching a container to extract the Swift SDK for the target triple...") 23 | try await withDockerContainer(fromImage: baseDockerImage) { containerID in 24 | try await inTemporaryDirectory { generator, _ in 25 | let sdkLibPath = sdkDirPath.appending("lib") 26 | let sdkUsrPath = sdkDirPath.appending("usr") 27 | try await generator.createDirectoryIfNeeded(at: sdkUsrPath) 28 | try await generator.copyFromDockerContainer( 29 | id: containerID, 30 | from: "/usr/include", 31 | to: sdkUsrPath.appending("include") 32 | ) 33 | 34 | if case .rhel = targetDistribution { 35 | try await generator.runOnDockerContainer( 36 | id: containerID, 37 | command: #""" 38 | sh -c ' 39 | chmod +w /usr/lib64 40 | cd /usr/lib64 41 | for n in *; do 42 | destination=$(readlink $n) 43 | echo $destination | grep "\.\." && \ 44 | rm -f $n && \ 45 | ln -s $(basename $destination) $n 46 | done 47 | rm -rf pm-utils 48 | ' 49 | """# 50 | ) 51 | } 52 | 53 | if case let containerLib64 = FilePath("/usr/lib64"), 54 | try await generator.doesPathExist(containerLib64, inContainer: containerID) 55 | { 56 | let sdkUsrLib64Path = sdkUsrPath.appending("lib64") 57 | // we already checked that the path exists above, so we don't pass `failIfNotExists: false` here. 58 | try await generator.copyFromDockerContainer( 59 | id: containerID, 60 | from: containerLib64, 61 | to: sdkUsrLib64Path 62 | ) 63 | try await createSymlink(at: sdkDirPath.appending("lib64"), pointingTo: "./usr/lib64") 64 | } else if case let containerLib64 = FilePath("/lib64"), 65 | try await generator.doesPathExist(containerLib64, inContainer: containerID) 66 | { 67 | let sdkLib64Path = sdkDirPath.appending("lib64") 68 | try await generator.copyFromDockerContainer( 69 | id: containerID, 70 | from: containerLib64, 71 | to: sdkLib64Path 72 | ) 73 | } 74 | 75 | let sdkUsrLibPath = sdkUsrPath.appending("lib") 76 | try await generator.createDirectoryIfNeeded(at: sdkUsrLibPath) 77 | var subpaths: [(subpath: String, failIfNotExists: Bool)] = [ 78 | ("clang", true), ("gcc", true), ("swift", true), ("swift_static", true), 79 | ] 80 | 81 | // Ubuntu's multiarch directory scheme puts some libraries in 82 | // architecture-specific directories: 83 | // https://wiki.ubuntu.com/MultiarchSpec 84 | // But not in all containers, so don't fail if it does not exist. 85 | if targetDistribution.name == .ubuntu || targetDistribution.name == .debian { 86 | var archSubpath = "\(targetTriple.archName)-linux-gnu" 87 | 88 | // armv7 with Debian uses a custom subpath for armhf 89 | if targetTriple.archName == "armv7" { 90 | archSubpath = "arm-linux-gnueabihf" 91 | } 92 | 93 | // Copy /lib/ for Debian 11 94 | if case let .debian(debian) = targetDistribution, debian == .bullseye { 95 | try await generator.createDirectoryIfNeeded(at: sdkLibPath) 96 | try await generator.copyFromDockerContainer( 97 | id: containerID, 98 | from: FilePath("/lib").appending(archSubpath), 99 | to: sdkLibPath.appending(archSubpath), 100 | failIfNotExists: false 101 | ) 102 | } 103 | 104 | subpaths += [(archSubpath, false)] 105 | } 106 | 107 | for (subpath, failIfNotExists) in subpaths { 108 | try await generator.copyFromDockerContainer( 109 | id: containerID, 110 | from: FilePath("/usr/lib").appending(subpath), 111 | to: sdkUsrLibPath.appending(subpath), 112 | failIfNotExists: failIfNotExists 113 | ) 114 | } 115 | 116 | // Symlink if we do not have a /lib directory in the SDK 117 | if await !generator.doesFileExist(at: sdkLibPath) { 118 | try await generator.createSymlink(at: sdkLibPath, pointingTo: "usr/lib") 119 | } 120 | 121 | // Look for 32-bit libraries to remove from RHEL-based distros 122 | // These are not needed, and the amazonlinux2 x86_64 symlinks are messed up 123 | if case .rhel = targetDistribution { 124 | for gccVersion in 7...13 { 125 | let removePath = "gcc/x86_64-redhat-linux/\(gccVersion)/32" 126 | if await doesFileExist(at: sdkUsrLibPath.appending(removePath)) { 127 | logger.warning( 128 | "Removing 32-bit libraries from RHEL imported sysroot", 129 | metadata: ["removePath": .stringConvertible(removePath)] 130 | ) 131 | try await removeRecursively(at: sdkUsrLibPath.appending(removePath)) 132 | } 133 | } 134 | } 135 | 136 | // Copy the ELF interpreter 137 | try await generator.copyFromDockerContainer( 138 | id: containerID, 139 | from: FilePath(targetTriple.interpreterPath), 140 | to: sdkDirPath.appending(targetTriple.interpreterPath), 141 | failIfNotExists: true 142 | ) 143 | 144 | // Python artifacts are redundant. 145 | try await generator.removeRecursively(at: sdkUsrLibPath.appending("python3.10")) 146 | 147 | try await generator.removeRecursively(at: sdkUsrLibPath.appending("ssl")) 148 | } 149 | } 150 | } 151 | 152 | func copyTargetSwift(from distributionPath: FilePath, sdkDirPath: FilePath) async throws { 153 | logger.info("Copying Swift core libraries for the target triple into Swift SDK bundle...") 154 | 155 | for (pathWithinPackage, pathWithinSwiftSDK, isOptional) in [ 156 | ("lib/swift", sdkDirPath.appending("usr/lib"), false), 157 | ("lib/swift_static", sdkDirPath.appending("usr/lib"), false), 158 | ("lib/clang", sdkDirPath.appending("usr/lib"), true), 159 | ("include", sdkDirPath.appending("usr"), false), 160 | ] { 161 | let fromPath = distributionPath.appending(pathWithinPackage) 162 | 163 | if isOptional && !doesFileExist(at: fromPath) { 164 | logger.debug( 165 | "Optional package path ignored since it does not exist", 166 | metadata: ["packagePath": .string(fromPath.string)] 167 | ) 168 | continue 169 | } 170 | 171 | try await rsync(from: fromPath, to: pathWithinSwiftSDK) 172 | } 173 | } 174 | } 175 | 176 | extension Triple { 177 | var interpreterPath: String { 178 | switch self.archName { 179 | case "x86_64": return "/lib64/ld-linux-x86-64.so.2" 180 | case "aarch64": return "/lib/ld-linux-aarch64.so.1" 181 | case "armv7": return "/lib/ld-linux-armhf.so.3" 182 | default: fatalError("unsupported architecture \(self.archName)") 183 | } 184 | } 185 | } 186 | -------------------------------------------------------------------------------- /Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Entrypoint.swift: -------------------------------------------------------------------------------- 1 | //===----------------------------------------------------------------------===// 2 | // 3 | // This source file is part of the Swift open source project 4 | // 5 | // Copyright (c) 2022-2023 Apple Inc. and the Swift project authors 6 | // Licensed under Apache License v2.0 with Runtime Library Exception 7 | // 8 | // See https://swift.org/LICENSE.txt for license information 9 | // See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 10 | // 11 | //===----------------------------------------------------------------------===// 12 | 13 | import AsyncAlgorithms 14 | import Foundation 15 | import Helpers 16 | import RegexBuilder 17 | import SystemPackage 18 | 19 | #if canImport(AsyncHTTPClient) 20 | import AsyncHTTPClient 21 | #endif 22 | 23 | extension Triple.Arch { 24 | /// Returns the value of `cpu` converted to a convention used in Debian package names 25 | public var debianConventionName: String { 26 | switch self { 27 | case .aarch64: return "arm64" 28 | case .x86_64: return "amd64" 29 | case .wasm32: return "wasm32" 30 | case .arm: return "armhf" 31 | default: fatalError("\(self) is not supported yet") 32 | } 33 | } 34 | } 35 | 36 | extension SwiftSDKGenerator { 37 | package func run(recipe: SwiftSDKRecipe) async throws { 38 | try await withQueryEngine(OSFileSystem(), self.logger, cacheLocation: self.engineCachePath) { 39 | engine in 40 | let httpClientType: HTTPClientProtocol.Type 41 | #if canImport(AsyncHTTPClient) 42 | httpClientType = HTTPClient.self 43 | #else 44 | httpClientType = OfflineHTTPClient.self 45 | #endif 46 | try await httpClientType.with { client in 47 | if !self.isIncremental { 48 | try await self.removeRecursively(at: pathsConfiguration.toolchainDirPath) 49 | } 50 | 51 | try await self.createDirectoryIfNeeded(at: pathsConfiguration.artifactsCachePath) 52 | 53 | let swiftSDKProduct = try await recipe.makeSwiftSDK( 54 | generator: self, 55 | engine: engine, 56 | httpClient: client 57 | ) 58 | 59 | let toolsetJSONPath = try await self.generateToolsetJSON(recipe: recipe) 60 | 61 | var artifacts = try await [ 62 | self.artifactID: generateSwiftSDKMetadata( 63 | toolsetPath: toolsetJSONPath, 64 | sdkDirPath: swiftSDKProduct.sdkDirPath, 65 | recipe: recipe 66 | ) 67 | ] 68 | 69 | if recipe.shouldSupportEmbeddedSwift { 70 | let toolsetJSONPath = try await self.generateToolsetJSON(recipe: recipe, isForEmbeddedSwift: true) 71 | 72 | artifacts["\(self.artifactID)-embedded"] = try await generateSwiftSDKMetadata( 73 | toolsetPath: toolsetJSONPath, 74 | sdkDirPath: swiftSDKProduct.sdkDirPath, 75 | recipe: recipe, 76 | isForEmbeddedSwift: true 77 | ) 78 | } 79 | 80 | try await generateArtifactBundleManifest( 81 | hostTriples: swiftSDKProduct.hostTriples, 82 | artifacts: artifacts 83 | ) 84 | 85 | // Extra spaces added for readability for the user 86 | print( 87 | """ 88 | 89 | All done! Install the newly generated SDK with this command: 90 | swift experimental-sdk install \(pathsConfiguration.artifactBundlePath) 91 | 92 | After that, use the newly installed SDK when building with this command: 93 | swift build --experimental-swift-sdk \(artifactID) 94 | 95 | """ 96 | ) 97 | } 98 | } 99 | } 100 | } 101 | -------------------------------------------------------------------------------- /Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Fixup.swift: -------------------------------------------------------------------------------- 1 | //===----------------------------------------------------------------------===// 2 | // 3 | // This source file is part of the Swift open source project 4 | // 5 | // Copyright (c) 2022-2023 Apple Inc. and the Swift project authors 6 | // Licensed under Apache License v2.0 with Runtime Library Exception 7 | // 8 | // See https://swift.org/LICENSE.txt for license information 9 | // See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 10 | // 11 | //===----------------------------------------------------------------------===// 12 | 13 | import RegexBuilder 14 | import SystemPackage 15 | 16 | import struct Foundation.Data 17 | 18 | extension SwiftSDKGenerator { 19 | func createLibSymlink(sdkDirPath: FilePath) throws { 20 | let libPath = sdkDirPath.appending("lib") 21 | if !doesFileExist(at: libPath) { 22 | logger.info("Adding lib symlink to usr/lib...") 23 | try createSymlink(at: libPath, pointingTo: "usr/lib") 24 | } 25 | } 26 | 27 | func fixAbsoluteSymlinks(sdkDirPath: FilePath) throws { 28 | logger.info("Fixing up absolute symlinks...") 29 | 30 | for (source, absoluteDestination) in try findSymlinks(at: sdkDirPath).filter({ 31 | $1.string.hasPrefix("/") 32 | }) { 33 | guard !absoluteDestination.string.hasPrefix("/etc") else { 34 | try removeFile(at: source) 35 | continue 36 | } 37 | var relativeSource = source 38 | var relativeDestination = FilePath() 39 | 40 | let isPrefixRemoved = relativeSource.removePrefix(sdkDirPath) 41 | precondition(isPrefixRemoved) 42 | for _ in relativeSource.removingLastComponent().components { 43 | relativeDestination.append("..") 44 | } 45 | 46 | relativeDestination.push(absoluteDestination.removingRoot()) 47 | try removeRecursively(at: source) 48 | try createSymlink(at: source, pointingTo: relativeDestination) 49 | 50 | guard doesFileExist(at: source) else { 51 | throw FileOperationError.symlinkFixupFailed( 52 | source: source, 53 | destination: absoluteDestination 54 | ) 55 | } 56 | } 57 | } 58 | 59 | func symlinkClangHeaders() throws { 60 | let swiftStaticClangPath = self.pathsConfiguration.toolchainDirPath.appending( 61 | "usr/lib/swift_static/clang" 62 | ) 63 | if !doesFileExist(at: swiftStaticClangPath) { 64 | logger.info("Symlinking clang headers...") 65 | try self.createSymlink( 66 | at: self.pathsConfiguration.toolchainDirPath.appending("usr/lib/swift_static/clang"), 67 | pointingTo: "../swift/clang" 68 | ) 69 | } 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Metadata.swift: -------------------------------------------------------------------------------- 1 | //===----------------------------------------------------------------------===// 2 | // 3 | // This source file is part of the Swift open source project 4 | // 5 | // Copyright (c) 2022-2023 Apple Inc. and the Swift project authors 6 | // Licensed under Apache License v2.0 with Runtime Library Exception 7 | // 8 | // See https://swift.org/LICENSE.txt for license information 9 | // See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 10 | // 11 | //===----------------------------------------------------------------------===// 12 | 13 | import SystemPackage 14 | 15 | import class Foundation.JSONEncoder 16 | 17 | private let encoder: JSONEncoder = { 18 | let encoder = JSONEncoder() 19 | encoder.outputFormatting = [.prettyPrinted, .withoutEscapingSlashes, .sortedKeys] 20 | return encoder 21 | }() 22 | 23 | extension SwiftSDKGenerator { 24 | func generateToolsetJSON(recipe: SwiftSDKRecipe, isForEmbeddedSwift: Bool = false) throws -> FilePath { 25 | logger.info("Generating toolset JSON file...") 26 | 27 | let toolsetJSONPath = pathsConfiguration.swiftSDKRootPath.appending( 28 | "\(isForEmbeddedSwift ? "embedded-" : "")toolset.json" 29 | ) 30 | 31 | var relativeToolchainBinDir = pathsConfiguration.toolchainBinDirPath 32 | 33 | guard 34 | relativeToolchainBinDir.removePrefix(pathsConfiguration.swiftSDKRootPath) 35 | else { 36 | fatalError( 37 | "`toolchainBinDirPath` is at an unexpected location that prevents computing a relative path" 38 | ) 39 | } 40 | 41 | var toolset = Toolset(rootPath: relativeToolchainBinDir.string) 42 | recipe.applyPlatformOptions( 43 | toolset: &toolset, 44 | targetTriple: self.targetTriple, 45 | isForEmbeddedSwift: isForEmbeddedSwift 46 | ) 47 | try writeFile(at: toolsetJSONPath, encoder.encode(toolset)) 48 | 49 | return toolsetJSONPath 50 | } 51 | 52 | func generateSwiftSDKMetadata( 53 | toolsetPath: FilePath, 54 | sdkDirPath: FilePath, 55 | recipe: SwiftSDKRecipe, 56 | isForEmbeddedSwift: Bool = false 57 | ) throws -> FilePath { 58 | logger.info("Generating Swift SDK metadata JSON file...") 59 | 60 | let destinationJSONPath = pathsConfiguration.swiftSDKRootPath.appending( 61 | "\(isForEmbeddedSwift ? "embedded-" : "")swift-sdk.json" 62 | ) 63 | 64 | var relativeToolchainBinDir = pathsConfiguration.toolchainBinDirPath 65 | var relativeSDKDir = sdkDirPath 66 | var relativeToolsetPath = toolsetPath 67 | 68 | guard 69 | relativeToolchainBinDir.removePrefix(pathsConfiguration.swiftSDKRootPath), 70 | relativeSDKDir.removePrefix(pathsConfiguration.swiftSDKRootPath), 71 | relativeToolsetPath.removePrefix(pathsConfiguration.swiftSDKRootPath) 72 | else { 73 | fatalError( 74 | """ 75 | `toolchainBinDirPath`, `sdkDirPath`, and `toolsetPath` are at unexpected locations that prevent computing \ 76 | relative paths 77 | """ 78 | ) 79 | } 80 | 81 | var metadata = SwiftSDKMetadataV4( 82 | targetTriples: [ 83 | self.targetTriple.triple: .init( 84 | sdkRootPath: relativeSDKDir.string, 85 | toolsetPaths: [relativeToolsetPath.string] 86 | ) 87 | ] 88 | ) 89 | 90 | recipe.applyPlatformOptions( 91 | metadata: &metadata, 92 | paths: pathsConfiguration, 93 | targetTriple: self.targetTriple, 94 | isForEmbeddedSwift: isForEmbeddedSwift 95 | ) 96 | 97 | try writeFile( 98 | at: destinationJSONPath, 99 | encoder.encode(metadata) 100 | ) 101 | 102 | return destinationJSONPath 103 | } 104 | 105 | func generateArtifactBundleManifest(hostTriples: [Triple]?, artifacts: [String: FilePath]) throws { 106 | logger.info("Generating .artifactbundle info JSON file...") 107 | 108 | let artifactBundleManifestPath = pathsConfiguration.artifactBundlePath.appending("info.json") 109 | 110 | try writeFile( 111 | at: artifactBundleManifestPath, 112 | encoder.encode( 113 | ArtifactsArchiveMetadata( 114 | schemaVersion: "1.0", 115 | artifacts: artifacts.mapValues { 116 | var relativePath = $0 117 | let prefixRemoved = relativePath.removePrefix(pathsConfiguration.artifactBundlePath) 118 | assert(prefixRemoved) 119 | 120 | return .init( 121 | type: .swiftSDK, 122 | version: self.bundleVersion, 123 | variants: [ 124 | .init( 125 | path: relativePath.string, 126 | supportedTriples: hostTriples.map { $0.map(\.triple) } 127 | ) 128 | ] 129 | ) 130 | } 131 | ) 132 | ) 133 | ) 134 | } 135 | 136 | struct SDKSettings: Codable { 137 | var DisplayName: String 138 | var Version: String 139 | var VersionMap: [String: String] = [:] 140 | var CanonicalName: String 141 | } 142 | 143 | /// Generates an `SDKSettings.json` file that looks like this: 144 | /// 145 | /// ```json 146 | /// { 147 | /// "CanonicalName" : "-swift-linux-[gnu|gnueabihf]", 148 | /// "DisplayName" : "Swift SDK for ()", 149 | /// "Version" : "0.0.1", 150 | /// "VersionMap" : { 151 | /// 152 | /// } 153 | /// } 154 | /// ``` 155 | func generateSDKSettingsFile(sdkDirPath: FilePath, distribution: LinuxDistribution) throws { 156 | logger.info("Generating SDKSettings.json file to silence cross-compilation warnings...") 157 | 158 | let sdkSettings = SDKSettings( 159 | DisplayName: "Swift SDK for \(distribution) (\(targetTriple.archName))", 160 | Version: bundleVersion, 161 | CanonicalName: targetTriple.triple.replacingOccurrences(of: "unknown", with: "swift") 162 | ) 163 | 164 | let sdkSettingsFilePath = sdkDirPath.appending("SDKSettings.json") 165 | try writeFile(at: sdkSettingsFilePath, encoder.encode(sdkSettings)) 166 | } 167 | } 168 | -------------------------------------------------------------------------------- /Sources/SwiftSDKGenerator/Generator/SwiftSDKGenerator+Unpack.swift: -------------------------------------------------------------------------------- 1 | //===----------------------------------------------------------------------===// 2 | // 3 | // This source file is part of the Swift open source project 4 | // 5 | // Copyright (c) 2022-2023 Apple Inc. and the Swift project authors 6 | // Licensed under Apache License v2.0 with Runtime Library Exception 7 | // 8 | // See https://swift.org/LICENSE.txt for license information 9 | // See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 10 | // 11 | //===----------------------------------------------------------------------===// 12 | 13 | import Helpers 14 | 15 | import struct SystemPackage.FilePath 16 | 17 | let unusedTargetPlatforms = [ 18 | "appletvos", 19 | "appletvsimulator", 20 | "embedded", 21 | "iphoneos", 22 | "iphonesimulator", 23 | "watchos", 24 | "watchsimulator", 25 | "xros", 26 | "xrsimulator", 27 | ] 28 | 29 | let unusedHostBinaries = [ 30 | "clangd", 31 | "docc", 32 | "dsymutil", 33 | "sourcekit-lsp", 34 | "swift-format", 35 | "swift-package", 36 | "swift-package-collection", 37 | "lldb*", 38 | ] 39 | 40 | let unusedHostLibraries = [ 41 | "sourcekitd.framework", 42 | "libsourcekitdInProc.so", 43 | "liblldb.so*", 44 | ] 45 | 46 | extension SwiftSDKGenerator { 47 | func unpackHostSwift(hostSwiftPackagePath: FilePath) async throws { 48 | logger.info("Unpacking and copying Swift binaries for the host triple...") 49 | let pathsConfiguration = self.pathsConfiguration 50 | 51 | try self.createDirectoryIfNeeded(at: pathsConfiguration.toolchainDirPath) 52 | 53 | let excludes = 54 | unusedTargetPlatforms.map { "--exclude usr/lib/swift/\($0)" } 55 | + unusedTargetPlatforms.map { "--exclude usr/lib/swift_static/\($0)" } 56 | + unusedHostBinaries.map { "--exclude usr/bin/\($0)" } 57 | + unusedHostLibraries.map { "--exclude usr/lib/\($0)" } 58 | 59 | if hostSwiftPackagePath.string.contains("tar.gz") { 60 | try await Shell.run( 61 | #""" 62 | tar -xzf "\#(hostSwiftPackagePath)" -C "\#(pathsConfiguration.toolchainDirPath)" -x \#(excludes.joined(separator: " ")) --strip-components=1 63 | """#, 64 | shouldLogCommands: isVerbose 65 | ) 66 | } else { 67 | try await Shell.run( 68 | #""" 69 | tar -x --to-stdout -f "\#(hostSwiftPackagePath)" \*.pkg/Payload | 70 | tar -C "\#(pathsConfiguration.toolchainDirPath)" -x \#(excludes.joined(separator: " ")) --include usr 71 | """#, 72 | shouldLogCommands: isVerbose 73 | ) 74 | } 75 | } 76 | 77 | func removeToolchainComponents( 78 | _ packagePath: FilePath, 79 | platforms: [String] = unusedTargetPlatforms, 80 | libraries: [String] = unusedHostLibraries, 81 | binaries: [String] = unusedHostBinaries 82 | ) async throws { 83 | // Remove libraries for platforms we don't intend cross-compiling to 84 | for platform in platforms { 85 | try self.removeRecursively(at: packagePath.appending("usr/lib/swift/\(platform)")) 86 | try self.removeRecursively(at: packagePath.appending("usr/lib/swift_static/\(platform)")) 87 | } 88 | for binary in binaries { 89 | try self.removeRecursively(at: packagePath.appending("usr/bin/\(binary)")) 90 | } 91 | for library in libraries { 92 | try self.removeRecursively(at: packagePath.appending("usr/lib/\(library)")) 93 | } 94 | } 95 | 96 | func unpackTargetSwiftPackage( 97 | targetSwiftPackagePath: FilePath, 98 | relativePathToRoot: [FilePath.Component], 99 | sdkDirPath: FilePath 100 | ) async throws { 101 | logger.info("Unpacking Swift distribution for the target triple...") 102 | 103 | try await inTemporaryDirectory { fs, tmpDir in 104 | try await fs.unpack(file: targetSwiftPackagePath, into: tmpDir) 105 | try await fs.copyTargetSwift( 106 | from: tmpDir.appending(relativePathToRoot).appending("usr"), 107 | sdkDirPath: sdkDirPath 108 | ) 109 | } 110 | } 111 | 112 | func prepareLLDLinker(_ engine: QueryEngine, llvmArtifact: DownloadableArtifacts.Item) 113 | async throws 114 | { 115 | logger.info("Unpacking and copying `lld` linker...") 116 | let pathsConfiguration = self.pathsConfiguration 117 | let targetOS = self.targetTriple.os 118 | 119 | let untarDestination = pathsConfiguration.artifactsCachePath.appending( 120 | FilePath.Component(llvmArtifact.localPath.stem!)!.stem 121 | ) 122 | try self.createDirectoryIfNeeded(at: untarDestination) 123 | 124 | let unpackedLLDPath: FilePath 125 | if llvmArtifact.isPrebuilt { 126 | unpackedLLDPath = try await engine[ 127 | TarExtractQuery( 128 | file: llvmArtifact.localPath, 129 | into: untarDestination, 130 | outputBinarySubpath: ["bin", "lld"], 131 | stripComponents: 1 132 | ) 133 | ].path 134 | } else { 135 | try await self.untar( 136 | file: llvmArtifact.localPath, 137 | into: untarDestination, 138 | stripComponents: 1 139 | ) 140 | unpackedLLDPath = try await engine[ 141 | CMakeBuildQuery( 142 | sourcesDirectory: untarDestination, 143 | outputBinarySubpath: ["bin", "lld"], 144 | options: "-DLLVM_ENABLE_PROJECTS=lld -DLLVM_TARGETS_TO_BUILD=''" 145 | ) 146 | ].path 147 | } 148 | 149 | let toolchainLLDPath: FilePath 150 | switch targetOS { 151 | case .linux: 152 | toolchainLLDPath = pathsConfiguration.toolchainBinDirPath.appending("ld.lld") 153 | case .wasi: 154 | toolchainLLDPath = pathsConfiguration.toolchainBinDirPath.appending("wasm-ld") 155 | default: 156 | fatalError("Unknown target OS to prepare lld \"\(targetOS?.rawValue ?? "nil")\"") 157 | } 158 | 159 | try self.copy(from: unpackedLLDPath, to: toolchainLLDPath) 160 | } 161 | } 162 | -------------------------------------------------------------------------------- /Sources/SwiftSDKGenerator/PathsConfiguration.swift: -------------------------------------------------------------------------------- 1 | //===----------------------------------------------------------------------===// 2 | // 3 | // This source file is part of the Swift open source project 4 | // 5 | // Copyright (c) 2022-2023 Apple Inc. and the Swift project authors 6 | // Licensed under Apache License v2.0 with Runtime Library Exception 7 | // 8 | // See https://swift.org/LICENSE.txt for license information 9 | // See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 10 | // 11 | //===----------------------------------------------------------------------===// 12 | 13 | import struct SystemPackage.FilePath 14 | 15 | public struct PathsConfiguration: Sendable { 16 | init(sourceRoot: FilePath, artifactID: String, targetTriple: Triple) { 17 | self.sourceRoot = sourceRoot 18 | self.artifactBundlePath = 19 | sourceRoot 20 | .appending("Bundles") 21 | .appending("\(artifactID).artifactbundle") 22 | self.artifactsCachePath = sourceRoot.appending("Artifacts") 23 | self.swiftSDKRootPath = self.artifactBundlePath 24 | .appending(artifactID) 25 | .appending(targetTriple.triple) 26 | self.toolchainDirPath = self.swiftSDKRootPath.appending("swift.xctoolchain") 27 | self.toolchainBinDirPath = self.toolchainDirPath.appending("usr/bin") 28 | } 29 | 30 | let sourceRoot: FilePath 31 | let artifactBundlePath: FilePath 32 | let artifactsCachePath: FilePath 33 | let swiftSDKRootPath: FilePath 34 | let toolchainDirPath: FilePath 35 | let toolchainBinDirPath: FilePath 36 | } 37 | -------------------------------------------------------------------------------- /Sources/SwiftSDKGenerator/PlatformModels/LinuxDistribution.swift: -------------------------------------------------------------------------------- 1 | //===----------------------------------------------------------------------===// 2 | // 3 | // This source file is part of the Swift open source project 4 | // 5 | // Copyright (c) 2022-2023 Apple Inc. and the Swift project authors 6 | // Licensed under Apache License v2.0 with Runtime Library Exception 7 | // 8 | // See https://swift.org/LICENSE.txt for license information 9 | // See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 10 | // 11 | //===----------------------------------------------------------------------===// 12 | 13 | public enum LinuxDistribution: Hashable, Sendable { 14 | public enum Name: String, Sendable { 15 | case rhel 16 | case ubuntu 17 | case debian 18 | } 19 | 20 | public enum RHEL: String, Sendable { 21 | case ubi9 22 | } 23 | 24 | public enum Ubuntu: String, Sendable { 25 | case focal 26 | case jammy 27 | case noble 28 | 29 | init(version: String) throws { 30 | switch version { 31 | case "20.04": 32 | self = .focal 33 | case "22.04": 34 | self = .jammy 35 | case "24.04": 36 | self = .noble 37 | default: 38 | throw GeneratorError.unknownLinuxDistribution( 39 | name: LinuxDistribution.Name.ubuntu.rawValue, 40 | version: version 41 | ) 42 | } 43 | } 44 | 45 | var version: String { 46 | switch self { 47 | case .focal: return "20.04" 48 | case .jammy: return "22.04" 49 | case .noble: return "24.04" 50 | } 51 | } 52 | 53 | public var requiredPackages: [String] { 54 | switch self { 55 | case .focal: 56 | return [ 57 | "libc6", 58 | "libc6-dev", 59 | "libgcc-s1", 60 | "libgcc-10-dev", 61 | "libicu66", 62 | "libicu-dev", 63 | "libstdc++-10-dev", 64 | "libstdc++6", 65 | "linux-libc-dev", 66 | "zlib1g", 67 | "zlib1g-dev", 68 | ] 69 | case .jammy: 70 | return [ 71 | "libc6", 72 | "libc6-dev", 73 | "libgcc-s1", 74 | "libgcc-12-dev", 75 | "libicu70", 76 | "libicu-dev", 77 | "libstdc++-12-dev", 78 | "libstdc++6", 79 | "linux-libc-dev", 80 | "zlib1g", 81 | "zlib1g-dev", 82 | ] 83 | case .noble: 84 | return [ 85 | "libc6", 86 | "libc6-dev", 87 | "libgcc-s1", 88 | "libgcc-13-dev", 89 | "libicu74", 90 | "libicu-dev", 91 | "libstdc++-13-dev", 92 | "libstdc++6", 93 | "linux-libc-dev", 94 | "zlib1g", 95 | "zlib1g-dev", 96 | ] 97 | } 98 | } 99 | } 100 | 101 | public enum Debian: String, Sendable { 102 | case bullseye 103 | case bookworm 104 | 105 | init(version: String) throws { 106 | switch version { 107 | case "11": self = .bullseye 108 | case "12": self = .bookworm 109 | default: 110 | throw GeneratorError.unknownLinuxDistribution( 111 | name: LinuxDistribution.Name.debian.rawValue, 112 | version: version 113 | ) 114 | } 115 | } 116 | 117 | var version: String { 118 | switch self { 119 | case .bullseye: return "11" 120 | case .bookworm: return "12" 121 | } 122 | } 123 | 124 | public var requiredPackages: [String] { 125 | switch self { 126 | case .bullseye: 127 | return [ 128 | "libc6", 129 | "libc6-dev", 130 | "libgcc-s1", 131 | "libgcc-10-dev", 132 | "libicu67", 133 | "libicu-dev", 134 | "libstdc++-10-dev", 135 | "libstdc++6", 136 | "linux-libc-dev", 137 | "zlib1g", 138 | "zlib1g-dev", 139 | ] 140 | case .bookworm: 141 | return [ 142 | "libc6", 143 | "libc6-dev", 144 | "libgcc-s1", 145 | "libgcc-12-dev", 146 | "libicu72", 147 | "libicu-dev", 148 | "libstdc++-12-dev", 149 | "libstdc++6", 150 | "linux-libc-dev", 151 | "zlib1g", 152 | "zlib1g-dev", 153 | ] 154 | } 155 | } 156 | } 157 | 158 | case rhel(RHEL) 159 | case ubuntu(Ubuntu) 160 | case debian(Debian) 161 | 162 | public init(name: Name, version: String) throws { 163 | switch name { 164 | case .rhel: 165 | guard let version = RHEL(rawValue: version) else { 166 | throw GeneratorError.unknownLinuxDistribution(name: name.rawValue, version: version) 167 | } 168 | self = .rhel(version) 169 | 170 | case .ubuntu: 171 | self = try .ubuntu(Ubuntu(version: version)) 172 | 173 | case .debian: 174 | self = try .debian(Debian(version: version)) 175 | } 176 | } 177 | 178 | var name: Name { 179 | switch self { 180 | case .rhel: return .rhel 181 | case .ubuntu: return .ubuntu 182 | case .debian: return .debian 183 | } 184 | } 185 | 186 | var release: String { 187 | switch self { 188 | case let .rhel(rhel): return rhel.rawValue 189 | case let .ubuntu(ubuntu): return ubuntu.rawValue 190 | case let .debian(debian): return debian.rawValue 191 | } 192 | } 193 | 194 | var swiftDockerImageSuffix: String { 195 | switch self { 196 | case let .rhel(rhel): return "rhel-\(rhel.rawValue)" 197 | case let .ubuntu(ubuntu): return ubuntu.rawValue 198 | case let .debian(debian): return debian.rawValue 199 | } 200 | } 201 | } 202 | 203 | extension LinuxDistribution.Name { 204 | public init(nameString: String) throws { 205 | guard let name = LinuxDistribution.Name(rawValue: nameString) else { 206 | throw GeneratorError.unknownLinuxDistribution(name: nameString, version: nil) 207 | } 208 | self = name 209 | } 210 | } 211 | 212 | extension LinuxDistribution: CustomStringConvertible { 213 | public var description: String { 214 | let versionComponent: String 215 | switch self { 216 | case .rhel: 217 | versionComponent = self.release.uppercased() 218 | case .ubuntu, .debian: 219 | versionComponent = self.release.capitalized 220 | } 221 | 222 | return "\(self.name) \(versionComponent)" 223 | } 224 | } 225 | 226 | extension LinuxDistribution.Name: CustomStringConvertible { 227 | public var description: String { 228 | switch self { 229 | case .rhel: return "RHEL" 230 | case .ubuntu: return "Ubuntu" 231 | case .debian: return "Debian" 232 | } 233 | } 234 | } 235 | -------------------------------------------------------------------------------- /Sources/SwiftSDKGenerator/PlatformModels/Triple.swift: -------------------------------------------------------------------------------- 1 | //===----------------------------------------------------------------------===// 2 | // 3 | // This source file is part of the Swift open source project 4 | // 5 | // Copyright (c) 2022-2023 Apple Inc. and the Swift project authors 6 | // Licensed under Apache License v2.0 with Runtime Library Exception 7 | // 8 | // See https://swift.org/LICENSE.txt for license information 9 | // See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 10 | // 11 | //===----------------------------------------------------------------------===// 12 | 13 | import Helpers 14 | 15 | public typealias Triple = Helpers.Triple 16 | 17 | extension Triple: @unchecked Sendable {} 18 | 19 | extension Triple { 20 | public init(arch: Arch, vendor: Vendor?, os: OS, environment: Environment) { 21 | self.init("\(arch)-\(vendor?.rawValue ?? "unknown")-\(os)-\(environment)", normalizing: true) 22 | } 23 | 24 | public init(arch: Arch, vendor: Vendor?, os: OS) { 25 | self.init("\(arch)-\(vendor?.rawValue ?? "unknown")-\(os)", normalizing: true) 26 | } 27 | } 28 | 29 | extension Triple.Arch { 30 | /// Returns the value of `cpu` converted to a convention used by Swift on Linux, i.e. `arm64` becomes `aarch64`. 31 | var linuxConventionName: String { 32 | switch self { 33 | case .aarch64: return "aarch64" 34 | case .x86_64: return "x86_64" 35 | case .wasm32: return "wasm32" 36 | case .arm: return "arm" 37 | default: fatalError("\(self) is not supported yet") 38 | } 39 | } 40 | 41 | /// Returns the value of `cpu` converted to a convention used by `LLVM_TARGETS_TO_BUILD` CMake setting. 42 | var llvmTargetConventionName: String { 43 | switch self { 44 | case .x86_64: return "X86" 45 | case .aarch64: return "AArch64" 46 | case .wasm32: return "WebAssembly" 47 | default: fatalError("\(self) is not supported yet") 48 | } 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /Sources/SwiftSDKGenerator/PlatformModels/VersionsConfiguration.swift: -------------------------------------------------------------------------------- 1 | //===----------------------------------------------------------------------===// 2 | // 3 | // This source file is part of the Swift open source project 4 | // 5 | // Copyright (c) 2022-2023 Apple Inc. and the Swift project authors 6 | // Licensed under Apache License v2.0 with Runtime Library Exception 7 | // 8 | // See https://swift.org/LICENSE.txt for license information 9 | // See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 10 | // 11 | //===----------------------------------------------------------------------===// 12 | 13 | import Logging 14 | 15 | import struct Foundation.URL 16 | 17 | public struct VersionsConfiguration: Sendable { 18 | init( 19 | swiftVersion: String, 20 | swiftBranch: String? = nil, 21 | lldVersion: String, 22 | linuxDistribution: LinuxDistribution, 23 | targetTriple: Triple, 24 | logger: Logger 25 | ) throws { 26 | self.swiftVersion = swiftVersion 27 | self.swiftBranch = swiftBranch ?? "swift-\(swiftVersion.lowercased())" 28 | self.lldVersion = lldVersion 29 | self.linuxDistribution = linuxDistribution 30 | self.linuxArchSuffix = 31 | targetTriple.arch == .aarch64 ? "-\(Triple.Arch.aarch64.linuxConventionName)" : "" 32 | self.logger = logger 33 | } 34 | 35 | let swiftVersion: String 36 | let swiftBranch: String 37 | let lldVersion: String 38 | let linuxDistribution: LinuxDistribution 39 | let linuxArchSuffix: String 40 | private let logger: Logger 41 | 42 | var swiftPlatform: String { 43 | switch self.linuxDistribution { 44 | case let .ubuntu(ubuntu): 45 | return "ubuntu\(ubuntu.version)" 46 | case let .debian(debian): 47 | if debian.version == "11" { 48 | // Ubuntu 20.04 toolchain is binary compatible with Debian 11 49 | return "ubuntu20.04" 50 | } else if self.swiftVersion.hasPrefix("5.9") || self.swiftVersion == "5.10" { 51 | // Ubuntu 22.04 toolchain is binary compatible with Debian 12 52 | return "ubuntu22.04" 53 | } 54 | return "debian\(debian.version)" 55 | case let .rhel(rhel): 56 | return rhel.rawValue 57 | } 58 | } 59 | 60 | var swiftPlatformAndSuffix: String { 61 | return "\(self.swiftPlatform)\(self.linuxArchSuffix)" 62 | } 63 | 64 | func swiftDistributionName(platform: String? = nil) -> String { 65 | return 66 | "swift-\(self.swiftVersion)-\(platform ?? self.swiftPlatformAndSuffix)" 67 | } 68 | 69 | func swiftDownloadURL( 70 | subdirectory: String? = nil, 71 | platform: String? = nil, 72 | fileExtension: String = "tar.gz" 73 | ) -> URL { 74 | let computedPlatform = platform ?? self.swiftPlatformAndSuffix 75 | let computedSubdirectory = 76 | subdirectory 77 | ?? computedPlatform.replacingOccurrences(of: ".", with: "") 78 | 79 | return URL( 80 | string: """ 81 | https://download.swift.org/\( 82 | self.swiftBranch 83 | )/\(computedSubdirectory)/\ 84 | swift-\(self.swiftVersion)/\(self.swiftDistributionName(platform: computedPlatform)).\(fileExtension) 85 | """ 86 | )! 87 | } 88 | 89 | var swiftBareSemVer: String { 90 | self.swiftVersion.components(separatedBy: "-")[0] 91 | } 92 | 93 | /// Name of a Docker image containing the Swift toolchain and SDK for this Linux distribution. 94 | var swiftBaseDockerImage: String { 95 | if self.swiftVersion.hasSuffix("-RELEASE") { 96 | return "swift:\(self.swiftBareSemVer)-\(self.linuxDistribution.swiftDockerImageSuffix)" 97 | } else { 98 | fatalError() 99 | } 100 | } 101 | } 102 | -------------------------------------------------------------------------------- /Sources/SwiftSDKGenerator/Queries/CMakeBuildQuery.swift: -------------------------------------------------------------------------------- 1 | //===----------------------------------------------------------------------===// 2 | // 3 | // This source file is part of the Swift open source project 4 | // 5 | // Copyright (c) 2022-2023 Apple Inc. and the Swift project authors 6 | // Licensed under Apache License v2.0 with Runtime Library Exception 7 | // 8 | // See https://swift.org/LICENSE.txt for license information 9 | // See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 10 | // 11 | //===----------------------------------------------------------------------===// 12 | 13 | import Helpers 14 | 15 | import struct SystemPackage.FilePath 16 | 17 | struct CMakeBuildQuery: CachingQuery { 18 | let sourcesDirectory: FilePath 19 | /// Path to the output binary relative to the CMake build directory. 20 | let outputBinarySubpath: [FilePath.Component] 21 | let options: String 22 | 23 | func run(engine: QueryEngine) async throws -> FilePath { 24 | try await Shell.run( 25 | """ 26 | cmake -S "\(self.sourcesDirectory)"/llvm -B "\( 27 | self 28 | .sourcesDirectory 29 | )"/build -G Ninja -DCMAKE_BUILD_TYPE=Release \(self.options) 30 | """, 31 | logStdout: true 32 | ) 33 | 34 | let buildDirectory = self.sourcesDirectory.appending("build") 35 | try await Shell.run( 36 | #"ninja -C "\#(buildDirectory)" "\#(FilePath(".").appending(self.outputBinarySubpath))""#, 37 | logStdout: true 38 | ) 39 | 40 | return buildDirectory.appending(self.outputBinarySubpath) 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /Sources/SwiftSDKGenerator/Queries/DownloadArtifactQuery.swift: -------------------------------------------------------------------------------- 1 | //===----------------------------------------------------------------------===// 2 | // 3 | // This source file is part of the Swift open source project 4 | // 5 | // Copyright (c) 2022-2023 Apple Inc. and the Swift project authors 6 | // Licensed under Apache License v2.0 with Runtime Library Exception 7 | // 8 | // See https://swift.org/LICENSE.txt for license information 9 | // See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 10 | // 11 | //===----------------------------------------------------------------------===// 12 | 13 | import Helpers 14 | import Logging 15 | 16 | import class Foundation.ByteCountFormatter 17 | import struct SystemPackage.FilePath 18 | 19 | struct DownloadArtifactQuery: Query { 20 | var cacheKey: some CacheKey { self.artifact } 21 | let artifact: DownloadableArtifacts.Item 22 | let httpClient: any HTTPClientProtocol 23 | let logger: Logger 24 | 25 | func run(engine: QueryEngine) async throws -> FilePath { 26 | logger.info( 27 | "Downloading remote artifact not available in local cache", 28 | metadata: ["remoteUrl": .string(self.artifact.remoteURL.absoluteString)] 29 | ) 30 | let stream = self.httpClient.streamDownloadProgress( 31 | from: self.artifact.remoteURL, 32 | to: self.artifact.localPath 33 | ) 34 | .removeDuplicates(by: didProgressChangeSignificantly) 35 | ._throttle(for: .seconds(1)) 36 | 37 | for try await progress in stream { 38 | report(progress: progress, for: self.artifact) 39 | } 40 | return self.artifact.localPath 41 | } 42 | 43 | private func report(progress: DownloadProgress, for artifact: DownloadableArtifacts.Item) { 44 | let byteCountFormatter = ByteCountFormatter() 45 | 46 | if let total = progress.totalBytes { 47 | logger.debug( 48 | """ 49 | \(artifact.remoteURL.lastPathComponent) \( 50 | byteCountFormatter 51 | .string(fromByteCount: Int64(progress.receivedBytes)) 52 | )/\( 53 | byteCountFormatter 54 | .string(fromByteCount: Int64(total)) 55 | ) 56 | """ 57 | ) 58 | } else { 59 | logger.debug( 60 | "\(artifact.remoteURL.lastPathComponent) \(byteCountFormatter.string(fromByteCount: Int64(progress.receivedBytes)))" 61 | ) 62 | } 63 | } 64 | } 65 | 66 | /// Checks whether two given progress value are different enough from each other. Used for filtering out progress 67 | /// values in async streams with `removeDuplicates` operator. 68 | /// - Parameters: 69 | /// - previous: Preceding progress value in the stream. 70 | /// - current: Currently processed progress value in the stream. 71 | /// - Returns: `true` if `totalBytes` value is different by any amount or if `receivedBytes` is different by amount 72 | /// larger than 1MiB. Returns `false` otherwise. 73 | @Sendable 74 | private func didProgressChangeSignificantly( 75 | previous: DownloadProgress, 76 | current: DownloadProgress 77 | ) -> Bool { 78 | guard previous.totalBytes == current.totalBytes else { 79 | return true 80 | } 81 | 82 | return current.receivedBytes - previous.receivedBytes > 1024 * 1024 * 1024 83 | } 84 | -------------------------------------------------------------------------------- /Sources/SwiftSDKGenerator/Queries/DownloadFileQuery.swift: -------------------------------------------------------------------------------- 1 | //===----------------------------------------------------------------------===// 2 | // 3 | // This source file is part of the Swift open source project 4 | // 5 | // Copyright (c) 2022-2023 Apple Inc. and the Swift project authors 6 | // Licensed under Apache License v2.0 with Runtime Library Exception 7 | // 8 | // See https://swift.org/LICENSE.txt for license information 9 | // See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 10 | // 11 | //===----------------------------------------------------------------------===// 12 | 13 | import Helpers 14 | 15 | import struct Foundation.URL 16 | import struct SystemPackage.FilePath 17 | 18 | struct DownloadFileQuery: Query { 19 | struct Key: CacheKey { 20 | let remoteURL: URL 21 | let localDirectory: FilePath 22 | } 23 | 24 | var cacheKey: Key { 25 | Key(remoteURL: self.remoteURL, localDirectory: self.localDirectory) 26 | } 27 | 28 | let remoteURL: URL 29 | let localDirectory: FilePath 30 | let httpClient: any HTTPClientProtocol 31 | 32 | func run(engine: QueryEngine) async throws -> FilePath { 33 | let downloadedFilePath = self.localDirectory.appending(self.remoteURL.lastPathComponent) 34 | _ = try await self.httpClient.downloadFile(from: self.remoteURL, to: downloadedFilePath) 35 | return downloadedFilePath 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /Sources/SwiftSDKGenerator/Queries/TarExtractQuery.swift: -------------------------------------------------------------------------------- 1 | //===----------------------------------------------------------------------===// 2 | // 3 | // This source file is part of the Swift open source project 4 | // 5 | // Copyright (c) 2022-2024 Apple Inc. and the Swift project authors 6 | // Licensed under Apache License v2.0 with Runtime Library Exception 7 | // 8 | // See https://swift.org/LICENSE.txt for license information 9 | // See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 10 | // 11 | //===----------------------------------------------------------------------===// 12 | 13 | import Helpers 14 | 15 | import struct SystemPackage.FilePath 16 | 17 | struct TarExtractQuery: CachingQuery { 18 | var file: FilePath // Archive to extract 19 | var into: FilePath // Destination for unpacked archive 20 | var outputBinarySubpath: [FilePath.Component] 21 | var stripComponents: Int? = nil 22 | 23 | func run(engine: QueryEngine) async throws -> FilePath { 24 | let stripComponentsOption = stripComponents.map { " --strip-components \($0)" } ?? "" 25 | let archivePath = self.file 26 | let destinationPath = self.into 27 | 28 | try await Shell.run( 29 | #"tar -C "\#(destinationPath)" -x -f "\#(archivePath)" \#(stripComponentsOption) \#(FilePath("*").appending(outputBinarySubpath))"#, 30 | logStdout: true 31 | ) 32 | 33 | // We must return a path to a single file which the cache will track 34 | return destinationPath.appending(outputBinarySubpath) 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /Sources/SwiftSDKGenerator/Serialization/SwiftSDKMetadata.swift: -------------------------------------------------------------------------------- 1 | //===----------------------------------------------------------------------===// 2 | // 3 | // This source file is part of the Swift open source project 4 | // 5 | // Copyright (c) 2022-2023 Apple Inc. and the Swift project authors 6 | // Licensed under Apache License v2.0 with Runtime Library Exception 7 | // 8 | // See https://swift.org/LICENSE.txt for license information 9 | // See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 10 | // 11 | //===----------------------------------------------------------------------===// 12 | 13 | struct DestinationV1: Encodable { 14 | enum CodingKeys: String, CodingKey { 15 | case version 16 | case sdk 17 | case toolchainBinDir = "toolchain-bin-dir" 18 | case target 19 | case extraCCFlags = "extra-cc-flags" 20 | case extraSwiftCFlags = "extra-swiftc-flags" 21 | case extraCPPFlags = "extra-cpp-flags" 22 | } 23 | 24 | let version = 1 25 | let sdk: String 26 | let toolchainBinDir: String 27 | let target: String 28 | let extraCCFlags: [String] 29 | let extraSwiftCFlags: [String] 30 | let extraCPPFlags: [String] 31 | } 32 | 33 | struct DestinationV2: Encodable { 34 | let version = 2 35 | 36 | let sdkRootDir: String 37 | let toolchainBinDir: String 38 | let hostTriples: [String] 39 | let targetTriples: [String] 40 | let extraCCFlags: [String] 41 | let extraSwiftCFlags: [String] 42 | let extraCXXFlags: [String] 43 | let extraLinkerFlags: [String] 44 | } 45 | 46 | /// Represents v3 schema of `destination.json` files used for cross-compilation. 47 | struct DestinationV3: Encodable { 48 | struct TripleProperties: Encodable { 49 | /// Path relative to `destination.json` containing SDK root. 50 | let sdkRootPath: String 51 | 52 | /// Path relative to `destination.json` containing Swift resources for dynamic linking. 53 | var swiftResourcesPath: String? 54 | 55 | /// Path relative to `destination.json` containing Swift resources for static linking. 56 | var swiftStaticResourcesPath: String? 57 | 58 | /// Array of paths relative to `destination.json` containing headers. 59 | var includeSearchPaths: [String]? 60 | 61 | /// Array of paths relative to `destination.json` containing libraries. 62 | var librarySearchPaths: [String]? 63 | 64 | /// Array of paths relative to `destination.json` containing toolset files. 65 | let toolsetPaths: [String]? 66 | } 67 | 68 | /// Version of the schema used when serializing the destination file. 69 | let schemaVersion = "3.0" 70 | 71 | /// Mapping of triple strings to corresponding properties of such target triple. 72 | let runTimeTriples: [String: TripleProperties] 73 | } 74 | 75 | /// Represents v4 schema of `swift-sdk.json` (previously `destination.json`) files used for cross-compilation. 76 | package struct SwiftSDKMetadataV4: Encodable { 77 | package struct TripleProperties: Encodable { 78 | /// Path relative to `swift-sdk.json` containing SDK root. 79 | var sdkRootPath: String 80 | 81 | /// Path relative to `swift-sdk.json` containing Swift resources for dynamic linking. 82 | var swiftResourcesPath: String? 83 | 84 | /// Path relative to `swift-sdk.json` containing Swift resources for static linking. 85 | var swiftStaticResourcesPath: String? 86 | 87 | /// Array of paths relative to `swift-sdk.json` containing headers. 88 | var includeSearchPaths: [String]? 89 | 90 | /// Array of paths relative to `swift-sdk.json` containing libraries. 91 | var librarySearchPaths: [String]? 92 | 93 | /// Array of paths relative to `swift-sdk.json` containing toolset files. 94 | var toolsetPaths: [String]? 95 | } 96 | 97 | /// Version of the schema used when serializing the destination file. 98 | let schemaVersion = "4.0" 99 | 100 | /// Mapping of triple strings to corresponding properties of such target triple. 101 | var targetTriples: [String: TripleProperties] 102 | } 103 | -------------------------------------------------------------------------------- /Sources/SwiftSDKGenerator/Serialization/Toolset.swift: -------------------------------------------------------------------------------- 1 | //===----------------------------------------------------------------------===// 2 | // 3 | // This source file is part of the Swift open source project 4 | // 5 | // Copyright (c) 2023 Apple Inc. and the Swift project authors 6 | // Licensed under Apache License v2.0 with Runtime Library Exception 7 | // 8 | // See https://swift.org/LICENSE.txt for license information 9 | // See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 10 | // 11 | //===----------------------------------------------------------------------===// 12 | 13 | /// A raw decoding of toolset configuration stored on disk. 14 | public struct Toolset: Encodable { 15 | /// Properties of a tool in a ``DecodedToolset``. 16 | struct ToolProperties: Encodable { 17 | /// Either a relative or an absolute path to the tool on the filesystem. 18 | var path: String? 19 | 20 | /// Command-line options to be passed to the tool when it's invoked. 21 | var extraCLIOptions: [String]? 22 | } 23 | 24 | /// Version of a toolset schema used for decoding a toolset file. 25 | let schemaVersion = "1.0" 26 | 27 | /// Root path of the toolset, if present. When filling in ``Toolset.ToolProperties/path``, if a raw path string in 28 | /// ``DecodedToolset`` is inferred to be relative, it's resolved as absolute path relatively to `rootPath`. 29 | var rootPath: String? 30 | 31 | // MARK: Tools currently known and used by SwiftPM. 32 | 33 | var swiftCompiler: ToolProperties? 34 | var cCompiler: ToolProperties? 35 | var cxxCompiler: ToolProperties? 36 | var linker: ToolProperties? 37 | var librarian: ToolProperties? 38 | var debugger: ToolProperties? 39 | var testRunner: ToolProperties? 40 | var xcbuild: ToolProperties? 41 | } 42 | -------------------------------------------------------------------------------- /Sources/SwiftSDKGenerator/SwiftSDKRecipes/SwiftSDKRecipe.swift: -------------------------------------------------------------------------------- 1 | //===----------------------------------------------------------------------===// 2 | // 3 | // This source file is part of the Swift open source project 4 | // 5 | // Copyright (c) 2022-2023 Apple Inc. and the Swift project authors 6 | // Licensed under Apache License v2.0 with Runtime Library Exception 7 | // 8 | // See https://swift.org/LICENSE.txt for license information 9 | // See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 10 | // 11 | //===----------------------------------------------------------------------===// 12 | 13 | import Helpers 14 | import Logging 15 | 16 | import struct SystemPackage.FilePath 17 | 18 | package struct SwiftSDKProduct { 19 | let sdkDirPath: FilePath 20 | /// Array of supported host triples. `nil` indicates the SDK can be universally used. 21 | let hostTriples: [Triple]? 22 | } 23 | 24 | /// A protocol describing a set of platform specific instructions to make a Swift SDK 25 | package protocol SwiftSDKRecipe: Sendable { 26 | /// Update the given toolset with platform specific options 27 | func applyPlatformOptions( 28 | toolset: inout Toolset, 29 | targetTriple: Triple, 30 | isForEmbeddedSwift: Bool 31 | ) 32 | func applyPlatformOptions( 33 | metadata: inout SwiftSDKMetadataV4, 34 | paths: PathsConfiguration, 35 | targetTriple: Triple, 36 | isForEmbeddedSwift: Bool 37 | ) 38 | 39 | /// The default identifier of the Swift SDK 40 | var defaultArtifactID: String { get } 41 | 42 | /// The logger to use for recipe generators. 43 | var logger: Logger { get } 44 | 45 | /// The main entrypoint of the recipe to make a Swift SDK 46 | func makeSwiftSDK( 47 | generator: SwiftSDKGenerator, 48 | engine: QueryEngine, 49 | httpClient: some HTTPClientProtocol 50 | ) async throws -> SwiftSDKProduct 51 | 52 | var shouldSupportEmbeddedSwift: Bool { get } 53 | } 54 | 55 | extension SwiftSDKRecipe { 56 | package func applyPlatformOptions( 57 | toolset: inout Toolset, 58 | targetTriple: Triple, 59 | isForEmbeddedSwift: Bool 60 | ) {} 61 | package func applyPlatformOptions( 62 | metadata: inout SwiftSDKMetadataV4.TripleProperties, 63 | paths: PathsConfiguration, 64 | targetTriple: Triple, 65 | isForEmbeddedSwift: Bool 66 | ) {} 67 | 68 | package var shouldSupportEmbeddedSwift: Bool { false } 69 | } 70 | -------------------------------------------------------------------------------- /Sources/SwiftSDKGenerator/SystemUtils/ByteBuffer+Utils.swift: -------------------------------------------------------------------------------- 1 | //===----------------------------------------------------------------------===// 2 | // 3 | // This source file is part of the Swift open source project 4 | // 5 | // Copyright (c) 2022-2023 Apple Inc. and the Swift project authors 6 | // Licensed under Apache License v2.0 with Runtime Library Exception 7 | // 8 | // See https://swift.org/LICENSE.txt for license information 9 | // See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 10 | // 11 | //===----------------------------------------------------------------------===// 12 | 13 | import AsyncProcess 14 | import Foundation 15 | import Logging 16 | import NIOCore 17 | 18 | extension ByteBuffer { 19 | public func unzip(zipPath: String, logger: Logger) async throws -> ByteBuffer? { 20 | let result = try await ProcessExecutor.runCollectingOutput( 21 | executable: zipPath, 22 | ["-cd"], 23 | standardInput: [self].async, 24 | collectStandardOutput: true, 25 | collectStandardError: false, 26 | perStreamCollectionLimitBytes: 100 * 1024 * 1024, 27 | logger: logger 28 | ) 29 | 30 | try result.exitReason.throwIfNonZero() 31 | 32 | return result.standardOutput 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /Sources/SwiftSDKGenerator/SystemUtils/FileOperationError.swift: -------------------------------------------------------------------------------- 1 | //===----------------------------------------------------------------------===// 2 | // 3 | // This source file is part of the Swift open source project 4 | // 5 | // Copyright (c) 2022-2023 Apple Inc. and the Swift project authors 6 | // Licensed under Apache License v2.0 with Runtime Library Exception 7 | // 8 | // See https://swift.org/LICENSE.txt for license information 9 | // See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 10 | // 11 | //===----------------------------------------------------------------------===// 12 | 13 | import struct Foundation.URL 14 | import struct SystemPackage.FilePath 15 | 16 | public enum FileOperationError: Error { 17 | case downloadFailed(URL, String) 18 | case directoryCreationFailed(FilePath) 19 | case downloadFailed(String) 20 | case unknownArchiveFormat(String?) 21 | case symlinkFixupFailed(source: FilePath, destination: FilePath) 22 | } 23 | -------------------------------------------------------------------------------- /Sources/SwiftSDKGenerator/SystemUtils/GeneratorError.swift: -------------------------------------------------------------------------------- 1 | //===----------------------------------------------------------------------===// 2 | // 3 | // This source file is part of the Swift open source project 4 | // 5 | // Copyright (c) 2023 Apple Inc. and the Swift project authors 6 | // Licensed under Apache License v2.0 with Runtime Library Exception 7 | // 8 | // See https://swift.org/LICENSE.txt for license information 9 | // See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 10 | // 11 | //===----------------------------------------------------------------------===// 12 | 13 | import struct Foundation.URL 14 | import struct SystemPackage.FilePath 15 | 16 | enum GeneratorError: Error { 17 | case noProcessOutput(String) 18 | case unhandledChildProcessSignal(CInt, CommandInfo) 19 | case nonZeroExitCode(CInt, CommandInfo) 20 | case unknownLinuxDistribution(name: String, version: String?) 21 | case unknownMacOSVersion(String) 22 | case unknownCPUArchitecture(String) 23 | case unknownLLDVersion(String) 24 | case distributionSupportsOnlyDockerGenerator(LinuxDistribution) 25 | case distributionDoesNotSupportArchitecture(LinuxDistribution, targetArchName: String) 26 | case fileDoesNotExist(FilePath) 27 | case fileDownloadFailed(URL, String) 28 | case debianPackagesListDownloadRequiresXz 29 | case packagesListDecompressionFailure 30 | case packagesListParsingFailure(expectedPackages: Int, actual: Int) 31 | } 32 | 33 | extension GeneratorError: CustomStringConvertible { 34 | var description: String { 35 | switch self { 36 | case let .noProcessOutput(process): 37 | return "Failed to read standard output of a launched process: \(process)" 38 | case let .unhandledChildProcessSignal(signal, commandInfo): 39 | return "Process launched with \(commandInfo) finished due to signal \(signal)" 40 | case let .nonZeroExitCode(exitCode, commandInfo): 41 | return "Process launched with \(commandInfo) failed with exit code \(exitCode)" 42 | case let .unknownLinuxDistribution(name, version): 43 | return 44 | "Linux distribution `\(name)`\(version.map { " with version \($0)" } ?? "")` is not supported by this generator." 45 | case let .unknownMacOSVersion(version): 46 | return "macOS version `\(version)` is not supported by this generator." 47 | case let .unknownCPUArchitecture(cpu): 48 | return "CPU architecture `\(cpu)` is not supported by this generator." 49 | case let .unknownLLDVersion(version): 50 | return "LLD version `\(version)` is not supported by this generator." 51 | case let .distributionSupportsOnlyDockerGenerator(linuxDistribution): 52 | return """ 53 | Target Linux distribution \(linuxDistribution) supports Swift SDK generation only when `--with-docker` flag is \ 54 | passed. 55 | """ 56 | case let .distributionDoesNotSupportArchitecture(linuxDistribution, targetArchName): 57 | return """ 58 | Target Linux distribution \(linuxDistribution) does not support the target architecture: \(targetArchName) 59 | """ 60 | case let .fileDoesNotExist(filePath): 61 | return "Expected to find a file at path `\(filePath)`." 62 | case let .fileDownloadFailed(url, status): 63 | return 64 | "File could not be downloaded from a URL `\(url)`, the server returned status `\(status)`." 65 | case .debianPackagesListDownloadRequiresXz: 66 | return "Downloading the Debian packages list requires xz, and it is not installed." 67 | case .packagesListDecompressionFailure: 68 | return "Failed to decompress the list of packages." 69 | case let .packagesListParsingFailure(expected, actual): 70 | return "Failed to parse packages manifest, expected \(expected), found \(actual) packages." 71 | } 72 | } 73 | } 74 | -------------------------------------------------------------------------------- /Sources/SwiftSDKGenerator/SystemUtils/HTTPClient+Download.swift: -------------------------------------------------------------------------------- 1 | //===----------------------------------------------------------------------===// 2 | // 3 | // This source file is part of the Swift open source project 4 | // 5 | // Copyright (c) 2022-2023 Apple Inc. and the Swift project authors 6 | // Licensed under Apache License v2.0 with Runtime Library Exception 7 | // 8 | // See https://swift.org/LICENSE.txt for license information 9 | // See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 10 | // 11 | //===----------------------------------------------------------------------===// 12 | 13 | import Foundation 14 | import Helpers 15 | import NIOCore 16 | import NIOHTTP1 17 | import SystemPackage 18 | 19 | public struct DownloadProgress: Sendable { 20 | public var totalBytes: Int? 21 | public var receivedBytes: Int 22 | } 23 | 24 | public protocol HTTPClientProtocol: Sendable { 25 | /// Perform an operation with a new HTTP client. 26 | /// NOTE: The client will be shutdown after the operation completes, so it 27 | /// should not be stored or used outside of the operation. 28 | static func with( 29 | http1Only: Bool, 30 | _ body: @Sendable (any HTTPClientProtocol) async throws -> Result 31 | ) async throws -> Result 32 | 33 | /// Download a file from the given URL to the given path. 34 | func downloadFile( 35 | from url: URL, 36 | to path: FilePath 37 | ) async throws 38 | 39 | /// Download a file from the given URL to the given path and report download 40 | /// progress as a stream. 41 | func streamDownloadProgress( 42 | from url: URL, 43 | to path: FilePath 44 | ) -> AsyncThrowingStream 45 | 46 | /// Perform GET request to the given URL. 47 | func get(url: String) async throws -> ( 48 | status: NIOHTTP1.HTTPResponseStatus, 49 | body: ByteBuffer? 50 | ) 51 | /// Perform HEAD request to the given URL. 52 | /// - Returns: `true` if the request returns a 200 status code, `false` otherwise. 53 | func head(url: String, headers: NIOHTTP1.HTTPHeaders) async throws -> Bool 54 | } 55 | 56 | extension HTTPClientProtocol { 57 | static func with( 58 | _ body: @Sendable (any HTTPClientProtocol) async throws -> Result 59 | ) async throws 60 | -> Result 61 | { 62 | try await self.with(http1Only: false, body) 63 | } 64 | } 65 | 66 | extension FilePath: @unchecked Sendable {} 67 | 68 | #if canImport(AsyncHTTPClient) 69 | import AsyncHTTPClient 70 | 71 | extension FileDownloadDelegate.Progress: @unchecked Sendable {} 72 | 73 | extension HTTPClient: HTTPClientProtocol { 74 | public static func with( 75 | http1Only: Bool, 76 | _ body: @Sendable (any HTTPClientProtocol) async throws -> Result 77 | ) async throws -> Result { 78 | var configuration = HTTPClient.Configuration( 79 | redirectConfiguration: .follow(max: 5, allowCycles: false) 80 | ) 81 | if http1Only { 82 | configuration.httpVersion = .http1Only 83 | } 84 | let client = HTTPClient(eventLoopGroupProvider: .singleton, configuration: configuration) 85 | return try await withAsyncThrowing { 86 | try await body(client) 87 | } defer: { 88 | try await client.shutdown() 89 | } 90 | } 91 | 92 | public func get(url: String) async throws -> ( 93 | status: NIOHTTP1.HTTPResponseStatus, body: NIOCore.ByteBuffer? 94 | ) { 95 | let response = try await self.get(url: url).get() 96 | return (status: response.status, body: response.body) 97 | } 98 | 99 | public func head(url: String, headers: NIOHTTP1.HTTPHeaders) async throws -> Bool { 100 | var headRequest = HTTPClientRequest(url: url) 101 | headRequest.method = .HEAD 102 | headRequest.headers = ["Accept": "*/*", "User-Agent": "Swift SDK Generator"] 103 | return try await self.execute(headRequest, deadline: .distantFuture).status == .ok 104 | } 105 | 106 | public func downloadFile( 107 | from url: URL, 108 | to path: FilePath 109 | ) async throws { 110 | try await withCheckedThrowingContinuation { (continuation: CheckedContinuation<(), Error>) in 111 | do { 112 | let delegate = try FileDownloadDelegate( 113 | path: path.string, 114 | reportHead: { task, responseHead in 115 | if responseHead.status != .ok { 116 | task.fail( 117 | reason: GeneratorError.fileDownloadFailed(url, responseHead.status.description) 118 | ) 119 | } 120 | } 121 | ) 122 | let request = try HTTPClient.Request(url: url) 123 | 124 | execute(request: request, delegate: delegate).futureResult.whenComplete { 125 | switch $0 { 126 | case let .failure(error): 127 | continuation.resume(throwing: error) 128 | case .success: 129 | continuation.resume(returning: ()) 130 | } 131 | } 132 | } catch { 133 | continuation.resume(throwing: error) 134 | } 135 | } 136 | } 137 | 138 | public func streamDownloadProgress( 139 | from url: URL, 140 | to path: FilePath 141 | ) -> AsyncThrowingStream { 142 | .init { continuation in 143 | do { 144 | let delegate = try FileDownloadDelegate( 145 | path: path.string, 146 | reportHead: { 147 | if $0.status != .ok { 148 | continuation 149 | .finish(throwing: FileOperationError.downloadFailed(url, $0.status.description)) 150 | } 151 | }, 152 | reportProgress: { 153 | continuation.yield( 154 | DownloadProgress(totalBytes: $0.totalBytes, receivedBytes: $0.receivedBytes) 155 | ) 156 | } 157 | ) 158 | let request = try HTTPClient.Request(url: url) 159 | 160 | execute(request: request, delegate: delegate).futureResult.whenComplete { 161 | switch $0 { 162 | case let .failure(error): 163 | continuation.finish(throwing: error) 164 | case let .success(finalProgress): 165 | continuation.yield( 166 | DownloadProgress( 167 | totalBytes: finalProgress.totalBytes, 168 | receivedBytes: finalProgress.receivedBytes 169 | ) 170 | ) 171 | continuation.finish() 172 | } 173 | } 174 | } catch { 175 | continuation.finish(throwing: error) 176 | } 177 | } 178 | } 179 | } 180 | #endif 181 | 182 | struct OfflineHTTPClient: HTTPClientProtocol { 183 | static func with( 184 | http1Only: Bool, 185 | _ body: @Sendable (any HTTPClientProtocol) async throws -> Result 186 | ) async throws -> Result { 187 | let client = OfflineHTTPClient() 188 | return try await body(client) 189 | } 190 | 191 | public func downloadFile(from url: URL, to path: SystemPackage.FilePath) async throws { 192 | throw FileOperationError.downloadFailed(url, "Cannot fetch file with offline client") 193 | } 194 | 195 | public func streamDownloadProgress( 196 | from url: URL, 197 | to path: SystemPackage.FilePath 198 | ) -> AsyncThrowingStream { 199 | AsyncThrowingStream { continuation in 200 | continuation.finish( 201 | throwing: FileOperationError.downloadFailed(url, "Cannot fetch file with offline client") 202 | ) 203 | } 204 | } 205 | 206 | public func get(url: String) async throws -> ( 207 | status: NIOHTTP1.HTTPResponseStatus, body: NIOCore.ByteBuffer? 208 | ) { 209 | throw FileOperationError.downloadFailed( 210 | URL(string: url)!, 211 | "Cannot fetch file with offline client" 212 | ) 213 | } 214 | 215 | public func head(url: String, headers: NIOHTTP1.HTTPHeaders) async throws -> Bool { 216 | throw FileOperationError.downloadFailed( 217 | URL(string: url)!, 218 | "Cannot fetch file with offline client" 219 | ) 220 | } 221 | } 222 | -------------------------------------------------------------------------------- /Sources/SwiftSDKGenerator/SystemUtils/Shell.swift: -------------------------------------------------------------------------------- 1 | //===----------------------------------------------------------------------===// 2 | // 3 | // This source file is part of the Swift open source project 4 | // 5 | // Copyright (c) 2022-2023 Apple Inc. and the Swift project authors 6 | // Licensed under Apache License v2.0 with Runtime Library Exception 7 | // 8 | // See https://swift.org/LICENSE.txt for license information 9 | // See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 10 | // 11 | //===----------------------------------------------------------------------===// 12 | 13 | import AsyncProcess 14 | 15 | import class Foundation.ProcessInfo 16 | import struct SystemPackage.FilePath 17 | 18 | public struct CommandInfo: Sendable { 19 | let command: String 20 | let file: String 21 | let line: Int 22 | } 23 | 24 | struct Shell { 25 | private let process: ProcessExecutor 26 | private let commandInfo: CommandInfo 27 | private let logStdout: Bool 28 | private let logStderr: Bool 29 | 30 | private init( 31 | _ command: String, 32 | shouldLogCommands: Bool, 33 | logStdout: Bool = false, 34 | logStderr: Bool = true, 35 | file: String = #file, 36 | line: Int = #line 37 | ) throws { 38 | self.process = ProcessExecutor( 39 | executable: "/bin/sh", 40 | ["-c", command], 41 | environment: ProcessInfo.processInfo.environment, 42 | standardOutput: logStdout ? .stream : .discard, 43 | standardError: logStderr ? .stream : .discard 44 | ) 45 | self.commandInfo = CommandInfo( 46 | command: command, 47 | file: file, 48 | line: line 49 | ) 50 | self.logStdout = logStdout 51 | self.logStderr = logStderr 52 | 53 | if shouldLogCommands { 54 | print(command) 55 | } 56 | } 57 | 58 | /// Wait for the process to exit in a non-blocking way. 59 | private func waitUntilExit() async throws { 60 | let result = try await withThrowingTaskGroup(of: Void.self) { group in 61 | group.addTask { 62 | if self.logStdout { 63 | try await self.process.standardOutput.printChunksAsStrings() 64 | } 65 | } 66 | group.addTask { 67 | if self.logStderr { 68 | try await self.process.standardError.printChunksAsStrings() 69 | } 70 | } 71 | return try await self.process.run() 72 | } 73 | do { 74 | try result.throwIfNonZero() 75 | } catch { 76 | switch result { 77 | case let .exit(code): 78 | throw GeneratorError.nonZeroExitCode(code, self.commandInfo) 79 | case .signal: 80 | fatalError() 81 | } 82 | } 83 | } 84 | 85 | /// Launch and wait until a shell command exists. Throws an error for non-zero exit codes. 86 | /// - Parameters: 87 | /// - command: the shell command to launch. 88 | /// - currentDirectory: current working directory for the command. 89 | static func run( 90 | _ command: String, 91 | shouldLogCommands: Bool = false, 92 | logStdout: Bool = false, 93 | logStderr: Bool = true, 94 | file: String = #file, 95 | line: Int = #line 96 | ) async throws { 97 | try await Shell( 98 | command, 99 | shouldLogCommands: shouldLogCommands, 100 | logStdout: logStdout, 101 | logStderr: logStderr, 102 | file: file, 103 | line: line 104 | ) 105 | .waitUntilExit() 106 | } 107 | 108 | static func readStdout( 109 | _ command: String, 110 | shouldLogCommands: Bool = false, 111 | file: String = #file, 112 | line: Int = #line 113 | ) async throws -> String { 114 | if shouldLogCommands { 115 | print(command) 116 | } 117 | 118 | let result = try await ProcessExecutor.runCollectingOutput( 119 | executable: "/bin/sh", 120 | ["-c", command], 121 | collectStandardOutput: true, 122 | collectStandardError: false, 123 | perStreamCollectionLimitBytes: 10 * 1024 * 1024, 124 | environment: ProcessInfo.processInfo.environment 125 | ) 126 | 127 | try result.exitReason.throwIfNonZero() 128 | 129 | guard let stdOutBuffer = result.standardOutput else { 130 | throw GeneratorError.noProcessOutput(command) 131 | } 132 | 133 | return String(buffer: stdOutBuffer) 134 | } 135 | } 136 | 137 | extension ChunkSequence { 138 | func printChunksAsStrings() async throws { 139 | for try await line in self.splitIntoLines(dropTerminator: true) { 140 | print(String(buffer: line)) 141 | } 142 | } 143 | } 144 | -------------------------------------------------------------------------------- /Sources/SwiftSDKGenerator/SystemUtils/UnixName.swift: -------------------------------------------------------------------------------- 1 | //===----------------------------------------------------------------------===// 2 | // 3 | // This source file is part of the Swift open source project 4 | // 5 | // Copyright (c) 2024-2025 Apple Inc. and the Swift project authors 6 | // Licensed under Apache License v2.0 with Runtime Library Exception 7 | // 8 | // See https://swift.org/LICENSE.txt for license information 9 | // See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 10 | // 11 | //===----------------------------------------------------------------------===// 12 | 13 | #if canImport(Darwin) 14 | import Darwin 15 | #elseif canImport(Glibc) 16 | import Glibc 17 | #endif 18 | 19 | /// libc's `uname` wrapper 20 | struct UnixName { 21 | let release: String 22 | let machine: String 23 | 24 | init(info: utsname) { 25 | var info = info 26 | 27 | func cloneCString(_ value: inout some Any) -> String { 28 | withUnsafePointer(to: &value) { 29 | String(cString: UnsafeRawPointer($0).assumingMemoryBound(to: CChar.self)) 30 | } 31 | } 32 | self.release = cloneCString(&info.release) 33 | self.machine = cloneCString(&info.machine) 34 | } 35 | 36 | static let current: UnixName! = { 37 | var info = utsname() 38 | guard uname(&info) == 0 else { return nil } 39 | return UnixName(info: info) 40 | }() 41 | } 42 | -------------------------------------------------------------------------------- /Sources/SwiftSDKGenerator/SystemUtils/which.swift: -------------------------------------------------------------------------------- 1 | //===----------------------------------------------------------------------===// 2 | // 3 | // This source file is part of the Swift open source project 4 | // 5 | // Copyright (c) 2022-2025 Apple Inc. and the Swift project authors 6 | // Licensed under Apache License v2.0 with Runtime Library Exception 7 | // 8 | // See https://swift.org/LICENSE.txt for license information 9 | // See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 10 | // 11 | //===----------------------------------------------------------------------===// 12 | 13 | import AsyncProcess 14 | import Foundation 15 | 16 | /// Look for an executable using the `which` utility. 17 | /// 18 | /// - Parameter executableName: The name of the executable to search for. 19 | /// - Throws: Any errors thrown by the ProcessExecutor. 20 | /// - Returns: The path to the executable if found, otherwise nil. 21 | func which(_ executableName: String) async throws -> String? { 22 | let result = try await ProcessExecutor.runCollectingOutput( 23 | executable: "/usr/bin/which", 24 | [executableName], 25 | collectStandardOutput: true, 26 | collectStandardError: false, 27 | environment: ProcessInfo.processInfo.environment 28 | ) 29 | 30 | guard result.exitReason == .exit(0) else { 31 | return nil 32 | } 33 | 34 | if let output = result.standardOutput { 35 | let path = String(buffer: output).trimmingCharacters(in: .whitespacesAndNewlines) 36 | return path.isEmpty ? nil : path 37 | } 38 | 39 | return nil 40 | } 41 | -------------------------------------------------------------------------------- /Sources/SystemSQLite/module.modulemap: -------------------------------------------------------------------------------- 1 | module SystemSQLite [system] { 2 | header "sqlite.h" 3 | link "sqlite3" 4 | export * 5 | } 6 | -------------------------------------------------------------------------------- /Sources/SystemSQLite/sqlite.h: -------------------------------------------------------------------------------- 1 | //===----------------------------------------------------------------------===// 2 | // 3 | // This source file is part of the Swift open source project 4 | // 5 | // Copyright (c) 2014-2020 Apple Inc. and the Swift project authors 6 | // Licensed under Apache License v2.0 with Runtime Library Exception 7 | // 8 | // See https://swift.org/LICENSE.txt for license information 9 | // See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 10 | // 11 | //===----------------------------------------------------------------------===// 12 | 13 | #pragma once 14 | #include 15 | -------------------------------------------------------------------------------- /Tests/AsyncProcessTests/AsyncByteBufferLineSequenceTests.swift: -------------------------------------------------------------------------------- 1 | //===----------------------------------------------------------------------===// 2 | // 3 | // This source file is part of the Swift open source project 4 | // 5 | // Copyright (c) 2022-2023 Apple Inc. and the Swift project authors 6 | // Licensed under Apache License v2.0 with Runtime Library Exception 7 | // 8 | // See https://swift.org/LICENSE.txt for license information 9 | // See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 10 | // 11 | //===----------------------------------------------------------------------===// 12 | 13 | import AsyncProcess 14 | import Foundation 15 | import NIO 16 | import XCTest 17 | 18 | final class AsyncByteBufferLineSequenceTests: XCTestCase { 19 | func testJustManyNewlines() async throws { 20 | for n in 0..<100 { 21 | let inputs: [ByteBuffer] = [ByteBuffer(repeating: UInt8(ascii: "\n"), count: n)] 22 | let lines = try await Array(inputs.async.splitIntoLines().strings) 23 | XCTAssertEqual(Array(repeating: "", count: n), lines) 24 | } 25 | } 26 | 27 | func testJustOneNewlineAtATime() async throws { 28 | for n in 0..<100 { 29 | let inputs: [ByteBuffer] = Array(repeating: ByteBuffer(integer: UInt8(ascii: "\n")), count: n) 30 | let lines = try await Array(inputs.async.splitIntoLines().strings) 31 | XCTAssertEqual(Array(repeating: "", count: n), lines) 32 | } 33 | } 34 | 35 | func testManyChunksNoNewlineDeliveringLastChunk() async throws { 36 | for n in 1..<100 { 37 | let inputs: [ByteBuffer] = [ByteBuffer(repeating: 0, count: n)] 38 | let lines = try await Array(inputs.async.splitIntoLines().strings) 39 | XCTAssertEqual([String(repeating: "\0", count: n)], lines) 40 | } 41 | } 42 | 43 | func testManyChunksNoNewlineNotDeliveringLastChunk() async throws { 44 | for n in 0..<100 { 45 | let inputs: [ByteBuffer] = [ByteBuffer(repeating: 0, count: n)] 46 | let lines = try await Array( 47 | inputs.async.splitIntoLines(dropLastChunkIfNoNewline: true).strings 48 | ) 49 | XCTAssertEqual([], lines) 50 | } 51 | } 52 | 53 | func testOverlyLongLineIsSplit() async throws { 54 | var inputs = Array(repeating: ByteBuffer(integer: UInt8(0)), count: 10) 55 | inputs.append(ByteBuffer(integer: UInt8(ascii: "\n"))) 56 | let lines = try await Array( 57 | inputs.async.splitIntoLines( 58 | maximumAllowableBufferSize: 3, 59 | dropLastChunkIfNoNewline: true 60 | ).strings 61 | ) 62 | XCTAssertEqual(["\0\0\0\0", "\0\0\0\0", "\0\0"], lines) 63 | } 64 | 65 | func testOverlyLongLineIsSplitByDefault() async throws { 66 | var inputs = [ByteBuffer(repeating: UInt8(0), count: 1024 * 1024 - 2)] // almost at the limit 67 | inputs.append(ByteBuffer(integer: UInt8(ascii: "\0"))) 68 | inputs.append(ByteBuffer(integer: UInt8(ascii: "\0"))) // hitting the limit 69 | inputs.append(ByteBuffer(integer: UInt8(ascii: "\0"))) // over the limit 70 | inputs.append(ByteBuffer(integer: UInt8(ascii: "\n"))) // too late 71 | let lines = try await Array( 72 | inputs.async.splitIntoLines( 73 | dropTerminator: false, 74 | dropLastChunkIfNoNewline: true 75 | ).strings 76 | ) 77 | XCTAssertEqual([String(repeating: "\0", count: 1024 * 1024 + 1), "\n"], lines) 78 | } 79 | } 80 | -------------------------------------------------------------------------------- /Tests/AsyncProcessTests/Helpers+LogRecorderHandler.swift: -------------------------------------------------------------------------------- 1 | //===----------------------------------------------------------------------===// 2 | // 3 | // This source file is part of the Swift open source project 4 | // 5 | // Copyright (c) 2022-2023 Apple Inc. and the Swift project authors 6 | // Licensed under Apache License v2.0 with Runtime Library Exception 7 | // 8 | // See https://swift.org/LICENSE.txt for license information 9 | // See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 10 | // 11 | //===----------------------------------------------------------------------===// 12 | 13 | import Logging 14 | import NIOConcurrencyHelpers 15 | 16 | final class LogRecorderHandler: LogHandler { 17 | let state = NIOLockedValueBox(State()) 18 | 19 | struct FullLogMessage: Equatable { 20 | var level: Logger.Level 21 | var message: Logger.Message 22 | var metadata: Logger.Metadata 23 | } 24 | 25 | struct State { 26 | var metadata: [String: Logger.Metadata.Value] = [:] 27 | var messages: [FullLogMessage] = [] 28 | var logLevel: Logger.Level = .trace 29 | } 30 | 31 | func makeLogger() -> Logger { 32 | Logger(label: "LogRecorder for tests", factory: { _ in self }) 33 | } 34 | 35 | func log( 36 | level: Logger.Level, 37 | message: Logger.Message, 38 | metadata: Logger.Metadata?, 39 | source: String, 40 | file: String, 41 | function: String, 42 | line: UInt 43 | ) { 44 | let fullMessage = FullLogMessage( 45 | level: level, 46 | message: message, 47 | metadata: self.metadata.merging(metadata ?? [:]) { _, r in r } 48 | ) 49 | self.state.withLockedValue { state in 50 | state.messages.append(fullMessage) 51 | } 52 | } 53 | 54 | var recordedMessages: [FullLogMessage] { 55 | self.state.withLockedValue { $0.messages } 56 | } 57 | 58 | subscript(metadataKey key: String) -> Logging.Logger.Metadata.Value? { 59 | get { 60 | self.state.withLockedValue { 61 | $0.metadata[key] 62 | } 63 | } 64 | set { 65 | self.state.withLockedValue { 66 | $0.metadata[key] = newValue 67 | } 68 | } 69 | } 70 | 71 | var metadata: Logging.Logger.Metadata { 72 | get { 73 | self.state.withLockedValue { 74 | $0.metadata 75 | } 76 | } 77 | 78 | set { 79 | self.state.withLockedValue { 80 | $0.metadata = newValue 81 | } 82 | } 83 | } 84 | 85 | var logLevel: Logging.Logger.Level { 86 | get { 87 | self.state.withLockedValue { 88 | $0.logLevel 89 | } 90 | } 91 | 92 | set { 93 | self.state.withLockedValue { 94 | $0.logLevel = newValue 95 | } 96 | } 97 | } 98 | } 99 | -------------------------------------------------------------------------------- /Tests/GeneratorEngineTests/EngineTests.swift: -------------------------------------------------------------------------------- 1 | //===----------------------------------------------------------------------===// 2 | // 3 | // This source file is part of the Swift open source project 4 | // 5 | // Copyright (c) 2023-2024 Apple Inc. and the Swift project authors 6 | // Licensed under Apache License v2.0 with Runtime Library Exception 7 | // 8 | // See https://swift.org/LICENSE.txt for license information 9 | // See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 10 | // 11 | //===----------------------------------------------------------------------===// 12 | 13 | import Crypto 14 | import XCTest 15 | 16 | import struct Foundation.Data 17 | import struct Logging.Logger 18 | import struct SystemPackage.FilePath 19 | 20 | @testable import Helpers 21 | 22 | private let encoder = JSONEncoder() 23 | private let decoder = JSONDecoder() 24 | 25 | extension AsyncFileSystem { 26 | fileprivate func read( 27 | _ path: FilePath, 28 | bufferLimit: Int = 10 * 1024 * 1024, 29 | as: V.Type 30 | ) async throws -> V { 31 | let data = try await self.withOpenReadableFile(path) { 32 | var data = Data() 33 | for try await chunk in try await $0.read() { 34 | data.append(contentsOf: chunk) 35 | 36 | assert(data.count < bufferLimit) 37 | } 38 | return data 39 | } 40 | 41 | return try decoder.decode(V.self, from: data) 42 | } 43 | 44 | fileprivate func write(_ path: FilePath, _ value: some Encodable) async throws { 45 | let data = try encoder.encode(value) 46 | try await self.withOpenWritableFile(path) { fileHandle in 47 | try await fileHandle.write(data) 48 | } 49 | } 50 | } 51 | 52 | struct Const: CachingQuery { 53 | let x: Int 54 | 55 | func run(engine: QueryEngine) async throws -> FilePath { 56 | let resultPath = FilePath("/Const-\(x)") 57 | try await engine.fileSystem.write(resultPath, self.x) 58 | return resultPath 59 | } 60 | } 61 | 62 | struct MultiplyByTwo: CachingQuery { 63 | let x: Int 64 | 65 | func run(engine: QueryEngine) async throws -> FilePath { 66 | let constPath = try await engine[Const(x: self.x)].path 67 | let constResult = try await engine.fileSystem.read(constPath, as: Int.self) 68 | 69 | let resultPath = FilePath("/MultiplyByTwo-\(constResult)") 70 | try await engine.fileSystem.write(resultPath, constResult * 2) 71 | return resultPath 72 | } 73 | } 74 | 75 | struct AddThirty: CachingQuery { 76 | let x: Int 77 | 78 | func run(engine: QueryEngine) async throws -> FilePath { 79 | let constPath = try await engine[Const(x: self.x)].path 80 | let constResult = try await engine.fileSystem.read(constPath, as: Int.self) 81 | 82 | let resultPath = FilePath("/AddThirty-\(constResult)") 83 | try await engine.fileSystem.write(resultPath, constResult + 30) 84 | return resultPath 85 | } 86 | } 87 | 88 | struct Expression: CachingQuery { 89 | let x: Int 90 | let y: Int 91 | 92 | func run(engine: QueryEngine) async throws -> FilePath { 93 | let multiplyPath = try await engine[MultiplyByTwo(x: self.x)].path 94 | let addThirtyPath = try await engine[AddThirty(x: self.y)].path 95 | 96 | let multiplyResult = try await engine.fileSystem.read(multiplyPath, as: Int.self) 97 | let addThirtyResult = try await engine.fileSystem.read(addThirtyPath, as: Int.self) 98 | 99 | let resultPath = FilePath("/Expression-\(multiplyResult)-\(addThirtyResult)") 100 | try await engine.fileSystem.write(resultPath, multiplyResult + addThirtyResult) 101 | return resultPath 102 | } 103 | } 104 | 105 | final class EngineTests: XCTestCase { 106 | func testFilePathHashing() throws { 107 | let path = "/root" 108 | 109 | let hashEncoder1 = HashEncoder() 110 | try hashEncoder1.encode(FilePath(path)) 111 | let digest1 = hashEncoder1.finalize() 112 | 113 | let hashEncoder2 = HashEncoder() 114 | try hashEncoder2.encode(String(reflecting: FilePath.self)) 115 | try hashEncoder2.encode(path) 116 | let digest2 = hashEncoder2.finalize() 117 | 118 | XCTAssertEqual(digest1, digest2) 119 | } 120 | 121 | func testSimpleCaching() async throws { 122 | let engine = QueryEngine( 123 | MockFileSystem(), 124 | Logger(label: "engine-tests") 125 | // cacheLocation: .memory 126 | ) 127 | 128 | var resultPath = try await engine[Expression(x: 1, y: 2)].path 129 | var result = try await engine.fileSystem.read(resultPath, as: Int.self) 130 | 131 | XCTAssertEqual(result, 34) 132 | 133 | var cacheMisses = await engine.cacheMisses 134 | XCTAssertEqual(cacheMisses, 5) 135 | 136 | var cacheHits = await engine.cacheHits 137 | XCTAssertEqual(cacheHits, 0) 138 | 139 | resultPath = try await engine[Expression(x: 1, y: 2)].path 140 | result = try await engine.fileSystem.read(resultPath, as: Int.self) 141 | XCTAssertEqual(result, 34) 142 | 143 | cacheMisses = await engine.cacheMisses 144 | XCTAssertEqual(cacheMisses, 5) 145 | 146 | cacheHits = await engine.cacheHits 147 | XCTAssertEqual(cacheHits, 1) 148 | 149 | resultPath = try await engine[Expression(x: 2, y: 1)].path 150 | result = try await engine.fileSystem.read(resultPath, as: Int.self) 151 | XCTAssertEqual(result, 35) 152 | 153 | cacheMisses = await engine.cacheMisses 154 | XCTAssertEqual(cacheMisses, 8) 155 | 156 | cacheHits = await engine.cacheHits 157 | XCTAssertEqual(cacheHits, 3) 158 | 159 | resultPath = try await engine[Expression(x: 2, y: 1)].path 160 | result = try await engine.fileSystem.read(resultPath, as: Int.self) 161 | XCTAssertEqual(result, 35) 162 | 163 | cacheMisses = await engine.cacheMisses 164 | XCTAssertEqual(cacheMisses, 8) 165 | 166 | cacheHits = await engine.cacheHits 167 | XCTAssertEqual(cacheHits, 4) 168 | 169 | try await engine.shutDown() 170 | } 171 | 172 | struct MyItem: Sendable, CacheKey { 173 | let remoteURL: URL 174 | var localPath: FilePath 175 | let isPrebuilt: Bool 176 | } 177 | 178 | func testQueryEncoding() throws { 179 | let item = MyItem( 180 | remoteURL: URL( 181 | string: 182 | "https://download.swift.org/swift-5.9.2-release/ubuntu2204-aarch64/swift-5.9.2-RELEASE/swift-5.9.2-RELEASE-ubuntu22.04-aarch64.tar.gz" 183 | )!, 184 | localPath: 185 | "/Users/katei/ghq/github.com/apple/swift-sdk-generator/Artifacts/target_swift_5.9.2-RELEASE_aarch64-unknown-linux-gnu.tar.gz", 186 | isPrebuilt: true 187 | ) 188 | func hashValue(of key: some CacheKey) throws -> SHA256Digest { 189 | let hasher = HashEncoder() 190 | try hasher.encode(key) 191 | return hasher.finalize() 192 | } 193 | // Ensure that hash key is stable across runs 194 | XCTAssertEqual( 195 | try hashValue(of: item).description, 196 | "SHA256 digest: 5178ba619e00da962d505954d33d0bceceeff29831bf5ee0c878dd1f2568b118" 197 | ) 198 | } 199 | } 200 | -------------------------------------------------------------------------------- /Tests/HelpersTests/ThrowingDeferTests.swift: -------------------------------------------------------------------------------- 1 | //===----------------------------------------------------------------------===// 2 | // 3 | // This source file is part of the Swift open source project 4 | // 5 | // Copyright (c) 2023 Apple Inc. and the Swift project authors 6 | // Licensed under Apache License v2.0 with Runtime Library Exception 7 | // 8 | // See https://swift.org/LICENSE.txt for license information 9 | // See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 10 | // 11 | //===----------------------------------------------------------------------===// 12 | 13 | import Foundation 14 | import XCTest 15 | 16 | @testable import Helpers 17 | 18 | final class ThrowingDeferTests: XCTestCase { 19 | struct EquatableError: Error, Equatable { 20 | let identifier = UUID() 21 | } 22 | 23 | final actor Worker { 24 | let error: EquatableError? 25 | private(set) var didRun = false 26 | 27 | init(error: EquatableError? = nil) { 28 | self.error = error 29 | } 30 | 31 | func run() throws { 32 | self.didRun = true 33 | if let error { 34 | throw error 35 | } 36 | } 37 | } 38 | 39 | // MARK: - Non-Async 40 | 41 | func testThrowingDeferWithoutThrowing() throws { 42 | var didRunWork = false 43 | var didRunCleanup = false 44 | 45 | try withThrowing { 46 | didRunWork = true 47 | } defer: { 48 | didRunCleanup = true 49 | } 50 | 51 | XCTAssertTrue(didRunWork) 52 | XCTAssertTrue(didRunCleanup) 53 | } 54 | 55 | func testThrowingDeferWhenThrowingFromWork() throws { 56 | let workError = EquatableError() 57 | var didRunCleanup = false 58 | 59 | XCTAssertThrowsError( 60 | try withThrowing { 61 | throw workError 62 | } defer: { 63 | didRunCleanup = true 64 | } 65 | ) { 66 | XCTAssertTrue($0 is EquatableError) 67 | XCTAssertEqual($0 as? EquatableError, workError) 68 | } 69 | XCTAssertTrue(didRunCleanup) 70 | } 71 | 72 | func testThrowingDeferWhenThrowingFromCleanup() throws { 73 | var didRunWork = false 74 | let cleanupError = EquatableError() 75 | 76 | XCTAssertThrowsError( 77 | try withThrowing { 78 | didRunWork = true 79 | } defer: { 80 | throw cleanupError 81 | } 82 | ) { 83 | XCTAssertTrue($0 is EquatableError) 84 | XCTAssertEqual($0 as? EquatableError, cleanupError) 85 | } 86 | XCTAssertTrue(didRunWork) 87 | } 88 | 89 | func testThrowingDeferWhenThrowingFromBothClosures() throws { 90 | var didRunWork = false 91 | let workError = EquatableError() 92 | let cleanupError = EquatableError() 93 | 94 | XCTAssertThrowsError( 95 | try withThrowing { 96 | didRunWork = true 97 | throw workError 98 | } defer: { 99 | throw cleanupError 100 | } 101 | ) { 102 | XCTAssertTrue($0 is EquatableError) 103 | XCTAssertEqual($0 as? EquatableError, cleanupError) 104 | } 105 | XCTAssertTrue(didRunWork) 106 | } 107 | 108 | // MARK: - Async 109 | 110 | func testAsyncThrowingDeferWithoutThrowing() async throws { 111 | let work = Worker() 112 | let cleanup = Worker() 113 | 114 | try await withAsyncThrowing { 115 | try await work.run() 116 | } defer: { 117 | try await cleanup.run() 118 | } 119 | 120 | let didRunWork = await work.didRun 121 | let didRunCleanup = await cleanup.didRun 122 | XCTAssertTrue(didRunWork) 123 | XCTAssertTrue(didRunCleanup) 124 | } 125 | 126 | func testAsyncThrowingDeferWhenThrowingFromWork() async throws { 127 | let workError = EquatableError() 128 | let work = Worker(error: workError) 129 | let cleanup = Worker() 130 | 131 | do { 132 | try await withAsyncThrowing { 133 | try await work.run() 134 | } defer: { 135 | try await cleanup.run() 136 | } 137 | XCTFail("No error was thrown!") 138 | } catch { 139 | XCTAssertTrue(error is EquatableError) 140 | XCTAssertEqual(error as? EquatableError, workError) 141 | } 142 | 143 | let didRunWork = await cleanup.didRun 144 | let didRunCleanup = await cleanup.didRun 145 | XCTAssertTrue(didRunWork) 146 | XCTAssertTrue(didRunCleanup) 147 | } 148 | 149 | func testAsyncThrowingDeferWhenThrowingFromCleanup() async throws { 150 | let cleanupError = EquatableError() 151 | let work = Worker() 152 | let cleanup = Worker(error: cleanupError) 153 | 154 | do { 155 | try await withAsyncThrowing { 156 | try await work.run() 157 | } defer: { 158 | try await cleanup.run() 159 | } 160 | XCTFail("No error was thrown!") 161 | } catch { 162 | XCTAssertTrue(error is EquatableError) 163 | XCTAssertEqual(error as? EquatableError, cleanupError) 164 | } 165 | 166 | let didRunWork = await work.didRun 167 | let didRunCleanup = await cleanup.didRun 168 | XCTAssertTrue(didRunWork) 169 | XCTAssertTrue(didRunCleanup) 170 | } 171 | 172 | func testAsyncThrowingDeferWhenThrowingFromBothClosures() async throws { 173 | let workError = EquatableError() 174 | let cleanupError = EquatableError() 175 | let work = Worker(error: workError) 176 | let cleanup = Worker(error: cleanupError) 177 | 178 | do { 179 | try await withAsyncThrowing { 180 | try await work.run() 181 | XCTFail("No error was thrown from work!") 182 | } defer: { 183 | try await cleanup.run() 184 | } 185 | XCTFail("No error was thrown!") 186 | } catch { 187 | XCTAssertTrue(error is EquatableError) 188 | XCTAssertEqual(error as? EquatableError, cleanupError) 189 | } 190 | 191 | let didRunWork = await work.didRun 192 | let didRunCleanup = await cleanup.didRun 193 | XCTAssertTrue(didRunWork) 194 | XCTAssertTrue(didRunCleanup) 195 | } 196 | } 197 | -------------------------------------------------------------------------------- /Tests/MacrosTests/MacrosTests.swift: -------------------------------------------------------------------------------- 1 | //===----------------------------------------------------------------------===// 2 | // 3 | // This source file is part of the Swift open source project 4 | // 5 | // Copyright (c) 2014 - 2023 Apple Inc. and the Swift project authors 6 | // Licensed under Apache License v2.0 with Runtime Library Exception 7 | // 8 | // See https://swift.org/LICENSE.txt for license information 9 | // See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 10 | // 11 | //===----------------------------------------------------------------------===// 12 | 13 | import Macros 14 | import SwiftSyntaxMacros 15 | import SwiftSyntaxMacrosTestSupport 16 | import XCTest 17 | 18 | final class MacrosTests: XCTestCase { 19 | private let macros: [String: Macro.Type] = [ 20 | "CacheKey": CacheKeyMacro.self, "Query": QueryMacro.self, 21 | ] 22 | 23 | func testCacheKeyDerived() { 24 | assertMacroExpansion( 25 | """ 26 | @CacheKey 27 | struct Message { 28 | let text: String 29 | let sender: String 30 | } 31 | 32 | @Query 33 | struct Q { 34 | let number: Int 35 | let text: String 36 | } 37 | """, 38 | expandedSource: """ 39 | struct Message { 40 | let text: String 41 | let sender: String 42 | } 43 | struct Q { 44 | let number: Int 45 | let text: String 46 | } 47 | 48 | extension Message: CacheKeyProtocol { 49 | func hash(with hashFunction: inout some HashFunction) { 50 | String(reflecting: Self.self).hash(with: &hashFunction) 51 | text.hash(with: &hashFunction) 52 | sender.hash(with: &hashFunction) 53 | } 54 | } 55 | 56 | extension Q: QueryProtocol { 57 | } 58 | 59 | extension Q: CacheKeyProtocol { 60 | func hash(with hashFunction: inout some HashFunction) { 61 | String(reflecting: Self.self).hash(with: &hashFunction) 62 | number.hash(with: &hashFunction) 63 | text.hash(with: &hashFunction) 64 | } 65 | } 66 | """, 67 | macros: self.macros, 68 | indentationWidth: .spaces(2) 69 | ) 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /Tests/SwiftSDKGeneratorTests/ArchitectureMappingTests.swift: -------------------------------------------------------------------------------- 1 | //===----------------------------------------------------------------------===// 2 | // 3 | // This source file is part of the Swift open source project 4 | // 5 | // Copyright (c) 2022-2023 Apple Inc. and the Swift project authors 6 | // Licensed under Apache License v2.0 with Runtime Library Exception 7 | // 8 | // See https://swift.org/LICENSE.txt for license information 9 | // See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 10 | // 11 | //===----------------------------------------------------------------------===// 12 | 13 | import Logging 14 | import XCTest 15 | 16 | @testable import SwiftSDKGenerator 17 | 18 | final class ArchitectureMappingTests: XCTestCase { 19 | let logger = Logger(label: "ArchitectureMappingTests") 20 | 21 | /// Swift on macOS, Swift on Linux and Debian packages all use 22 | /// different names for the x86 and Arm architectures: 23 | /// 24 | /// | x86_64 arm64 25 | /// ------------------------------------ 26 | /// Swift macOS | x86_64 arm64 27 | /// Swift Linux | x86_64 aarch64 28 | /// Debian packages | amd64 arm64 29 | /// 30 | /// The right names must be used in the right places, such as 31 | /// in download URLs and paths within the SDK bundle. These 32 | /// tests check several paths and URLs for each combination 33 | /// of host and target architecture. 34 | /// 35 | /// At present macOS is the only supported build environment 36 | /// and Linux is the only supported target environment. 37 | 38 | public func verifySDKSpec( 39 | bundleVersion: String, 40 | hostTriple: Triple, 41 | targetTriple: Triple, 42 | 43 | artifactID: String, // Base name of the generated bundle 44 | hostLLVMDownloadURL: String, // URL of the host LLVM package 45 | targetSwiftDownloadURL: String, // URL of the target Swift SDK 46 | 47 | artifactBundlePathSuffix: String, // Path to the generated bundle 48 | sdkDirPathSuffix: String // Path of the SDK within the bundle 49 | ) async throws { 50 | let recipe = try LinuxRecipe( 51 | targetTriple: targetTriple, 52 | hostTriple: hostTriple, 53 | linuxDistribution: .ubuntu(.jammy), 54 | swiftVersion: "5.8-RELEASE", 55 | swiftBranch: nil, 56 | lldVersion: "16.0.4", 57 | withDocker: false, 58 | fromContainerImage: nil, 59 | hostSwiftPackagePath: nil, 60 | targetSwiftPackagePath: nil, 61 | logger: logger 62 | ) 63 | // LocalSwiftSDKGenerator constructs URLs and paths which depend on architectures 64 | let sdk = try await SwiftSDKGenerator( 65 | bundleVersion: bundleVersion, 66 | 67 | // Linux is currently the only supported runtime environment 68 | targetTriple: targetTriple, 69 | artifactID: recipe.defaultArtifactID, 70 | // Remaining fields are placeholders which are the same for all 71 | // combinations of build and runtime architecture 72 | isIncremental: false, 73 | isVerbose: false, 74 | logger: Logger(label: "org.swift.swift-sdk-generator") 75 | ) 76 | 77 | let sdkArtifactID = await sdk.artifactID 78 | XCTAssertEqual(sdkArtifactID, artifactID, "Unexpected artifactID") 79 | 80 | // Verify download URLs 81 | let artifacts = try await DownloadableArtifacts( 82 | hostTriple: hostTriple, 83 | targetTriple: sdk.targetTriple, 84 | recipe.versionsConfiguration, 85 | sdk.pathsConfiguration 86 | ) 87 | 88 | // The build-time Swift SDK is a multiarch package and so is always the same 89 | XCTAssertEqual( 90 | artifacts.hostSwift.remoteURL.absoluteString, 91 | "https://download.swift.org/swift-5.8-release/xcode/swift-5.8-RELEASE/swift-5.8-RELEASE-osx.pkg", 92 | "Unexpected build-time Swift SDK URL" 93 | ) 94 | 95 | // LLVM provides ld.lld 96 | XCTAssertEqual( 97 | artifacts.hostLLVM.remoteURL.absoluteString, 98 | hostLLVMDownloadURL, 99 | "Unexpected llvmDownloadURL" 100 | ) 101 | 102 | // The Swift runtime must match the target architecture 103 | XCTAssertEqual( 104 | artifacts.targetSwift.remoteURL.absoluteString, 105 | targetSwiftDownloadURL, 106 | "Unexpected runtimeSwiftDownloadURL" 107 | ) 108 | 109 | // Verify paths within the bundle 110 | let paths = await sdk.pathsConfiguration 111 | 112 | // The bundle path is not critical - it uses Swift's name 113 | // for the target architecture 114 | XCTAssertEqual( 115 | paths.artifactBundlePath.string, 116 | paths.sourceRoot.string + artifactBundlePathSuffix, 117 | "Unexpected artifactBundlePathSuffix" 118 | ) 119 | 120 | // The SDK path must use Swift's name for the architecture 121 | XCTAssertEqual( 122 | recipe.sdkDirPath(paths: paths).string, 123 | paths.artifactBundlePath.string + sdkDirPathSuffix, 124 | "Unexpected sdkDirPathSuffix" 125 | ) 126 | } 127 | 128 | func testX86ToX86SDKGenerator() async throws { 129 | try await self.verifySDKSpec( 130 | bundleVersion: "0.0.1", 131 | hostTriple: Triple("x86_64-apple-macosx13"), 132 | targetTriple: Triple("x86_64-unknown-linux-gnu"), 133 | artifactID: "5.8-RELEASE_ubuntu_jammy_x86_64", 134 | hostLLVMDownloadURL: 135 | "https://github.com/llvm/llvm-project/releases/download/llvmorg-16.0.4/clang+llvm-16.0.4-x86_64-apple-darwin22.0.tar.xz", 136 | targetSwiftDownloadURL: 137 | "https://download.swift.org/swift-5.8-release/ubuntu2204/swift-5.8-RELEASE/swift-5.8-RELEASE-ubuntu22.04.tar.gz", 138 | artifactBundlePathSuffix: "/Bundles/5.8-RELEASE_ubuntu_jammy_x86_64.artifactbundle", 139 | sdkDirPathSuffix: "/5.8-RELEASE_ubuntu_jammy_x86_64/x86_64-unknown-linux-gnu/ubuntu-jammy.sdk" 140 | ) 141 | } 142 | 143 | func testX86ToArmSDKGenerator() async throws { 144 | try await self.verifySDKSpec( 145 | bundleVersion: "0.0.1", 146 | hostTriple: Triple("x86_64-apple-macosx13"), 147 | targetTriple: Triple("aarch64-unknown-linux-gnu"), 148 | artifactID: "5.8-RELEASE_ubuntu_jammy_aarch64", 149 | hostLLVMDownloadURL: 150 | "https://github.com/llvm/llvm-project/releases/download/llvmorg-16.0.4/clang+llvm-16.0.4-x86_64-apple-darwin22.0.tar.xz", 151 | targetSwiftDownloadURL: 152 | "https://download.swift.org/swift-5.8-release/ubuntu2204-aarch64/swift-5.8-RELEASE/swift-5.8-RELEASE-ubuntu22.04-aarch64.tar.gz", 153 | artifactBundlePathSuffix: "/Bundles/5.8-RELEASE_ubuntu_jammy_aarch64.artifactbundle", 154 | sdkDirPathSuffix: 155 | "/5.8-RELEASE_ubuntu_jammy_aarch64/aarch64-unknown-linux-gnu/ubuntu-jammy.sdk" 156 | ) 157 | } 158 | 159 | func testArmToArmSDKGenerator() async throws { 160 | try await self.verifySDKSpec( 161 | bundleVersion: "0.0.1", 162 | hostTriple: Triple("arm64-apple-macosx13"), 163 | targetTriple: Triple("aarch64-unknown-linux-gnu"), 164 | artifactID: "5.8-RELEASE_ubuntu_jammy_aarch64", 165 | hostLLVMDownloadURL: 166 | "https://github.com/llvm/llvm-project/releases/download/llvmorg-16.0.4/clang+llvm-16.0.4-arm64-apple-darwin22.0.tar.xz", 167 | targetSwiftDownloadURL: 168 | "https://download.swift.org/swift-5.8-release/ubuntu2204-aarch64/swift-5.8-RELEASE/swift-5.8-RELEASE-ubuntu22.04-aarch64.tar.gz", 169 | artifactBundlePathSuffix: "/Bundles/5.8-RELEASE_ubuntu_jammy_aarch64.artifactbundle", 170 | sdkDirPathSuffix: 171 | "/5.8-RELEASE_ubuntu_jammy_aarch64/aarch64-unknown-linux-gnu/ubuntu-jammy.sdk" 172 | ) 173 | } 174 | 175 | func testArmToX86SDKGenerator() async throws { 176 | try await self.verifySDKSpec( 177 | bundleVersion: "0.0.1", 178 | hostTriple: Triple("arm64-apple-macosx13"), 179 | targetTriple: Triple("x86_64-unknown-linux-gnu"), 180 | artifactID: "5.8-RELEASE_ubuntu_jammy_x86_64", 181 | hostLLVMDownloadURL: 182 | "https://github.com/llvm/llvm-project/releases/download/llvmorg-16.0.4/clang+llvm-16.0.4-arm64-apple-darwin22.0.tar.xz", 183 | targetSwiftDownloadURL: 184 | "https://download.swift.org/swift-5.8-release/ubuntu2204/swift-5.8-RELEASE/swift-5.8-RELEASE-ubuntu22.04.tar.gz", 185 | artifactBundlePathSuffix: "/Bundles/5.8-RELEASE_ubuntu_jammy_x86_64.artifactbundle", 186 | sdkDirPathSuffix: "/5.8-RELEASE_ubuntu_jammy_x86_64/x86_64-unknown-linux-gnu/ubuntu-jammy.sdk" 187 | ) 188 | } 189 | } 190 | -------------------------------------------------------------------------------- /Tests/SwiftSDKGeneratorTests/Generator/SwiftSDKGenerator+MetadataTests.swift: -------------------------------------------------------------------------------- 1 | //===----------------------------------------------------------------------===// 2 | // 3 | // This source file is part of the Swift open source project 4 | // 5 | // Copyright (c) 2022-2025 Apple Inc. and the Swift project authors 6 | // Licensed under Apache License v2.0 with Runtime Library Exception 7 | // 8 | // See https://swift.org/LICENSE.txt for license information 9 | // See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 10 | // 11 | //===----------------------------------------------------------------------===// 12 | 13 | import Logging 14 | import SystemPackage 15 | import XCTest 16 | 17 | @testable import SwiftSDKGenerator 18 | 19 | #if canImport(FoundationEssentials) 20 | import FoundationEssentials 21 | #else 22 | import Foundation 23 | #endif 24 | 25 | final class SwiftSDKGeneratorMetadataTests: XCTestCase { 26 | let logger = Logger(label: "SwiftSDKGeneratorMetadataTests") 27 | 28 | func testGenerateSDKSettingsFile() async throws { 29 | let testCases = [ 30 | ( 31 | bundleVersion: "0.0.1", 32 | targetTriple: Triple("x86_64-unknown-linux-gnu"), 33 | expectedCanonicalName: "x86_64-swift-linux-gnu" 34 | ), 35 | ( 36 | bundleVersion: "0.0.2", 37 | targetTriple: Triple("aarch64-unknown-linux-gnu"), 38 | expectedCanonicalName: "aarch64-swift-linux-gnu" 39 | ), 40 | ( 41 | bundleVersion: "0.0.3", 42 | targetTriple: Triple("armv7-unknown-linux-gnueabihf"), 43 | expectedCanonicalName: "armv7-swift-linux-gnueabihf" 44 | ), 45 | ] 46 | 47 | for testCase in testCases { 48 | let sdk = try await SwiftSDKGenerator( 49 | bundleVersion: testCase.bundleVersion, 50 | targetTriple: testCase.targetTriple, 51 | artifactID: "6.0.3-RELEASE_ubuntu_jammy_\(testCase.targetTriple.archName)", 52 | isIncremental: false, 53 | isVerbose: false, 54 | logger: logger 55 | ) 56 | let linuxDistribution = try LinuxDistribution(name: .ubuntu, version: "22.04") 57 | 58 | let sdkDirPath = FilePath(".") 59 | try await sdk.generateSDKSettingsFile(sdkDirPath: sdkDirPath, distribution: linuxDistribution) 60 | 61 | // Make sure the file exists 62 | let sdkSettingsFile = sdkDirPath.appending("SDKSettings.json") 63 | var fileExists = await sdk.doesFileExist(at: sdkSettingsFile) 64 | XCTAssertTrue(fileExists) 65 | 66 | // Read back file, make sure it contains the expected data 67 | let maybeString = String(data: try await sdk.readFile(at: sdkSettingsFile), encoding: .utf8) 68 | let string = try XCTUnwrap(maybeString) 69 | XCTAssertTrue(string.contains(testCase.bundleVersion)) 70 | XCTAssertTrue(string.contains("(\(testCase.targetTriple.archName))")) 71 | XCTAssertTrue(string.contains(linuxDistribution.description)) 72 | XCTAssertTrue(string.contains(testCase.expectedCanonicalName)) 73 | 74 | // Cleanup 75 | try await sdk.removeFile(at: sdkSettingsFile) 76 | 77 | try await sdk.createDirectoryIfNeeded(at: sdk.pathsConfiguration.artifactBundlePath) 78 | 79 | // Generate bundle metadata 80 | try await sdk.generateArtifactBundleManifest( 81 | hostTriples: [sdk.targetTriple], 82 | artifacts: ["foo": sdk.pathsConfiguration.artifactBundlePath.appending("bar.json")] 83 | ) 84 | 85 | // Make sure the file exists 86 | let archiveMetadataFile = await sdk.pathsConfiguration.artifactBundlePath.appending("info.json") 87 | fileExists = await sdk.doesFileExist(at: archiveMetadataFile) 88 | XCTAssertTrue(fileExists) 89 | 90 | // Read back file, make sure it contains the expected data 91 | let data = try await sdk.readFile(at: archiveMetadataFile) 92 | let decodedMetadata = try JSONDecoder().decode(ArtifactsArchiveMetadata.self, from: data) 93 | XCTAssertEqual(decodedMetadata.artifacts.count, 1) 94 | for (id, artifact) in decodedMetadata.artifacts { 95 | XCTAssertEqual(id, "foo") 96 | XCTAssertEqual(artifact.variants, [.init(path: "bar.json", supportedTriples: [testCase.targetTriple.triple])]) 97 | } 98 | 99 | // Cleanup 100 | try await sdk.removeFile(at: archiveMetadataFile) 101 | } 102 | } 103 | } 104 | -------------------------------------------------------------------------------- /Tests/SwiftSDKGeneratorTests/SwiftSDKRecipes/WebAssemblyRecipe.swift: -------------------------------------------------------------------------------- 1 | //===----------------------------------------------------------------------===// 2 | // 3 | // This source file is part of the Swift open source project 4 | // 5 | // Copyright (c) 2022-2024 Apple Inc. and the Swift project authors 6 | // Licensed under Apache License v2.0 with Runtime Library Exception 7 | // 8 | // See https://swift.org/LICENSE.txt for license information 9 | // See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 10 | // 11 | //===----------------------------------------------------------------------===// 12 | 13 | import Logging 14 | import XCTest 15 | 16 | @testable import SwiftSDKGenerator 17 | 18 | final class WebAssemblyRecipeTests: XCTestCase { 19 | let logger = Logger(label: "WebAssemblyRecipeTests") 20 | 21 | func createRecipe() -> WebAssemblyRecipe { 22 | WebAssemblyRecipe( 23 | hostSwiftPackage: nil, 24 | targetSwiftPackagePath: "./target-toolchain", 25 | wasiSysroot: "./wasi-sysroot", 26 | swiftVersion: "5.10", 27 | logger: logger 28 | ) 29 | } 30 | 31 | func testToolOptions() { 32 | let recipe = self.createRecipe() 33 | var toolset = Toolset(rootPath: nil) 34 | recipe.applyPlatformOptions( 35 | toolset: &toolset, 36 | targetTriple: Triple("wasm32-unknown-wasi"), 37 | isForEmbeddedSwift: false 38 | ) 39 | XCTAssertEqual(toolset.swiftCompiler?.extraCLIOptions, ["-static-stdlib"]) 40 | XCTAssertNil(toolset.cCompiler) 41 | XCTAssertNil(toolset.cxxCompiler) 42 | XCTAssertNil(toolset.linker) 43 | } 44 | 45 | func testEmbeddedToolOptions() { 46 | let recipe = self.createRecipe() 47 | var toolset = Toolset(rootPath: nil) 48 | recipe.applyPlatformOptions( 49 | toolset: &toolset, 50 | targetTriple: Triple("wasm32-unknown-wasi"), 51 | isForEmbeddedSwift: true 52 | ) 53 | XCTAssertEqual( 54 | toolset.swiftCompiler?.extraCLIOptions, 55 | [ 56 | "-static-stdlib", 57 | "-enable-experimental-feature", "Embedded", "-wmo", 58 | ] 59 | + ["-lc++", "-lswift_Concurrency", "-lswift_ConcurrencyDefaultExecutor"].flatMap { 60 | ["-Xlinker", $0] 61 | } 62 | ) 63 | XCTAssertEqual(toolset.cCompiler?.extraCLIOptions, ["-D__EMBEDDED_SWIFT__"]) 64 | XCTAssertEqual(toolset.cxxCompiler?.extraCLIOptions, ["-D__EMBEDDED_SWIFT__"]) 65 | XCTAssertNil(toolset.linker) 66 | } 67 | 68 | func testToolOptionsWithThreads() { 69 | let recipe = self.createRecipe() 70 | var toolset = Toolset(rootPath: nil) 71 | recipe.applyPlatformOptions( 72 | toolset: &toolset, 73 | targetTriple: Triple("wasm32-unknown-wasip1-threads"), 74 | isForEmbeddedSwift: false 75 | ) 76 | XCTAssertEqual( 77 | toolset.swiftCompiler?.extraCLIOptions, 78 | [ 79 | "-static-stdlib", 80 | "-Xcc", "-matomics", 81 | "-Xcc", "-mbulk-memory", 82 | "-Xcc", "-mthread-model", 83 | "-Xcc", "posix", 84 | "-Xcc", "-pthread", 85 | "-Xcc", "-ftls-model=local-exec", 86 | ] 87 | ) 88 | 89 | let ccOptions = [ 90 | "-matomics", "-mbulk-memory", "-mthread-model", "posix", 91 | "-pthread", "-ftls-model=local-exec", 92 | ] 93 | XCTAssertEqual(toolset.cCompiler?.extraCLIOptions, ccOptions) 94 | XCTAssertEqual(toolset.cxxCompiler?.extraCLIOptions, ccOptions) 95 | XCTAssertEqual( 96 | toolset.linker?.extraCLIOptions, 97 | [ 98 | "--import-memory", "--export-memory", "--shared-memory", "--max-memory=1073741824", 99 | ] 100 | ) 101 | } 102 | } 103 | -------------------------------------------------------------------------------- /Utilities/git.commit.template: -------------------------------------------------------------------------------- 1 | One line description of your change 2 | 3 | Motivation: 4 | 5 | Explain here the context, and why you're making that change. 6 | What is the problem you're trying to solve. 7 | 8 | Modifications: 9 | 10 | Describe the modifications you've done. 11 | 12 | Result: 13 | 14 | After your change, what will change. 15 | --------------------------------------------------------------------------------