├── .devcontainer ├── Dockerfile └── devcontainer.json ├── .gitignore ├── .gitmodules ├── .swiftpm └── configuration │ └── Package.resolved ├── CODEOWNERS ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── Docs ├── Development.md ├── action_graph.md ├── index.md ├── llcastool.md └── serialization.md ├── Examples └── GameOfLife │ ├── Package.resolved │ ├── Package.swift │ ├── README.md │ └── Sources │ └── GameOfLife │ ├── BuildSystemDelegate.swift │ ├── Configuration │ └── Configuration.swift │ ├── ConwayUI.swift │ ├── Functions │ └── Generation.swift │ ├── Rules │ ├── Board.swift │ └── Cell.swift │ ├── SwiftUIApplication.swift │ └── main.swift ├── LICENSE.txt ├── Makefile ├── Package.resolved ├── Package.swift ├── Protos ├── BazelRemoteAPI │ ├── googleapis │ │ ├── LICENSE │ │ └── google │ │ │ ├── api │ │ │ ├── annotations.proto │ │ │ ├── client.proto │ │ │ ├── http.proto │ │ │ └── launch_stage.proto │ │ │ ├── bytestream │ │ │ └── bytestream.proto │ │ │ ├── longrunning │ │ │ └── operations.proto │ │ │ └── rpc │ │ │ ├── code.proto │ │ │ ├── error_details.proto │ │ │ ├── http.proto │ │ │ └── status.proto │ └── remote-apis │ │ ├── LICENSE │ │ └── build │ │ └── bazel │ │ ├── remote │ │ ├── asset │ │ │ └── v1 │ │ │ │ └── remote_asset.proto │ │ ├── execution │ │ │ └── v2 │ │ │ │ └── remote_execution.proto │ │ └── logstream │ │ │ └── v1 │ │ │ └── remote_logstream.proto │ │ └── semver │ │ └── semver.proto ├── EngineProtocol │ └── any_serializable.proto └── module_map.asciipb ├── README.md ├── Sources ├── BazelRemoteAPI │ ├── Generated │ │ ├── build │ │ │ └── bazel │ │ │ │ ├── remote │ │ │ │ ├── asset │ │ │ │ │ └── v1 │ │ │ │ │ │ ├── remote_asset.grpc.swift │ │ │ │ │ │ └── remote_asset.pb.swift │ │ │ │ ├── execution │ │ │ │ │ └── v2 │ │ │ │ │ │ ├── remote_execution.grpc.swift │ │ │ │ │ │ └── remote_execution.pb.swift │ │ │ │ └── logstream │ │ │ │ │ └── v1 │ │ │ │ │ ├── remote_logstream.grpc.swift │ │ │ │ │ └── remote_logstream.pb.swift │ │ │ │ └── semver │ │ │ │ └── semver.pb.swift │ │ └── google │ │ │ ├── api │ │ │ ├── annotations.pb.swift │ │ │ ├── client.pb.swift │ │ │ ├── http.pb.swift │ │ │ └── launch_stage.pb.swift │ │ │ ├── bytestream │ │ │ ├── bytestream.grpc.swift │ │ │ └── bytestream.pb.swift │ │ │ ├── longrunning │ │ │ ├── operations.grpc.swift │ │ │ └── operations.pb.swift │ │ │ └── rpc │ │ │ ├── code.pb.swift │ │ │ ├── error_details.pb.swift │ │ │ ├── http.pb.2.swift │ │ │ └── status.pb.swift │ └── Typealiases.swift ├── LLBBazelBackend │ └── CAS │ │ ├── BazelCASDatabase.swift │ │ └── Digest.swift ├── LLBCASTool │ ├── CASTool.swift │ └── Options.swift ├── Tools │ └── llcastool │ │ ├── CASCommands.swift │ │ ├── Capabilities.swift │ │ ├── CommonOptions.swift │ │ ├── main.swift │ │ └── misc.swift ├── llbuild2 │ └── Reexport.swift └── llbuild2fx │ ├── Action.swift │ ├── ActionCache │ ├── FileBackedFunctionCache.swift │ └── InMemoryFunctionCache.swift │ ├── Coding.swift │ ├── CommandLineArgsCoder.swift │ ├── Deadline.swift │ ├── Diagnostics.swift │ ├── Engine.swift │ ├── Environment.swift │ ├── Errors.swift │ ├── Executor.swift │ ├── FunctionCache.swift │ ├── FunctionInterface.swift │ ├── Generated │ └── EngineProtocol │ │ └── any_serializable.pb.swift │ ├── Key.swift │ ├── KeyConfiguration.swift │ ├── KeyDependencyGraph.swift │ ├── LocalExecutor.swift │ ├── NullExecutor.swift │ ├── Predicate │ ├── Expression.swift │ └── Predicate.swift │ ├── Resources.swift │ ├── Reëxport.swift │ ├── Ruleset.swift │ ├── Service.swift │ ├── SortedSet.swift │ ├── SpawnProcess.swift │ ├── Stats.swift │ ├── Support │ ├── AnySerializable.swift │ ├── CommonCodables.swift │ ├── Logging.swift │ └── Protobuf+Extensions.swift │ ├── TreeMaterialization.swift │ ├── Value.swift │ ├── Versioning.swift │ └── WrappedDataID.swift ├── Tests └── llbuild2fxTests │ ├── CommandLineArgsCoderTests.swift │ ├── DeadlineTests.swift │ ├── EngineTests.swift │ ├── ExpressionTests.swift │ ├── FunctionCacheTests.swift │ ├── KeyDependencyGraphTests.swift │ ├── KeyTests.swift │ ├── PredicateTests.swift │ ├── RequirementTests.swift │ ├── ResourcesTests.swift │ └── VersioningTests.swift └── Utilities └── build_proto_toolchain.sh /.devcontainer/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM swift:focal 2 | 3 | RUN apt-get update 4 | 5 | RUN apt-get install -y \ 6 | python \ 7 | libsqlite3-dev \ 8 | libncurses5-dev 9 | 10 | CMD ["zsh"] -------------------------------------------------------------------------------- /.devcontainer/devcontainer.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "llbuild2Container", 3 | "dockerFile": "Dockerfile", 4 | "appPort": 3503, 5 | "extensions": [], 6 | "settings": { 7 | "terminal.integrated.shell.linux": "/bin/bash" 8 | } 9 | } -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | #==============================================================================# 2 | # File extensions to be ignored anywhere in the tree. 3 | #==============================================================================# 4 | # Temp files created by most text editors. 5 | *~ 6 | # Merge files created by git. 7 | *.orig 8 | # Byte compiled python modules. 9 | *.pyc 10 | # vim swap files 11 | .*.swp 12 | .sw? 13 | # OS X specific files. 14 | .DS_Store 15 | # Notes files. 16 | notes.txt 17 | 18 | #==============================================================================# 19 | # Directories to ignore (do not add trailing '/'s, they skip symlinks). 20 | #==============================================================================# 21 | .build 22 | .swiftpm/xcode 23 | ExternalRepositories 24 | Packages 25 | Utilities/tools 26 | xcuserdata 27 | Examples/*/.build 28 | Examples/*/.swiftpm 29 | /.vscode 30 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "ThirdParty/BLAKE3"] 2 | path = ThirdParty/BLAKE3 3 | url = https://github.com/BLAKE3-team/BLAKE3.git 4 | -------------------------------------------------------------------------------- /.swiftpm/configuration/Package.resolved: -------------------------------------------------------------------------------- 1 | { 2 | "object": { 3 | "pins": [ 4 | { 5 | "package": "grpc-swift", 6 | "repositoryURL": "https://github.com/grpc/grpc-swift.git", 7 | "state": { 8 | "branch": null, 9 | "revision": "6a90b7e77e29f9bda6c2b3a4165a40d6c02cfda1", 10 | "version": "1.23.0" 11 | } 12 | }, 13 | { 14 | "package": "swift-argument-parser", 15 | "repositoryURL": "https://github.com/apple/swift-argument-parser.git", 16 | "state": { 17 | "branch": null, 18 | "revision": "41982a3656a71c768319979febd796c6fd111d5c", 19 | "version": "1.5.0" 20 | } 21 | }, 22 | { 23 | "package": "swift-atomics", 24 | "repositoryURL": "https://github.com/apple/swift-atomics.git", 25 | "state": { 26 | "branch": null, 27 | "revision": "cd142fd2f64be2100422d658e7411e39489da985", 28 | "version": "1.2.0" 29 | } 30 | }, 31 | { 32 | "package": "swift-collections", 33 | "repositoryURL": "https://github.com/apple/swift-collections.git", 34 | "state": { 35 | "branch": null, 36 | "revision": "3d2dc41a01f9e49d84f0a3925fb858bed64f702d", 37 | "version": "1.1.2" 38 | } 39 | }, 40 | { 41 | "package": "swift-crypto", 42 | "repositoryURL": "https://github.com/apple/swift-crypto.git", 43 | "state": { 44 | "branch": null, 45 | "revision": "a53a7e8f858902659d4784322bede34f4e49097e", 46 | "version": "3.6.1" 47 | } 48 | }, 49 | { 50 | "package": "swift-http-types", 51 | "repositoryURL": "https://github.com/apple/swift-http-types", 52 | "state": { 53 | "branch": null, 54 | "revision": "ae67c8178eb46944fd85e4dc6dd970e1f3ed6ccd", 55 | "version": "1.3.0" 56 | } 57 | }, 58 | { 59 | "package": "llbuild", 60 | "repositoryURL": "https://github.com/apple/swift-llbuild.git", 61 | "state": { 62 | "branch": null, 63 | "revision": "5cd4df550b31301508a77064e3dfaa5c5628780e", 64 | "version": "0.5.0" 65 | } 66 | }, 67 | { 68 | "package": "swift-log", 69 | "repositoryURL": "https://github.com/apple/swift-log.git", 70 | "state": { 71 | "branch": null, 72 | "revision": "9cb486020ebf03bfa5b5df985387a14a98744537", 73 | "version": "1.6.1" 74 | } 75 | }, 76 | { 77 | "package": "swift-nio", 78 | "repositoryURL": "https://github.com/apple/swift-nio.git", 79 | "state": { 80 | "branch": null, 81 | "revision": "4c4453b489cf76e6b3b0f300aba663eb78182fad", 82 | "version": "2.70.0" 83 | } 84 | }, 85 | { 86 | "package": "swift-nio-extras", 87 | "repositoryURL": "https://github.com/apple/swift-nio-extras.git", 88 | "state": { 89 | "branch": null, 90 | "revision": "d1ead62745cc3269e482f1c51f27608057174379", 91 | "version": "1.24.0" 92 | } 93 | }, 94 | { 95 | "package": "swift-nio-http2", 96 | "repositoryURL": "https://github.com/apple/swift-nio-http2.git", 97 | "state": { 98 | "branch": null, 99 | "revision": "b5f7062b60e4add1e8c343ba4eb8da2e324b3a94", 100 | "version": "1.34.0" 101 | } 102 | }, 103 | { 104 | "package": "swift-nio-ssl", 105 | "repositoryURL": "https://github.com/apple/swift-nio-ssl.git", 106 | "state": { 107 | "branch": null, 108 | "revision": "7b84abbdcef69cc3be6573ac12440220789dcd69", 109 | "version": "2.27.2" 110 | } 111 | }, 112 | { 113 | "package": "swift-nio-transport-services", 114 | "repositoryURL": "https://github.com/apple/swift-nio-transport-services.git", 115 | "state": { 116 | "branch": null, 117 | "revision": "38ac8221dd20674682148d6451367f89c2652980", 118 | "version": "1.21.0" 119 | } 120 | }, 121 | { 122 | "package": "SwiftProtobuf", 123 | "repositoryURL": "https://github.com/apple/swift-protobuf.git", 124 | "state": { 125 | "branch": null, 126 | "revision": "e17d61f26df0f0e06f58f6977ba05a097a720106", 127 | "version": "1.27.1" 128 | } 129 | }, 130 | { 131 | "package": "swift-system", 132 | "repositoryURL": "https://github.com/apple/swift-system.git", 133 | "state": { 134 | "branch": null, 135 | "revision": "d2ba781702a1d8285419c15ee62fd734a9437ff5", 136 | "version": "1.3.2" 137 | } 138 | }, 139 | { 140 | "package": "swift-tools-support-async", 141 | "repositoryURL": "https://github.com/apple/swift-tools-support-async.git", 142 | "state": { 143 | "branch": null, 144 | "revision": "611f98e85f986ad4bad2cfbc1f4a7537613e680c", 145 | "version": "0.11.1" 146 | } 147 | }, 148 | { 149 | "package": "swift-tools-support-core", 150 | "repositoryURL": "https://github.com/apple/swift-tools-support-core.git", 151 | "state": { 152 | "branch": null, 153 | "revision": "62ba237eda0f7ef15264ef3468ae4cda1866b7c8", 154 | "version": "0.5.3" 155 | } 156 | } 157 | ] 158 | }, 159 | "version": 1 160 | } 161 | -------------------------------------------------------------------------------- /CODEOWNERS: -------------------------------------------------------------------------------- 1 | # This file is a list of the people responsible for ensuring that patches for a 2 | # particular part of Swift are reviewed, either by themself or by someone else. 3 | # They are also the gatekeepers for their part of Swift, with the final word on 4 | # what goes in or not. 5 | 6 | # The list is sorted by surname and formatted to allow easy grepping and 7 | # beautification by scripts. The fields are: name (N), email (E), web-address 8 | # (W), PGP key ID and fingerprint (P), description (D), and snail-mail address 9 | # (S). 10 | 11 | # N: David M. Bryson 12 | # E: dmbryson@apple.com 13 | # D: Everything in llbuild2 14 | 15 | ### 16 | 17 | # The following lines are used by GitHub to automatically recommend reviewers. 18 | 19 | * @dmbryson 20 | 21 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Code of Conduct 2 | To be a truly great community, Swift.org needs to welcome developers from all walks of life, 3 | with different backgrounds, and with a wide range of experience. A diverse and friendly 4 | community will have more great ideas, more unique perspectives, and produce more great 5 | code. We will work diligently to make the Swift community welcoming to everyone. 6 | 7 | To give clarity of what is expected of our members, Swift.org has adopted the code of conduct 8 | defined by [contributor-covenant.org](https://www.contributor-covenant.org). This document is used across many open source 9 | communities, and we think it articulates our values well. The full text is copied below: 10 | 11 | ### Contributor Code of Conduct v1.3 12 | As contributors and maintainers of this project, and in the interest of fostering an open and 13 | welcoming community, we pledge to respect all people who contribute through reporting 14 | issues, posting feature requests, updating documentation, submitting pull requests or patches, 15 | and other activities. 16 | 17 | We are committed to making participation in this project a harassment-free experience for 18 | everyone, regardless of level of experience, gender, gender identity and expression, sexual 19 | orientation, disability, personal appearance, body size, race, ethnicity, age, religion, or 20 | nationality. 21 | 22 | Examples of unacceptable behavior by participants include: 23 | - The use of sexualized language or imagery 24 | - Personal attacks 25 | - Trolling or insulting/derogatory comments 26 | - Public or private harassment 27 | - Publishing other’s private information, such as physical or electronic addresses, without explicit permission 28 | - Other unethical or unprofessional conduct 29 | 30 | Project maintainers have the right and responsibility to remove, edit, or reject comments, 31 | commits, code, wiki edits, issues, and other contributions that are not aligned to this Code of 32 | Conduct, or to ban temporarily or permanently any contributor for other behaviors that they 33 | deem inappropriate, threatening, offensive, or harmful. 34 | 35 | By adopting this Code of Conduct, project maintainers commit themselves to fairly and 36 | consistently applying these principles to every aspect of managing this project. Project 37 | maintainers who do not follow or enforce the Code of Conduct may be permanently removed 38 | from the project team. 39 | 40 | This code of conduct applies both within project spaces and in public spaces when an 41 | individual is representing the project or its community. 42 | 43 | Instances of abusive, harassing, or otherwise unacceptable behavior may be reported by 44 | contacting a project maintainer at [conduct@swift.org](mailto:conduct@swift.org). All complaints will be reviewed and 45 | investigated and will result in a response that is deemed necessary and appropriate to the 46 | circumstances. Maintainers are obligated to maintain confidentiality with regard to the reporter 47 | of an incident. 48 | 49 | *This policy is adapted from the Contributor Code of Conduct [version 1.3.0](http://contributor-covenant.org/version/1/3/0/).* 50 | 51 | ### Reporting 52 | A working group of community members is committed to promptly addressing any [reported 53 | issues](mailto:conduct@swift.org). Working group members are volunteers appointed by the project lead, with a 54 | preference for individuals with varied backgrounds and perspectives. Membership is expected 55 | to change regularly, and may grow or shrink. 56 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | By submitting a pull request, you represent that you have the right to license 2 | your contribution to Apple and the community, and agree by submitting the patch 3 | that your contributions are licensed under the [Swift 4 | license](https://swift.org/LICENSE.txt). 5 | 6 | --- 7 | 8 | Before submitting the pull request, please make sure you have tested your 9 | changes and that they follow the Swift project [guidelines for contributing 10 | code](https://swift.org/contributing/#contributing-code). 11 | -------------------------------------------------------------------------------- /Docs/Development.md: -------------------------------------------------------------------------------- 1 | # Development 2 | 3 | This document contains information on how to develop `llbuild2`. 4 | 5 | # RE2 Server 6 | 7 | `llbuild2` can perform execution on build servers that implement [Bazel's RE2 APIs](https://github.com/bazelbuild/remote-apis). There are [many](https://github.com/bazelbuild/remote-apis#servers) OSS build servers that you can stand up for development. 8 | 9 | [Buildbarn](https://github.com/buildbarn) is one such build server which can be run locally with Docker. Follow the instructions on [this](https://github.com/buildbarn/bb-deployments#recommended-setup) page to setup your server for development and then test the connection using [`retool`](./retool.md): 10 | 11 | ```sh 12 | $ swift run llcastool capabilities --url bazel://localhost:8980/remote-execution 13 | 14 | BazelRemoteAPI.Build_Bazel_Remote_Execution_V2_ServerCapabilities: 15 | cache_capabilities { 16 | digest_function: [MD5, SHA1, SHA256, SHA384, SHA512] 17 | action_cache_update_capabilities { 18 | } 19 | symlink_absolute_path_strategy: ALLOWED 20 | } 21 | execution_capabilities { 22 | digest_function: SHA256 23 | exec_enabled: true 24 | execution_priority_capabilities { 25 | priorities { 26 | min_priority: -2147483648 27 | max_priority: 2147483647 28 | } 29 | } 30 | } 31 | low_api_version { 32 | major: 2 33 | } 34 | high_api_version { 35 | major: 2 36 | } 37 | ``` 38 | -------------------------------------------------------------------------------- /Docs/action_graph.md: -------------------------------------------------------------------------------- 1 | # llbuild2's Action Graph 2 | 3 | This document explains how the action graph is modeled in llbuild2. There are 2 4 | important pieces in the action graph, `Artifact`s and `ActionKey`s. 5 | 6 | ## `Artifact` 7 | 8 | Artifacts represent a handle to a file system entity expected to be produced, 9 | consumed, or both, during the execution of a build. In llbuild2, there are 2 10 | main categories for artifacts: source artifacts and derived artifacts. 11 | 12 | Source artifacts refer to the artifacts that are inputs into the build, and may 13 | be any kind of file system entity that is part of your project. Some examples 14 | for source artifacts include source code, like `.swift` files; resources, like 15 | `xcassets` bundles; or in general, any kind of entity that would be checked in 16 | into a git repository that can't be derived in any way. Source artifacts in 17 | llbuild2 are represented by the CAS data ID as returned by a CAS database. This 18 | data ID acts as both a uniqueness identifier (2 artifacts with the same dataID 19 | contain the same contents) and as a handle to retrieve the data from the 20 | database (`database.get(dataID) -> data`). 21 | 22 | Derived artifacts on the other hand, are artifacts that are produced during the 23 | build, which may be the final artifacts that you expect from the build (like a 24 | compiled and linked executable) or intermediate artifacts that are not as useful 25 | by themselves, but are combined into other derived artifacts to produce more 26 | useful results. In llbuild2, most derived artifacts are represented by the 27 | dataID of the producing entity, like an ActionKey. 28 | 29 | ## `ActionKey` 30 | 31 | Action keys represent the transformation of an arbitrary set of artifacts inputs 32 | into a new set of artifact outputs. llbuild2 currently supports command line 33 | based action keys, where the transformation is effectively the execution of a 34 | command line invocation on some execution environment, but action keys are 35 | designed in an extensible manner to allow other types of transformations. 36 | 37 | ## Action Graph 38 | 39 | In llbuild2, there are no specialized graph data structures to manage the action 40 | graph, instead, it is the relationships between `Artifact` and `ActionKey` that 41 | make up an implicit action graph. Because artifacts and action keys are 42 | serialized into CAS databases, the scalability of the action graph is determined 43 | by the available CAS database storage. 44 | 45 | Take for example the following action graph: 46 | 47 | ``` 48 | +------------+ 49 | | executable | 50 | +------------+ 51 | | 52 | +--------------+ 53 | | Link Action | 54 | +--------------+ 55 | | | 56 | +--------+ +--------+ 57 | | main.o | | shrd.o | 58 | +--------+ +--------+ 59 | | | 60 | +----------+ +----------+ 61 | | Compiler | | Compiler | 62 | +----------+ +----------+ 63 | | | 64 | +--------+ +--------+ 65 | | main.c | | shrd.c | 66 | +--------+ +--------+ 67 | ``` 68 | 69 | In llbuild2, this action graph would be represented as: 70 | 71 | ``` 72 | +------------+ 73 | | Artifact | .derived(0~abc...) 74 | +------------+ 75 | | 76 | +--------------+ 77 | | Action Key | 0~abc... 78 | +--------------+ 79 | | | 80 | +----------+ +----------+ 81 | .derived(0~xnu..., 0) | Artifact | | Artifact | .derived(0~blm..., 0) 82 | +----------+ +----------+ 83 | | | 84 | +------------+ +------------+ 85 | 0~xnu... | Action Key | | Action Key | 0~blm... 86 | +------------+ +------------+ 87 | | | 88 | +----------+ +----------+ 89 | .source(0~llb...) | Artifact | | Artifact | .source(0~ahb) 90 | +----------+ +----------+ 91 | ``` 92 | 93 | Where the derived artifacts have a data ID pointer to the action key that 94 | produces the artifact, and the action keys have artifact pointers to the input 95 | artifacts. It is interesting to notice that since the graph references are data 96 | IDs, which represent a digest of the contents of the graphs' nodes, the action 97 | graph actually represents a Merkle tree. 98 | 99 | If for example main.c has changed, then it's data ID would necessarily be 100 | different, and that would result in a completely new action graph with a new 101 | root. But since the sub-graph that depends on shrd.c hasn't changed, that 102 | sub-graph would be shared among the old and the new action graphs. This allows 103 | for minimizing the CAS storage usage since we only need to store the pieces that 104 | have changed, and not a completely new action graph. 105 | 106 | ## Action execution 107 | 108 | There is another aspect of the action graph which is its computation in order 109 | to build the requested artifact. Since any source change invalidates all of the 110 | ActionKeys that transitively depend on it, that would mean that the ActionKeys 111 | for downstream dependents would also be evaluated during a build. In order to 112 | avoid reëxecuting actions excessively, the `ActionFunction` requests the 113 | evaluation of all the inputs for the action, and constructs an 114 | `ActionExecutionKey`, in which the inputs are specified as the actual data ID 115 | for the contents of the artifact. 116 | 117 | At this stage, llbuild2 can check its cache to check whether that particular 118 | `ActionExecutionKey` has been evaluated before, and return the cached results if 119 | so. This architecture allows llbuild2 to avoid excessive recomputations of the 120 | inputs for an action haven't changed. 121 | 122 | As an example, if we modify `main.c` above to add a comment, the newly compiled 123 | `main.o` would be no different from the previosly compiled `main.o` (assuming 124 | the compilation action is deterministic). That would mean that the 125 | `ActionExecutionKey`s being evaluated downstream would effectively be cached, 126 | resulting in faster builds. -------------------------------------------------------------------------------- /Docs/index.md: -------------------------------------------------------------------------------- 1 | # Overview 2 | 3 | ## What is llbuild2? 4 | 5 | llbuild2 is a functional, artifact-based, and extensible build system framework designed for large scale projects, 6 | focusing on maximizing the reusability of past evaluations. llbuild2 provides general abstractions related to build 7 | system problems and is designed with extensibility in mind. While llbuild2 by itself is not a build system in itself, 8 | you can build (hah) custom build systems with it. 9 | 10 | ### Functional build systems 11 | 12 | Functional systems are characterized by enforcing that the results (values) of evaluations (functions) are only affected 13 | by the declared inputs (keys). Systems defined in such way can be smarter about the way evaluations are scheduled and 14 | cached. Some of the benefits of functional systems include: 15 | 16 | * If an evaluation is only affected by the input key, then the result can be memoized (cached) for future requests of 17 | the same key. 18 | * If the value provided by an evaluation is not used as input to any other evaluation, then it can be skipped. 19 | * If there are no dependencies between evaluations, they can be reordered and/or evaluated in parallel. 20 | 21 | It is easy to see how this functional thinking maps into the actions executed as part of a build. If the outputs of an 22 | action are only affected by the action's inputs (i.e. command-line arguments, environment variables, input artifacts), 23 | then the build system can cache the results of that action for future requests of that same action. 24 | 25 | This functional architecture can also be applied to the build graph itself, by considering the action graph as the value 26 | of an evaluation where the key is the project sources plus the requested configuration. By modeling the different pieces 27 | required to represent a build graph in their most granular versions, the construction of the action graph can also be 28 | cached. 29 | 30 | ### Build phases 31 | 32 | llbuild2 is architected around the 2 phases that happen in any build system implementation: 33 | 34 | 1. Evaluation: During this phase, project description files are parsed and evaluated to construct an action 35 | graph. 36 | 1. Execution: The action graph is evaluated in a way where only the actions affecting the requested outputs are 37 | executed. The ordering of these actions is implicit in the relationships between input and output artifacts. 38 | 39 | ### CAS usage 40 | 41 | llbuild2 makes heavy use of CAS (Content Addressable Storage) technologies. With CAS, data, file and directory 42 | structures can be represented and accessed by the digest (or hash) of its contents. Using CAS identifiers, llbuild2 can 43 | detect when changes have occurred to any portion of the build graph, and only reevaluate the pieces that have never been 44 | evaluated before. 45 | 46 | With shared CAS services, it's even possible to reüse evaluation results across different development or CI machines. 47 | 48 | ### Remote execution 49 | 50 | llbuild2 provides data structures that enforce that action specifications are completely defined. This allows clients of 51 | llbuild2 to implement any kind of execution engine to power the action graph resolution. Through these interfaces, it 52 | would be possible to support common remote execution APIs such as Bazel's RE2 protocol. 53 | -------------------------------------------------------------------------------- /Docs/llcastool.md: -------------------------------------------------------------------------------- 1 | # `llcastool` - CAS Database manipulation tool 2 | 3 | `llcastool` is a low-level helper tool for interacting with the cas databases. 4 | 5 | ## capabilities 6 | 7 | Dump server capabilities. This command is also useful for testing connection with the remote server. 8 | 9 | ```sh 10 | $ llcastool capabilities --url bazel://localhost:8980/remote-execution 11 | ``` 12 | 13 | ## put/get 14 | 15 | The put/get a single file into or out of the CAS database specified by the URL. 16 | 17 | For example, to store a file in a file-backed CAS database and then retreive it using its CAS data id. 18 | 19 | ```sh 20 | $ cd "$(mktemp -d /tmp/llb2_XXXXXX)" 21 | $ echo foo > foo.txt 22 | 23 | # Put the file in the database. 24 | $ llcastool put --url file://$PWD/cas foo.txt 25 | 0~SdyHDfHef9YHlM685En1zNrlda_6pnoktirLA-A525I= 26 | 27 | # Retreive the previously stored file from the database. 28 | $ llcastool get --url file://$PWD/cas --id 0~SdyHDfHef9YHlM685En1zNrlda_6pnoktirLA-A525I= bar.txt 29 | $ cat bar.txt 30 | foo 31 | ``` 32 | -------------------------------------------------------------------------------- /Docs/serialization.md: -------------------------------------------------------------------------------- 1 | # Serialization 2 | 3 | llbuild2 makes heavy use of serialization for storing keys and values in the CAS. For this purpose, we're making use of 4 | SwiftProtobuf for a couple of reasons: 5 | 6 | 1. Data structure definitions are defined externally to the code. This makes it easier to navigate the codebase understanding 7 | the basic building blocks for the build system, leaving Swift code to only encode the logic en how the data structures are processed. 8 | 1. Support for polymorphic codables through `google.protobuf.Any`. llbuild2 is designed around extensibility, so there are places where we'd like to encode client data without knowing what the types are, but with the flexibility to access runtime information for inspection, which is available through `SwiftProtobuf.Google_Protobuf_Any`'s static type registry. 9 | 10 | There are downsides to using protocol buffers though, such as: 11 | 12 | * The protocol buffers specification requires that fields have default values, which doesn't play well with Swift Optional support. For message types, the fields will generate some `hasX` properties that can be used for this purpose. 13 | -------------------------------------------------------------------------------- /Examples/GameOfLife/Package.resolved: -------------------------------------------------------------------------------- 1 | { 2 | "object": { 3 | "pins": [ 4 | { 5 | "package": "grpc-swift", 6 | "repositoryURL": "https://github.com/grpc/grpc-swift.git", 7 | "state": { 8 | "branch": null, 9 | "revision": "9e464a75079928366aa7041769a271fac89271bf", 10 | "version": "1.0.0" 11 | } 12 | }, 13 | { 14 | "package": "swift-argument-parser", 15 | "repositoryURL": "https://github.com/apple/swift-argument-parser.git", 16 | "state": { 17 | "branch": null, 18 | "revision": "9564d61b08a5335ae0a36f789a7d71493eacadfc", 19 | "version": "0.3.2" 20 | } 21 | }, 22 | { 23 | "package": "swift-crypto", 24 | "repositoryURL": "https://github.com/apple/swift-crypto.git", 25 | "state": { 26 | "branch": null, 27 | "revision": "296d3308b4b2fa355cfe0de4ca411bf7a1cd8cf8", 28 | "version": "1.1.4" 29 | } 30 | }, 31 | { 32 | "package": "llbuild", 33 | "repositoryURL": "https://github.com/apple/swift-llbuild.git", 34 | "state": { 35 | "branch": null, 36 | "revision": "53d12bc6e70e4ae6c0dec634fd48ec0cfff17f2d", 37 | "version": "0.4.0" 38 | } 39 | }, 40 | { 41 | "package": "swift-log", 42 | "repositoryURL": "https://github.com/apple/swift-log.git", 43 | "state": { 44 | "branch": null, 45 | "revision": "12d3a8651d32295794a850307f77407f95b8c881", 46 | "version": "1.4.1" 47 | } 48 | }, 49 | { 50 | "package": "swift-nio", 51 | "repositoryURL": "https://github.com/apple/swift-nio.git", 52 | "state": { 53 | "branch": null, 54 | "revision": "6d3ca7e54e06a69d0f2612c2ce8bb8b7319085a4", 55 | "version": "2.26.0" 56 | } 57 | }, 58 | { 59 | "package": "swift-nio-extras", 60 | "repositoryURL": "https://github.com/apple/swift-nio-extras.git", 61 | "state": { 62 | "branch": null, 63 | "revision": "de1c80ad1fdff1ba772bcef6b392c3ef735f39a6", 64 | "version": "1.8.0" 65 | } 66 | }, 67 | { 68 | "package": "swift-nio-http2", 69 | "repositoryURL": "https://github.com/apple/swift-nio-http2.git", 70 | "state": { 71 | "branch": null, 72 | "revision": "f4736a3b78a2bbe3feb7fc0f33f6683a8c27974c", 73 | "version": "1.16.3" 74 | } 75 | }, 76 | { 77 | "package": "swift-nio-ssl", 78 | "repositoryURL": "https://github.com/apple/swift-nio-ssl.git", 79 | "state": { 80 | "branch": null, 81 | "revision": "4c933e955b8797f5a5a90bd2a0fb411fdb11bb94", 82 | "version": "2.10.3" 83 | } 84 | }, 85 | { 86 | "package": "swift-nio-transport-services", 87 | "repositoryURL": "https://github.com/apple/swift-nio-transport-services.git", 88 | "state": { 89 | "branch": null, 90 | "revision": "1d28d48e071727f4558a8a4bb1894472abc47a58", 91 | "version": "1.9.2" 92 | } 93 | }, 94 | { 95 | "package": "SwiftProtobuf", 96 | "repositoryURL": "https://github.com/apple/swift-protobuf.git", 97 | "state": { 98 | "branch": null, 99 | "revision": "e1904bf5a5f79cb7e0ff68a427a53a93b652fcd1", 100 | "version": "1.15.0" 101 | } 102 | }, 103 | { 104 | "package": "swift-tools-support-async", 105 | "repositoryURL": "https://github.com/apple/swift-tools-support-async.git", 106 | "state": { 107 | "branch": null, 108 | "revision": "6cceee235f16ce5d732ebeb4c188463ba1cd0ec3", 109 | "version": "0.1.9" 110 | } 111 | }, 112 | { 113 | "package": "swift-tools-support-core", 114 | "repositoryURL": "https://github.com/apple/swift-tools-support-core.git", 115 | "state": { 116 | "branch": null, 117 | "revision": "2954e55faee5bfee928e844bb09e97fcfa8d24af", 118 | "version": "0.2.0" 119 | } 120 | } 121 | ] 122 | }, 123 | "version": 1 124 | } 125 | -------------------------------------------------------------------------------- /Examples/GameOfLife/Package.swift: -------------------------------------------------------------------------------- 1 | // swift-tools-version:5.1 2 | // The swift-tools-version declares the minimum version of Swift required to build this package. 3 | 4 | import PackageDescription 5 | 6 | let package = Package( 7 | name: "GameOfLife", 8 | platforms: [ 9 | .macOS(.v10_15) 10 | ], 11 | products: [ 12 | .executable( 13 | name: "game_of_life", 14 | targets: ["GameOfLife"]), 15 | ], 16 | dependencies: [ 17 | .package(path: "../.."), 18 | ], 19 | targets: [ 20 | .target( 21 | name: "GameOfLife", 22 | dependencies: ["llbuild2BuildSystem", "llbuild2Util"]), 23 | ] 24 | ) 25 | -------------------------------------------------------------------------------- /Examples/GameOfLife/README.md: -------------------------------------------------------------------------------- 1 | # GameOfLife 2 | 3 | An implementation of Conway's Game of Life using llbuild2's Build System 4 | component. 5 | 6 | To run, just open the project in Xcode and hit run. Alternatively, 7 | `cd Examples/GameOfLife; swift run` should also work. This should open a SwiftUI 8 | based application with a view of the board. 9 | 10 | At generation 0, you can edit which cells are alive or dead by clicking on them, 11 | and evolving the board will evaluate the dependency graph of cells to produce 12 | the board state at the next generation. 13 | 14 | This implementation of Conway's Game of Life is based on declaring cell 15 | dependencies between surrounding cells at different generations, where the 16 | computation for the next cell state is done using a bash command line 17 | invocation. 18 | 19 | ## Notes 20 | 21 | * There might be a bug in the blake3 implementation when building game_of_life 22 | with the `swift` command line tool which results in a segmentation fault. If it 23 | happens to you, please use Xcode to open the project instead, since that 24 | should avoid the issue. 25 | -------------------------------------------------------------------------------- /Examples/GameOfLife/Sources/GameOfLife/BuildSystemDelegate.swift: -------------------------------------------------------------------------------- 1 | // This source file is part of the Swift.org open source project 2 | // 3 | // Copyright (c) 2020 Apple Inc. and the Swift project authors 4 | // Licensed under Apache License v2.0 with Runtime Library Exception 5 | // 6 | // See http://swift.org/LICENSE.txt for license information 7 | // See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 8 | 9 | import LLBBuildSystem 10 | import llbuild2 11 | 12 | enum GameOfLifeConfiguredTargetError: Error { 13 | case notFound 14 | } 15 | 16 | /// Implementation of the LLBBuildEngine delegates for extending the engine with 17 | /// our custom functions, rules and targets. 18 | class GameOfLifeBuildSystemDelegate { 19 | /// Registry of available rules in the GameOfLife build system. This means 20 | /// that CellTarget targets are evaluated using the CellRule implementation. 21 | /// Same for the BoardTarget and BoardRule. Rules are used mostly for 22 | /// processing artifact related computations, since it has access to APIs for 23 | /// managing inputs, outputs and action registration. 24 | let rules: [String: LLBRule] = [ 25 | CellTarget.identifier: CellRule(), 26 | BoardTarget.identifier: BoardRule(), 27 | ] 28 | 29 | /// Registry of key identifiers to the functions that evaluate them. 30 | /// Functions are used to access the raw llbuild2 engine capabilities for 31 | /// implementing generic functional computations that are not necesarily 32 | /// artifact related. 33 | let functions: [LLBBuildKeyIdentifier: LLBFunction] = [ 34 | GameOfLifeConfigurationKey.identifier: GameOfLifeConfigurationFunction(), 35 | GenerationKey.identifier: GenerationFunction(), 36 | ] 37 | } 38 | 39 | extension GameOfLifeBuildSystemDelegate: LLBConfiguredTargetDelegate { 40 | func configuredTarget( 41 | for key: LLBConfiguredTargetKey, 42 | _ fi: LLBBuildFunctionInterface, 43 | _ ctx: Context 44 | ) throws -> LLBFuture { 45 | let label = key.label 46 | 47 | if label.logicalPathComponents[0] == "cell" { 48 | // Cell targets are identified using the `//cell/:-` scheme. 49 | return try CellTarget.with(key: key, fi, ctx).map { $0 } 50 | } else if label.logicalPathComponents[0] == "board" { 51 | // Board targets are identified using the `//board:` scheme. 52 | return try BoardTarget.with(key: key, fi, ctx).map { $0 } 53 | } 54 | 55 | // Only cell and board targets are supported. 56 | throw GameOfLifeConfiguredTargetError.notFound 57 | } 58 | } 59 | 60 | extension GameOfLifeBuildSystemDelegate: LLBRuleLookupDelegate { 61 | func rule(for configuredTargetType: LLBConfiguredTarget.Type) -> LLBRule? { 62 | return rules[configuredTargetType.identifier] 63 | } 64 | } 65 | 66 | extension GameOfLifeBuildSystemDelegate: LLBBuildFunctionLookupDelegate { 67 | func lookupBuildFunction(for identifier: LLBBuildKeyIdentifier) -> LLBFunction? { 68 | return functions[identifier] 69 | } 70 | } 71 | 72 | extension GameOfLifeBuildSystemDelegate: LLBSerializableRegistrationDelegate { 73 | func registerTypes(registry: LLBSerializableRegistry) { 74 | registry.register(type: GameOfLifeConfigurationKey.self) 75 | registry.register(type: GameOfLifeConfigurationFragment.self) 76 | registry.register(type: CellTarget.self) 77 | registry.register(type: BoardTarget.self) 78 | } 79 | } 80 | -------------------------------------------------------------------------------- /Examples/GameOfLife/Sources/GameOfLife/Configuration/Configuration.swift: -------------------------------------------------------------------------------- 1 | // This source file is part of the Swift.org open source project 2 | // 3 | // Copyright (c) 2020 Apple Inc. and the Swift project authors 4 | // Licensed under Apache License v2.0 with Runtime Library Exception 5 | // 6 | // See http://swift.org/LICENSE.txt for license information 7 | // See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 8 | 9 | import LLBBuildSystem 10 | import Foundation 11 | import llbuild2 12 | 13 | /// A simple structure containing the initial state for the board at generation 0. 14 | struct InitialBoard: Codable, Hashable { 15 | let points: [Point] 16 | 17 | func isAlive(_ point: Point) -> Bool { 18 | return Set(points).contains(point) 19 | } 20 | } 21 | 22 | /// The configuration key represents the minimum amount of data needed to 23 | /// construct a full configuration fragment. For GameOfLife purpose, we just need 24 | /// to store the initial board state and the size of the board. All cell and 25 | /// board targets at any generation derived from to this initial state. 26 | struct GameOfLifeConfigurationKey: LLBConfigurationFragmentKey, Codable, Hashable { 27 | static let identifier = String(describing: Self.self) 28 | 29 | let initialBoard: InitialBoard 30 | let size: Point 31 | 32 | init(initialBoard: InitialBoard, size: Point) { 33 | self.initialBoard = initialBoard 34 | self.size = size 35 | } 36 | } 37 | 38 | /// The configuration fragment for the configuration key. For GameOfLife, since 39 | /// the key is mostly static data, the fragment just copies the values from the 40 | /// keys. 41 | struct GameOfLifeConfigurationFragment: LLBConfigurationFragment, Codable { 42 | let initialBoard: InitialBoard 43 | let size: Point 44 | 45 | init(initialBoard: InitialBoard, size: Point) { 46 | self.initialBoard = initialBoard 47 | self.size = size 48 | } 49 | 50 | /// Convenience constructor for the fragment from the key. 51 | static func fromKey(_ key: GameOfLifeConfigurationKey) -> GameOfLifeConfigurationFragment { 52 | return Self.init(initialBoard: key.initialBoard, size: key.size) 53 | } 54 | } 55 | 56 | /// The configuration key to configuration fragment function. Since the key has 57 | /// mostly static data, the function just copies the results into the fragment. 58 | class GameOfLifeConfigurationFunction: LLBBuildFunction { 59 | override func evaluate(key: GameOfLifeConfigurationKey, _ fi: LLBBuildFunctionInterface, _ ctx: Context) -> LLBFuture { 60 | return ctx.group.next().makeSucceededFuture(.fromKey(key)) 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /Examples/GameOfLife/Sources/GameOfLife/ConwayUI.swift: -------------------------------------------------------------------------------- 1 | // This source file is part of the Swift.org open source project 2 | // 3 | // Copyright (c) 2020 Apple Inc. and the Swift project authors 4 | // Licensed under Apache License v2.0 with Runtime Library Exception 5 | // 6 | // See http://swift.org/LICENSE.txt for license information 7 | // See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 8 | 9 | import SwiftUI 10 | import LLBBuildSystem 11 | import Dispatch 12 | import llbuild2 13 | 14 | /// Environment object with a reference to the engine and database. 15 | class GameOfLifeEnvironment: ObservableObject { 16 | let engine: LLBBuildEngine 17 | 18 | let ctx: Context 19 | 20 | /// Storage of the initial board to use when constructing the GenerationKey for each generation. 21 | var initialBoard: InitialBoard? = nil 22 | 23 | init(engine: LLBBuildEngine, _ ctx: Context) { 24 | self.engine = engine 25 | self.ctx = ctx 26 | } 27 | } 28 | 29 | struct GameOfLifeView: View { 30 | @EnvironmentObject 31 | var environment: GameOfLifeEnvironment 32 | 33 | @State 34 | var boardState = Set() 35 | 36 | @State 37 | var updateTime = 0.0 38 | 39 | @State 40 | var generation = 0 41 | 42 | @State 43 | var gridSize = 10 44 | 45 | @State 46 | var loading = false 47 | 48 | var body: some View { 49 | VStack(spacing: 0) { 50 | HStack { 51 | Text("Generation \(self.generation)\(self.boardState.isEmpty ? " (No Life)" : "")") 52 | .padding() 53 | Text(self.generation == 0 ? "(Edit Mode)" : "") 54 | } 55 | ForEach(0.. 0 { 106 | generation -= 1 107 | } 108 | if generation == 0 { 109 | // If devolving to generation 0, restore the initial state. 110 | boardState = Set(environment.initialBoard!.points) 111 | } else { 112 | updateBoard() 113 | } 114 | } 115 | 116 | private func updateBoard() { 117 | // Construct the GenerationKey to request to the build engine. 118 | let generationKey = GenerationKey( 119 | initialBoard: environment.initialBoard!, 120 | size: Point(gridSize, gridSize), 121 | generation: generation 122 | ) 123 | 124 | self.loading = true 125 | DispatchQueue.global(qos: .userInteractive).async { [self] in 126 | defer { 127 | DispatchQueue.main.async { 128 | self.loading = false 129 | } 130 | } 131 | 132 | let generationValue: Set 133 | let elapsedS: Double 134 | do { 135 | let start = DispatchTime.now() 136 | generationValue = try environment.engine.build(generationKey, environment.ctx).flatMap { (value: GenerationValue) in 137 | // Read the output data from the CAS. 138 | return LLBCASFSClient(environment.ctx.db).load(value.boardID, environment.ctx).flatMap { node in 139 | return node.blob!.read(environment.ctx).map { Data($0) } 140 | } 141 | }.map { (data: Data) in 142 | // Process the data as a 0s and 1s matrix where 1s signal alive cells, so add them to the 143 | // boardState. 144 | let boardString = String(data: data, encoding: .utf8)! 145 | let lines = boardString.split(separator: "\n") 146 | var boardState = Set() 147 | for y in 0 ..< self.gridSize { 148 | for x in 0 ..< self.gridSize { 149 | if Array(lines[y])[x] == "1" { 150 | boardState.insert(Point(x, y)) 151 | } 152 | } 153 | } 154 | return boardState 155 | }.wait() 156 | let finish = DispatchTime.now() 157 | elapsedS = Double(finish.uptimeNanoseconds - start.uptimeNanoseconds) / 1_000_000_000 158 | 159 | DispatchQueue.main.async { 160 | self.boardState = generationValue 161 | self.updateTime = elapsedS 162 | } 163 | } catch { 164 | print("Error evaluating: \(error)") 165 | } 166 | } 167 | } 168 | } 169 | -------------------------------------------------------------------------------- /Examples/GameOfLife/Sources/GameOfLife/Functions/Generation.swift: -------------------------------------------------------------------------------- 1 | // This source file is part of the Swift.org open source project 2 | // 3 | // Copyright (c) 2020 Apple Inc. and the Swift project authors 4 | // Licensed under Apache License v2.0 with Runtime Library Exception 5 | // 6 | // See http://swift.org/LICENSE.txt for license information 7 | // See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 8 | 9 | import llbuild2 10 | import LLBBuildSystem 11 | 12 | /// GenerationKey represents the request for a board at a particular generation 13 | /// for a given board size and initial state. 14 | struct GenerationKey: LLBBuildKey, Codable, Hashable { 15 | let initialBoard: InitialBoard 16 | let size: Point 17 | let generation: Int 18 | 19 | init(initialBoard: InitialBoard, size: Point, generation: Int) { 20 | self.initialBoard = initialBoard 21 | self.size = size 22 | self.generation = generation 23 | } 24 | } 25 | 26 | /// GenerationValue corresponds to the result of evaluating a GenerationKey, and 27 | /// contains a data ID pointer to the data that represents the board for the 28 | /// specified generation key. 29 | struct GenerationValue: LLBBuildValue, Codable { 30 | let boardID: LLBDataID 31 | 32 | init(boardID: LLBDataID) { 33 | self.boardID = boardID 34 | } 35 | } 36 | 37 | class GenerationFunction: LLBBuildFunction { 38 | override func evaluate(key: GenerationKey, _ fi: LLBBuildFunctionInterface, _ ctx: Context) -> LLBFuture { 39 | do { 40 | // Construct the GameOfLife configuration key with the initial board and board size. 41 | let configurationKey = try LLBConfigurationKey( 42 | fragmentKeys: [GameOfLifeConfigurationKey(initialBoard: key.initialBoard, size: key.size)] 43 | ) 44 | 45 | // Request the BoardTarget for the specified generation, with the configuration key. 46 | let configuredTargetKey = LLBConfiguredTargetKey( 47 | rootID: LLBDataID(), 48 | label: try LLBLabel("//board:\(key.generation)"), 49 | configurationKey: configurationKey 50 | ) 51 | 52 | // Request the evaluation of the board target and retrieve the 53 | // BoardProvider's board artifact. 54 | return fi.requestDependency(configuredTargetKey, ctx).flatMap { providerMap in 55 | do { 56 | let artifact = try providerMap.get(BoardProvider.self).board 57 | // Request the board artifact to evaluate and trigger action execution. 58 | return fi.requestArtifact(artifact, ctx) 59 | } catch { 60 | return ctx.group.next().makeFailedFuture(error) 61 | } 62 | }.map { (artifactValue: LLBArtifactValue) in 63 | // With the data ID for the artifact, return the GenerationValue. 64 | return GenerationValue(boardID: artifactValue.dataID) 65 | } 66 | } catch { 67 | return ctx.group.next().makeFailedFuture(error) 68 | } 69 | } 70 | } 71 | 72 | -------------------------------------------------------------------------------- /Examples/GameOfLife/Sources/GameOfLife/Rules/Board.swift: -------------------------------------------------------------------------------- 1 | // This source file is part of the Swift.org open source project 2 | // 3 | // Copyright (c) 2020 Apple Inc. and the Swift project authors 4 | // Licensed under Apache License v2.0 with Runtime Library Exception 5 | // 6 | // See http://swift.org/LICENSE.txt for license information 7 | // See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 8 | 9 | import LLBBuildSystem 10 | import Foundation 11 | import llbuild2 12 | 13 | /// A BoardTarget represents a target for a complete board. The size of the board is specified in the configuration for 14 | /// the build, which is passed to the target lookup method in `with(key:_:)`. This target will be populated with all of 15 | /// the cell targets for the requested generation. 16 | struct BoardTarget: LLBConfiguredTarget, Codable { 17 | /// The provider map for each of the dependencies declared for this target. Provider maps are the interface for 18 | /// reading the evaluation outputs of dependency targets. 19 | let dependencies: [LLBLabel] 20 | 21 | init(dependencies: [LLBLabel]) { 22 | self.dependencies = dependencies 23 | } 24 | 25 | var targetDependencies: [String : LLBTargetDependency] { 26 | return ["dependencies": .list(dependencies)] 27 | } 28 | 29 | /// Constructor for a BoardTarget from the ConfigurationTargetKey. 30 | static func with(key: LLBConfiguredTargetKey, _ fi: LLBBuildFunctionInterface, _ ctx: Context) throws -> LLBFuture { 31 | // Board labels are defined as //board:, since that's all that needed to reference a board at a 32 | // specific generation (i.e. the target). 33 | let label = key.label 34 | let generation = Int(label.targetName)! 35 | 36 | // This target is evaluated with a GameOfLifeConfigurationKey containing the global parameters of the build, like the 37 | // size of the board and the initial state. 38 | let boardSize = try key.configurationKey.get(GameOfLifeConfigurationKey.self).size 39 | 40 | var dependencies = [LLBLabel]() 41 | 42 | // Request the cell dependendency for each of the points in the board. This will effectively trigger the 43 | // evaluation of those targets and rules in order to provided the ProviderMap for each of the cell targets. 44 | // Once those are evaluated, the build system will be unblocked to evaluate this target. 45 | for x in 0 ..< boardSize.x { 46 | for y in 0 ..< boardSize.y { 47 | let dependencyLabel = try LLBLabel("//cell/\(generation):\(x)-\(y)") 48 | dependencies.append(dependencyLabel) 49 | } 50 | } 51 | 52 | return ctx.group.next().makeSucceededFuture(BoardTarget(dependencies: dependencies)) 53 | } 54 | } 55 | 56 | /// BoardProvider is how BoardTargets communicate their state to dependents. It contains a single artifact reference 57 | /// which when resolved will contain a matrix of `0` for dead cells and `1` for alive cells. 58 | struct BoardProvider: LLBProvider, Codable { 59 | let board: LLBArtifact 60 | 61 | init(board: LLBArtifact) { 62 | self.board = board 63 | } 64 | } 65 | 66 | /// The rule implementation that processes a BoardTarget under the given configuration. Because cell dependencies were 67 | /// already resolved at the BoardTarget creation time, we just need to read the CellProviders for each cell, order them 68 | /// in a matrix form and then register an action that reads all the files and concatenates them into the matrix form 69 | /// expected in the output. 70 | class BoardRule: LLBBuildRule { 71 | override func evaluate(configuredTarget: BoardTarget, _ ruleContext: LLBRuleContext) throws -> LLBFuture<[LLBProvider]> { 72 | let dependencies: [CellProvider] = try ruleContext.getProviders(for: "dependencies") 73 | 74 | // Make a dictionary lookup of the cell's point to the artifact containing that state. 75 | let boardMap = dependencies.reduce(into: [:]) { (dict, dep) in dict[dep.position] = dep.state } 76 | 77 | let boardSize = try ruleContext.getFragment(GameOfLifeConfigurationFragment.self).size 78 | 79 | var matrix = [[LLBArtifact]]() 80 | 81 | // Go over the complete board finding the state artifact for each point, and add them as rows into the matrix. 82 | // Since we requested a dependency on the cell for each point, the boardMap will contain an artifact for each 83 | // point. 84 | for y in 0 ..< boardSize.y { 85 | matrix.append((0 ..< boardSize.x).map { x in boardMap[Point(x, y)]! }) 86 | } 87 | 88 | // Declare the output artifact for the board. Declared artifacts are namespaced to the active configuration and 89 | // label, so there is no risk of collision for this artifact between other BoardTargets. 90 | let board = try ruleContext.declareArtifact("board.txt") 91 | 92 | // Create a "matrix" of `cat` commands that output each of the cells' state. 93 | let catCommands = matrix.map { row in 94 | "cat \(row.map(\.path).joined(separator: " ")) >> \(board.path); echo \"\" >> \(board.path)" 95 | }.joined(separator: "\n") 96 | 97 | // Register the action that populates the output board artifact. 98 | try ruleContext.registerAction( 99 | arguments: [ 100 | "/bin/bash", "-c", 101 | """ 102 | echo "" > \(board.path) 103 | \(catCommands) 104 | """ 105 | ], 106 | inputs: Array(boardMap.values), 107 | outputs: [board], 108 | mnemonic: "BoardTask", 109 | description: "Evaluating board \(ruleContext.label.canonical)..." 110 | ) 111 | 112 | // Return the BoardProvider containing the board artifact so that dependents can read it and use it accordingly. 113 | return ruleContext.group.next().makeSucceededFuture( 114 | [BoardProvider(board: board)] 115 | ) 116 | } 117 | } 118 | -------------------------------------------------------------------------------- /Examples/GameOfLife/Sources/GameOfLife/SwiftUIApplication.swift: -------------------------------------------------------------------------------- 1 | // This source file is part of the Swift.org open source project 2 | // 3 | // Copyright (c) 2020 Apple Inc. and the Swift project authors 4 | // Licensed under Apache License v2.0 with Runtime Library Exception 5 | // 6 | // See http://swift.org/LICENSE.txt for license information 7 | // See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 8 | 9 | import Cocoa 10 | import SwiftUI 11 | 12 | /// Simple application wrapper to create a command line UI application based on the excelent post in 13 | /// https://www.objc.io/blog/2020/05/19/swiftui-without-an-xcodeproj 14 | class SwiftUIApplication: NSObject, NSApplicationDelegate, NSWindowDelegate { 15 | var window: NSWindow! 16 | 17 | let contentView: V 18 | let observable: O 19 | 20 | init(_ contentView: V, observable: O) { 21 | self.contentView = contentView 22 | self.observable = observable 23 | } 24 | 25 | func run() { 26 | let app = NSApplication.shared 27 | NSApp.setActivationPolicy(.regular) 28 | app.mainMenu = NSMenu(title: "Main Menu") 29 | app.delegate = self 30 | app.run() 31 | } 32 | 33 | func applicationDidFinishLaunching(_ notification: Notification) { 34 | window = NSWindow( 35 | contentRect: NSRect(x: 0, y: 0, width: 480, height: 300), 36 | styleMask: [.titled, .closable, .resizable, .fullSizeContentView, .borderless], 37 | backing: .buffered, defer: false) 38 | window.center() 39 | window.setFrameAutosaveName("Main Window") 40 | window.contentView = NSHostingView(rootView: contentView.environmentObject(observable)) 41 | window.makeKeyAndOrderFront(nil) 42 | window.delegate = self 43 | 44 | NSApp.activate(ignoringOtherApps: true) 45 | } 46 | 47 | func applicationShouldTerminateAfterLastWindowClosed(_ sender: NSApplication) -> Bool { 48 | return true 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /Examples/GameOfLife/Sources/GameOfLife/main.swift: -------------------------------------------------------------------------------- 1 | // This source file is part of the Swift.org open source project 2 | // 3 | // Copyright (c) 2020 Apple Inc. and the Swift project authors 4 | // Licensed under Apache License v2.0 with Runtime Library Exception 5 | // 6 | // See http://swift.org/LICENSE.txt for license information 7 | // See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 8 | 9 | 10 | import LLBBuildSystem 11 | import llbuild2 12 | import NIO 13 | import Foundation 14 | import LLBUtil 15 | import LLBBuildSystemUtil 16 | import Logging 17 | import TSCBasic 18 | 19 | let gameOfLifeDirectory = AbsolutePath("/tmp/game_of_life") 20 | try localFileSystem.createDirectory(gameOfLifeDirectory, recursive: true) 21 | 22 | var ctx = Context() 23 | ctx.group = MultiThreadedEventLoopGroup(numberOfThreads: ProcessInfo.processInfo.processorCount) 24 | ctx.db = LLBFileBackedCASDatabase(group: ctx.group, path: gameOfLifeDirectory.appending(component: "cas")) 25 | 26 | var logger = Logger(label: "org.swift.llbuild2.game_of_life") 27 | logger.logLevel = .debug 28 | ctx.logger = logger 29 | 30 | // Create the build engine's dependencies. 31 | let executor = LLBLocalExecutor(outputBase: gameOfLifeDirectory.appending(component: "executor_output")) 32 | let functionCache = LLBFileBackedFunctionCache( 33 | group: ctx.group, 34 | path: gameOfLifeDirectory.appending(component: "function_cache"), 35 | version: "1" 36 | ) 37 | 38 | let buildSystemDelegate = GameOfLifeBuildSystemDelegate() 39 | 40 | class GameOfLifeBuildEventDelegate: LLBBuildEventDelegate { 41 | func targetEvaluationRequested(label: LLBLabel) { 42 | logger.debug("Target \(label.canonical) being evaluated") 43 | } 44 | 45 | func targetEvaluationCompleted(label: LLBLabel) { 46 | logger.debug("Target \(label.canonical) evaluated") 47 | } 48 | 49 | func actionScheduled(action: LLBBuildEventActionDescription) { 50 | logger.debug("Action scheduled: \(action.description)") 51 | } 52 | 53 | func actionCompleted(action: LLBBuildEventActionDescription, result: LLBActionResult) { 54 | logger.debug("Action completed: \(action.description) - \(result)") 55 | } 56 | 57 | func actionExecutionStarted(action: LLBBuildEventActionDescription) { 58 | logger.debug("Action execution started: \(action.description)") 59 | } 60 | 61 | func actionExecutionCompleted(action: LLBBuildEventActionDescription) { 62 | logger.debug("Action action execution completed: \(action.description)") 63 | } 64 | } 65 | 66 | ctx.buildEventDelegate = GameOfLifeBuildEventDelegate() 67 | 68 | // Construct the build engine instance and 69 | let engine = LLBBuildEngine( 70 | group: ctx.group, 71 | db: ctx.db, 72 | buildFunctionLookupDelegate: buildSystemDelegate, 73 | configuredTargetDelegate: buildSystemDelegate, 74 | ruleLookupDelegate: buildSystemDelegate, 75 | registrationDelegate: buildSystemDelegate, 76 | executor: executor, 77 | functionCache: functionCache 78 | ) 79 | 80 | // Construct the SwiftUI environment object to access the build engine and database. 81 | let environment = GameOfLifeEnvironment(engine: engine, ctx) 82 | 83 | // Run the UI. 84 | SwiftUIApplication(GameOfLifeView(), observable: environment).run() 85 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | # This source file is part of the Swift.org open source project 2 | # 3 | # Copyright (c) 2020 Apple Inc. and the Swift project authors 4 | # Licensed under Apache License v2.0 with Runtime Library Exception 5 | # 6 | # See http://swift.org/LICENSE.txt for license information 7 | # See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 8 | 9 | .PHONY: 10 | generate: clean generate-protos 11 | 12 | .PHONY: 13 | update: clone-external-protos 14 | 15 | # Clone external repositories. 16 | .PHONY: 17 | clone-external-repos: 18 | mkdir -p ExternalRepositories 19 | if [ ! -d ExternalRepositories/remote-apis ]; then \ 20 | git -C ExternalRepositories clone --depth 1 https://github.com/bazelbuild/remote-apis; \ 21 | else \ 22 | git -C ExternalRepositories/remote-apis pull --rebase; \ 23 | fi 24 | if [ ! -d ExternalRepositories/googleapis ]; then \ 25 | git -C ExternalRepositories clone --depth 1 https://github.com/googleapis/googleapis; \ 26 | else \ 27 | git -C ExternalRepositories/googleapis pull --rebase; \ 28 | fi 29 | 30 | .PHONY: 31 | clone-external-protos: clone-external-repos 32 | rm -rf Protos/BazelRemoteAPI 33 | mkdir -p Protos/BazelRemoteAPI 34 | rsync -arv --prune-empty-dirs --include \*/ \ 35 | --include LICENSE \ 36 | --include \*.proto \ 37 | --exclude \* \ 38 | ExternalRepositories/remote-apis Protos/BazelRemoteAPI 39 | rsync -arv --prune-empty-dirs --include \*/ \ 40 | --include LICENSE \ 41 | --include api/annotations.proto \ 42 | --include api/client.proto \ 43 | --include api/http.proto \ 44 | --include api/launch_stage.proto \ 45 | --include bytestream/\*.proto \ 46 | --include longrunning/\*.proto \ 47 | --include rpc/\*.proto \ 48 | --exclude \* \ 49 | ExternalRepositories/googleapis Protos/BazelRemoteAPI 50 | 51 | # These command should be executed any time the proto definitions change. It is 52 | # not required to be generated as part of a regular `swift build` since we're 53 | # checking in the generated sources. 54 | .PHONY: 55 | generate-protos: proto-toolchain Protos/BazelRemoteAPI 56 | mkdir -p Sources/llbuild2fx/Generated 57 | Utilities/tools/bin/protoc \ 58 | -I=Protos \ 59 | -I=.build/checkouts/swift-tools-support-async/Protos \ 60 | --plugin=Utilities/tools/bin/protoc-gen-swift \ 61 | --swift_out=Sources/llbuild2fx/Generated \ 62 | --swift_opt=Visibility=Public \ 63 | --swift_opt=ProtoPathModuleMappings=Protos/module_map.asciipb \ 64 | $$(find Protos/EngineProtocol -name \*.proto) 65 | mkdir -p Sources/BazelRemoteAPI/Generated 66 | Utilities/tools/bin/protoc \ 67 | -I=Protos/BazelRemoteAPI/googleapis \ 68 | -I=Protos/BazelRemoteAPI/remote-apis \ 69 | --plugin=Utilities/tools/bin/protoc-gen-swift \ 70 | --plugin=Utilities/tools/bin/protoc-gen-grpc-swift \ 71 | --swift_out=Sources/BazelRemoteAPI/Generated \ 72 | --swift_opt=Visibility=Public \ 73 | --grpc-swift_opt=Visibility=Public \ 74 | --grpc-swift_out=Sources/BazelRemoteAPI/Generated \ 75 | --experimental_allow_proto3_optional \ 76 | $$(find Protos/BazelRemoteAPI -name \*.proto) 77 | 78 | .PHONY: 79 | proto-toolchain: 80 | Utilities/build_proto_toolchain.sh 81 | 82 | .PHONY: 83 | clean: 84 | rm -rf Sources/BazelRemoteAPI/Generated 85 | rm -rf Sources/llbuild2fx/Generated 86 | 87 | Protos/BazelRemoteAPI: clone-external-protos 88 | -------------------------------------------------------------------------------- /Package.swift: -------------------------------------------------------------------------------- 1 | // swift-tools-version:5.1 2 | 3 | import PackageDescription 4 | 5 | let package = Package( 6 | name: "llbuild2", 7 | platforms: [ 8 | .macOS(.v10_15) 9 | ], 10 | products: [ 11 | .library(name: "llbuild2", targets: ["llbuild2"]), 12 | .library(name: "llbuild2fx", targets: ["llbuild2fx"]), 13 | ], 14 | dependencies: [ 15 | .package(url: "https://github.com/apple/swift-argument-parser.git", from: "1.0.0"), 16 | .package(url: "https://github.com/apple/swift-crypto.git", "1.1.4" ..< "4.0.0"), 17 | .package(url: "https://github.com/apple/swift-tools-support-async.git", from: "0.10.0"), 18 | .package(url: "https://github.com/apple/swift-tools-support-core.git", from: "0.2.7"), 19 | .package(url: "https://github.com/apple/swift-protobuf.git", from: "1.17.0"), 20 | .package(url: "https://github.com/grpc/grpc-swift.git", from: "1.4.1"), 21 | .package(url: "https://github.com/apple/swift-log.git", from: "1.4.2"), 22 | .package(url: "https://github.com/apple/swift-distributed-tracing", from: "1.1.2"), 23 | ], 24 | targets: [ 25 | // FX build engine 26 | .target( 27 | name: "llbuild2fx", 28 | dependencies: [ 29 | "SwiftProtobuf", 30 | "SwiftToolsSupportCAS", 31 | "Logging", 32 | .product(name: "Tracing", package: "swift-distributed-tracing"), 33 | .product(name: "Instrumentation", package: "swift-distributed-tracing") 34 | ] 35 | ), 36 | .testTarget( 37 | name: "llbuild2fxTests", 38 | dependencies: ["llbuild2fx"] 39 | ), 40 | 41 | // Bazel RemoteAPI Protocol 42 | .target( 43 | name: "BazelRemoteAPI", 44 | dependencies: [ 45 | "GRPC", 46 | "SwiftProtobuf", 47 | "SwiftProtobufPluginLibrary", 48 | ] 49 | ), 50 | 51 | // Compatibility/convenience wrapper library 52 | .target( 53 | name: "llbuild2", 54 | dependencies: ["llbuild2fx"] 55 | ), 56 | 57 | // Bazel CAS/Execution Backend 58 | .target( 59 | name: "LLBBazelBackend", 60 | dependencies: [ 61 | "BazelRemoteAPI", 62 | "Crypto", 63 | "GRPC", 64 | "SwiftToolsSupportCAS", 65 | ] 66 | ), 67 | 68 | // llcastool implementation 69 | .target( 70 | name: "LLBCASTool", 71 | dependencies: [ 72 | "GRPC", 73 | "SwiftToolsSupport-auto", 74 | "BazelRemoteAPI", 75 | "LLBBazelBackend", 76 | ] 77 | ), 78 | 79 | // `llcastool` executable. 80 | .target( 81 | name: "llcastool", 82 | dependencies: ["LLBCASTool", "ArgumentParser"], 83 | path: "Sources/Tools/llcastool" 84 | ), 85 | ] 86 | ) 87 | -------------------------------------------------------------------------------- /Protos/BazelRemoteAPI/googleapis/google/api/annotations.proto: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | syntax = "proto3"; 16 | 17 | package google.api; 18 | 19 | import "google/api/http.proto"; 20 | import "google/protobuf/descriptor.proto"; 21 | 22 | option go_package = "google.golang.org/genproto/googleapis/api/annotations;annotations"; 23 | option java_multiple_files = true; 24 | option java_outer_classname = "AnnotationsProto"; 25 | option java_package = "com.google.api"; 26 | option objc_class_prefix = "GAPI"; 27 | 28 | extend google.protobuf.MethodOptions { 29 | // See `HttpRule`. 30 | HttpRule http = 72295728; 31 | } 32 | -------------------------------------------------------------------------------- /Protos/BazelRemoteAPI/googleapis/google/api/launch_stage.proto: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | syntax = "proto3"; 16 | 17 | package google.api; 18 | 19 | option go_package = "google.golang.org/genproto/googleapis/api;api"; 20 | option java_multiple_files = true; 21 | option java_outer_classname = "LaunchStageProto"; 22 | option java_package = "com.google.api"; 23 | option objc_class_prefix = "GAPI"; 24 | 25 | // The launch stage as defined by [Google Cloud Platform 26 | // Launch Stages](https://cloud.google.com/terms/launch-stages). 27 | enum LaunchStage { 28 | // Do not use this default value. 29 | LAUNCH_STAGE_UNSPECIFIED = 0; 30 | 31 | // The feature is not yet implemented. Users can not use it. 32 | UNIMPLEMENTED = 6; 33 | 34 | // Prelaunch features are hidden from users and are only visible internally. 35 | PRELAUNCH = 7; 36 | 37 | // Early Access features are limited to a closed group of testers. To use 38 | // these features, you must sign up in advance and sign a Trusted Tester 39 | // agreement (which includes confidentiality provisions). These features may 40 | // be unstable, changed in backward-incompatible ways, and are not 41 | // guaranteed to be released. 42 | EARLY_ACCESS = 1; 43 | 44 | // Alpha is a limited availability test for releases before they are cleared 45 | // for widespread use. By Alpha, all significant design issues are resolved 46 | // and we are in the process of verifying functionality. Alpha customers 47 | // need to apply for access, agree to applicable terms, and have their 48 | // projects allowlisted. Alpha releases don't have to be feature complete, 49 | // no SLAs are provided, and there are no technical support obligations, but 50 | // they will be far enough along that customers can actually use them in 51 | // test environments or for limited-use tests -- just like they would in 52 | // normal production cases. 53 | ALPHA = 2; 54 | 55 | // Beta is the point at which we are ready to open a release for any 56 | // customer to use. There are no SLA or technical support obligations in a 57 | // Beta release. Products will be complete from a feature perspective, but 58 | // may have some open outstanding issues. Beta releases are suitable for 59 | // limited production use cases. 60 | BETA = 3; 61 | 62 | // GA features are open to all developers and are considered stable and 63 | // fully qualified for production use. 64 | GA = 4; 65 | 66 | // Deprecated features are scheduled to be shut down and removed. For more 67 | // information, see the "Deprecation Policy" section of our [Terms of 68 | // Service](https://cloud.google.com/terms/) 69 | // and the [Google Cloud Platform Subject to the Deprecation 70 | // Policy](https://cloud.google.com/terms/deprecation) documentation. 71 | DEPRECATED = 5; 72 | } 73 | -------------------------------------------------------------------------------- /Protos/BazelRemoteAPI/googleapis/google/rpc/http.proto: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | syntax = "proto3"; 16 | 17 | package google.rpc; 18 | 19 | option go_package = "google.golang.org/genproto/googleapis/rpc/http;http"; 20 | option java_multiple_files = true; 21 | option java_outer_classname = "HttpProto"; 22 | option java_package = "com.google.rpc"; 23 | option objc_class_prefix = "RPC"; 24 | 25 | // Represents an HTTP request. 26 | message HttpRequest { 27 | // The HTTP request method. 28 | string method = 1; 29 | 30 | // The HTTP request URI. 31 | string uri = 2; 32 | 33 | // The HTTP request headers. The ordering of the headers is significant. 34 | // Multiple headers with the same key may present for the request. 35 | repeated HttpHeader headers = 3; 36 | 37 | // The HTTP request body. If the body is not expected, it should be empty. 38 | bytes body = 4; 39 | } 40 | 41 | // Represents an HTTP response. 42 | message HttpResponse { 43 | // The HTTP status code, such as 200 or 404. 44 | int32 status = 1; 45 | 46 | // The HTTP reason phrase, such as "OK" or "Not Found". 47 | string reason = 2; 48 | 49 | // The HTTP response headers. The ordering of the headers is significant. 50 | // Multiple headers with the same key may present for the response. 51 | repeated HttpHeader headers = 3; 52 | 53 | // The HTTP response body. If the body is not expected, it should be empty. 54 | bytes body = 4; 55 | } 56 | 57 | // Represents an HTTP header. 58 | message HttpHeader { 59 | // The HTTP header key. It is case insensitive. 60 | string key = 1; 61 | 62 | // The HTTP header value. 63 | string value = 2; 64 | } 65 | -------------------------------------------------------------------------------- /Protos/BazelRemoteAPI/googleapis/google/rpc/status.proto: -------------------------------------------------------------------------------- 1 | // Copyright 2025 Google LLC 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | syntax = "proto3"; 16 | 17 | package google.rpc; 18 | 19 | import "google/protobuf/any.proto"; 20 | 21 | option cc_enable_arenas = true; 22 | option go_package = "google.golang.org/genproto/googleapis/rpc/status;status"; 23 | option java_multiple_files = true; 24 | option java_outer_classname = "StatusProto"; 25 | option java_package = "com.google.rpc"; 26 | option objc_class_prefix = "RPC"; 27 | 28 | // The `Status` type defines a logical error model that is suitable for 29 | // different programming environments, including REST APIs and RPC APIs. It is 30 | // used by [gRPC](https://github.com/grpc). Each `Status` message contains 31 | // three pieces of data: error code, error message, and error details. 32 | // 33 | // You can find out more about this error model and how to work with it in the 34 | // [API Design Guide](https://cloud.google.com/apis/design/errors). 35 | message Status { 36 | // The status code, which should be an enum value of 37 | // [google.rpc.Code][google.rpc.Code]. 38 | int32 code = 1; 39 | 40 | // A developer-facing error message, which should be in English. Any 41 | // user-facing error message should be localized and sent in the 42 | // [google.rpc.Status.details][google.rpc.Status.details] field, or localized 43 | // by the client. 44 | string message = 2; 45 | 46 | // A list of messages that carry the error details. There is a common set of 47 | // message types for APIs to use. 48 | repeated google.protobuf.Any details = 3; 49 | } 50 | -------------------------------------------------------------------------------- /Protos/BazelRemoteAPI/remote-apis/build/bazel/semver/semver.proto: -------------------------------------------------------------------------------- 1 | // Copyright 2018 The Bazel Authors. 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | syntax = "proto3"; 16 | 17 | package build.bazel.semver; 18 | 19 | option csharp_namespace = "Build.Bazel.Semver"; 20 | option go_package = "github.com/bazelbuild/remote-apis/build/bazel/semver"; 21 | option java_multiple_files = true; 22 | option java_outer_classname = "SemverProto"; 23 | option java_package = "build.bazel.semver"; 24 | option objc_class_prefix = "SMV"; 25 | 26 | // The full version of a given tool. 27 | message SemVer { 28 | // The major version, e.g 10 for 10.2.3. 29 | int32 major = 1; 30 | 31 | // The minor version, e.g. 2 for 10.2.3. 32 | int32 minor = 2; 33 | 34 | // The patch version, e.g 3 for 10.2.3. 35 | int32 patch = 3; 36 | 37 | // The pre-release version. Either this field or major/minor/patch fields 38 | // must be filled. They are mutually exclusive. Pre-release versions are 39 | // assumed to be earlier than any released versions. 40 | string prerelease = 4; 41 | } 42 | -------------------------------------------------------------------------------- /Protos/EngineProtocol/any_serializable.proto: -------------------------------------------------------------------------------- 1 | // This source file is part of the Swift.org open source project 2 | // 3 | // Copyright (c) 2020 Apple Inc. and the Swift project authors 4 | // Licensed under Apache License v2.0 with Runtime Library Exception 5 | // 6 | // See http://swift.org/LICENSE.txt for license information 7 | // See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 8 | 9 | syntax = "proto3"; 10 | 11 | option java_package = "com.apple.llbuild2.engineprotocol"; 12 | 13 | // AnyCodable is a wrapper type for polymorphic codables, in which the type of the serialized data is not known at 14 | // compile time. 15 | message LLBAnySerializable { 16 | // A string identifier to map the runtime bytes into a specific type, for deserialization. 17 | string typeIdentifier = 1; 18 | 19 | // The serialized bytes of the underlying data structure. 20 | bytes serializedBytes = 2; 21 | } 22 | -------------------------------------------------------------------------------- /Protos/module_map.asciipb: -------------------------------------------------------------------------------- 1 | mapping { 2 | module_name: "TSFCAS" 3 | proto_file_path: "CASProtocol/cas_object.proto" 4 | proto_file_path: "CASProtocol/data_id.proto" 5 | } 6 | mapping { 7 | module_name: "TSFCASFileTree" 8 | proto_file_path: "CASFileTreeProtocol/file_tree.proto" 9 | } 10 | mapping { 11 | module_name: "llbuild2fx" 12 | proto_file_path: "EngineProtocol/any_serializable.proto" 13 | } 14 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # llbuild2 2 | 3 | **llbuild2** is an experimental, Swift native, fully async, NIO futures-based 4 | low level build system. Started as the **cevobuild** experiment in [**llbuild**](https://github.com/apple/swift-llbuild), 5 | this repository aims to continue that exploration. 6 | 7 | Check the [docs](Docs/index.md) for more information about llbuild2. 8 | 9 | # Development 10 | 11 | Development documentation is available [here](Docs/Development.md). 12 | 13 | ## License 14 | 15 | Copyright (c) 2020 Apple Inc. and the Swift project authors. 16 | Licensed under Apache License v2.0 with Runtime Library Exception. 17 | 18 | See https://www.swift.org/LICENSE.txt for license information. 19 | 20 | See https://www.swift.org/CONTRIBUTORS.txt for Swift project authors. 21 | -------------------------------------------------------------------------------- /Sources/BazelRemoteAPI/Generated/build/bazel/semver/semver.pb.swift: -------------------------------------------------------------------------------- 1 | // DO NOT EDIT. 2 | // swift-format-ignore-file 3 | // 4 | // Generated by the Swift generator plugin for the protocol buffer compiler. 5 | // Source: build/bazel/semver/semver.proto 6 | // 7 | // For information on using the generated types, please see the documentation: 8 | // https://github.com/apple/swift-protobuf/ 9 | 10 | // Copyright 2018 The Bazel Authors. 11 | // 12 | // Licensed under the Apache License, Version 2.0 (the "License"); 13 | // you may not use this file except in compliance with the License. 14 | // You may obtain a copy of the License at 15 | // 16 | // http://www.apache.org/licenses/LICENSE-2.0 17 | // 18 | // Unless required by applicable law or agreed to in writing, software 19 | // distributed under the License is distributed on an "AS IS" BASIS, 20 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 21 | // See the License for the specific language governing permissions and 22 | // limitations under the License. 23 | 24 | import Foundation 25 | import SwiftProtobuf 26 | 27 | // If the compiler emits an error on this type, it is because this file 28 | // was generated by a version of the `protoc` Swift plug-in that is 29 | // incompatible with the version of SwiftProtobuf to which you are linking. 30 | // Please ensure that you are building against the same version of the API 31 | // that was used to generate this file. 32 | fileprivate struct _GeneratedWithProtocGenSwiftVersion: SwiftProtobuf.ProtobufAPIVersionCheck { 33 | struct _2: SwiftProtobuf.ProtobufAPIVersion_2 {} 34 | typealias Version = _2 35 | } 36 | 37 | /// The full version of a given tool. 38 | public struct Build_Bazel_Semver_SemVer { 39 | // SwiftProtobuf.Message conformance is added in an extension below. See the 40 | // `Message` and `Message+*Additions` files in the SwiftProtobuf library for 41 | // methods supported on all messages. 42 | 43 | /// The major version, e.g 10 for 10.2.3. 44 | public var major: Int32 = 0 45 | 46 | /// The minor version, e.g. 2 for 10.2.3. 47 | public var minor: Int32 = 0 48 | 49 | /// The patch version, e.g 3 for 10.2.3. 50 | public var patch: Int32 = 0 51 | 52 | /// The pre-release version. Either this field or major/minor/patch fields 53 | /// must be filled. They are mutually exclusive. Pre-release versions are 54 | /// assumed to be earlier than any released versions. 55 | public var prerelease: String = String() 56 | 57 | public var unknownFields = SwiftProtobuf.UnknownStorage() 58 | 59 | public init() {} 60 | } 61 | 62 | #if swift(>=5.5) && canImport(_Concurrency) 63 | extension Build_Bazel_Semver_SemVer: @unchecked Sendable {} 64 | #endif // swift(>=5.5) && canImport(_Concurrency) 65 | 66 | // MARK: - Code below here is support for the SwiftProtobuf runtime. 67 | 68 | fileprivate let _protobuf_package = "build.bazel.semver" 69 | 70 | extension Build_Bazel_Semver_SemVer: SwiftProtobuf.Message, SwiftProtobuf._MessageImplementationBase, SwiftProtobuf._ProtoNameProviding { 71 | public static let protoMessageName: String = _protobuf_package + ".SemVer" 72 | public static let _protobuf_nameMap: SwiftProtobuf._NameMap = [ 73 | 1: .same(proto: "major"), 74 | 2: .same(proto: "minor"), 75 | 3: .same(proto: "patch"), 76 | 4: .same(proto: "prerelease"), 77 | ] 78 | 79 | public mutating func decodeMessage(decoder: inout D) throws { 80 | while let fieldNumber = try decoder.nextFieldNumber() { 81 | // The use of inline closures is to circumvent an issue where the compiler 82 | // allocates stack space for every case branch when no optimizations are 83 | // enabled. https://github.com/apple/swift-protobuf/issues/1034 84 | switch fieldNumber { 85 | case 1: try { try decoder.decodeSingularInt32Field(value: &self.major) }() 86 | case 2: try { try decoder.decodeSingularInt32Field(value: &self.minor) }() 87 | case 3: try { try decoder.decodeSingularInt32Field(value: &self.patch) }() 88 | case 4: try { try decoder.decodeSingularStringField(value: &self.prerelease) }() 89 | default: break 90 | } 91 | } 92 | } 93 | 94 | public func traverse(visitor: inout V) throws { 95 | if self.major != 0 { 96 | try visitor.visitSingularInt32Field(value: self.major, fieldNumber: 1) 97 | } 98 | if self.minor != 0 { 99 | try visitor.visitSingularInt32Field(value: self.minor, fieldNumber: 2) 100 | } 101 | if self.patch != 0 { 102 | try visitor.visitSingularInt32Field(value: self.patch, fieldNumber: 3) 103 | } 104 | if !self.prerelease.isEmpty { 105 | try visitor.visitSingularStringField(value: self.prerelease, fieldNumber: 4) 106 | } 107 | try unknownFields.traverse(visitor: &visitor) 108 | } 109 | 110 | public static func ==(lhs: Build_Bazel_Semver_SemVer, rhs: Build_Bazel_Semver_SemVer) -> Bool { 111 | if lhs.major != rhs.major {return false} 112 | if lhs.minor != rhs.minor {return false} 113 | if lhs.patch != rhs.patch {return false} 114 | if lhs.prerelease != rhs.prerelease {return false} 115 | if lhs.unknownFields != rhs.unknownFields {return false} 116 | return true 117 | } 118 | } 119 | -------------------------------------------------------------------------------- /Sources/BazelRemoteAPI/Generated/google/api/annotations.pb.swift: -------------------------------------------------------------------------------- 1 | // DO NOT EDIT. 2 | // swift-format-ignore-file 3 | // 4 | // Generated by the Swift generator plugin for the protocol buffer compiler. 5 | // Source: google/api/annotations.proto 6 | // 7 | // For information on using the generated types, please see the documentation: 8 | // https://github.com/apple/swift-protobuf/ 9 | 10 | // Copyright 2025 Google LLC 11 | // 12 | // Licensed under the Apache License, Version 2.0 (the "License"); 13 | // you may not use this file except in compliance with the License. 14 | // You may obtain a copy of the License at 15 | // 16 | // http://www.apache.org/licenses/LICENSE-2.0 17 | // 18 | // Unless required by applicable law or agreed to in writing, software 19 | // distributed under the License is distributed on an "AS IS" BASIS, 20 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 21 | // See the License for the specific language governing permissions and 22 | // limitations under the License. 23 | 24 | import Foundation 25 | import SwiftProtobuf 26 | 27 | // If the compiler emits an error on this type, it is because this file 28 | // was generated by a version of the `protoc` Swift plug-in that is 29 | // incompatible with the version of SwiftProtobuf to which you are linking. 30 | // Please ensure that you are building against the same version of the API 31 | // that was used to generate this file. 32 | fileprivate struct _GeneratedWithProtocGenSwiftVersion: SwiftProtobuf.ProtobufAPIVersionCheck { 33 | struct _2: SwiftProtobuf.ProtobufAPIVersion_2 {} 34 | typealias Version = _2 35 | } 36 | 37 | // MARK: - Extension support defined in annotations.proto. 38 | 39 | // MARK: - Extension Properties 40 | 41 | // Swift Extensions on the exteneded Messages to add easy access to the declared 42 | // extension fields. The names are based on the extension field name from the proto 43 | // declaration. To avoid naming collisions, the names are prefixed with the name of 44 | // the scope where the extend directive occurs. 45 | 46 | extension SwiftProtobuf.Google_Protobuf_MethodOptions { 47 | 48 | /// See `HttpRule`. 49 | public var Google_Api_http: Google_Api_HttpRule { 50 | get {return getExtensionValue(ext: Google_Api_Extensions_http) ?? Google_Api_HttpRule()} 51 | set {setExtensionValue(ext: Google_Api_Extensions_http, value: newValue)} 52 | } 53 | /// Returns true if extension `Google_Api_Extensions_http` 54 | /// has been explicitly set. 55 | public var hasGoogle_Api_http: Bool { 56 | return hasExtensionValue(ext: Google_Api_Extensions_http) 57 | } 58 | /// Clears the value of extension `Google_Api_Extensions_http`. 59 | /// Subsequent reads from it will return its default value. 60 | public mutating func clearGoogle_Api_http() { 61 | clearExtensionValue(ext: Google_Api_Extensions_http) 62 | } 63 | 64 | } 65 | 66 | // MARK: - File's ExtensionMap: Google_Api_Annotations_Extensions 67 | 68 | /// A `SwiftProtobuf.SimpleExtensionMap` that includes all of the extensions defined by 69 | /// this .proto file. It can be used any place an `SwiftProtobuf.ExtensionMap` is needed 70 | /// in parsing, or it can be combined with other `SwiftProtobuf.SimpleExtensionMap`s to create 71 | /// a larger `SwiftProtobuf.SimpleExtensionMap`. 72 | public let Google_Api_Annotations_Extensions: SwiftProtobuf.SimpleExtensionMap = [ 73 | Google_Api_Extensions_http 74 | ] 75 | 76 | // Extension Objects - The only reason these might be needed is when manually 77 | // constructing a `SimpleExtensionMap`, otherwise, use the above _Extension Properties_ 78 | // accessors for the extension fields on the messages directly. 79 | 80 | /// See `HttpRule`. 81 | public let Google_Api_Extensions_http = SwiftProtobuf.MessageExtension, SwiftProtobuf.Google_Protobuf_MethodOptions>( 82 | _protobuf_fieldNumber: 72295728, 83 | fieldName: "google.api.http" 84 | ) 85 | -------------------------------------------------------------------------------- /Sources/BazelRemoteAPI/Generated/google/api/launch_stage.pb.swift: -------------------------------------------------------------------------------- 1 | // DO NOT EDIT. 2 | // swift-format-ignore-file 3 | // 4 | // Generated by the Swift generator plugin for the protocol buffer compiler. 5 | // Source: google/api/launch_stage.proto 6 | // 7 | // For information on using the generated types, please see the documentation: 8 | // https://github.com/apple/swift-protobuf/ 9 | 10 | // Copyright 2025 Google LLC 11 | // 12 | // Licensed under the Apache License, Version 2.0 (the "License"); 13 | // you may not use this file except in compliance with the License. 14 | // You may obtain a copy of the License at 15 | // 16 | // http://www.apache.org/licenses/LICENSE-2.0 17 | // 18 | // Unless required by applicable law or agreed to in writing, software 19 | // distributed under the License is distributed on an "AS IS" BASIS, 20 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 21 | // See the License for the specific language governing permissions and 22 | // limitations under the License. 23 | 24 | import Foundation 25 | import SwiftProtobuf 26 | 27 | // If the compiler emits an error on this type, it is because this file 28 | // was generated by a version of the `protoc` Swift plug-in that is 29 | // incompatible with the version of SwiftProtobuf to which you are linking. 30 | // Please ensure that you are building against the same version of the API 31 | // that was used to generate this file. 32 | fileprivate struct _GeneratedWithProtocGenSwiftVersion: SwiftProtobuf.ProtobufAPIVersionCheck { 33 | struct _2: SwiftProtobuf.ProtobufAPIVersion_2 {} 34 | typealias Version = _2 35 | } 36 | 37 | /// The launch stage as defined by [Google Cloud Platform 38 | /// Launch Stages](https://cloud.google.com/terms/launch-stages). 39 | public enum Google_Api_LaunchStage: SwiftProtobuf.Enum { 40 | public typealias RawValue = Int 41 | 42 | /// Do not use this default value. 43 | case unspecified // = 0 44 | 45 | /// The feature is not yet implemented. Users can not use it. 46 | case unimplemented // = 6 47 | 48 | /// Prelaunch features are hidden from users and are only visible internally. 49 | case prelaunch // = 7 50 | 51 | /// Early Access features are limited to a closed group of testers. To use 52 | /// these features, you must sign up in advance and sign a Trusted Tester 53 | /// agreement (which includes confidentiality provisions). These features may 54 | /// be unstable, changed in backward-incompatible ways, and are not 55 | /// guaranteed to be released. 56 | case earlyAccess // = 1 57 | 58 | /// Alpha is a limited availability test for releases before they are cleared 59 | /// for widespread use. By Alpha, all significant design issues are resolved 60 | /// and we are in the process of verifying functionality. Alpha customers 61 | /// need to apply for access, agree to applicable terms, and have their 62 | /// projects allowlisted. Alpha releases don't have to be feature complete, 63 | /// no SLAs are provided, and there are no technical support obligations, but 64 | /// they will be far enough along that customers can actually use them in 65 | /// test environments or for limited-use tests -- just like they would in 66 | /// normal production cases. 67 | case alpha // = 2 68 | 69 | /// Beta is the point at which we are ready to open a release for any 70 | /// customer to use. There are no SLA or technical support obligations in a 71 | /// Beta release. Products will be complete from a feature perspective, but 72 | /// may have some open outstanding issues. Beta releases are suitable for 73 | /// limited production use cases. 74 | case beta // = 3 75 | 76 | /// GA features are open to all developers and are considered stable and 77 | /// fully qualified for production use. 78 | case ga // = 4 79 | 80 | /// Deprecated features are scheduled to be shut down and removed. For more 81 | /// information, see the "Deprecation Policy" section of our [Terms of 82 | /// Service](https://cloud.google.com/terms/) 83 | /// and the [Google Cloud Platform Subject to the Deprecation 84 | /// Policy](https://cloud.google.com/terms/deprecation) documentation. 85 | case deprecated // = 5 86 | case UNRECOGNIZED(Int) 87 | 88 | public init() { 89 | self = .unspecified 90 | } 91 | 92 | public init?(rawValue: Int) { 93 | switch rawValue { 94 | case 0: self = .unspecified 95 | case 1: self = .earlyAccess 96 | case 2: self = .alpha 97 | case 3: self = .beta 98 | case 4: self = .ga 99 | case 5: self = .deprecated 100 | case 6: self = .unimplemented 101 | case 7: self = .prelaunch 102 | default: self = .UNRECOGNIZED(rawValue) 103 | } 104 | } 105 | 106 | public var rawValue: Int { 107 | switch self { 108 | case .unspecified: return 0 109 | case .earlyAccess: return 1 110 | case .alpha: return 2 111 | case .beta: return 3 112 | case .ga: return 4 113 | case .deprecated: return 5 114 | case .unimplemented: return 6 115 | case .prelaunch: return 7 116 | case .UNRECOGNIZED(let i): return i 117 | } 118 | } 119 | 120 | } 121 | 122 | #if swift(>=4.2) 123 | 124 | extension Google_Api_LaunchStage: CaseIterable { 125 | // The compiler won't synthesize support with the UNRECOGNIZED case. 126 | public static var allCases: [Google_Api_LaunchStage] = [ 127 | .unspecified, 128 | .unimplemented, 129 | .prelaunch, 130 | .earlyAccess, 131 | .alpha, 132 | .beta, 133 | .ga, 134 | .deprecated, 135 | ] 136 | } 137 | 138 | #endif // swift(>=4.2) 139 | 140 | #if swift(>=5.5) && canImport(_Concurrency) 141 | extension Google_Api_LaunchStage: @unchecked Sendable {} 142 | #endif // swift(>=5.5) && canImport(_Concurrency) 143 | 144 | // MARK: - Code below here is support for the SwiftProtobuf runtime. 145 | 146 | extension Google_Api_LaunchStage: SwiftProtobuf._ProtoNameProviding { 147 | public static let _protobuf_nameMap: SwiftProtobuf._NameMap = [ 148 | 0: .same(proto: "LAUNCH_STAGE_UNSPECIFIED"), 149 | 1: .same(proto: "EARLY_ACCESS"), 150 | 2: .same(proto: "ALPHA"), 151 | 3: .same(proto: "BETA"), 152 | 4: .same(proto: "GA"), 153 | 5: .same(proto: "DEPRECATED"), 154 | 6: .same(proto: "UNIMPLEMENTED"), 155 | 7: .same(proto: "PRELAUNCH"), 156 | ] 157 | } 158 | -------------------------------------------------------------------------------- /Sources/BazelRemoteAPI/Generated/google/rpc/status.pb.swift: -------------------------------------------------------------------------------- 1 | // DO NOT EDIT. 2 | // swift-format-ignore-file 3 | // 4 | // Generated by the Swift generator plugin for the protocol buffer compiler. 5 | // Source: google/rpc/status.proto 6 | // 7 | // For information on using the generated types, please see the documentation: 8 | // https://github.com/apple/swift-protobuf/ 9 | 10 | // Copyright 2025 Google LLC 11 | // 12 | // Licensed under the Apache License, Version 2.0 (the "License"); 13 | // you may not use this file except in compliance with the License. 14 | // You may obtain a copy of the License at 15 | // 16 | // http://www.apache.org/licenses/LICENSE-2.0 17 | // 18 | // Unless required by applicable law or agreed to in writing, software 19 | // distributed under the License is distributed on an "AS IS" BASIS, 20 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 21 | // See the License for the specific language governing permissions and 22 | // limitations under the License. 23 | 24 | import Foundation 25 | import SwiftProtobuf 26 | 27 | // If the compiler emits an error on this type, it is because this file 28 | // was generated by a version of the `protoc` Swift plug-in that is 29 | // incompatible with the version of SwiftProtobuf to which you are linking. 30 | // Please ensure that you are building against the same version of the API 31 | // that was used to generate this file. 32 | fileprivate struct _GeneratedWithProtocGenSwiftVersion: SwiftProtobuf.ProtobufAPIVersionCheck { 33 | struct _2: SwiftProtobuf.ProtobufAPIVersion_2 {} 34 | typealias Version = _2 35 | } 36 | 37 | /// The `Status` type defines a logical error model that is suitable for 38 | /// different programming environments, including REST APIs and RPC APIs. It is 39 | /// used by [gRPC](https://github.com/grpc). Each `Status` message contains 40 | /// three pieces of data: error code, error message, and error details. 41 | /// 42 | /// You can find out more about this error model and how to work with it in the 43 | /// [API Design Guide](https://cloud.google.com/apis/design/errors). 44 | public struct Google_Rpc_Status { 45 | // SwiftProtobuf.Message conformance is added in an extension below. See the 46 | // `Message` and `Message+*Additions` files in the SwiftProtobuf library for 47 | // methods supported on all messages. 48 | 49 | /// The status code, which should be an enum value of 50 | /// [google.rpc.Code][google.rpc.Code]. 51 | public var code: Int32 = 0 52 | 53 | /// A developer-facing error message, which should be in English. Any 54 | /// user-facing error message should be localized and sent in the 55 | /// [google.rpc.Status.details][google.rpc.Status.details] field, or localized 56 | /// by the client. 57 | public var message: String = String() 58 | 59 | /// A list of messages that carry the error details. There is a common set of 60 | /// message types for APIs to use. 61 | public var details: [SwiftProtobuf.Google_Protobuf_Any] = [] 62 | 63 | public var unknownFields = SwiftProtobuf.UnknownStorage() 64 | 65 | public init() {} 66 | } 67 | 68 | #if swift(>=5.5) && canImport(_Concurrency) 69 | extension Google_Rpc_Status: @unchecked Sendable {} 70 | #endif // swift(>=5.5) && canImport(_Concurrency) 71 | 72 | // MARK: - Code below here is support for the SwiftProtobuf runtime. 73 | 74 | fileprivate let _protobuf_package = "google.rpc" 75 | 76 | extension Google_Rpc_Status: SwiftProtobuf.Message, SwiftProtobuf._MessageImplementationBase, SwiftProtobuf._ProtoNameProviding { 77 | public static let protoMessageName: String = _protobuf_package + ".Status" 78 | public static let _protobuf_nameMap: SwiftProtobuf._NameMap = [ 79 | 1: .same(proto: "code"), 80 | 2: .same(proto: "message"), 81 | 3: .same(proto: "details"), 82 | ] 83 | 84 | public mutating func decodeMessage(decoder: inout D) throws { 85 | while let fieldNumber = try decoder.nextFieldNumber() { 86 | // The use of inline closures is to circumvent an issue where the compiler 87 | // allocates stack space for every case branch when no optimizations are 88 | // enabled. https://github.com/apple/swift-protobuf/issues/1034 89 | switch fieldNumber { 90 | case 1: try { try decoder.decodeSingularInt32Field(value: &self.code) }() 91 | case 2: try { try decoder.decodeSingularStringField(value: &self.message) }() 92 | case 3: try { try decoder.decodeRepeatedMessageField(value: &self.details) }() 93 | default: break 94 | } 95 | } 96 | } 97 | 98 | public func traverse(visitor: inout V) throws { 99 | if self.code != 0 { 100 | try visitor.visitSingularInt32Field(value: self.code, fieldNumber: 1) 101 | } 102 | if !self.message.isEmpty { 103 | try visitor.visitSingularStringField(value: self.message, fieldNumber: 2) 104 | } 105 | if !self.details.isEmpty { 106 | try visitor.visitRepeatedMessageField(value: self.details, fieldNumber: 3) 107 | } 108 | try unknownFields.traverse(visitor: &visitor) 109 | } 110 | 111 | public static func ==(lhs: Google_Rpc_Status, rhs: Google_Rpc_Status) -> Bool { 112 | if lhs.code != rhs.code {return false} 113 | if lhs.message != rhs.message {return false} 114 | if lhs.details != rhs.details {return false} 115 | if lhs.unknownFields != rhs.unknownFields {return false} 116 | return true 117 | } 118 | } 119 | -------------------------------------------------------------------------------- /Sources/BazelRemoteAPI/Typealiases.swift: -------------------------------------------------------------------------------- 1 | // This source file is part of the Swift.org open source project 2 | // 3 | // Copyright (c) 2020 Apple Inc. and the Swift project authors 4 | // Licensed under Apache License v2.0 with Runtime Library Exception 5 | // 6 | // See http://swift.org/LICENSE.txt for license information 7 | // See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 8 | 9 | // Type aliases to drop the autogenerated prefix which is not really required since we're keeping the generated code in its own target. 10 | 11 | public typealias CapabilitiesClient = Build_Bazel_Remote_Execution_V2_CapabilitiesClient 12 | public typealias GetCapabilitiesRequest = Build_Bazel_Remote_Execution_V2_GetCapabilitiesRequest 13 | public typealias ContentAddressableStorageClient = Build_Bazel_Remote_Execution_V2_ContentAddressableStorageClient 14 | public typealias ServerCapabilities = Build_Bazel_Remote_Execution_V2_ServerCapabilities 15 | 16 | 17 | public typealias FindMissingBlobsRequest = Build_Bazel_Remote_Execution_V2_FindMissingBlobsRequest 18 | -------------------------------------------------------------------------------- /Sources/LLBBazelBackend/CAS/Digest.swift: -------------------------------------------------------------------------------- 1 | // This source file is part of the Swift.org open source project 2 | // 3 | // Copyright (c) 2020 Apple Inc. and the Swift project authors 4 | // Licensed under Apache License v2.0 with Runtime Library Exception 5 | // 6 | // See http://swift.org/LICENSE.txt for license information 7 | // See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 8 | 9 | import Foundation 10 | 11 | import BazelRemoteAPI 12 | import Crypto 13 | import SwiftProtobuf 14 | import TSCUtility 15 | import TSFCAS 16 | 17 | typealias Digest = Build_Bazel_Remote_Execution_V2_Digest 18 | 19 | /// A Bazel digest. 20 | extension Digest { 21 | public init(with bytes: D) where D : DataProtocol { 22 | // Translate to SHA256. 23 | var hashFunction = Crypto.SHA256() 24 | hashFunction.update(data: bytes) 25 | let cryptoDigest = hashFunction.finalize() 26 | 27 | var hashBytes = Data() 28 | cryptoDigest.withUnsafeBytes { ptr in 29 | hashBytes.append(contentsOf: ptr) 30 | } 31 | 32 | self = .with { 33 | $0.hash = hexEncode(hashBytes) 34 | $0.sizeBytes = Int64(bytes.count) 35 | } 36 | } 37 | 38 | func asDataID() throws -> LLBDataID { 39 | return LLBDataID(directHash: Array(try self.serializedData())) 40 | } 41 | } 42 | 43 | extension LLBDataID { 44 | func asBazelDigest() throws -> Digest { 45 | return try bytes.dropFirst().withUnsafeBytes { 46 | try Digest.init(serializedData: Data(bytesNoCopy: UnsafeMutableRawPointer(mutating: $0.baseAddress!), count: $0.count, deallocator: .none)) } 47 | } 48 | } 49 | 50 | extension SwiftProtobuf.Message { 51 | func asDigest() throws -> Digest { 52 | let bytes = [UInt8](try self.serializedData()) 53 | return Digest(with: bytes) 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /Sources/LLBCASTool/CASTool.swift: -------------------------------------------------------------------------------- 1 | // This source file is part of the Swift.org open source project 2 | // 3 | // Copyright (c) 2020 Apple Inc. and the Swift project authors 4 | // Licensed under Apache License v2.0 with Runtime Library Exception 5 | // 6 | // See http://swift.org/LICENSE.txt for license information 7 | // See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 8 | 9 | import Foundation 10 | 11 | import NIO 12 | import GRPC 13 | import SwiftProtobuf 14 | import BazelRemoteAPI 15 | import TSCBasic 16 | import TSFCAS 17 | import TSCUtility 18 | 19 | import LLBBazelBackend 20 | 21 | 22 | /// Frontend to the remote execution tool. 23 | public final class LLBCASTool { 24 | 25 | /// The tool options. 26 | public let options: LLBCASToolOptions 27 | 28 | public let group: LLBFuturesDispatchGroup 29 | private let db: LLBCASDatabase 30 | 31 | let threadPool: NIOThreadPool 32 | let fileIO: NonBlockingFileIO 33 | 34 | public enum Error: Swift.Error { 35 | case unsupported 36 | } 37 | 38 | public init(group: LLBFuturesDispatchGroup, _ options: LLBCASToolOptions) throws { 39 | self.group = group 40 | self.options = options 41 | 42 | self.db = try LLBCASDatabaseSpec(options.url).open(group: group) 43 | 44 | let threadPool = NIOThreadPool(numberOfThreads: 6) 45 | self.threadPool = threadPool 46 | threadPool.start() 47 | self.fileIO = NonBlockingFileIO(threadPool: threadPool) 48 | } 49 | 50 | deinit { 51 | try? threadPool.syncShutdownGracefully() 52 | } 53 | 54 | /// Put the given file into the CAS database. 55 | public func casPut(file: AbsolutePath, _ ctx: Context) -> LLBFuture { 56 | let handleAndRegion = fileIO.openFile( 57 | path: file.pathString, eventLoop: group.next() 58 | ) 59 | 60 | let buffer: LLBFuture = handleAndRegion.flatMap { (handle, region) in 61 | let allocator = ByteBufferAllocator() 62 | return self.fileIO.read( 63 | fileRegion: region, 64 | allocator: allocator, 65 | eventLoop: self.group.next() 66 | ).flatMapThrowing { buffer in 67 | try handle.close() 68 | return buffer 69 | } 70 | } 71 | 72 | return buffer.flatMap { buf in 73 | self.db.put(refs: [], data: buf, ctx) 74 | } 75 | } 76 | 77 | /// Get the contents of the given data id from CAS database. 78 | public func casGet( 79 | id: LLBDataID, 80 | to outputFile: AbsolutePath, 81 | _ ctx: Context 82 | ) -> LLBFuture { 83 | let object = db.get(id, ctx) 84 | 85 | let data: LLBFuture = object.flatMapThrowing { 86 | guard let data = $0?.data else { 87 | throw StringError("No object in CAS with id \(id)") 88 | } 89 | return data 90 | } 91 | 92 | let handle = fileIO.openFile( 93 | path: outputFile.pathString, 94 | mode: .write, 95 | flags: .allowFileCreation(), 96 | eventLoop: group.next() 97 | ) 98 | 99 | return handle.and(data).flatMap { (handle, data) in 100 | self.fileIO.write( 101 | fileHandle: handle, 102 | buffer: data, 103 | eventLoop: self.group.next() 104 | ).flatMapThrowing { _ in 105 | try handle.close() 106 | } 107 | } 108 | } 109 | 110 | 111 | /// Get the server capabilities of the remote endpoint. 112 | public func getCapabilities() -> LLBFuture { 113 | guard let bazelDB = self.db as? LLBBazelCASDatabase else { 114 | return group.next().makeFailedFuture(Error.unsupported) 115 | } 116 | 117 | return bazelDB.serverCapabilities() 118 | } 119 | } 120 | 121 | -------------------------------------------------------------------------------- /Sources/LLBCASTool/Options.swift: -------------------------------------------------------------------------------- 1 | // This source file is part of the Swift.org open source project 2 | // 3 | // Copyright (c) 2020 Apple Inc. and the Swift project authors 4 | // Licensed under Apache License v2.0 with Runtime Library Exception 5 | // 6 | // See http://swift.org/LICENSE.txt for license information 7 | // See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 8 | 9 | import Foundation 10 | 11 | public struct LLBCASToolOptions { 12 | /// The frontend URL for the CAS 13 | public var url: URL 14 | 15 | public init( 16 | url: URL 17 | ) { 18 | self.url = url 19 | } 20 | } 21 | 22 | -------------------------------------------------------------------------------- /Sources/Tools/llcastool/CASCommands.swift: -------------------------------------------------------------------------------- 1 | // This source file is part of the Swift.org open source project 2 | // 3 | // Copyright (c) 2020 Apple Inc. and the Swift project authors 4 | // Licensed under Apache License v2.0 with Runtime Library Exception 5 | // 6 | // See http://swift.org/LICENSE.txt for license information 7 | // See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 8 | 9 | import Foundation 10 | 11 | import ArgumentParser 12 | import TSCBasic 13 | import TSFCAS 14 | import TSCUtility 15 | 16 | import LLBCASTool 17 | 18 | 19 | struct CASPut: ParsableCommand { 20 | static let configuration: CommandConfiguration = CommandConfiguration( 21 | commandName: "put", 22 | abstract: "Put the given file into the CAS database" 23 | ) 24 | 25 | @OptionGroup() 26 | var options: CommonOptions 27 | 28 | @Argument() 29 | var path: AbsolutePath 30 | 31 | func run() throws { 32 | let fileSize = try localFileSystem.getFileInfo(path).size 33 | stderrStream <<< "importing \(path.basename), \(prettyFileSize(fileSize))\n" 34 | stderrStream.flush() 35 | 36 | let group = LLBMakeDefaultDispatchGroup() 37 | let ctx = Context() 38 | let toolOptions = self.options.toToolOptions() 39 | let tool = try LLBCASTool(group: group, toolOptions) 40 | let dataID = try tool.casPut(file: path, ctx).wait() 41 | print(dataID) 42 | } 43 | } 44 | 45 | struct CASGet: ParsableCommand { 46 | static let configuration: CommandConfiguration = CommandConfiguration( 47 | commandName: "get", 48 | abstract: "Get a file from the CAS database given a data id" 49 | ) 50 | 51 | @OptionGroup() 52 | var options: CommonOptions 53 | 54 | @Option() 55 | var id: String 56 | 57 | @Argument() 58 | var path: AbsolutePath 59 | 60 | func run() throws { 61 | guard let id = LLBDataID(string: self.id) else { 62 | throw StringError("Invalid data id \(self.id)") 63 | } 64 | 65 | let group = LLBMakeDefaultDispatchGroup() 66 | let ctx = Context() 67 | let toolOptions = self.options.toToolOptions() 68 | let tool = try LLBCASTool(group: group, toolOptions) 69 | try tool.casGet(id: id, to: path, ctx).wait() 70 | } 71 | } 72 | 73 | func prettyFileSize(_ size: UInt64) -> String { 74 | if size < 100_000 { 75 | return "\(size) bytes" 76 | } else if size < 100_000_000 { 77 | return String(format: "%.1f MB", Double(size) / 1_000_000) 78 | } else { 79 | return String(format: "%.1f GB", Double(size) / 1_000_000_000) 80 | } 81 | } 82 | -------------------------------------------------------------------------------- /Sources/Tools/llcastool/Capabilities.swift: -------------------------------------------------------------------------------- 1 | // This source file is part of the Swift.org open source project 2 | // 3 | // Copyright (c) 2020 Apple Inc. and the Swift project authors 4 | // Licensed under Apache License v2.0 with Runtime Library Exception 5 | // 6 | // See http://swift.org/LICENSE.txt for license information 7 | // See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 8 | 9 | import Foundation 10 | 11 | import ArgumentParser 12 | import GRPC 13 | import TSCBasic 14 | import TSFFutures 15 | 16 | import LLBCASTool 17 | 18 | 19 | struct Capabilities: ParsableCommand { 20 | static let configuration: CommandConfiguration = CommandConfiguration( 21 | abstract: "Get the server capabilities of the remote server" 22 | ) 23 | 24 | @OptionGroup() 25 | var options: CommonOptions 26 | 27 | func run() throws { 28 | let group = LLBMakeDefaultDispatchGroup() 29 | let toolOptions = self.options.toToolOptions() 30 | let tool = try LLBCASTool(group: group, toolOptions) 31 | 32 | let response = try tool.getCapabilities().wait() 33 | print(response) 34 | } 35 | } 36 | 37 | -------------------------------------------------------------------------------- /Sources/Tools/llcastool/CommonOptions.swift: -------------------------------------------------------------------------------- 1 | // This source file is part of the Swift.org open source project 2 | // 3 | // Copyright (c) 2020 Apple Inc. and the Swift project authors 4 | // Licensed under Apache License v2.0 with Runtime Library Exception 5 | // 6 | // See http://swift.org/LICENSE.txt for license information 7 | // See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 8 | 9 | import ArgumentParser 10 | import Foundation 11 | import LLBCASTool 12 | import TSCUtility 13 | 14 | struct CommonOptions: ParsableArguments { 15 | @Option(help: "The CAS database URL to use") 16 | var url: Foundation.URL 17 | } 18 | 19 | extension CommonOptions { 20 | func toToolOptions() -> LLBCASToolOptions { 21 | return LLBCASToolOptions( 22 | url: url 23 | ) 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /Sources/Tools/llcastool/main.swift: -------------------------------------------------------------------------------- 1 | // This source file is part of the Swift.org open source project 2 | // 3 | // Copyright (c) 2020 Apple Inc. and the Swift project authors 4 | // Licensed under Apache License v2.0 with Runtime Library Exception 5 | // 6 | // See http://swift.org/LICENSE.txt for license information 7 | // See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 8 | 9 | import ArgumentParser 10 | 11 | import LLBBazelBackend 12 | 13 | 14 | struct llcastoolCommand: ParsableCommand { 15 | static var configuration = CommandConfiguration( 16 | commandName: "llcastool", 17 | abstract: "llcastool — cas manipulation tools", 18 | subcommands: [ 19 | Capabilities.self, 20 | CASGet.self, 21 | CASPut.self, 22 | ] 23 | ) 24 | } 25 | 26 | LLBBazelBackend.registerCASSchemes() 27 | 28 | llcastoolCommand.main() 29 | -------------------------------------------------------------------------------- /Sources/Tools/llcastool/misc.swift: -------------------------------------------------------------------------------- 1 | // This source file is part of the Swift.org open source project 2 | // 3 | // Copyright (c) 2020 Apple Inc. and the Swift project authors 4 | // Licensed under Apache License v2.0 with Runtime Library Exception 5 | // 6 | // See http://swift.org/LICENSE.txt for license information 7 | // See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 8 | 9 | import Foundation 10 | import ArgumentParser 11 | import GRPC 12 | import TSCBasic 13 | 14 | extension URL: ExpressibleByArgument { 15 | public init?(argument: String) { 16 | if let parsed = URL(string: argument) { 17 | self = parsed 18 | } else { 19 | return nil 20 | } 21 | } 22 | } 23 | 24 | extension AbsolutePath: ExpressibleByArgument { 25 | public init?(argument: String) { 26 | if let path = try? AbsolutePath(validating: argument) { 27 | self = path 28 | } else if let cwd = localFileSystem.currentWorkingDirectory, 29 | let path = try? AbsolutePath(validating: argument, relativeTo: cwd) { 30 | self = path 31 | } else { 32 | return nil 33 | } 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /Sources/llbuild2/Reexport.swift: -------------------------------------------------------------------------------- 1 | // This source file is part of the Swift.org open source project 2 | // 3 | // Copyright (c) 2025 Apple Inc. and the Swift project authors 4 | // Licensed under Apache License v2.0 with Runtime Library Exception 5 | // 6 | // See http://swift.org/LICENSE.txt for license information 7 | // See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 8 | 9 | @_exported import llbuild2fx 10 | -------------------------------------------------------------------------------- /Sources/llbuild2fx/Action.swift: -------------------------------------------------------------------------------- 1 | // This source file is part of the Swift.org open source project 2 | // 3 | // Copyright (c) 2021 Apple Inc. and the Swift project authors 4 | // Licensed under Apache License v2.0 with Runtime Library Exception 5 | // 6 | // See http://swift.org/LICENSE.txt for license information 7 | // See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 8 | 9 | import Foundation 10 | import NIOCore 11 | import TSCUtility 12 | import TSFFutures 13 | 14 | public struct FXActionRequirements { 15 | public let predicate: (any Predicate)? 16 | 17 | public init(predicate: (any Predicate)? = nil) { 18 | self.predicate = predicate 19 | } 20 | } 21 | 22 | public protocol FXAction: FXValue { 23 | associatedtype ValueType: FXValue 24 | 25 | static var name: String { get } 26 | static var version: Int { get } 27 | 28 | var requirements: FXActionRequirements { get } 29 | 30 | func run(_ ctx: Context) -> LLBFuture 31 | } 32 | 33 | extension FXAction { 34 | public static var name: String { String(describing: self) } 35 | public static var version: Int { 0 } 36 | 37 | public var requirements: FXActionRequirements { 38 | FXActionRequirements() 39 | } 40 | } 41 | 42 | public protocol AsyncFXAction: FXAction { 43 | func run(_ ctx: Context) async throws -> ValueType 44 | } 45 | 46 | extension AsyncFXAction { 47 | public func run(_ ctx: Context) -> LLBFuture { 48 | ctx.group.any().makeFutureWithTask { 49 | try await run(ctx) 50 | } 51 | } 52 | } 53 | 54 | -------------------------------------------------------------------------------- /Sources/llbuild2fx/ActionCache/FileBackedFunctionCache.swift: -------------------------------------------------------------------------------- 1 | // This source file is part of the Swift.org open source project 2 | // 3 | // Copyright (c) 2020 Apple Inc. and the Swift project authors 4 | // Licensed under Apache License v2.0 with Runtime Library Exception 5 | // 6 | // See http://swift.org/LICENSE.txt for license information 7 | // See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 8 | 9 | import NIO 10 | import NIOConcurrencyHelpers 11 | import TSCBasic 12 | 13 | /// A simple in-memory implementation of the `FXFunctionCache` protocol. 14 | public final class FXFileBackedFunctionCache: FXFunctionCache { 15 | /// The content root path. 16 | public let path: AbsolutePath 17 | 18 | /// Threads capable of running futures. 19 | public let group: LLBFuturesDispatchGroup 20 | 21 | let threadPool: NIOThreadPool 22 | let fileIO: NonBlockingFileIO 23 | 24 | /// Create an in-memory database. 25 | public init(group: LLBFuturesDispatchGroup, path: AbsolutePath, version: String = "default") { 26 | self.group = group 27 | self.threadPool = NIOThreadPool(numberOfThreads: 6) 28 | threadPool.start() 29 | self.fileIO = NonBlockingFileIO(threadPool: threadPool) 30 | self.path = path.appending(component: version) 31 | try? localFileSystem.createDirectory(self.path, recursive: true) 32 | } 33 | 34 | deinit { 35 | try? threadPool.syncShutdownGracefully() 36 | } 37 | 38 | private func filePath(key: FXRequestKey) -> AbsolutePath { 39 | return path.appending(component: "\(key.stableHashValue)") 40 | } 41 | 42 | public func get(key: FXRequestKey, props: any FXKeyProperties, _ ctx: Context) -> LLBFuture { 43 | let file = filePath(key: key) 44 | let handleAndRegion = fileIO.openFile( 45 | path: file.pathString, eventLoop: group.next() 46 | ) 47 | 48 | let data: LLBFuture = handleAndRegion.flatMap { (handle, region) in 49 | let allocator = ByteBufferAllocator() 50 | return self.fileIO.read( 51 | fileRegion: region, 52 | allocator: allocator, 53 | eventLoop: self.group.next() 54 | ) 55 | } 56 | 57 | return handleAndRegion.and(data).flatMapThrowing { (handle, data) in 58 | try handle.0.close() 59 | return try LLBDataID(from: data) 60 | }.recover { _ in 61 | return nil 62 | } 63 | } 64 | 65 | public func update(key: FXRequestKey, props: any FXKeyProperties, value: LLBDataID, _ ctx: Context) -> LLBFuture { 66 | let file = filePath(key: key) 67 | let handle = fileIO.openFile( 68 | path: file.pathString, 69 | mode: .write, 70 | flags: .allowFileCreation(), 71 | eventLoop: group.next() 72 | ) 73 | 74 | let result = handle.flatMap { handle -> LLBFuture in 75 | do { 76 | return self.fileIO.write( 77 | fileHandle: handle, 78 | buffer: try value.toBytes(), 79 | eventLoop: self.group.next() 80 | ) 81 | } catch { 82 | return self.group.next().makeFailedFuture(error) 83 | } 84 | } 85 | 86 | return handle.and(result).flatMapThrowing { (handle, _) in 87 | try handle.close() 88 | } 89 | } 90 | } 91 | -------------------------------------------------------------------------------- /Sources/llbuild2fx/ActionCache/InMemoryFunctionCache.swift: -------------------------------------------------------------------------------- 1 | // This source file is part of the Swift.org open source project 2 | // 3 | // Copyright (c) 2020 Apple Inc. and the Swift project authors 4 | // Licensed under Apache License v2.0 with Runtime Library Exception 5 | // 6 | // See http://swift.org/LICENSE.txt for license information 7 | // See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 8 | 9 | import NIOConcurrencyHelpers 10 | import NIOCore 11 | import TSFFutures 12 | 13 | /// A simple in-memory implementation of the `FXFunctionCache` protocol. 14 | public final class FXInMemoryFunctionCache: FXFunctionCache { 15 | /// The cache. 16 | private let cache = NIOLockedValueBox([HashableKey: LLBDataID]()) 17 | 18 | /// Threads capable of running futures. 19 | public let group: LLBFuturesDispatchGroup 20 | 21 | /// The lock protecting content. 22 | let lock = NIOConcurrencyHelpers.NIOLock() 23 | 24 | /// Create an in-memory database. 25 | public init(group: LLBFuturesDispatchGroup) { 26 | self.group = group 27 | } 28 | 29 | public func get(key: FXRequestKey, props: any FXKeyProperties, _ ctx: Context) -> LLBFuture { 30 | return group.next().makeSucceededFuture(cache.withLockedValue { $0[HashableKey(key: key)] }) 31 | } 32 | 33 | public func update(key: FXRequestKey, props: any FXKeyProperties, value: LLBDataID, _ ctx: Context) -> LLBFuture { 34 | return group.next().makeSucceededFuture(cache.withLockedValue { $0[HashableKey(key: key)] = value }) 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /Sources/llbuild2fx/Coding.swift: -------------------------------------------------------------------------------- 1 | // This source file is part of the Swift.org open source project 2 | // 3 | // Copyright (c) 2021 Apple Inc. and the Swift project authors 4 | // Licensed under Apache License v2.0 with Runtime Library Exception 5 | // 6 | // See http://swift.org/LICENSE.txt for license information 7 | // See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 8 | 9 | import Foundation 10 | import SwiftProtobuf 11 | 12 | public struct FXEncoder { 13 | private let encoder: JSONEncoder 14 | 15 | public init() { 16 | encoder = JSONEncoder() 17 | encoder.dateEncodingStrategy = .iso8601 18 | encoder.outputFormatting = [.sortedKeys] 19 | } 20 | 21 | public func encode(_ value: T) throws -> Data { 22 | try encoder.encode(value) 23 | } 24 | } 25 | 26 | public struct FXDecoder { 27 | private let decoder: JSONDecoder 28 | 29 | public init() { 30 | decoder = JSONDecoder() 31 | decoder.dateDecodingStrategy = .iso8601 32 | } 33 | 34 | public func decode(_ type: T.Type, from data: Data) throws -> T { 35 | try decoder.decode(type, from: data) 36 | } 37 | } 38 | 39 | extension Encoder { 40 | public func fxEncodeHash(of value: V) throws { 41 | let data = try FXEncoder().encode(value) 42 | try encodeHash(of: ArraySlice(data)) 43 | } 44 | 45 | func encodeHash(of data: ArraySlice) throws { 46 | var container = singleValueContainer() 47 | 48 | let hash = LLBDataID(blake3hash: data) 49 | // We don't need the whole ID to avoid key collisions. 50 | let str = ArraySlice(hash.bytes.dropFirst().prefix(9)).base64URL() 51 | try container.encode(str) 52 | } 53 | } 54 | 55 | extension Encodable { 56 | public func fxEncodeJSON() throws -> String { 57 | let encoder = FXEncoder() 58 | return try String(decoding: encoder.encode(self), as: UTF8.self) 59 | } 60 | } 61 | 62 | extension Encodable where Self: SwiftProtobuf.Message { 63 | public func encode(to encoder: Swift.Encoder) throws { 64 | var container = encoder.singleValueContainer() 65 | try container.encode(self.serializedData()) 66 | } 67 | } 68 | 69 | extension Decodable where Self: SwiftProtobuf.Message { 70 | public init(from decoder: Swift.Decoder) throws { 71 | let container = try decoder.singleValueContainer() 72 | try self.init(serializedBytes: container.decode(Data.self)) 73 | } 74 | } 75 | 76 | // Convenience constraint so that SwiftProtobuf serialization is preferred over Codable. 77 | extension LLBSerializableIn where Self: Decodable, Self: SwiftProtobuf.Message { 78 | public init(from bytes: LLBByteBuffer) throws { 79 | let data = Data(bytes.readableBytesView) 80 | self = try Self.init(serializedBytes: data) 81 | } 82 | } 83 | 84 | // Convenience constraint so that SwiftProtobuf serialization is preferred over Codable. 85 | extension LLBSerializableOut where Self: Encodable, Self: SwiftProtobuf.Message { 86 | public func toBytes(into buffer: inout LLBByteBuffer) throws { 87 | buffer.writeBytes(try self.serializedData()) 88 | } 89 | } 90 | -------------------------------------------------------------------------------- /Sources/llbuild2fx/Deadline.swift: -------------------------------------------------------------------------------- 1 | // This source file is part of the Swift.org open source project 2 | // 3 | // Copyright (c) 2021 Apple Inc. and the Swift project authors 4 | // Licensed under Apache License v2.0 with Runtime Library Exception 5 | // 6 | // See http://swift.org/LICENSE.txt for license information 7 | // See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 8 | 9 | import Foundation 10 | import NIO 11 | import TSCUtility 12 | import TSFFutures 13 | 14 | private class DeadlineKey {} 15 | 16 | extension Context { 17 | public var fxDeadline: Date? { 18 | get { 19 | self[ObjectIdentifier(DeadlineKey.self), as: Date.self] 20 | } 21 | set { 22 | self[ObjectIdentifier(DeadlineKey.self)] = newValue 23 | } 24 | } 25 | 26 | public var nioDeadline: NIODeadline? { 27 | guard let foundationDeadline = fxDeadline else { 28 | return nil 29 | } 30 | 31 | guard foundationDeadline != .distantFuture else { 32 | return nil 33 | } 34 | 35 | let then = NIODeadline.now() 36 | let now = Date() 37 | let timeLeft: TimeInterval = foundationDeadline.timeIntervalSince(now) 38 | let milliseconds = timeLeft * 1000 39 | let microseconds = milliseconds * 1000 40 | let nanoseconds = Int64(microseconds * 1000) 41 | return then + TimeAmount.nanoseconds(nanoseconds) 42 | } 43 | 44 | public func fxReducingDeadline(to atLeast: Date) -> Self { 45 | var ctx = self 46 | 47 | if let existing = fxDeadline { 48 | if existing > atLeast { 49 | ctx.fxDeadline = atLeast 50 | } 51 | } else { 52 | ctx.fxDeadline = atLeast 53 | } 54 | 55 | return ctx 56 | } 57 | 58 | public func fxApplyDeadline(_ cancellable: LLBCancellableFuture) { 59 | if let actualDeadline = nioDeadline { 60 | let timer = cancellable.future.eventLoop.scheduleTask(deadline: actualDeadline) { 61 | cancellable.cancel(reason: "timeout") 62 | } 63 | 64 | cancellable.future.whenComplete { 65 | _ in timer.cancel() 66 | } 67 | } 68 | } 69 | } 70 | -------------------------------------------------------------------------------- /Sources/llbuild2fx/Diagnostics.swift: -------------------------------------------------------------------------------- 1 | // This source file is part of the Swift.org open source project 2 | // 3 | // Copyright (c) 2022 Apple Inc. and the Swift project authors 4 | // Licensed under Apache License v2.0 with Runtime Library Exception 5 | // 6 | // See http://swift.org/LICENSE.txt for license information 7 | // See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 8 | 9 | import NIOCore 10 | 11 | public struct FXDiagnostics: FXThinEncodedSingleDataIDValue, FXTreeID { 12 | public let dataID: LLBDataID 13 | public init(dataID: LLBDataID) { 14 | self.dataID = dataID 15 | } 16 | } 17 | 18 | public protocol FXDiagnosticsGathering: Sendable { 19 | func gatherDiagnostics(pid: Int32?, _ ctx: Context) -> LLBFuture 20 | } 21 | 22 | private final class ContextDiagnosticsGatherer: Sendable {} 23 | 24 | extension Context { 25 | public var fxDiagnosticsGatherer: FXDiagnosticsGathering? { 26 | get { 27 | guard let value = self[ObjectIdentifier(ContextDiagnosticsGatherer.self), as: FXDiagnosticsGathering.self] else { 28 | return nil 29 | } 30 | 31 | return value 32 | } 33 | set { 34 | self[ObjectIdentifier(ContextDiagnosticsGatherer.self)] = newValue 35 | } 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /Sources/llbuild2fx/Engine.swift: -------------------------------------------------------------------------------- 1 | // This source file is part of the Swift.org open source project 2 | // 3 | // Copyright (c) 2021 Apple Inc. and the Swift project authors 4 | // Licensed under Apache License v2.0 with Runtime Library Exception 5 | // 6 | // See http://swift.org/LICENSE.txt for license information 7 | // See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 8 | 9 | import Dispatch 10 | import Foundation 11 | import Logging 12 | import NIOCore 13 | @preconcurrency import TSFFutures 14 | import Tracing 15 | import Instrumentation 16 | 17 | public protocol FXStablyHashable: Sendable { 18 | var stableHashValue: LLBDataID { get } 19 | } 20 | 21 | public protocol FXRequestKey: FXStablyHashable { 22 | func hash(into hasher: inout Hasher) 23 | var hashValue: Int { get } 24 | 25 | /// Use for debugging purposes, should not be invoked by the engine unless the logger is configured for the trace 26 | /// level. 27 | func logDescription() -> String 28 | } 29 | 30 | public extension FXRequestKey { 31 | /// Default implementation for all keys. Keys can implement their own method if they want to display more relevant 32 | /// information. 33 | func logDescription() -> String { 34 | String(describing: type(of: self)) 35 | } 36 | } 37 | 38 | 39 | public protocol FXResult: LLBCASObjectRepresentable {} 40 | 41 | internal struct HashableKey { 42 | let key: FXRequestKey 43 | } 44 | 45 | extension HashableKey: Hashable { 46 | func hash(into hasher: inout Hasher) { 47 | key.hash(into: &hasher) 48 | } 49 | static func ==(lhs: HashableKey, rhs: HashableKey) -> Bool { 50 | return lhs.key.hashValue == rhs.key.hashValue 51 | } 52 | } 53 | 54 | internal protocol CallableKey { 55 | func function() -> GenericFunction 56 | } 57 | internal protocol GenericFunction { 58 | func compute(key: FXRequestKey, _ fi: FunctionInterface, _ ctx: Context) -> LLBFuture 59 | } 60 | 61 | internal final class FunctionInterface: Sendable { 62 | @usableFromInline 63 | let engine: FXEngine 64 | 65 | let key: FXRequestKey 66 | 67 | init(engine: FXEngine, key: FXRequestKey) { 68 | self.engine = engine 69 | self.key = key 70 | } 71 | 72 | func request(_ key: FXRequestKey, as type: V.Type = V.self, _ ctx: Context) -> LLBFuture { 73 | do { 74 | try engine.keyDependencyGraph.addEdge(from: self.key, to: key) 75 | } catch { 76 | return ctx.group.next().makeFailedFuture(error) 77 | } 78 | let future = engine.build(key: key, as: type, ctx) 79 | future.whenComplete { _ in 80 | self.engine.keyDependencyGraph.removeEdge(from: self.key, to: key) 81 | } 82 | return future 83 | } 84 | } 85 | 86 | public typealias FXBuildID = Foundation.UUID 87 | 88 | public final class FXEngine: Sendable { 89 | internal let group: LLBFuturesDispatchGroup 90 | internal let db: LLBCASDatabase 91 | @usableFromInline internal let cache: FXFunctionCache 92 | internal let resources: [ResourceKey: FXResource] 93 | internal let executor: FXExecutor 94 | internal let stats: FXBuildEngineStats 95 | internal let logger: Logger? 96 | 97 | internal let cacheRequestOnly: Bool 98 | 99 | fileprivate let pendingResults: LLBEventualResultsCache 100 | fileprivate let keyDependencyGraph = FXKeyDependencyGraph() 101 | 102 | public let buildID: FXBuildID 103 | 104 | public init( 105 | group: LLBFuturesDispatchGroup, 106 | db: LLBCASDatabase, 107 | functionCache: FXFunctionCache?, 108 | executor: FXExecutor, 109 | resources: [ResourceKey: FXResource] = [:], 110 | buildID: FXBuildID = FXBuildID(), 111 | stats: FXBuildEngineStats? = nil, 112 | logger: Logger? = nil, 113 | partialResultExpiration: DispatchTimeInterval = .seconds(300) 114 | ) { 115 | self.group = group 116 | self.db = db 117 | self.cache = functionCache ?? FXInMemoryFunctionCache(group: group) 118 | self.resources = resources 119 | self.executor = executor 120 | self.stats = stats ?? .init() 121 | self.logger = logger 122 | 123 | self.pendingResults = LLBEventualResultsCache(group: group, partialResultExpiration: partialResultExpiration) 124 | 125 | self.buildID = buildID 126 | 127 | var cacheRequestOnly = false 128 | for res in resources.values { 129 | if case .requestOnly = res.lifetime { 130 | cacheRequestOnly = true 131 | break 132 | } 133 | } 134 | self.cacheRequestOnly = cacheRequestOnly 135 | } 136 | 137 | /// Populate context with engine provided values 138 | private func engineContext(_ ctx: Context) -> Context { 139 | var ctx = ctx 140 | ctx.group = self.group 141 | ctx.db = self.db 142 | 143 | if let logger = self.logger { 144 | ctx.logger = logger 145 | } 146 | 147 | return ctx 148 | } 149 | 150 | internal func build(key: FXRequestKey, _ ctx: Context) -> LLBFuture { 151 | let ctx = engineContext(ctx) 152 | return self.pendingResults.value(for: HashableKey(key: key)) { _ in 153 | guard let ikey = key as? CallableKey else { 154 | return ctx.group.any().makeFailedFuture(FXError.nonCallableKey) 155 | } 156 | let fn = ikey.function() 157 | let fi = FunctionInterface(engine: self, key: key) 158 | return fn.compute(key: key, fi, ctx) 159 | } 160 | } 161 | 162 | internal func build(key: FXRequestKey, as: V.Type, _ ctx: Context) -> LLBFuture { 163 | return self.build(key: key, ctx).flatMapThrowing { 164 | guard let value = $0 as? V else { 165 | throw FXError.invalidValueType("Expected value of type \(V.self)") 166 | } 167 | return value 168 | } 169 | } 170 | 171 | public func build( 172 | key: K, 173 | _ ctx: Context 174 | ) -> LLBFuture { 175 | let ctx = engineContext(ctx) 176 | return self.build(key: key.internalKey(self, ctx), as: InternalValue.self, ctx).map { internalValue in 177 | internalValue.value 178 | } 179 | } 180 | } 181 | -------------------------------------------------------------------------------- /Sources/llbuild2fx/Environment.swift: -------------------------------------------------------------------------------- 1 | // This source file is part of the Swift.org open source project 2 | // 3 | // Copyright (c) 2022 Apple Inc. and the Swift project authors 4 | // Licensed under Apache License v2.0 with Runtime Library Exception 5 | // 6 | // See http://swift.org/LICENSE.txt for license information 7 | // See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 8 | 9 | import TSCUtility 10 | 11 | public typealias FXActionExecutionEnvironment = Context 12 | -------------------------------------------------------------------------------- /Sources/llbuild2fx/Errors.swift: -------------------------------------------------------------------------------- 1 | // This source file is part of the Swift.org open source project 2 | // 3 | // Copyright (c) 2025 Apple Inc. and the Swift project authors 4 | // Licensed under Apache License v2.0 with Runtime Library Exception 5 | // 6 | // See http://swift.org/LICENSE.txt for license information 7 | // See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 8 | 9 | public enum FXError: Swift.Error { 10 | case nonCallableKey 11 | case cycleDetected([FXRequestKey]) 12 | 13 | case valueComputationError(keyPrefix: String, key: String, error: Swift.Error, requestedCacheKeyPaths: FXSortedSet) 14 | case keyEncodingError(keyPrefix: String, encodingError: Swift.Error, underlyingError: Swift.Error) 15 | 16 | case missingRequiredCacheEntry(cachePath: String) 17 | case unexpressedKeyDependency(from: String, to: String) 18 | case executorCannotSatisfyRequirements 19 | case noExecutable 20 | case invalidValueType(String) 21 | case unexpectedKeyType(String) 22 | case inconsistentValue(String) 23 | case resourceNotFound(ResourceKey) 24 | } 25 | 26 | func unwrapFXError(_ error: Swift.Error) -> Swift.Error { 27 | guard case FXError.valueComputationError( 28 | keyPrefix: _, 29 | key: _, 30 | error: let underlyingError, 31 | requestedCacheKeyPaths: _ 32 | ) = error else { 33 | return error 34 | } 35 | 36 | // May need to continue unwrapping 37 | return unwrapFXError(underlyingError) 38 | } 39 | 40 | /// Overall result that this error implies 41 | public enum FXErrorStatus: Sendable { 42 | /// A non-terminal error 43 | case warning 44 | 45 | /// A generic terminal error 46 | case failure 47 | 48 | /// An implementation specific string 49 | case custom(String) 50 | } 51 | 52 | public enum FXErrorClassification: String, Sendable { 53 | /// A user caused failure (such as bad input, config, etc.) 54 | case user 55 | 56 | /// An internal failure 57 | case infrastructure 58 | } 59 | 60 | public struct FXErrorDetails: Sendable { 61 | public var status: FXErrorStatus 62 | public var classification: FXErrorClassification 63 | public var details: String 64 | 65 | public init( 66 | status: FXErrorStatus, 67 | classification: FXErrorClassification, 68 | details: String 69 | ) { 70 | self.status = status 71 | self.classification = classification 72 | self.details = details 73 | } 74 | } 75 | 76 | public protocol FXErrorClassifier { 77 | func tryClassifyError(_ error: Swift.Error) -> FXErrorDetails? 78 | } 79 | -------------------------------------------------------------------------------- /Sources/llbuild2fx/Executor.swift: -------------------------------------------------------------------------------- 1 | // This source file is part of the Swift.org open source project 2 | // 3 | // Copyright (c) 2021 Apple Inc. and the Swift project authors 4 | // Licensed under Apache License v2.0 with Runtime Library Exception 5 | // 6 | // See http://swift.org/LICENSE.txt for license information 7 | // See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 8 | 9 | import TSCUtility 10 | import TSFFutures 11 | import NIOCore 12 | 13 | public protocol FXExecutor: Sendable { 14 | func perform( 15 | _ action: ActionType, 16 | _ ctx: Context 17 | ) -> LLBFuture 18 | 19 | func canSatisfy(requirements: P) -> Bool where P.EvaluatedType == FXActionExecutionEnvironment 20 | 21 | @available(*, deprecated, message: "use self-resolving perform") 22 | func perform( 23 | action: ActionType, 24 | with executable: LLBFuture, 25 | requirements: P, 26 | _ ctx: Context 27 | ) -> LLBFuture where P.EvaluatedType == FXActionExecutionEnvironment 28 | } 29 | 30 | extension FXExecutor { 31 | func canSatisfy(requirements: P) -> Bool where P.EvaluatedType == FXActionExecutionEnvironment { 32 | true 33 | } 34 | } 35 | 36 | public struct FXExecutableID: FXSingleDataIDValue, FXFileID { 37 | public let dataID: LLBDataID 38 | public init(dataID: LLBDataID) { 39 | self.dataID = dataID 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /Sources/llbuild2fx/FunctionCache.swift: -------------------------------------------------------------------------------- 1 | // This source file is part of the Swift.org open source project 2 | // 3 | // Copyright (c) 2021 Apple Inc. and the Swift project authors 4 | // Licensed under Apache License v2.0 with Runtime Library Exception 5 | // 6 | // See http://swift.org/LICENSE.txt for license information 7 | // See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 8 | 9 | import Logging 10 | import NIOCore 11 | import TSCUtility 12 | 13 | public protocol FXKeyProperties { 14 | var volatile: Bool { get } 15 | 16 | var cachePath: String { get } 17 | } 18 | 19 | public protocol FXFunctionCache : Sendable { 20 | func get(key: FXRequestKey, props: FXKeyProperties, _ ctx: Context) -> LLBFuture 21 | func update(key: FXRequestKey, props: FXKeyProperties, value: LLBDataID, _ ctx: Context) -> LLBFuture 22 | } 23 | -------------------------------------------------------------------------------- /Sources/llbuild2fx/FunctionInterface.swift: -------------------------------------------------------------------------------- 1 | // This source file is part of the Swift.org open source project 2 | // 3 | // Copyright (c) 2021 Apple Inc. and the Swift project authors 4 | // Licensed under Apache License v2.0 with Runtime Library Exception 5 | // 6 | // See http://swift.org/LICENSE.txt for license information 7 | // See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 8 | 9 | import NIOConcurrencyHelpers 10 | import NIOCore 11 | 12 | public final class FXFunctionInterface: Sendable { 13 | private let key: K 14 | private let fi: FunctionInterface 15 | private let requestedKeyCachePaths = NIOLockedValueBox(FXSortedSet()) 16 | private let lock = NIOLock() 17 | var requestedCacheKeyPathsSnapshot: FXSortedSet { 18 | return requestedKeyCachePaths.withLockedValue() { return $0 } 19 | } 20 | 21 | init(_ key: K, _ fi: FunctionInterface) { 22 | self.key = key 23 | self.fi = fi 24 | } 25 | 26 | public func request(_ x: X, requireCacheHit: Bool = false, _ ctx: Context) -> LLBFuture { 27 | do { 28 | let realX = x.internalKey(fi.engine, ctx) 29 | 30 | // Check that the key dependency is either explicity declared or 31 | // recursive/self-referential. 32 | guard K.versionDependencies.contains(where: { $0 == X.self }) || X.self == K.self else { 33 | throw FXError.unexpressedKeyDependency( 34 | from: key.internalKey(fi.engine, ctx).logDescription(), 35 | to: realX.logDescription() 36 | ) 37 | } 38 | 39 | requestedKeyCachePaths.withLockedValue { 40 | $0.insert(realX.cachePath) 41 | return 42 | } 43 | 44 | let cacheCheck: LLBFuture 45 | if requireCacheHit { 46 | cacheCheck = self.fi.engine.cache.get(key: realX, props: realX, ctx).flatMapThrowing { maybeValue in 47 | guard maybeValue != nil else { 48 | throw FXError.missingRequiredCacheEntry(cachePath: realX.cachePath) 49 | } 50 | 51 | return 52 | } 53 | } else { 54 | cacheCheck = ctx.group.next().makeSucceededFuture(()) 55 | } 56 | 57 | return cacheCheck.flatMap { 58 | self.fi.request(realX, as: InternalValue.self, ctx) 59 | }.map { internalValue in 60 | internalValue.value 61 | } 62 | } catch { 63 | return ctx.group.next().makeFailedFuture(error) 64 | } 65 | } 66 | 67 | public func spawn( 68 | _ action: ActionType, 69 | _ ctx: Context 70 | ) -> LLBFuture { 71 | guard K.actionDependencies.contains(where: { $0 == ActionType.self }) else { 72 | return ctx.group.any().makeFailedFuture( 73 | FXError.unexpressedKeyDependency( 74 | from: key.internalKey(fi.engine, ctx).logDescription(), 75 | to: "action: \(ActionType.name)" 76 | ) 77 | ) 78 | } 79 | 80 | fi.engine.stats.add(action: ActionType.name) 81 | 82 | ctx.logger?.debug("Will perform action: \(action)") 83 | let result = fi.engine.executor.perform(action, ctx) 84 | 85 | return result.always { _ in 86 | self.fi.engine.stats.remove(action: ActionType.name) 87 | } 88 | } 89 | 90 | 91 | @available(*, deprecated, message: "use spawn, with registered actions") 92 | public func execute( 93 | action: ActionType, 94 | with executable: LLBFuture? = nil, 95 | requirements: P, 96 | _ ctx: Context 97 | ) -> LLBFuture where P.EvaluatedType == FXActionExecutionEnvironment { 98 | let actionName = String(describing: ActionType.self) 99 | 100 | fi.engine.stats.add(action: actionName) 101 | 102 | let executor = fi.engine.executor 103 | let result: LLBFuture 104 | 105 | if executor.canSatisfy(requirements: requirements) { 106 | ctx.logger?.debug("Will perform action: \(action)") 107 | let exe = executable ?? ctx.group.next().makeFailedFuture(FXError.noExecutable) 108 | result = executor.perform(action: action, with: exe, requirements: requirements, ctx) 109 | } else { 110 | result = ctx.group.next().makeFailedFuture(FXError.executorCannotSatisfyRequirements) 111 | } 112 | 113 | return result.always { _ in 114 | self.fi.engine.stats.remove(action: actionName) 115 | } 116 | } 117 | 118 | @available(*, deprecated, message: "use spawn, with registered actions") 119 | public func execute( 120 | action: ActionType, 121 | with executable: LLBFuture? = nil, 122 | _ ctx: Context 123 | ) -> LLBFuture { 124 | execute( 125 | action: action, 126 | with: executable, 127 | requirements: ConstantPredicate(value: true), 128 | ctx 129 | ) 130 | } 131 | 132 | public func resource(_ key: ResourceKey) -> T? { 133 | if !K.resourceEntitlements.contains(key) { 134 | return nil 135 | } 136 | 137 | return fi.engine.resources[key] as? T 138 | } 139 | } 140 | 141 | extension FXFunctionInterface { 142 | public func request(_ x: X, requireCacheHit: Bool = false, _ ctx: Context) async throws -> X.ValueType { 143 | return try await request(x, requireCacheHit: requireCacheHit, ctx).get() 144 | } 145 | 146 | public func spawn( 147 | _ action: ActionType, 148 | _ ctx: Context 149 | ) async throws -> ActionType.ValueType { 150 | return try await spawn(action, ctx).get() 151 | } 152 | } 153 | -------------------------------------------------------------------------------- /Sources/llbuild2fx/Generated/EngineProtocol/any_serializable.pb.swift: -------------------------------------------------------------------------------- 1 | // DO NOT EDIT. 2 | // swift-format-ignore-file 3 | // 4 | // Generated by the Swift generator plugin for the protocol buffer compiler. 5 | // Source: EngineProtocol/any_serializable.proto 6 | // 7 | // For information on using the generated types, please see the documentation: 8 | // https://github.com/apple/swift-protobuf/ 9 | 10 | // This source file is part of the Swift.org open source project 11 | // 12 | // Copyright (c) 2020 Apple Inc. and the Swift project authors 13 | // Licensed under Apache License v2.0 with Runtime Library Exception 14 | // 15 | // See http://swift.org/LICENSE.txt for license information 16 | // See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 17 | 18 | import Foundation 19 | import SwiftProtobuf 20 | 21 | // If the compiler emits an error on this type, it is because this file 22 | // was generated by a version of the `protoc` Swift plug-in that is 23 | // incompatible with the version of SwiftProtobuf to which you are linking. 24 | // Please ensure that you are building against the same version of the API 25 | // that was used to generate this file. 26 | fileprivate struct _GeneratedWithProtocGenSwiftVersion: SwiftProtobuf.ProtobufAPIVersionCheck { 27 | struct _2: SwiftProtobuf.ProtobufAPIVersion_2 {} 28 | typealias Version = _2 29 | } 30 | 31 | /// AnyCodable is a wrapper type for polymorphic codables, in which the type of the serialized data is not known at 32 | /// compile time. 33 | public struct LLBAnySerializable { 34 | // SwiftProtobuf.Message conformance is added in an extension below. See the 35 | // `Message` and `Message+*Additions` files in the SwiftProtobuf library for 36 | // methods supported on all messages. 37 | 38 | /// A string identifier to map the runtime bytes into a specific type, for deserialization. 39 | public var typeIdentifier: String = String() 40 | 41 | /// The serialized bytes of the underlying data structure. 42 | public var serializedBytes: Data = Data() 43 | 44 | public var unknownFields = SwiftProtobuf.UnknownStorage() 45 | 46 | public init() {} 47 | } 48 | 49 | #if swift(>=5.5) && canImport(_Concurrency) 50 | extension LLBAnySerializable: @unchecked Sendable {} 51 | #endif // swift(>=5.5) && canImport(_Concurrency) 52 | 53 | // MARK: - Code below here is support for the SwiftProtobuf runtime. 54 | 55 | extension LLBAnySerializable: SwiftProtobuf.Message, SwiftProtobuf._MessageImplementationBase, SwiftProtobuf._ProtoNameProviding { 56 | public static let protoMessageName: String = "LLBAnySerializable" 57 | public static let _protobuf_nameMap: SwiftProtobuf._NameMap = [ 58 | 1: .same(proto: "typeIdentifier"), 59 | 2: .same(proto: "serializedBytes"), 60 | ] 61 | 62 | public mutating func decodeMessage(decoder: inout D) throws { 63 | while let fieldNumber = try decoder.nextFieldNumber() { 64 | // The use of inline closures is to circumvent an issue where the compiler 65 | // allocates stack space for every case branch when no optimizations are 66 | // enabled. https://github.com/apple/swift-protobuf/issues/1034 67 | switch fieldNumber { 68 | case 1: try { try decoder.decodeSingularStringField(value: &self.typeIdentifier) }() 69 | case 2: try { try decoder.decodeSingularBytesField(value: &self.serializedBytes) }() 70 | default: break 71 | } 72 | } 73 | } 74 | 75 | public func traverse(visitor: inout V) throws { 76 | if !self.typeIdentifier.isEmpty { 77 | try visitor.visitSingularStringField(value: self.typeIdentifier, fieldNumber: 1) 78 | } 79 | if !self.serializedBytes.isEmpty { 80 | try visitor.visitSingularBytesField(value: self.serializedBytes, fieldNumber: 2) 81 | } 82 | try unknownFields.traverse(visitor: &visitor) 83 | } 84 | 85 | public static func ==(lhs: LLBAnySerializable, rhs: LLBAnySerializable) -> Bool { 86 | if lhs.typeIdentifier != rhs.typeIdentifier {return false} 87 | if lhs.serializedBytes != rhs.serializedBytes {return false} 88 | if lhs.unknownFields != rhs.unknownFields {return false} 89 | return true 90 | } 91 | } 92 | -------------------------------------------------------------------------------- /Sources/llbuild2fx/KeyConfiguration.swift: -------------------------------------------------------------------------------- 1 | // This source file is part of the Swift.org open source project 2 | // 3 | // Copyright (c) 2022 Apple Inc. and the Swift project authors 4 | // Licensed under Apache License v2.0 with Runtime Library Exception 5 | // 6 | // See http://swift.org/LICENSE.txt for license information 7 | // See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 8 | 9 | import Foundation 10 | 11 | extension FXVersioning { 12 | public func getConfigurationInputs(_ ctx: Context) -> KeyConfiguration { 13 | KeyConfiguration(inputs: ctx.fxConfigurationInputs) 14 | } 15 | } 16 | 17 | public typealias FXConfigurationInputs = [String: Encodable] 18 | 19 | extension Context { 20 | public var fxConfigurationInputs: FXConfigurationInputs { 21 | get { 22 | return (self[ObjectIdentifier(FXConfigurationInputs.self), as: FXConfigurationInputs.self]) ?? [:] 23 | } 24 | set { 25 | self[ObjectIdentifier(FXConfigurationInputs.self)] = newValue 26 | } 27 | } 28 | } 29 | 30 | /// KeyConfiguration represents a grab-bag of information that could inform various steps of 31 | /// the build. For example, it's the ideal container for information from an A/B testing or 32 | /// feature-flag system. It ensures that only the configuration explicitly requested by a 33 | /// given FXKey is visible to that key, and that the cache key is set up appropriately 34 | /// so that if a downstream key starts requesting more configuration, we actually recalculate 35 | /// appropriately. 36 | public final class KeyConfiguration: Encodable { 37 | private let inputs: FXConfigurationInputs 38 | private let allowedInputs: FXSortedSet 39 | 40 | init(inputs: FXConfigurationInputs) { 41 | self.inputs = inputs 42 | self.allowedInputs = Self.resolveKeys(K.configurationKeys, universe: Array(inputs.keys)) 43 | } 44 | 45 | static func resolveKeys(_ keys: [ConfigurationKey], universe: [String]) -> FXSortedSet { 46 | return FXSortedSet(keys.reduce(into: [String](), { (result, key) in 47 | switch key { 48 | case let .literal(val): 49 | result.append(val) 50 | case let .prefix(prefix): 51 | result.append(contentsOf: universe.filter { $0.hasPrefix(prefix)} ) 52 | } 53 | })) 54 | } 55 | 56 | public func get(_ key: String) -> T? { 57 | if !allowedInputs.contains(key) { 58 | return nil 59 | } 60 | return inputs[key] as? T 61 | } 62 | 63 | public func encode(to encoder: Encoder) throws { 64 | let allPossibleAllowed = Self.resolveKeys(Array(K.aggregatedConfigurationKeys), universe: Array(self.inputs.keys)) 65 | 66 | try encoder.fxEncodeHash(of: try inputs.filter{ (k, _) in allPossibleAllowed.contains(k) }.mapValues { (val: Encodable) -> String in try val.fxEncodeJSON() }) 67 | } 68 | 69 | func isNoop() -> Bool { 70 | return self.allowedInputs.isEmpty && K.aggregatedConfigurationKeys.allSatisfy({ key in 71 | switch key { 72 | case .literal(_): 73 | return false 74 | case let .prefix(prefix): 75 | return self.inputs.keys.filter { $0.hasPrefix(prefix) }.isEmpty 76 | } 77 | }) 78 | } 79 | } 80 | -------------------------------------------------------------------------------- /Sources/llbuild2fx/LocalExecutor.swift: -------------------------------------------------------------------------------- 1 | // This source file is part of the Swift.org open source project 2 | // 3 | // Copyright (c) 2022 Apple Inc. and the Swift project authors 4 | // Licensed under Apache License v2.0 with Runtime Library Exception 5 | // 6 | // See http://swift.org/LICENSE.txt for license information 7 | // See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 8 | 9 | import NIOCore 10 | 11 | extension FXActionExecutionEnvironment { 12 | private final class ContextKey {} 13 | private static let key = ContextKey() 14 | public var isLocal: Bool { 15 | get { 16 | guard let value = self[ObjectIdentifier(Self.key), as: Bool.self] else { 17 | return false 18 | } 19 | 20 | return value 21 | } 22 | set { 23 | self[ObjectIdentifier(Self.key)] = newValue 24 | } 25 | } 26 | } 27 | 28 | extension FXActionExecutionEnvironment { 29 | public static var local: EqualityPredicate, ConstantExpression> { 30 | EqualityPredicate( 31 | leftExpression: KeyPathExpression(keyPath: \FXActionExecutionEnvironment.isLocal), 32 | rightExpression: ConstantExpression(value: true) 33 | ) 34 | } 35 | } 36 | 37 | public final class FXLocalExecutor: FXExecutor { 38 | private let environment: FXActionExecutionEnvironment 39 | 40 | public init(environment: FXActionExecutionEnvironment = .init()) { 41 | var env = environment 42 | env.isLocal = true 43 | self.environment = env 44 | } 45 | 46 | public func perform( 47 | _ action: ActionType, _ ctx: Context 48 | ) -> LLBFuture { 49 | return action.run(ctx) 50 | } 51 | 52 | public func canSatisfy(requirements: P) -> Bool where P.EvaluatedType == FXActionExecutionEnvironment { 53 | requirements.evaluate(with: environment) 54 | } 55 | 56 | public func perform( 57 | action: ActionType, 58 | with executable: LLBFuture, 59 | requirements: P, 60 | _ ctx: Context 61 | ) -> LLBFuture where P.EvaluatedType == FXActionExecutionEnvironment { 62 | action.run(ctx) 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /Sources/llbuild2fx/NullExecutor.swift: -------------------------------------------------------------------------------- 1 | // This source file is part of the Swift.org open source project 2 | // 3 | // Copyright (c) 2022 Apple Inc. and the Swift project authors 4 | // Licensed under Apache License v2.0 with Runtime Library Exception 5 | // 6 | // See http://swift.org/LICENSE.txt for license information 7 | // See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 8 | 9 | import NIOCore 10 | 11 | public final class FXNullExecutor: FXExecutor { 12 | public init() {} 13 | 14 | public func perform( 15 | _ action: ActionType, 16 | _ ctx: Context 17 | ) -> LLBFuture { 18 | return ctx.group.any().makeFailedFuture(Error.nullExecutor) 19 | } 20 | 21 | public func canSatisfy(requirements: P) -> Bool where P.EvaluatedType == FXActionExecutionEnvironment { 22 | false 23 | } 24 | 25 | enum Error: Swift.Error { 26 | case nullExecutor 27 | } 28 | 29 | public func perform( 30 | action: ActionType, 31 | with executable: LLBFuture, 32 | requirements: P, 33 | _ ctx: Context 34 | ) -> LLBFuture where P.EvaluatedType == FXActionExecutionEnvironment { 35 | return ctx.group.any().makeFailedFuture(Error.nullExecutor) 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /Sources/llbuild2fx/Predicate/Expression.swift: -------------------------------------------------------------------------------- 1 | // This source file is part of the Swift.org open source project 2 | // 3 | // Copyright (c) 2022 Apple Inc. and the Swift project authors 4 | // Licensed under Apache License v2.0 with Runtime Library Exception 5 | // 6 | // See http://swift.org/LICENSE.txt for license information 7 | // See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 8 | 9 | public protocol Expression { 10 | associatedtype EvaluatedType 11 | associatedtype Value 12 | 13 | func value(with object: EvaluatedType) -> Value 14 | } 15 | 16 | public struct ConstantExpression: Expression { 17 | public let value: Value 18 | 19 | public init(value: Value) { 20 | self.value = value 21 | } 22 | 23 | public func value(with object: EvaluatedType) -> Value { 24 | value 25 | } 26 | } 27 | 28 | public struct KeyPathExpression: Expression { 29 | public let keyPath: KeyPath 30 | 31 | public init(keyPath: KeyPath) { 32 | self.keyPath = keyPath 33 | } 34 | 35 | public func value(with object: EvaluatedType) -> Value { 36 | object[keyPath: keyPath] 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /Sources/llbuild2fx/Predicate/Predicate.swift: -------------------------------------------------------------------------------- 1 | // This source file is part of the Swift.org open source project 2 | // 3 | // Copyright (c) 2022 Apple Inc. and the Swift project authors 4 | // Licensed under Apache License v2.0 with Runtime Library Exception 5 | // 6 | // See http://swift.org/LICENSE.txt for license information 7 | // See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 8 | 9 | public protocol Predicate { 10 | associatedtype EvaluatedType 11 | func evaluate(with object: EvaluatedType) -> Bool 12 | } 13 | 14 | public struct AnyPredicate: Predicate { 15 | public let predicate: Any 16 | private let evaluator: (EvaluatedType) -> Bool 17 | 18 | public init(_ predicate: P) where P.EvaluatedType == EvaluatedType { 19 | self.predicate = predicate 20 | evaluator = predicate.evaluate 21 | } 22 | 23 | public func evaluate(with object: EvaluatedType) -> Bool { 24 | evaluator(object) 25 | } 26 | } 27 | 28 | public struct ConstantPredicate: Predicate { 29 | public let value: Bool 30 | 31 | public init(value: Bool) { 32 | self.value = value 33 | } 34 | 35 | public func evaluate(with object: EvaluatedType) -> Bool { 36 | value 37 | } 38 | } 39 | 40 | public struct EqualityPredicate: Predicate where 41 | LHS: Expression, 42 | RHS: Expression, 43 | LHS.EvaluatedType == RHS.EvaluatedType, 44 | LHS.Value == RHS.Value, 45 | LHS.Value: Equatable 46 | { 47 | public let leftExpression: LHS 48 | public let rightExpression: RHS 49 | 50 | public init(leftExpression lhs: LHS, rightExpression rhs: RHS) { 51 | leftExpression = lhs 52 | rightExpression = rhs 53 | } 54 | 55 | public func evaluate(with object: LHS.EvaluatedType) -> Bool { 56 | let lhs = leftExpression.value(with: object) 57 | let rhs = rightExpression.value(with: object) 58 | 59 | return lhs == rhs 60 | } 61 | } 62 | 63 | public struct NotPredicate: Predicate { 64 | public let subpredicate: P 65 | 66 | public init(subpredicate: P) { 67 | self.subpredicate = subpredicate 68 | } 69 | 70 | public func evaluate(with object: P.EvaluatedType) -> Bool { 71 | !subpredicate.evaluate(with: object) 72 | } 73 | } 74 | 75 | public struct AndPredicate: Predicate { 76 | public let subpredicates: [AnyPredicate] 77 | 78 | public init(subpredicates: [AnyPredicate]) { 79 | self.subpredicates = subpredicates 80 | } 81 | 82 | public func evaluate(with object: EvaluatedType) -> Bool { 83 | subpredicates.reduce(true) { 84 | $0 && $1.evaluate(with: object) 85 | } 86 | } 87 | } 88 | 89 | public struct OrPredicate: Predicate { 90 | public let subpredicates: [AnyPredicate] 91 | 92 | public init(subpredicates: [AnyPredicate]) { 93 | self.subpredicates = subpredicates 94 | } 95 | 96 | public func evaluate(with object: EvaluatedType) -> Bool { 97 | subpredicates.reduce(false) { 98 | $0 || $1.evaluate(with: object) 99 | } 100 | } 101 | } 102 | -------------------------------------------------------------------------------- /Sources/llbuild2fx/Resources.swift: -------------------------------------------------------------------------------- 1 | // This source file is part of the Swift.org open source project 2 | // 3 | // Copyright (c) 2025 Apple Inc. and the Swift project authors 4 | // Licensed under Apache License v2.0 with Runtime Library Exception 5 | // 6 | // See http://swift.org/LICENSE.txt for license information 7 | // See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 8 | 9 | public enum ResourceLifetime: Sendable { 10 | case idempotent 11 | case versioned 12 | case requestOnly 13 | } 14 | 15 | public protocol FXResource: Sendable { 16 | var name: String { get } 17 | var version: Int? { get } 18 | var lifetime: ResourceLifetime { get } 19 | } 20 | 21 | 22 | final class ResourceVersions: Encodable { 23 | private let versionedResources: [ResourceKey: Int] 24 | 25 | init(resources: [ResourceKey: FXResource]) { 26 | var versionedResources: [ResourceKey: Int] = [:] 27 | for key in K.resourceEntitlements { 28 | if let res = resources[key], case .versioned = res.lifetime, let version = res.version { 29 | versionedResources[key] = version 30 | } 31 | } 32 | self.versionedResources = versionedResources 33 | } 34 | 35 | func isNoop() -> Bool { 36 | return versionedResources.isEmpty 37 | } 38 | 39 | func encode(to encoder: Encoder) throws { 40 | try encoder.fxEncodeHash(of: versionedResources.fxEncodeJSON()) 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /Sources/llbuild2fx/Reëxport.swift: -------------------------------------------------------------------------------- 1 | // This source file is part of the Swift.org open source project 2 | // 3 | // Copyright (c) 2021 Apple Inc. and the Swift project authors 4 | // Licensed under Apache License v2.0 with Runtime Library Exception 5 | // 6 | // See http://swift.org/LICENSE.txt for license information 7 | // See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 8 | 9 | import TSCUtility 10 | @_exported import TSFCAS 11 | @_exported import TSFCASFileTree 12 | import TSFFutures 13 | 14 | @_exported import struct TSCUtility.Context 15 | @_exported import class TSFFutures.LLBFuture 16 | -------------------------------------------------------------------------------- /Sources/llbuild2fx/Ruleset.swift: -------------------------------------------------------------------------------- 1 | // This source file is part of the Swift.org open source project 2 | // 3 | // Copyright (c) 2025 Apple Inc. and the Swift project authors 4 | // Licensed under Apache License v2.0 with Runtime Library Exception 5 | // 6 | // See http://swift.org/LICENSE.txt for license information 7 | // See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 8 | 9 | import TSFFutures 10 | 11 | public protocol FXEntrypoint: FXKey { 12 | init(withEntrypointPayload casObject: LLBCASObject) throws 13 | init(withEntrypointPayload buffer: LLBByteBuffer) throws 14 | } 15 | 16 | public class FXRuleset { 17 | public let name: String 18 | public let entrypoints: [String : any FXEntrypoint.Type] 19 | public let actionDependencies: [any FXAction.Type] 20 | 21 | let aggregatedResourceEntitlements: FXSortedSet 22 | 23 | public init(name: String, entrypoints: [String: any FXEntrypoint.Type]) { 24 | self.name = name 25 | self.entrypoints = entrypoints 26 | 27 | aggregatedResourceEntitlements = FXSortedSet(entrypoints.values.map { $0.aggregatedResourceEntitlements }.reduce([], +)) 28 | 29 | var actionDeps: [String: any FXAction.Type] = [:] 30 | for ep in entrypoints.values { 31 | for ad in ep.aggregatedActionDependencies { 32 | actionDeps[ad.name] = ad 33 | } 34 | } 35 | actionDependencies = Array(actionDeps.values) 36 | } 37 | 38 | public func constrainResources(_ resources: [ResourceKey: FXResource]) throws -> [ResourceKey: FXResource] { 39 | var constrained: [ResourceKey: FXResource] = [:] 40 | for key in aggregatedResourceEntitlements { 41 | guard let resource = resources[key] else { 42 | throw FXError.resourceNotFound(key) 43 | } 44 | constrained[key] = resource 45 | } 46 | return constrained 47 | } 48 | } 49 | 50 | public protocol FXResourceAuthenticator: Sendable { 51 | // stub protocol for passing an authenticator object for resource creation 52 | } 53 | 54 | public protocol FXRulesetPackage { 55 | associatedtype Config: Sendable 56 | 57 | // Create all the rulesets supported by this package 58 | static func createRulesets() -> [FXRuleset] 59 | 60 | // Given the configuration for the package, construct all external resources 61 | // that may be used by rulesets provided by this package and initialize any 62 | // other resident facilities, such as logging handlers. Implementations 63 | // using package(s) MUST call this method exactly once per process lifetime. 64 | static func createExternalResources( 65 | _ config: Config, 66 | group: LLBFuturesDispatchGroup, 67 | authenticator: FXResourceAuthenticator, 68 | _ ctx: Context 69 | ) async throws -> [FXResource] 70 | 71 | static func createErrorClassifier() -> FXErrorClassifier? 72 | } 73 | 74 | public extension FXRulesetPackage { 75 | static func createExternalResources( 76 | _ config: Config, 77 | group: LLBFuturesDispatchGroup, 78 | authenticator: FXResourceAuthenticator, 79 | _ ctx: Context 80 | ) async throws -> [FXResource] { 81 | return [] 82 | } 83 | 84 | static func createErrorClassifier() -> FXErrorClassifier? { 85 | return nil 86 | } 87 | } 88 | -------------------------------------------------------------------------------- /Sources/llbuild2fx/Service.swift: -------------------------------------------------------------------------------- 1 | // This source file is part of the Swift.org open source project 2 | // 3 | // Copyright (c) 2025 Apple Inc. and the Swift project authors 4 | // Licensed under Apache License v2.0 with Runtime Library Exception 5 | // 6 | // See http://swift.org/LICENSE.txt for license information 7 | // See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 8 | 9 | import NIOConcurrencyHelpers 10 | import TSFFutures 11 | 12 | public class FXService: FXErrorClassifier { 13 | public let group: LLBFuturesDispatchGroup 14 | 15 | public enum Error: Swift.Error { 16 | case duplicateResource(String) 17 | } 18 | 19 | private let _resources = NIOLockedValueBox([ResourceKey: FXResource]()) 20 | private let _rulesets = NIOLockedValueBox([String: FXRuleset]()) 21 | private let _errorClassifiers = NIOLockedValueBox([FXErrorClassifier]()) 22 | 23 | public init(group: LLBFuturesDispatchGroup) { 24 | self.group = group 25 | } 26 | 27 | public func ruleset(_ name: String) -> FXRuleset? { 28 | return _rulesets.withLockedValue { $0[name] } 29 | } 30 | 31 | public func resources(for ruleset: FXRuleset) throws -> [ResourceKey: FXResource] { 32 | return try _resources.withLockedValue { 33 | return try ruleset.constrainResources($0) 34 | } 35 | } 36 | 37 | public func registerResource(_ resource: FXResource) throws { 38 | try _resources.withLockedValue { resources in 39 | guard !resources.keys.contains(.external(resource.name)) else { 40 | throw Error.duplicateResource(resource.name) 41 | } 42 | resources[.external(resource.name)] = resource 43 | } 44 | } 45 | 46 | public func registerPackage(_ pkg: T.Type, with config: T.Config, authenticator: FXResourceAuthenticator, _ ctx: Context) async throws { 47 | 48 | let newResources = try await pkg.createExternalResources(config, group: group, authenticator: authenticator, ctx) 49 | try _resources.withLockedValue { resources in 50 | // check all resources first, so that we don't leave anything dangling on failure 51 | for r in newResources { 52 | if resources.keys.contains(.external(r.name)) { 53 | throw Error.duplicateResource(r.name) 54 | } 55 | } 56 | 57 | for r in newResources { 58 | resources[.external(r.name)] = r 59 | } 60 | } 61 | 62 | let newRulesets = pkg.createRulesets() 63 | _rulesets.withLockedValue { 64 | for ruleset in newRulesets { 65 | $0[ruleset.name] = ruleset 66 | } 67 | } 68 | 69 | if let classifier = pkg.createErrorClassifier() { 70 | _errorClassifiers.withLockedValue { 71 | $0.append(classifier) 72 | } 73 | } 74 | } 75 | 76 | public func tryClassifyError(_ error: Swift.Error) -> FXErrorDetails? { 77 | return _errorClassifiers.withLockedValue { classifiers in 78 | for classifier in classifiers { 79 | if let details = classifier.tryClassifyError(error) { 80 | return details 81 | } 82 | } 83 | return nil 84 | } 85 | } 86 | } 87 | -------------------------------------------------------------------------------- /Sources/llbuild2fx/SortedSet.swift: -------------------------------------------------------------------------------- 1 | // This source file is part of the Swift.org open source project 2 | // 3 | // Copyright (c) 2021 Apple Inc. and the Swift project authors 4 | // Licensed under Apache License v2.0 with Runtime Library Exception 5 | // 6 | // See http://swift.org/LICENSE.txt for license information 7 | // See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 8 | 9 | 10 | public struct FXSortedSet: Sendable { 11 | fileprivate var uniqueElements: Set 12 | fileprivate var sortedElements: [Element] 13 | 14 | public init() { 15 | uniqueElements = Set() 16 | sortedElements = Array() 17 | } 18 | 19 | public init(_ elements: [Element]) { 20 | uniqueElements = Set(elements) 21 | sortedElements = uniqueElements.sorted() 22 | } 23 | 24 | public init(_ elements: Set) { 25 | uniqueElements = elements 26 | sortedElements = uniqueElements.sorted() 27 | } 28 | 29 | public var isEmpty: Bool { 30 | uniqueElements.isEmpty 31 | } 32 | } 33 | 34 | extension FXSortedSet: Equatable { 35 | public static func == (lhs: Self, rhs: Self) -> Bool { 36 | lhs.sortedElements == rhs.sortedElements 37 | } 38 | } 39 | 40 | extension FXSortedSet: Hashable { 41 | public func hash(into hasher: inout Hasher) { 42 | hasher.combine(uniqueElements) 43 | } 44 | } 45 | 46 | extension FXSortedSet: Collection { 47 | public typealias Index = Array.Index 48 | 49 | public var startIndex: Index { 50 | sortedElements.startIndex 51 | } 52 | 53 | public var endIndex: Index { 54 | sortedElements.endIndex 55 | } 56 | 57 | public subscript(position: Index) -> Element { 58 | get { 59 | sortedElements[position] 60 | } 61 | } 62 | 63 | public func index(after i: Index) -> Index { 64 | sortedElements.index(after: i) 65 | } 66 | } 67 | 68 | extension FXSortedSet: BidirectionalCollection { 69 | public func index(before i: Index) -> Index { 70 | sortedElements.index(before: i) 71 | } 72 | } 73 | 74 | extension FXSortedSet: Sequence { 75 | public typealias Iterator = Array.Iterator 76 | 77 | public func makeIterator() -> Iterator { 78 | sortedElements.makeIterator() 79 | } 80 | } 81 | 82 | extension FXSortedSet: SetAlgebra { 83 | public func contains(_ member: Element) -> Bool { 84 | uniqueElements.contains(member) 85 | } 86 | 87 | public func union(_ other: Self) -> Self { 88 | Self(uniqueElements.union(other.uniqueElements)) 89 | } 90 | 91 | public func intersection(_ other: Self) -> Self { 92 | Self(uniqueElements.intersection(other.uniqueElements)) 93 | } 94 | 95 | public func symmetricDifference(_ other: Self) -> Self { 96 | Self(uniqueElements.symmetricDifference(other.uniqueElements)) 97 | } 98 | 99 | @discardableResult 100 | public mutating func insert( 101 | _ newMember: Element 102 | ) -> (inserted: Bool, memberAfterInsert: Element) { 103 | defer { sortedElements = uniqueElements.sorted() } 104 | return uniqueElements.insert(newMember) 105 | } 106 | 107 | @discardableResult 108 | public mutating func remove(_ member: Element) -> Element? { 109 | defer { sortedElements = uniqueElements.sorted() } 110 | return uniqueElements.remove(member) 111 | } 112 | 113 | @discardableResult 114 | public mutating func update(with newMember: Element) -> Element? { 115 | defer { sortedElements = uniqueElements.sorted() } 116 | return uniqueElements.update(with: newMember) 117 | } 118 | 119 | public mutating func formUnion(_ other: Self) { 120 | uniqueElements.formUnion(other.uniqueElements) 121 | sortedElements = uniqueElements.sorted() 122 | } 123 | 124 | public mutating func formIntersection(_ other: Self) { 125 | uniqueElements.formIntersection(other.uniqueElements) 126 | sortedElements = uniqueElements.sorted() 127 | } 128 | 129 | public mutating func formSymmetricDifference(_ other: Self) { 130 | uniqueElements.formSymmetricDifference(other.uniqueElements) 131 | sortedElements = uniqueElements.sorted() 132 | } 133 | } 134 | 135 | extension FXSortedSet: ExpressibleByArrayLiteral { 136 | public init(arrayLiteral elements: Element...) { 137 | self = Self(elements) 138 | } 139 | } 140 | 141 | extension FXSortedSet: Encodable where Element: Encodable { 142 | public func encode(to encoder: Encoder) throws { 143 | try sortedElements.encode(to: encoder) 144 | } 145 | } 146 | 147 | extension FXSortedSet: Decodable where Element: Decodable { 148 | public init(from decoder: Decoder) throws { 149 | let elements: [Element] = try .init(from: decoder) 150 | 151 | self = Self(elements) 152 | } 153 | } 154 | -------------------------------------------------------------------------------- /Sources/llbuild2fx/Stats.swift: -------------------------------------------------------------------------------- 1 | // This source file is part of the Swift.org open source project 2 | // 3 | // Copyright (c) 2021 Apple Inc. and the Swift project authors 4 | // Licensed under Apache License v2.0 with Runtime Library Exception 5 | // 6 | // See http://swift.org/LICENSE.txt for license information 7 | // See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 8 | 9 | import NIOConcurrencyHelpers 10 | import TSCUtility 11 | 12 | public struct FXBuildEngineStatsSnapshot { 13 | public let currentKeys: [String: Int] 14 | public let totalKeys: [String: Int] 15 | public let currentActions: [String: Int] 16 | public let totalActions: [String: Int] 17 | } 18 | 19 | public final class FXBuildEngineStats: @unchecked Sendable { 20 | private let lock = NIOLock() 21 | 22 | private var currentKeyCounts: [String: Int] = [:] 23 | private var totalKeyCounts: [String: Int] = [:] 24 | private var currentActionCounts: [String: Int] = [:] 25 | private var totalActionCounts: [String: Int] = [:] 26 | 27 | public init() {} 28 | 29 | public var snapshot: FXBuildEngineStatsSnapshot { 30 | lock.withLock { 31 | FXBuildEngineStatsSnapshot( 32 | currentKeys: currentKeyCounts.filter { $0.1 > 0 }, 33 | totalKeys: totalKeyCounts.filter { $0.1 > 0 }, 34 | currentActions: currentActionCounts.filter { $0.1 > 0 }, 35 | totalActions: totalActionCounts.filter { $0.1 > 0 } 36 | ) 37 | } 38 | } 39 | 40 | func add(key: String) { 41 | lock.withLock { 42 | currentKeyCounts[key] = (currentKeyCounts[key] ?? 0) + 1 43 | totalKeyCounts[key] = (totalKeyCounts[key] ?? 0) + 1 44 | } 45 | } 46 | 47 | func remove(key: String) { 48 | lock.withLock { 49 | currentKeyCounts[key] = currentKeyCounts[key]! - 1 50 | } 51 | } 52 | 53 | func add(action: String) { 54 | lock.withLock { 55 | currentActionCounts[action] = (currentActionCounts[action] ?? 0) + 1 56 | totalActionCounts[action] = (totalActionCounts[action] ?? 0) + 1 57 | } 58 | } 59 | 60 | func remove(action: String) { 61 | lock.withLock { 62 | currentActionCounts[action] = currentActionCounts[action]! - 1 63 | } 64 | } 65 | } 66 | -------------------------------------------------------------------------------- /Sources/llbuild2fx/Support/AnySerializable.swift: -------------------------------------------------------------------------------- 1 | // This source file is part of the Swift.org open source project 2 | // 3 | // Copyright (c) 2020 Apple Inc. and the Swift project authors 4 | // Licensed under Apache License v2.0 with Runtime Library Exception 5 | // 6 | // See http://swift.org/LICENSE.txt for license information 7 | // See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 8 | 9 | import Foundation 10 | 11 | import NIOConcurrencyHelpers 12 | import SwiftProtobuf 13 | import TSFUtility 14 | 15 | // MARK:- PolymorphicSerializable - 16 | 17 | /// Types conforming to LLBPolymorphicSerializable are allowed to be serialized into 18 | /// LLBAnySerializable. They need to be registered in order to be deserialized 19 | /// at runtime without compile-time type information. 20 | public protocol LLBPolymorphicSerializable: LLBSerializable { 21 | static var polymorphicIdentifier: String { get } 22 | } 23 | 24 | /// Make all LLBSerializbles automatically conform to this by retrieving it's 25 | /// described type. This is not ideal since it doesn't return the module name. 26 | /// Later on, we might be able to migrate this logic to use _mangledTypeName 27 | /// instead to make this more robust. 28 | extension LLBPolymorphicSerializable { 29 | public static var polymorphicIdentifier: String { 30 | return String(describing: Self.self) 31 | } 32 | } 33 | 34 | // Convenience internal initializer. 35 | extension LLBAnySerializable { 36 | public init(from polymorphicSerializable: LLBPolymorphicSerializable) throws { 37 | self.typeIdentifier = type(of: polymorphicSerializable).polymorphicIdentifier 38 | self.serializedBytes = try Data(polymorphicSerializable.toBytes().readableBytesView) 39 | } 40 | } 41 | 42 | 43 | // MARK:- SerializableRegistry - 44 | 45 | public protocol LLBSerializableLookup { 46 | func lookupType(identifier: String) -> LLBPolymorphicSerializable.Type? 47 | } 48 | 49 | /// Container for mapping registered identifiers to their runtime types. 50 | public class LLBSerializableRegistry: LLBSerializableLookup { 51 | /// Types registered at runtime that are allowed to be deserialized. 52 | private var registeredTypes: [String: LLBPolymorphicSerializable.Type] = [:] 53 | 54 | public init() { } 55 | 56 | /// Register a new type for use in polymorphic serialization 57 | /// 58 | /// If a type has already been registered for this identifier, no change is 59 | /// made. 60 | /// 61 | /// CONCURRENCY: *NOT* Thread-safe 62 | public func register(type: LLBPolymorphicSerializable.Type) { 63 | if registeredTypes[type.polymorphicIdentifier] == nil { 64 | registeredTypes[type.polymorphicIdentifier] = type 65 | } 66 | } 67 | 68 | /// Lookup the register runtime type for a given type identifier 69 | public func lookupType(identifier: String) -> LLBPolymorphicSerializable.Type? { 70 | return registeredTypes[identifier] 71 | } 72 | } 73 | 74 | 75 | // MARK:- AnySerializable deserialization support - 76 | 77 | public enum LLBAnySerializableError: Swift.Error { 78 | case unknownType(String) 79 | case typeMismatch(String) 80 | } 81 | 82 | extension LLBAnySerializable { 83 | public func deserialize(registry: LLBSerializableLookup) throws -> T { 84 | guard let serializableType = registry.lookupType(identifier: typeIdentifier) else { 85 | throw LLBAnySerializableError.unknownType(typeIdentifier) 86 | } 87 | 88 | // FIXME: this extra buffer copy is unfortunate 89 | let buffer = LLBByteBuffer.withBytes(ArraySlice(serializedBytes)) 90 | guard let deserialized = try serializableType.init(from: buffer) as? T else { 91 | throw LLBAnySerializableError.typeMismatch("\(typeIdentifier) not convertible to \(T.Type.self)") 92 | } 93 | 94 | return deserialized 95 | } 96 | } 97 | 98 | 99 | // MARK:- CASObjectRepresentable for any Serializable via AnySerializable 100 | 101 | extension LLBAnySerializable: LLBSerializable {} 102 | 103 | extension LLBPolymorphicSerializable { 104 | init(from casObject: LLBCASObject, registry: LLBSerializableLookup) throws { 105 | let any = try LLBAnySerializable(from: casObject.data) 106 | guard let objType = registry.lookupType(identifier: any.typeIdentifier) else { 107 | throw LLBAnySerializableError.unknownType(any.typeIdentifier) 108 | } 109 | // FIXME: this extra buffer copy is unfortunate 110 | let buffer = LLBByteBuffer.withBytes(ArraySlice(any.serializedBytes)) 111 | self = try objType.init(from: buffer) as! Self 112 | } 113 | } 114 | 115 | extension LLBCASObjectRepresentable where Self: LLBPolymorphicSerializable { 116 | public func asCASObject() throws -> LLBCASObject { 117 | let any = try LLBAnySerializable(from: self) 118 | return LLBCASObject(refs: [], data: try any.toBytes()) 119 | } 120 | } 121 | 122 | extension LLBCASObjectRepresentable where Self: LLBSerializable { 123 | public func asCASObject() throws -> LLBCASObject { 124 | return LLBCASObject(refs: [], data: try self.toBytes()) 125 | } 126 | } 127 | extension LLBCASObjectConstructable where Self: LLBSerializable { 128 | public init(from casObject: LLBCASObject) throws { 129 | try self.init(from: casObject.data) 130 | } 131 | } 132 | 133 | extension LLBAnySerializable: LLBCASObjectConstructable { 134 | public init(from casObject: LLBCASObject) throws { 135 | self = try LLBAnySerializable.init(from: casObject.data) 136 | } 137 | } 138 | 139 | -------------------------------------------------------------------------------- /Sources/llbuild2fx/Support/CommonCodables.swift: -------------------------------------------------------------------------------- 1 | // This source file is part of the Swift.org open source project 2 | // 3 | // Copyright (c) 2020 Apple Inc. and the Swift project authors 4 | // Licensed under Apache License v2.0 with Runtime Library Exception 5 | // 6 | // See http://swift.org/LICENSE.txt for license information 7 | // See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 8 | 9 | import NIOCore 10 | import TSFUtility 11 | 12 | // LLBCodable support for common types used in llbuild2. Should be expanded as more types are needed. This is not 13 | // meant ot be a full featured serialization library support, so there's no need to be eager and add support for most of 14 | // the POD types in Swift, just the ones that we are interested in in the short term, or that clients of llbuild2 15 | // request support of. 16 | 17 | extension String: LLBPolymorphicSerializable { 18 | public func toBytes(into buffer: inout LLBByteBuffer) throws { 19 | buffer.writeString(self) 20 | } 21 | 22 | public init(from bytes: LLBByteBuffer) throws { 23 | var mutableBytes = bytes 24 | guard let decoded = mutableBytes.readString(length: bytes.readableBytes) else { 25 | throw LLBSerializableError.unknownError("could not decode String bytes") 26 | } 27 | self = decoded 28 | 29 | } 30 | } 31 | 32 | // Int currently support doesn't handle endian-ness. These are mostly used as basic data types for testing on local 33 | // systems. For more complex data types that use Ints, each type should account for the serialization mechanism. 34 | extension Int: LLBPolymorphicSerializable { 35 | public func toBytes(into buffer: inout LLBByteBuffer) throws { 36 | buffer.writeInteger(self) 37 | } 38 | 39 | public init(from bytes: LLBByteBuffer) throws { 40 | var mutableBytes = bytes 41 | guard let decoded: Int = mutableBytes.readInteger() else { 42 | throw LLBSerializableError.unknownError("could not decode Int bytes") 43 | } 44 | self = decoded 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /Sources/llbuild2fx/Support/Logging.swift: -------------------------------------------------------------------------------- 1 | // This source file is part of the Swift.org open source project 2 | // 3 | // Copyright (c) 2020 Apple Inc. and the Swift project authors 4 | // Licensed under Apache License v2.0 with Runtime Library Exception 5 | // 6 | // See http://swift.org/LICENSE.txt for license information 7 | // See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 8 | 9 | import Logging 10 | 11 | // Support protocol for recording events during FXKey evaluations 12 | public protocol FXMetricsSink { 13 | func event( 14 | _ message: Logger.Message, 15 | metadata: @autoclosure () -> Logger.Metadata, 16 | _ ctx: Context, 17 | file: String, 18 | function: String, 19 | line: UInt 20 | ) 21 | } 22 | 23 | extension FXMetricsSink { 24 | public func event( 25 | _ message: Logger.Message, 26 | metadata: @autoclosure () -> Logger.Metadata = [:], 27 | _ ctx: Context = .init(), 28 | file_ file: String = #file, 29 | function: String = #function, 30 | line: UInt = #line 31 | ) { 32 | self.event(message, metadata: metadata(), ctx, file: file, function: function, line: line) 33 | } 34 | } 35 | 36 | // Support storing and retrieving logger and metrics instances from a Context. 37 | public extension Context { 38 | var logger: Logger? { 39 | get { 40 | return self[ObjectIdentifier(Logger.self), as: Logger.self] 41 | } 42 | set { 43 | self[ObjectIdentifier(Logger.self)] = newValue 44 | } 45 | } 46 | 47 | var metrics: FXMetricsSink? { 48 | get { 49 | return self[ObjectIdentifier(FXMetricsSink.self), as: FXMetricsSink.self] 50 | } 51 | set { 52 | self[ObjectIdentifier(FXMetricsSink.self)] = newValue 53 | } 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /Sources/llbuild2fx/Support/Protobuf+Extensions.swift: -------------------------------------------------------------------------------- 1 | // This source file is part of the Swift.org open source project 2 | // 3 | // Copyright (c) 2020 Apple Inc. and the Swift project authors 4 | // Licensed under Apache License v2.0 with Runtime Library Exception 5 | // 6 | // See http://swift.org/LICENSE.txt for license information 7 | // See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 8 | 9 | import SwiftProtobuf 10 | import Foundation 11 | import NIOCore 12 | 13 | /// Convenience implementation for types that extend SwiftProtobuf.Message. 14 | extension LLBSerializableOut where Self: SwiftProtobuf.Message { 15 | public func toBytes(into buffer: inout LLBByteBuffer) throws { 16 | buffer.writeBytes(try self.serializedData()) 17 | } 18 | } 19 | 20 | /// Convenience implementation for types that extend SwiftProtobuf.Message. 21 | extension LLBSerializableIn where Self: SwiftProtobuf.Message { 22 | public init(from bytes: LLBByteBuffer) throws { 23 | let data = Data(bytes.readableBytesView) 24 | self = try Self.init(serializedBytes: data) 25 | } 26 | } 27 | 28 | /// Convenience implementation for types that extend SwiftProtobuf.Message 29 | extension FXRequestKey where Self: SwiftProtobuf.Message { 30 | public var stableHashValue: LLBDataID { 31 | return LLBDataID(blake3hash: ArraySlice(try! self.serializedData())) 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /Sources/llbuild2fx/TreeMaterialization.swift: -------------------------------------------------------------------------------- 1 | // This source file is part of the Swift.org open source project 2 | // 3 | // Copyright (c) 2021 Apple Inc. and the Swift project authors 4 | // Licensed under Apache License v2.0 with Runtime Library Exception 5 | // 6 | // See http://swift.org/LICENSE.txt for license information 7 | // See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 8 | 9 | import Foundation 10 | import NIOCore 11 | import TSCBasic 12 | import TSCUtility 13 | 14 | public protocol FXTreeMaterializer { 15 | func materialize(tree: FXTreeID) -> AbsolutePath? 16 | } 17 | 18 | private class ContextTreeMaterializer {} 19 | 20 | extension Context { 21 | public var fxTreeMaterializer: FXTreeMaterializer? { 22 | get { 23 | guard let value = self[ObjectIdentifier(ContextTreeMaterializer.self), as: FXTreeMaterializer.self] else { 24 | return nil 25 | } 26 | 27 | return value 28 | } 29 | set { 30 | self[ObjectIdentifier(ContextTreeMaterializer.self)] = newValue 31 | } 32 | } 33 | } 34 | 35 | public func withTemporaryDirectory(_ ctx: Context, _ body: (AbsolutePath) -> LLBFuture) -> LLBFuture { 36 | do { 37 | return try withTemporaryDirectory(removeTreeOnDeinit: false) { path in 38 | body(path).always { _ in 39 | _ = try? FileManager.default.removeItem(atPath: path.pathString) 40 | } 41 | } 42 | } catch { 43 | return ctx.group.next().makeFailedFuture(error) 44 | } 45 | } 46 | 47 | struct UntypedTreeID: FXSingleDataIDValue, FXTreeID { 48 | let dataID: LLBDataID 49 | } 50 | 51 | extension FXFileID { 52 | public func materialize(filename: String, _ ctx: Context, _ body: @escaping (AbsolutePath) -> LLBFuture) -> LLBFuture { 53 | load(ctx).flatMap { blob in 54 | let files: [LLBDirectoryEntryID] = [ 55 | blob.asDirectoryEntry(filename: filename) 56 | ] 57 | 58 | return LLBCASFileTree.create(files: files, in: ctx.db, ctx) 59 | }.map { (tree: LLBCASFileTree) in 60 | UntypedTreeID(dataID: tree.id) 61 | }.flatMap { treeID in 62 | treeID.materialize(ctx) { treePath in 63 | body(treePath.appending(component: filename)) 64 | } 65 | } 66 | } 67 | } 68 | 69 | extension FXTreeID { 70 | public func materialize(_ ctx: Context, _ body: @escaping (AbsolutePath) -> LLBFuture) -> LLBFuture { 71 | if let path = ctx.fxTreeMaterializer?.materialize(tree: self) { 72 | return body(path) 73 | } 74 | 75 | return withTemporaryDirectory(ctx) { tmp in 76 | LLBCASFileTree.export(self.dataID, from: ctx.db, to: tmp, stats: LLBCASFileTree.ExportProgressStatsInt64(), ctx).flatMap { 77 | body(tmp) 78 | } 79 | } 80 | } 81 | } 82 | -------------------------------------------------------------------------------- /Sources/llbuild2fx/Versioning.swift: -------------------------------------------------------------------------------- 1 | // This source file is part of the Swift.org open source project 2 | // 3 | // Copyright (c) 2021 Apple Inc. and the Swift project authors 4 | // Licensed under Apache License v2.0 with Runtime Library Exception 5 | // 6 | // See http://swift.org/LICENSE.txt for license information 7 | // See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 8 | 9 | public enum ConfigurationKey: Hashable, Comparable, Codable, Sendable { 10 | case literal(String) 11 | case prefix(String) 12 | } 13 | 14 | public enum ResourceKey: Hashable, Comparable, Codable, Sendable { 15 | case external(String) 16 | } 17 | 18 | public protocol FXVersioning: Sendable { 19 | static var name: String { get } 20 | static var version: Int { get } 21 | static var versionDependencies: [FXVersioning.Type] { get } 22 | static var actionDependencies: [any FXAction.Type] { get } 23 | static var configurationKeys: [ConfigurationKey] { get } 24 | static var resourceEntitlements: [ResourceKey] { get } 25 | } 26 | 27 | extension FXKey { 28 | public static var name: String { String(describing: self) } 29 | public static var version: Int { 0 } 30 | public static var versionDependencies: [FXVersioning.Type] { 31 | [FXVersioning.Type]() 32 | } 33 | public static var actionDependencies: [any FXAction.Type] { 34 | [any FXAction.Type]() 35 | } 36 | public static var configurationKeys: [ConfigurationKey] { 37 | [ConfigurationKey]() 38 | } 39 | public static var resourceEntitlements: [ResourceKey] { 40 | [ResourceKey]() 41 | } 42 | } 43 | 44 | extension FXVersioning { 45 | public static var actionDependencies: [any FXAction.Type] { 46 | [any FXAction.Type]() 47 | } 48 | public static var configurationKeys: [ConfigurationKey] { 49 | [ConfigurationKey]() 50 | } 51 | public static var resourceEntitlements: [ResourceKey] { 52 | [ResourceKey]() 53 | } 54 | } 55 | 56 | extension FXVersioning { 57 | private static func aggregateGraph(_ transform: (FXVersioning.Type) -> [FXVersioning.Type]) 58 | -> [FXVersioning.Type] 59 | { 60 | var settled = [ObjectIdentifier: FXVersioning.Type]() 61 | var frontier: [ObjectIdentifier: FXVersioning.Type] = [ObjectIdentifier(self): self] 62 | 63 | while !frontier.isEmpty { 64 | settled.merge(frontier) { a, _ in a } 65 | 66 | let nestedFrontierCandidates: [[FXVersioning.Type]] = frontier.values.map(transform) 67 | 68 | let frontierCandidates: [FXVersioning.Type] = nestedFrontierCandidates.flatMap { $0 } 69 | 70 | let unsettled: [FXVersioning.Type] = frontierCandidates.filter { 71 | settled[ObjectIdentifier($0)] == nil 72 | } 73 | 74 | let tuples: [(ObjectIdentifier, FXVersioning.Type)] = unsettled.map { 75 | (ObjectIdentifier($0), $0) 76 | } 77 | 78 | frontier = Dictionary(tuples) { a, _ in a } 79 | } 80 | 81 | return Array(settled.values) 82 | } 83 | 84 | static var aggregatedVersionDependencies: [FXVersioning.Type] { 85 | aggregateGraph { 86 | $0.versionDependencies 87 | } 88 | } 89 | 90 | static var aggregatedVersion: Int { 91 | return aggregatedVersionDependencies.map { $0.version }.reduce(0, +) 92 | } 93 | 94 | public static var aggregatedConfigurationKeys: FXSortedSet { 95 | return FXSortedSet(aggregatedVersionDependencies.map { $0.configurationKeys }.reduce([], +)) 96 | } 97 | 98 | public static var aggregatedResourceEntitlements: FXSortedSet { 99 | return FXSortedSet(aggregatedVersionDependencies.map { $0.resourceEntitlements }.reduce([], +)) 100 | } 101 | 102 | public static var aggregatedActionDependencies: [any FXAction.Type] { 103 | var actionDeps: [String: any FXAction.Type] = [:] 104 | for dep in aggregatedVersionDependencies { 105 | for ad in dep.actionDependencies { 106 | actionDeps[ad.name] = ad 107 | } 108 | } 109 | return Array(actionDeps.values) 110 | } 111 | 112 | public static var cacheKeyPrefix: String { 113 | [ 114 | "\(name)", 115 | "\(aggregatedVersion)", 116 | ].joined(separator: "/") 117 | } 118 | } 119 | -------------------------------------------------------------------------------- /Sources/llbuild2fx/WrappedDataID.swift: -------------------------------------------------------------------------------- 1 | // This source file is part of the Swift.org open source project 2 | // 3 | // Copyright (c) 2021 Apple Inc. and the Swift project authors 4 | // Licensed under Apache License v2.0 with Runtime Library Exception 5 | // 6 | // See http://swift.org/LICENSE.txt for license information 7 | // See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 8 | 9 | import NIOCore 10 | 11 | public protocol FXWrappedDataID { 12 | var dataID: LLBDataID { get } 13 | } 14 | 15 | public protocol FXSingleDataIDValue: FXValue, FXWrappedDataID, Encodable, Hashable, Comparable { 16 | init(dataID: LLBDataID) 17 | } 18 | 19 | public protocol FXThinEncodedSingleDataIDValue: FXSingleDataIDValue { 20 | 21 | } 22 | 23 | extension FXSingleDataIDValue { 24 | public init(_ dataID: LLBDataID) { 25 | self = Self(dataID: dataID) 26 | } 27 | } 28 | 29 | public struct FXNullCodableValue: Codable {} 30 | 31 | extension FXSingleDataIDValue { 32 | public var refs: [LLBDataID] { [dataID] } 33 | public var codableValue: FXNullCodableValue { FXNullCodableValue() } 34 | public init(refs: [LLBDataID], codableValue: FXNullCodableValue) { 35 | self.init(refs[0]) 36 | } 37 | } 38 | 39 | extension FXSingleDataIDValue { 40 | public static func < (lhs: Self, rhs: Self) -> Bool { 41 | lhs.dataID < rhs.dataID 42 | } 43 | } 44 | 45 | extension FXSingleDataIDValue { 46 | public func encode(to encoder: Encoder) throws { 47 | try encoder.encodeHash(of: ArraySlice(dataID.bytes)) 48 | } 49 | } 50 | 51 | extension FXThinEncodedSingleDataIDValue { 52 | public func encode(to encoder: Encoder) throws { 53 | // Since we already have a hash value, directly encode a short prefix of it 54 | var container = encoder.singleValueContainer() 55 | let str = ArraySlice(dataID.bytes.dropFirst().prefix(9)).base64URL() 56 | try container.encode(str) 57 | } 58 | } 59 | 60 | enum WrappedDataIDError: Swift.Error { 61 | case noRefs 62 | case wrongNodeType(id: LLBDataID, expected: LLBFileType, actual: LLBFileType) 63 | } 64 | 65 | extension FXSingleDataIDValue { 66 | public init(from casObject: LLBCASObject) throws { 67 | let refs = casObject.refs 68 | guard !refs.isEmpty else { 69 | throw WrappedDataIDError.noRefs 70 | } 71 | 72 | let dataID = refs[0] 73 | 74 | self = Self(dataID) 75 | } 76 | } 77 | 78 | extension FXSingleDataIDValue { 79 | public func asCASObject() throws -> LLBCASObject { 80 | LLBCASObject(refs: [dataID], data: LLBByteBuffer()) 81 | } 82 | } 83 | 84 | public protocol FXNodeID: FXWrappedDataID { 85 | func load(_ ctx: Context) -> LLBFuture 86 | } 87 | 88 | extension FXNodeID { 89 | public func load(_ ctx: Context) -> LLBFuture { 90 | let client = LLBCASFSClient(ctx.db) 91 | return client.load(self.dataID, ctx) 92 | } 93 | } 94 | 95 | public protocol FXTreeID: FXNodeID { 96 | func load(_ ctx: Context) -> LLBFuture 97 | } 98 | 99 | extension FXTreeID { 100 | public func load(_ ctx: Context) -> LLBFuture { 101 | let client = LLBCASFSClient(ctx.db) 102 | let dataID = self.dataID 103 | return client.load(dataID, type: .directory, ctx).flatMapThrowing { node in 104 | let type = node.type() 105 | guard type == .directory else { 106 | throw WrappedDataIDError.wrongNodeType(id: dataID, expected: .directory, actual: type) 107 | } 108 | 109 | return node.tree! 110 | } 111 | } 112 | } 113 | 114 | public protocol FXFileID: FXNodeID { 115 | func load(_ ctx: Context) -> LLBFuture 116 | } 117 | 118 | extension FXFileID { 119 | public func load(_ ctx: Context) -> LLBFuture { 120 | let client = LLBCASFSClient(ctx.db) 121 | let dataID = self.dataID 122 | return client.load(dataID, type: .plainFile, ctx).flatMapThrowing { node in 123 | let type = node.type() 124 | guard type == .plainFile else { 125 | throw WrappedDataIDError.wrongNodeType(id: dataID, expected: .plainFile, actual: type) 126 | } 127 | 128 | return node.blob! 129 | } 130 | } 131 | } 132 | 133 | public protocol FXExecutableFileID: FXNodeID { 134 | func load(_ ctx: Context) -> LLBFuture 135 | } 136 | 137 | extension FXExecutableFileID { 138 | public func load(_ ctx: Context) -> LLBFuture { 139 | let client = LLBCASFSClient(ctx.db) 140 | let dataID = self.dataID 141 | return client.load(dataID, type: .executable, ctx).flatMapThrowing { node in 142 | let type = node.type() 143 | guard type == .executable else { 144 | throw WrappedDataIDError.wrongNodeType(id: dataID, expected: .executable, actual: type) 145 | } 146 | 147 | return node.blob! 148 | } 149 | } 150 | } 151 | -------------------------------------------------------------------------------- /Tests/llbuild2fxTests/CommandLineArgsCoderTests.swift: -------------------------------------------------------------------------------- 1 | // This source file is part of the Swift.org open source project 2 | // 3 | // Copyright (c) 2020 Apple Inc. and the Swift project authors 4 | // Licensed under Apache License v2.0 with Runtime Library Exception 5 | // 6 | // See http://swift.org/LICENSE.txt for license information 7 | // See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 8 | 9 | import XCTest 10 | import llbuild2fx 11 | 12 | class CommandLineArgsCoderTests: XCTestCase { 13 | 14 | enum Error: Swift.Error { 15 | case someError 16 | } 17 | 18 | struct SubA: Codable, Equatable { 19 | var x: String 20 | var y: UInt 21 | } 22 | struct A: Codable, Equatable { 23 | var a: String 24 | var b: String? 25 | var c: Int? 26 | var d: [Double] 27 | var e: SubA 28 | var f: [String: String] 29 | } 30 | struct B: Codable, Equatable { 31 | var a: Bool 32 | } 33 | 34 | // CommandLineArgsCoder tests 35 | func testCommandLineArgsEncoder() throws { 36 | let a = A( 37 | a: "abc", b: nil, c: 12, d: [2.3, Double.infinity, -Double.zero, 0.112], 38 | e: SubA(x: "hello world", y: 12345), f: ["one": "two", "three": "four"]) 39 | let e = CommandLineArgsEncoder() 40 | XCTAssertNoThrow(try e.encode(a)) 41 | let aE = try e.encode(a) 42 | XCTAssertEqual( 43 | aE, 44 | [ 45 | "--a=abc", "--c=12", "--d.0=2.3", "--d.1=inf", "--d.2=-0.0", "--d.3=0.112", "--e.x=hello world", 46 | "--e.y=12345", "--f.one=two", "--f.three=four", 47 | ]) 48 | } 49 | 50 | func testCommandLineArgsDecoder() throws { 51 | let str = [ 52 | "--a=abc", "--c=12", "--d.0=2.3", "--d.1=inf", "--d.2=-0.0", "--d.3=0.112", "--e.x=hello world", 53 | "--e.y=12345", "--f.one=two", "--f.three=four", 54 | ] 55 | let d = CommandLineArgsDecoder() 56 | XCTAssertNoThrow(try d.decode(from: str) as A) 57 | let aD: A = try d.decode(from: str) 58 | XCTAssertEqual( 59 | aD, 60 | A( 61 | a: "abc", b: nil, c: 12, d: [2.3, Double.infinity, -Double.zero, 0.112], 62 | e: SubA(x: "hello world", y: 12345), f: ["one": "two", "three": "four"])) 63 | } 64 | 65 | func testCommandLineArgsTypeRoundTrip() throws { 66 | let a = A( 67 | a: "abc", b: nil, c: 12, d: [2.3, Double.infinity, -Double.zero, 0.112], 68 | e: SubA(x: "hello world", y: 12345), f: ["one": "two", "three": "four"]) 69 | let e = CommandLineArgsEncoder() 70 | let d = CommandLineArgsDecoder() 71 | XCTAssertNoThrow(try e.encode(a)) 72 | let aE = try e.encode(a) 73 | XCTAssertNoThrow(try d.decode(from: aE) as A) 74 | let aD: A = try d.decode(from: aE) 75 | XCTAssertEqual(a, aD) 76 | } 77 | 78 | func testCommandLineArgsArgsRoundTrip() throws { 79 | let str = [ 80 | "--a=abc", "--c=12", "--d.0=2.3", "--d.1=inf", "--d.2=-0.0", "--d.3=0.112", "--e.x=hello world", 81 | "--e.y=12345", "--f.one=two", "--f.three=four", 82 | ] 83 | let d = CommandLineArgsDecoder() 84 | let e = CommandLineArgsEncoder() 85 | XCTAssertNoThrow(try d.decode(from: str) as A) 86 | let aD: A = try d.decode(from: str) 87 | XCTAssertNoThrow(try e.encode(aD)) 88 | let aE = try e.encode(aD) 89 | XCTAssertEqual(aE, str) 90 | } 91 | 92 | func testCommandLineArgsFlexibleDecoder() throws { 93 | let str = [ 94 | "--a", "abc", "--c=12", "--d", "2.3", "--d", "inf", "--d", "-0.0", "--d", "0.112", "--e.x=hello world", 95 | "--e.y=12345", "--f.one", "two", "--f.three", "four", 96 | ] 97 | let d = CommandLineArgsDecoder() 98 | XCTAssertNoThrow(try d.decode(from: str) as A) 99 | let aD: A = try d.decode(from: str) 100 | XCTAssertEqual( 101 | aD, 102 | A( 103 | a: "abc", b: nil, c: 12, d: [2.3, Double.infinity, -Double.zero, 0.112], 104 | e: SubA(x: "hello world", y: 12345), f: ["one": "two", "three": "four"])) 105 | let str2 = ["--a"] 106 | XCTAssertNoThrow(try d.decode(from: str2) as B) 107 | let bD: B = try d.decode(from: str2) 108 | XCTAssertEqual(bD, B(a: true)) 109 | } 110 | 111 | } 112 | -------------------------------------------------------------------------------- /Tests/llbuild2fxTests/DeadlineTests.swift: -------------------------------------------------------------------------------- 1 | import Foundation 2 | import TSCUtility 3 | import XCTest 4 | import llbuild2fx 5 | 6 | final class DeadlineTests: XCTestCase { 7 | func testDistantFutureDeadlineIsNil() { 8 | var ctx = Context() 9 | ctx.fxDeadline = Date.distantFuture 10 | 11 | let deadline = ctx.nioDeadline 12 | 13 | XCTAssertNil(deadline) 14 | } 15 | } 16 | -------------------------------------------------------------------------------- /Tests/llbuild2fxTests/ExpressionTests.swift: -------------------------------------------------------------------------------- 1 | // This source file is part of the Swift.org open source project 2 | // 3 | // Copyright (c) 2022 Apple Inc. and the Swift project authors 4 | // Licensed under Apache License v2.0 with Runtime Library Exception 5 | // 6 | // See http://swift.org/LICENSE.txt for license information 7 | // See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 8 | 9 | import XCTest 10 | import llbuild2fx 11 | 12 | private struct TestableValue { 13 | let value: Bool 14 | } 15 | 16 | final class ExpressionTests: XCTestCase { 17 | func testConstantExpression() { 18 | let expr = ConstantExpression(value: true) 19 | 20 | let value = expr.value(with: nil) 21 | 22 | XCTAssertEqual(value, true) 23 | } 24 | 25 | func testKeyPathExpression() { 26 | let obj = TestableValue(value: true) 27 | let expr = KeyPathExpression(keyPath: \TestableValue.value) 28 | 29 | let value = expr.value(with: obj) 30 | 31 | XCTAssertEqual(value, true) 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /Tests/llbuild2fxTests/FunctionCacheTests.swift: -------------------------------------------------------------------------------- 1 | // This source file is part of the Swift.org open source project 2 | // 3 | // Copyright (c) 2020 Apple Inc. and the Swift project authors 4 | // Licensed under Apache License v2.0 with Runtime Library Exception 5 | // 6 | // See http://swift.org/LICENSE.txt for license information 7 | // See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 8 | 9 | import XCTest 10 | 11 | import llbuild2fx 12 | import TSCBasic 13 | 14 | extension String: @retroactive FXRequestKey { 15 | public var stableHashValue: LLBDataID { 16 | return LLBDataID(blake3hash: self) 17 | } 18 | } 19 | 20 | final class FunctionCacheTests: XCTestCase { 21 | let group = LLBMakeDefaultDispatchGroup() 22 | 23 | /// ${TMPDIR} or just "/tmp", expressed as AbsolutePath 24 | private var temporaryPath: AbsolutePath { 25 | return try! AbsolutePath(validating: ProcessInfo.processInfo.environment["TMPDIR", default: "/tmp"]) 26 | } 27 | 28 | func doFunctionCacheTests(cache: FXFunctionCache) throws { 29 | struct FakeProps: FXKeyProperties { 30 | var volatile: Bool = false 31 | var cachePath: String = "keypath" 32 | } 33 | let p = FakeProps() 34 | 35 | let ctx = Context() 36 | XCTAssertNil(try cache.get(key: "key1", props: p, ctx).wait()) 37 | 38 | let id1 = LLBDataID(blake3hash: LLBByteBuffer.withBytes(ArraySlice("value1".utf8)), refs: []) 39 | try cache.update(key: "key1", props: p, value: id1, ctx).wait() 40 | XCTAssertEqual(id1, try cache.get(key: "key1", props: p, ctx).wait()) 41 | } 42 | 43 | func testInMemoryFunctionCache() throws { 44 | let cache = FXInMemoryFunctionCache(group: group) 45 | try doFunctionCacheTests(cache: cache) 46 | } 47 | 48 | func testFileBackedFunctionCache() throws { 49 | try withTemporaryDirectory(dir: temporaryPath, prefix: "FXFunctionCacheTests" + #function, removeTreeOnDeinit: true) { tmpDir in 50 | let cache = FXFileBackedFunctionCache(group: group, path: tmpDir) 51 | try doFunctionCacheTests(cache: cache) 52 | } 53 | 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /Tests/llbuild2fxTests/KeyDependencyGraphTests.swift: -------------------------------------------------------------------------------- 1 | // This source file is part of the Swift.org open source project 2 | // 3 | // Copyright (c) 2020 Apple Inc. and the Swift project authors 4 | // Licensed under Apache License v2.0 with Runtime Library Exception 5 | // 6 | // See http://swift.org/LICENSE.txt for license information 7 | // See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 8 | 9 | import XCTest 10 | 11 | import llbuild2fx 12 | 13 | extension Int: @retroactive FXRequestKey { 14 | public var stableHashValue: LLBDataID { 15 | var v = self 16 | let s: Int = MemoryLayout.size 17 | return withUnsafePointer(to: &v) { p in 18 | p.withMemoryRebound(to: UInt8.self, capacity: s) { pb in 19 | LLBDataID(directHash: Array(UnsafeBufferPointer(start: pb, count: s))) 20 | } 21 | } 22 | } 23 | } 24 | 25 | final class KeyDependencyGraphTests: XCTestCase { 26 | func testSimpleCycle() throws { 27 | let keyDependencyGraph = FXKeyDependencyGraph() 28 | 29 | XCTAssertNoThrow(try keyDependencyGraph.addEdge(from: 1, to: 2)) 30 | XCTAssertNoThrow(try keyDependencyGraph.addEdge(from: 2, to: 3)) 31 | XCTAssertNoThrow(try keyDependencyGraph.addEdge(from: 3, to: 4)) 32 | 33 | XCTAssertThrowsError(try keyDependencyGraph.addEdge(from: 4, to: 1)) { error in 34 | guard case let FXError.cycleDetected(cycle) = error else { 35 | XCTFail("Unexpected error type") 36 | return 37 | } 38 | 39 | XCTAssertEqual([4, 1, 2, 3, 4], cycle as! [Int]) 40 | } 41 | } 42 | } 43 | 44 | -------------------------------------------------------------------------------- /Tests/llbuild2fxTests/KeyTests.swift: -------------------------------------------------------------------------------- 1 | // This source file is part of the Swift.org open source project 2 | // 3 | // Copyright (c) 2021 Apple Inc. and the Swift project authors 4 | // Licensed under Apache License v2.0 with Runtime Library Exception 5 | // 6 | // See http://swift.org/LICENSE.txt for license information 7 | // See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 8 | 9 | import llbuild2fx 10 | import XCTest 11 | 12 | final class FXKeyTests: XCTestCase { 13 | private struct TestValue: FXSingleDataIDValue { 14 | let dataID: LLBDataID 15 | } 16 | 17 | func testDataIDKeyEncoding() throws { 18 | struct TestKey: FXKey { 19 | let dataID: LLBDataID 20 | func computeValue(_ fi: FXFunctionInterface, _ ctx: Context) -> LLBFuture { 21 | ctx.group.next().makeSucceededFuture(TestValue(dataID)) 22 | } 23 | } 24 | 25 | let sensitive = "0~AA==" 26 | let key = TestKey(dataID: LLBDataID(string: sensitive)!) 27 | 28 | let encoded = try CommandLineArgsEncoder().encode(key) 29 | 30 | XCTAssertEqual(encoded, ["--dataID=\(sensitive)"]) 31 | } 32 | 33 | func testWrappedKeyEncoding() throws { 34 | struct TestKey: FXKey { 35 | let value: TestValue 36 | func computeValue(_ fi: FXFunctionInterface, _ ctx: Context) -> LLBFuture { 37 | ctx.group.next().makeSucceededFuture(value) 38 | } 39 | } 40 | 41 | let sensitive = "0~AA==" 42 | let key = TestKey(value: TestValue(LLBDataID(string: sensitive)!)) 43 | 44 | let encoded = try CommandLineArgsEncoder().encode(key) 45 | 46 | XCTAssertNotEqual(encoded, ["--value=\(sensitive)"]) 47 | XCTAssertEqual(encoded, ["--value=GtSPSWJwedgG"]) 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /Tests/llbuild2fxTests/PredicateTests.swift: -------------------------------------------------------------------------------- 1 | // This source file is part of the Swift.org open source project 2 | // 3 | // Copyright (c) 2022 Apple Inc. and the Swift project authors 4 | // Licensed under Apache License v2.0 with Runtime Library Exception 5 | // 6 | // See http://swift.org/LICENSE.txt for license information 7 | // See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 8 | 9 | import XCTest 10 | import llbuild2fx 11 | 12 | final class EqualityPredicateTests: XCTestCase { 13 | func testTrueTrue() { 14 | let lhs = ConstantExpression(value: true) 15 | let rhs = ConstantExpression(value: true) 16 | 17 | let predicate = EqualityPredicate(leftExpression: lhs, rightExpression: rhs) 18 | 19 | XCTAssertTrue(predicate.evaluate(with: nil)) 20 | } 21 | 22 | func testTrueFalse() { 23 | let lhs = ConstantExpression(value: true) 24 | let rhs = ConstantExpression(value: false) 25 | 26 | let predicate = EqualityPredicate(leftExpression: lhs, rightExpression: rhs) 27 | 28 | XCTAssertFalse(predicate.evaluate(with: nil)) 29 | } 30 | 31 | func testFalseTrue() { 32 | let lhs = ConstantExpression(value: false) 33 | let rhs = ConstantExpression(value: true) 34 | 35 | let predicate = EqualityPredicate(leftExpression: lhs, rightExpression: rhs) 36 | 37 | XCTAssertFalse(predicate.evaluate(with: nil)) 38 | } 39 | 40 | func testFalseFalse() { 41 | let lhs = ConstantExpression(value: false) 42 | let rhs = ConstantExpression(value: false) 43 | 44 | let predicate = EqualityPredicate(leftExpression: lhs, rightExpression: rhs) 45 | 46 | XCTAssertTrue(predicate.evaluate(with: nil)) 47 | } 48 | } 49 | 50 | final class ContantPredicateTests: XCTestCase { 51 | func testTrue() { 52 | let predicate = ConstantPredicate(value: true) 53 | 54 | XCTAssertTrue(predicate.evaluate(with: nil)) 55 | } 56 | 57 | func testFalse() { 58 | let predicate = ConstantPredicate(value: false) 59 | 60 | XCTAssertFalse(predicate.evaluate(with: nil)) 61 | } 62 | } 63 | 64 | final class NotPredicateTests { 65 | func testTrue() { 66 | let predicate = NotPredicate(subpredicate: ConstantPredicate(value: true)) 67 | 68 | XCTAssertFalse(predicate.evaluate(with: nil)) 69 | } 70 | 71 | func testFalse() { 72 | let predicate = NotPredicate(subpredicate: ConstantPredicate(value: false)) 73 | 74 | XCTAssertTrue(predicate.evaluate(with: nil)) 75 | } 76 | } 77 | 78 | final class AndPredicateTests: XCTestCase { 79 | func testEmpty() { 80 | let predicate = AndPredicate(subpredicates: []) 81 | 82 | XCTAssertTrue(predicate.evaluate(with: nil)) 83 | } 84 | 85 | func testTrue() { 86 | let predicate = AndPredicate(subpredicates: [AnyPredicate(ConstantPredicate(value: true))]) 87 | 88 | XCTAssertTrue(predicate.evaluate(with: nil)) 89 | } 90 | 91 | func testTrueTrue() { 92 | let predicate = AndPredicate(subpredicates: [ 93 | AnyPredicate(ConstantPredicate(value: true)), 94 | AnyPredicate(ConstantPredicate(value: true)), 95 | ]) 96 | 97 | XCTAssertTrue(predicate.evaluate(with: nil)) 98 | } 99 | 100 | func testFalse() { 101 | let predicate = AndPredicate(subpredicates: [AnyPredicate(ConstantPredicate(value: false))]) 102 | 103 | XCTAssertFalse(predicate.evaluate(with: nil)) 104 | } 105 | 106 | func testTrueFalse() { 107 | let predicate = AndPredicate(subpredicates: [ 108 | AnyPredicate(ConstantPredicate(value: true)), 109 | AnyPredicate(ConstantPredicate(value: false)), 110 | ]) 111 | 112 | XCTAssertFalse(predicate.evaluate(with: nil)) 113 | } 114 | 115 | func testFalseTrue() { 116 | let predicate = AndPredicate(subpredicates: [ 117 | AnyPredicate(ConstantPredicate(value: false)), 118 | AnyPredicate(ConstantPredicate(value: true)), 119 | ]) 120 | 121 | XCTAssertFalse(predicate.evaluate(with: nil)) 122 | } 123 | 124 | func testFalseFalse() { 125 | let predicate = AndPredicate(subpredicates: [ 126 | AnyPredicate(ConstantPredicate(value: false)), 127 | AnyPredicate(ConstantPredicate(value: false)), 128 | ]) 129 | 130 | XCTAssertFalse(predicate.evaluate(with: nil)) 131 | } 132 | } 133 | 134 | final class OrPredicateTests: XCTestCase { 135 | func testEmpty() { 136 | let predicate = OrPredicate(subpredicates: []) 137 | 138 | XCTAssertFalse(predicate.evaluate(with: nil)) 139 | } 140 | 141 | func testTrue() { 142 | let predicate = OrPredicate(subpredicates: [AnyPredicate(ConstantPredicate(value: true))]) 143 | 144 | XCTAssertTrue(predicate.evaluate(with: nil)) 145 | } 146 | 147 | func testTrueTrue() { 148 | let predicate = OrPredicate(subpredicates: [ 149 | AnyPredicate(ConstantPredicate(value: true)), 150 | AnyPredicate(ConstantPredicate(value: true)), 151 | ]) 152 | 153 | XCTAssertTrue(predicate.evaluate(with: nil)) 154 | } 155 | 156 | func testTrueFalse() { 157 | let predicate = OrPredicate(subpredicates: [ 158 | AnyPredicate(ConstantPredicate(value: true)), 159 | AnyPredicate(ConstantPredicate(value: false)), 160 | ]) 161 | 162 | XCTAssertTrue(predicate.evaluate(with: nil)) 163 | } 164 | 165 | func testFalseTrue() { 166 | let predicate = OrPredicate(subpredicates: [ 167 | AnyPredicate(ConstantPredicate(value: false)), 168 | AnyPredicate(ConstantPredicate(value: true)), 169 | ]) 170 | 171 | XCTAssertTrue(predicate.evaluate(with: nil)) 172 | } 173 | 174 | func testFalseFalse() { 175 | let predicate = OrPredicate(subpredicates: [ 176 | AnyPredicate(ConstantPredicate(value: false)), 177 | AnyPredicate(ConstantPredicate(value: false)), 178 | ]) 179 | 180 | XCTAssertFalse(predicate.evaluate(with: nil)) 181 | } 182 | 183 | func testFalse() { 184 | let predicate = OrPredicate(subpredicates: [AnyPredicate(ConstantPredicate(value: false))]) 185 | 186 | XCTAssertFalse(predicate.evaluate(with: nil)) 187 | } 188 | } 189 | -------------------------------------------------------------------------------- /Tests/llbuild2fxTests/RequirementTests.swift: -------------------------------------------------------------------------------- 1 | // This source file is part of the Swift.org open source project 2 | // 3 | // Copyright (c) 2022 Apple Inc. and the Swift project authors 4 | // Licensed under Apache License v2.0 with Runtime Library Exception 5 | // 6 | // See http://swift.org/LICENSE.txt for license information 7 | // See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 8 | 9 | import XCTest 10 | import llbuild2fx 11 | 12 | final class RequirementTests: XCTestCase { 13 | func testLocalExecutorCanSatisfyRequirement() throws { 14 | let executor = FXLocalExecutor() 15 | 16 | XCTAssertTrue(executor.canSatisfy(requirements: FXActionExecutionEnvironment.local)) 17 | } 18 | 19 | func testLocalExecutorFailsToSatisfyProhibition() throws { 20 | let executor = FXLocalExecutor() 21 | let requirement = NotPredicate(subpredicate: FXActionExecutionEnvironment.local) 22 | 23 | XCTAssertFalse(executor.canSatisfy(requirements: requirement)) 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /Tests/llbuild2fxTests/VersioningTests.swift: -------------------------------------------------------------------------------- 1 | // 2 | // VersioningTests.swift 3 | // llbuild2 4 | // 5 | // Created by David Bryson on 5/14/25. 6 | // 7 | 8 | @testable import llbuild2fx 9 | import XCTest 10 | 11 | 12 | struct ExampleActionResult: Codable, Sendable { 13 | } 14 | 15 | extension ExampleActionResult: FXValue { 16 | public typealias CodableValueType = String 17 | public var refs: [TSFCAS.LLBDataID] { [] } 18 | 19 | public var codableValue: String { return String() } 20 | 21 | public init(refs: [TSFCAS.LLBDataID], codableValue: String) throws { 22 | } 23 | } 24 | 25 | struct ExampleAction: AsyncFXAction { 26 | typealias ValueType = ExampleActionResult 27 | 28 | func run(_ ctx: Context) async throws -> ExampleActionResult { 29 | return ExampleActionResult() 30 | } 31 | } 32 | 33 | extension ExampleAction: FXValue { 34 | typealias CodableValueType = String 35 | var refs: [TSFCAS.LLBDataID] { [] } 36 | 37 | var codableValue: String { return String() } 38 | 39 | public init(refs: [TSFCAS.LLBDataID], codableValue: String) throws { 40 | } 41 | } 42 | 43 | extension String: @retroactive FXValue { 44 | 45 | } 46 | 47 | 48 | struct Hello: AsyncFXKey { 49 | typealias ValueType = String 50 | 51 | static let version = 1 52 | static let versionDependencies: [FXVersioning.Type] = [] 53 | static let actionDependencies: [any FXAction.Type] = [ExampleAction.self] 54 | static let resourceEntitlements: [ResourceKey] = [] 55 | 56 | func computeValue(_ fi: FXFunctionInterface, _ ctx: Context) async throws -> String { 57 | _ = try await fi.spawn(ExampleAction(), ctx) 58 | return String() 59 | } 60 | } 61 | 62 | struct World: AsyncFXKey { 63 | typealias ValueType = String 64 | 65 | static let version = 1 66 | static let versionDependencies: [FXVersioning.Type] = [] 67 | static let actionDependencies: [any FXAction.Type] = [ExampleAction.self] 68 | static let resourceEntitlements: [ResourceKey] = [] 69 | 70 | func computeValue(_ fi: FXFunctionInterface, _ ctx: Context) async throws -> String { 71 | _ = try await fi.spawn(ExampleAction(), ctx) 72 | return String() 73 | } 74 | } 75 | 76 | struct Concat: AsyncFXKey { 77 | typealias ValueType = String 78 | 79 | static let version = 1 80 | static let versionDependencies: [FXVersioning.Type] = [Hello.self, World.self] 81 | static let resourceEntitlements: [ResourceKey] = [] 82 | 83 | func computeValue(_ fi: FXFunctionInterface, _ ctx: Context) async throws -> String { 84 | async let helloResult = fi.request(Hello(), ctx) 85 | async let worldResult = fi.request(World(), ctx) 86 | 87 | return "\(try await helloResult) \(try await worldResult)!" 88 | } 89 | } 90 | 91 | 92 | final class VersioningTests: XCTestCase { 93 | func testAggregatedVersion() { 94 | XCTAssertEqual(Concat.aggregatedVersion, 3) 95 | } 96 | 97 | func testAggregatedActionDependencies() { 98 | let actionDeps = Concat.aggregatedActionDependencies 99 | XCTAssertEqual(actionDeps.count, 1) 100 | XCTAssertNotNil(actionDeps[0] as? ExampleAction.Type) 101 | } 102 | } 103 | -------------------------------------------------------------------------------- /Utilities/build_proto_toolchain.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -eu 2 | # 3 | # This source file is part of the Swift.org open source project 4 | # 5 | # Copyright (c) 2020 Apple Inc. and the Swift project authors 6 | # Licensed under Apache License v2.0 with Runtime Library Exception 7 | # 8 | # See http://swift.org/LICENSE.txt for license information 9 | # See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 10 | 11 | PROTOC_ZIP=protoc-3.14.0-osx-x86_64.zip 12 | PROTOC_URL="https://github.com/protocolbuffers/protobuf/releases/download/v3.14.0/$PROTOC_ZIP" 13 | 14 | UTILITIES_DIR="$(dirname "$0")" 15 | TOOLS_DIR="$UTILITIES_DIR/tools" 16 | 17 | mkdir -p "$TOOLS_DIR" 18 | 19 | if [[ ! -f "$UTILITIES_DIR/tools/$PROTOC_ZIP" ]]; then 20 | curl -L "$PROTOC_URL" --output "$TOOLS_DIR/$PROTOC_ZIP" 21 | unzip -o "$TOOLS_DIR/$PROTOC_ZIP" -d "$TOOLS_DIR" 22 | fi 23 | 24 | # Use swift build instead of cloning the repo to make sure that the generated code matches the SwiftProtobuf library 25 | # being used as a dependency in the build. This might be a bit slower, but it's correct. 26 | swift build -c release --product protoc-gen-swift --package-path "$UTILITIES_DIR/.." 27 | swift build -c release --product protoc-gen-grpc-swift --package-path "$UTILITIES_DIR/.." 28 | 29 | cp "$UTILITIES_DIR"/../.build/release/protoc-gen{-grpc,}-swift "$TOOLS_DIR/bin" 30 | --------------------------------------------------------------------------------