├── .gitignore ├── .gitmodules ├── .swift-format ├── CODEOWNERS ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── LICENSE.txt ├── Makefile ├── Package.resolved ├── Package.swift ├── Protos ├── CASFileTreeProtocol │ └── file_tree.proto ├── CASProtocol │ ├── cas_object.proto │ └── data_id.proto └── module_map.asciipb ├── README.md ├── Sources ├── AsyncProcess │ ├── ChunkSequence.swift │ ├── EOFSequence.swift │ ├── FileContentStream.swift │ ├── NIOAsyncPipeWriter.swift │ ├── ProcessExecutor+Convenience.swift │ ├── ProcessExecutor.swift │ ├── ProcessExit.swift │ └── StructuredConcurrencyHelpers.swift ├── CBLAKE3 │ ├── blake3.c │ ├── blake3_avx2.c │ ├── blake3_avx512.c │ ├── blake3_impl.h │ ├── blake3_sse41.c │ └── include │ │ └── blake3.h ├── CProcessSpawnSync │ ├── include │ │ ├── CProcessSpawnSync.h │ │ └── ps-api.h │ ├── internal-helpers.h │ └── spawner.c ├── ProcessSpawnSync │ └── ProcessSpawner.swift ├── TSFCAS │ ├── DataID.swift │ ├── Database.swift │ ├── DatabaseSpec.swift │ ├── Generated │ │ └── CASProtocol │ │ │ ├── cas_object.pb.swift │ │ │ └── data_id.pb.swift │ ├── Implementations │ │ ├── Blake3DataID.swift │ │ ├── FileBackedCASDatabase.swift │ │ └── InMemoryCASDatabase.swift │ └── Object.swift ├── TSFCASFileTree │ ├── BinarySearch.swift │ ├── CASBlob.swift │ ├── CASFSClient.swift │ ├── CASFSNode.swift │ ├── ConcurrentFileTreeWalker.swift │ ├── Context.swift │ ├── DeclFileTree.swift │ ├── DirectoryEntry.swift │ ├── Errors.swift │ ├── FileInfo.swift │ ├── FileTree.swift │ ├── FileTreeExport.swift │ ├── FileTreeImport.swift │ ├── FilesystemObject.swift │ ├── Generated │ │ └── CASFileTreeProtocol │ │ │ └── file_tree.pb.swift │ ├── Internal │ │ ├── ConcurrentFilesystemScanner.swift │ │ ├── FileSegmenter.swift │ │ └── FileTreeParser.swift │ └── TSCCASFileSystem.swift ├── TSFCASUtilities │ ├── BufferedStreamWriter.swift │ ├── LinkedListStream.swift │ └── StreamReader.swift ├── TSFFutures │ ├── BatchingFutureOperationQueue.swift │ ├── CancellableFuture.swift │ ├── CancellablePromise.swift │ ├── Canceller.swift │ ├── EventualResultsCache.swift │ ├── FutureDeduplicator.swift │ ├── FutureOperationQueue.swift │ ├── Futures.swift │ ├── OperationQueue+Extensions.swift │ └── OrderManager.swift └── TSFUtility │ ├── ByteBuffer.swift │ ├── FastData.swift │ ├── FutureFileSystem.swift │ └── Serializable.swift ├── Tests ├── AsyncProcessTests │ ├── AsyncByteBufferLineSequenceTests.swift │ ├── Helpers+LogRecorderHandler.swift │ └── IntegrationTests.swift ├── TSFCASFileTreeTests │ ├── CASBlobTests.swift │ ├── FileTreeImportExportTests.swift │ └── FileTreeTests.swift ├── TSFCASTests │ ├── DataIDTests.swift │ ├── FileBackedCASDatabaseTests.swift │ └── InMemoryCASDatabaseTests.swift ├── TSFCASUtilitiesTests │ ├── BufferedStreamWriterTests.swift │ └── LinkedListStreamTests.swift └── TSFFuturesTests │ ├── BatchingFutureOperationQueue.swift │ ├── CancellableFutureTests.swift │ ├── CancellablePromiseTests.swift │ ├── CancellerTests.swift │ ├── EventualResultsCacheTests.swift │ ├── FutureDeduplicatorTests.swift │ ├── FutureOperationQueueTests.swift │ └── OrderManagerTests.swift └── Utilities └── build_proto_toolchain.sh /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | /.build 3 | /Packages 4 | /*.xcodeproj 5 | xcuserdata/ 6 | DerivedData/ 7 | .swiftpm/xcode/package.xcworkspace/contents.xcworkspacedata 8 | /Utilities/tools 9 | 10 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "ThirdParty/BLAKE3"] 2 | path = ThirdParty/BLAKE3 3 | url = https://github.com/BLAKE3-team/BLAKE3.git 4 | -------------------------------------------------------------------------------- /.swift-format: -------------------------------------------------------------------------------- 1 | { 2 | "fileScopedDeclarationPrivacy" : { 3 | "accessLevel" : "private" 4 | }, 5 | "indentConditionalCompilationBlocks" : true, 6 | "indentSwitchCaseLabels" : false, 7 | "indentation" : { 8 | "spaces" : 4 9 | }, 10 | "lineBreakAroundMultilineExpressionChainComponents" : false, 11 | "lineBreakBeforeControlFlowKeywords" : false, 12 | "lineBreakBeforeEachArgument" : false, 13 | "lineBreakBeforeEachGenericRequirement" : false, 14 | "lineBreakBetweenDeclarationAttributes" : false, 15 | "lineLength" : 100, 16 | "maximumBlankLines" : 1, 17 | "multiElementCollectionTrailingCommas" : true, 18 | "noAssignmentInExpressions" : { 19 | "allowedFunctions" : [ 20 | "XCTAssertNoThrow" 21 | ] 22 | }, 23 | "prioritizeKeepingFunctionOutputTogether" : false, 24 | "reflowMultilineStringLiterals" : "never", 25 | "respectsExistingLineBreaks" : true, 26 | "rules" : { 27 | "AllPublicDeclarationsHaveDocumentation" : false, 28 | "AlwaysUseLiteralForEmptyCollectionInit" : false, 29 | "AlwaysUseLowerCamelCase" : true, 30 | "AmbiguousTrailingClosureOverload" : true, 31 | "AvoidRetroactiveConformances" : true, 32 | "BeginDocumentationCommentWithOneLineSummary" : false, 33 | "DoNotUseSemicolons" : true, 34 | "DontRepeatTypeInStaticProperties" : true, 35 | "FileScopedDeclarationPrivacy" : true, 36 | "FullyIndirectEnum" : true, 37 | "GroupNumericLiterals" : true, 38 | "IdentifiersMustBeASCII" : true, 39 | "NeverForceUnwrap" : false, 40 | "NeverUseForceTry" : false, 41 | "NeverUseImplicitlyUnwrappedOptionals" : false, 42 | "NoAccessLevelOnExtensionDeclaration" : true, 43 | "NoAssignmentInExpressions" : true, 44 | "NoBlockComments" : true, 45 | "NoCasesWithOnlyFallthrough" : true, 46 | "NoEmptyLinesOpeningClosingBraces" : false, 47 | "NoEmptyTrailingClosureParentheses" : true, 48 | "NoLabelsInCasePatterns" : true, 49 | "NoLeadingUnderscores" : false, 50 | "NoParensAroundConditions" : true, 51 | "NoPlaygroundLiterals" : true, 52 | "NoVoidReturnOnFunctionSignature" : true, 53 | "OmitExplicitReturns" : false, 54 | "OneCasePerLine" : true, 55 | "OneVariableDeclarationPerLine" : true, 56 | "OnlyOneTrailingClosureArgument" : true, 57 | "OrderedImports" : true, 58 | "ReplaceForEachWithForLoop" : true, 59 | "ReturnVoidInsteadOfEmptyTuple" : true, 60 | "TypeNamesShouldBeCapitalized" : true, 61 | "UseEarlyExits" : false, 62 | "UseExplicitNilCheckInConditions" : true, 63 | "UseLetInEveryBoundCaseVariable" : true, 64 | "UseShorthandTypeNames" : true, 65 | "UseSingleLinePropertyGetter" : true, 66 | "UseSynthesizedInitializer" : true, 67 | "UseTripleSlashForDocumentationComments" : true, 68 | "UseWhereClausesInForLoops" : false, 69 | "ValidateDocumentationComments" : false 70 | }, 71 | "spacesAroundRangeFormationOperators" : false, 72 | "spacesBeforeEndOfLineComments" : 2, 73 | "tabWidth" : 8, 74 | "version" : 1 75 | } 76 | -------------------------------------------------------------------------------- /CODEOWNERS: -------------------------------------------------------------------------------- 1 | # This file is a list of the people responsible for ensuring that patches for a 2 | # particular part of Swift are reviewed, either by themself or by someone else. 3 | # They are also the gatekeepers for their part of Swift, with the final word on 4 | # what goes in or not. 5 | 6 | # The list is sorted by surname and formatted to allow easy grepping and 7 | # beautification by scripts. The fields are: name (N), email (E), web-address 8 | # (W), PGP key ID and fingerprint (P), description (D), and snail-mail address 9 | # (S). 10 | 11 | # N: David M. Bryson 12 | # E: dmbryson@apple.com 13 | # D: Everything in tools-support-async not covered by someone else 14 | 15 | ### 16 | 17 | # The following lines are used by GitHub to automatically recommend reviewers. 18 | 19 | * @dmbryson 20 | 21 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Code of Conduct 2 | To be a truly great community, Swift.org needs to welcome developers from all walks of life, 3 | with different backgrounds, and with a wide range of experience. A diverse and friendly 4 | community will have more great ideas, more unique perspectives, and produce more great 5 | code. We will work diligently to make the Swift community welcoming to everyone. 6 | 7 | To give clarity of what is expected of our members, Swift.org has adopted the code of conduct 8 | defined by [contributor-covenant.org](https://www.contributor-covenant.org). This document is used across many open source 9 | communities, and we think it articulates our values well. The full text is copied below: 10 | 11 | ### Contributor Code of Conduct v1.3 12 | As contributors and maintainers of this project, and in the interest of fostering an open and 13 | welcoming community, we pledge to respect all people who contribute through reporting 14 | issues, posting feature requests, updating documentation, submitting pull requests or patches, 15 | and other activities. 16 | 17 | We are committed to making participation in this project a harassment-free experience for 18 | everyone, regardless of level of experience, gender, gender identity and expression, sexual 19 | orientation, disability, personal appearance, body size, race, ethnicity, age, religion, or 20 | nationality. 21 | 22 | Examples of unacceptable behavior by participants include: 23 | - The use of sexualized language or imagery 24 | - Personal attacks 25 | - Trolling or insulting/derogatory comments 26 | - Public or private harassment 27 | - Publishing other’s private information, such as physical or electronic addresses, without explicit permission 28 | - Other unethical or unprofessional conduct 29 | 30 | Project maintainers have the right and responsibility to remove, edit, or reject comments, 31 | commits, code, wiki edits, issues, and other contributions that are not aligned to this Code of 32 | Conduct, or to ban temporarily or permanently any contributor for other behaviors that they 33 | deem inappropriate, threatening, offensive, or harmful. 34 | 35 | By adopting this Code of Conduct, project maintainers commit themselves to fairly and 36 | consistently applying these principles to every aspect of managing this project. Project 37 | maintainers who do not follow or enforce the Code of Conduct may be permanently removed 38 | from the project team. 39 | 40 | This code of conduct applies both within project spaces and in public spaces when an 41 | individual is representing the project or its community. 42 | 43 | Instances of abusive, harassing, or otherwise unacceptable behavior may be reported by 44 | contacting a project maintainer at [conduct@swift.org](mailto:conduct@swift.org). All complaints will be reviewed and 45 | investigated and will result in a response that is deemed necessary and appropriate to the 46 | circumstances. Maintainers are obligated to maintain confidentiality with regard to the reporter 47 | of an incident. 48 | 49 | *This policy is adapted from the Contributor Code of Conduct [version 1.3.0](http://contributor-covenant.org/version/1/3/0/).* 50 | 51 | ### Reporting 52 | A working group of community members is committed to promptly addressing any [reported 53 | issues](mailto:conduct@swift.org). Working group members are volunteers appointed by the project lead, with a 54 | preference for individuals with varied backgrounds and perspectives. Membership is expected 55 | to change regularly, and may grow or shrink. 56 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | By submitting a pull request, you represent that you have the right to license 2 | your contribution to Apple and the community, and agree by submitting the patch 3 | that your contributions are licensed under the [Swift 4 | license](https://swift.org/LICENSE.txt). 5 | 6 | --- 7 | 8 | Before submitting the pull request, please make sure you have tested your 9 | changes and that they follow the Swift project [guidelines for contributing 10 | code](https://swift.org/contributing/#contributing-code). 11 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | # This source file is part of the Swift.org open source project 2 | # 3 | # Copyright (c) 2020 Apple Inc. and the Swift project authors 4 | # Licensed under Apache License v2.0 with Runtime Library Exception 5 | # 6 | # See http://swift.org/LICENSE.txt for license information 7 | # See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 8 | 9 | .PHONY: 10 | generate: clean generate-protos 11 | 12 | # These command should be executed any time the proto definitions change. It is 13 | # not required to be generated as part of a regular `swift build` since we're 14 | # checking in the generated sources. 15 | .PHONY: 16 | generate-protos: proto-toolchain 17 | mkdir -p Sources/TSFCAS/Generated 18 | Utilities/tools/bin/protoc \ 19 | -I=Protos \ 20 | --plugin=Utilities/tools/bin/protoc-gen-swift \ 21 | --swift_out=Sources/TSFCAS/Generated \ 22 | --swift_opt=Visibility=Public \ 23 | --swift_opt=ProtoPathModuleMappings=Protos/module_map.asciipb \ 24 | $$(find Protos/CASProtocol -name \*.proto) 25 | mkdir -p Sources/TSFCASFileTree/Generated 26 | Utilities/tools/bin/protoc \ 27 | -I=Protos \ 28 | --plugin=Utilities/tools/bin/protoc-gen-swift \ 29 | --swift_out=Sources/TSFCASFileTree/Generated \ 30 | --swift_opt=Visibility=Public \ 31 | --swift_opt=ProtoPathModuleMappings=Protos/module_map.asciipb \ 32 | $$(find Protos/CASFileTreeProtocol -name \*.proto) 33 | 34 | .PHONY: 35 | proto-toolchain: 36 | Utilities/build_proto_toolchain.sh 37 | 38 | .PHONY: 39 | clean: 40 | rm -rf Sources/TSFCAS/Generated 41 | rm -rf Sources/TSFCASFileTree/Generated 42 | -------------------------------------------------------------------------------- /Package.resolved: -------------------------------------------------------------------------------- 1 | { 2 | "object": { 3 | "pins": [ 4 | { 5 | "package": "swift-async-algorithms", 6 | "repositoryURL": "https://github.com/apple/swift-async-algorithms.git", 7 | "state": { 8 | "branch": null, 9 | "revision": "042e1c4d9d19748c9c228f8d4ebc97bb1e339b0b", 10 | "version": "1.0.4" 11 | } 12 | }, 13 | { 14 | "package": "swift-atomics", 15 | "repositoryURL": "https://github.com/apple/swift-atomics.git", 16 | "state": { 17 | "branch": null, 18 | "revision": "cd142fd2f64be2100422d658e7411e39489da985", 19 | "version": "1.2.0" 20 | } 21 | }, 22 | { 23 | "package": "swift-collections", 24 | "repositoryURL": "https://github.com/apple/swift-collections.git", 25 | "state": { 26 | "branch": null, 27 | "revision": "671108c96644956dddcd89dd59c203dcdb36cec7", 28 | "version": "1.1.4" 29 | } 30 | }, 31 | { 32 | "package": "swift-log", 33 | "repositoryURL": "https://github.com/apple/swift-log.git", 34 | "state": { 35 | "branch": null, 36 | "revision": "ce592ae52f982c847a4efc0dd881cc9eb32d29f2", 37 | "version": "1.6.4" 38 | } 39 | }, 40 | { 41 | "package": "swift-nio", 42 | "repositoryURL": "https://github.com/apple/swift-nio.git", 43 | "state": { 44 | "branch": null, 45 | "revision": "f7dc3f527576c398709b017584392fb58592e7f5", 46 | "version": "2.75.0" 47 | } 48 | }, 49 | { 50 | "package": "SwiftProtobuf", 51 | "repositoryURL": "https://github.com/apple/swift-protobuf.git", 52 | "state": { 53 | "branch": null, 54 | "revision": "ebc7251dd5b37f627c93698e4374084d98409633", 55 | "version": "1.28.2" 56 | } 57 | }, 58 | { 59 | "package": "swift-system", 60 | "repositoryURL": "https://github.com/apple/swift-system.git", 61 | "state": { 62 | "branch": null, 63 | "revision": "c8a44d836fe7913603e246acab7c528c2e780168", 64 | "version": "1.4.0" 65 | } 66 | }, 67 | { 68 | "package": "swift-tools-support-core", 69 | "repositoryURL": "https://github.com/apple/swift-tools-support-core.git", 70 | "state": { 71 | "branch": null, 72 | "revision": "ad2fc22a00b898c7af5d8c74a555666f67a06720", 73 | "version": "0.7.0" 74 | } 75 | } 76 | ] 77 | }, 78 | "version": 1 79 | } 80 | -------------------------------------------------------------------------------- /Package.swift: -------------------------------------------------------------------------------- 1 | // swift-tools-version:5.5 2 | import PackageDescription 3 | 4 | import class Foundation.ProcessInfo 5 | 6 | let macOSPlatform: SupportedPlatform 7 | let iOSPlatform: SupportedPlatform 8 | if let deploymentTarget = ProcessInfo.processInfo.environment["SWIFTTSC_MACOS_DEPLOYMENT_TARGET"] { 9 | macOSPlatform = .macOS(deploymentTarget) 10 | } else { 11 | macOSPlatform = .macOS(.v10_15) 12 | } 13 | if let deploymentTarget = ProcessInfo.processInfo.environment["SWIFTTSC_IOS_DEPLOYMENT_TARGET"] { 14 | iOSPlatform = .iOS(deploymentTarget) 15 | } else { 16 | iOSPlatform = .iOS(.v13) 17 | } 18 | 19 | let package = Package( 20 | name: "swift-tools-support-async", 21 | platforms: [ 22 | macOSPlatform, 23 | iOSPlatform, 24 | ], 25 | products: [ 26 | .library( 27 | name: "SwiftToolsSupportAsync", 28 | targets: ["TSFFutures", "TSFUtility", "TSFAsyncProcess"]), 29 | .library( 30 | name: "SwiftToolsSupportCAS", 31 | targets: ["TSFCAS", "TSFCASFileTree", "TSFCASUtilities"]), 32 | ], 33 | dependencies: [ 34 | .package(url: "https://github.com/apple/swift-async-algorithms.git", from: "1.0.4"), 35 | .package(url: "https://github.com/apple/swift-atomics.git", from: "1.2.0"), 36 | .package(url: "https://github.com/apple/swift-collections.git", from: "1.0.0"), 37 | .package(url: "https://github.com/apple/swift-log.git", from: "1.4.2"), 38 | .package(url: "https://github.com/apple/swift-nio.git", from: "2.68.0"), 39 | .package(url: "https://github.com/apple/swift-protobuf.git", from: "1.28.0"), 40 | .package(url: "https://github.com/apple/swift-system.git", from: "1.1.1"), 41 | .package(url: "https://github.com/apple/swift-tools-support-core.git", "0.5.8"..<"0.8.0"), 42 | ], 43 | targets: [ 44 | // BLAKE3 hash support 45 | .target( 46 | name: "CBLAKE3", 47 | dependencies: [], 48 | cSettings: [ 49 | .headerSearchPath("./") 50 | ] 51 | ), 52 | 53 | // Async Process vendored library 54 | .target( 55 | name: "TSFAsyncProcess", 56 | dependencies: [ 57 | "TSFProcessSpawnSync", 58 | .product(name: "Atomics", package: "swift-atomics"), 59 | .product(name: "AsyncAlgorithms", package: "swift-async-algorithms"), 60 | .product(name: "Logging", package: "swift-log"), 61 | .product(name: "NIO", package: "swift-nio"), 62 | .product(name: "DequeModule", package: "swift-collections"), 63 | .product(name: "SystemPackage", package: "swift-system"), 64 | ], 65 | path: "Sources/AsyncProcess" 66 | ), 67 | .testTarget( 68 | name: "TSFAsyncProcessTests", 69 | dependencies: [ 70 | "TSFAsyncProcess", 71 | .product(name: "Atomics", package: "swift-atomics"), 72 | .product(name: "AsyncAlgorithms", package: "swift-async-algorithms"), 73 | .product(name: "NIO", package: "swift-nio"), 74 | .product(name: "NIOConcurrencyHelpers", package: "swift-nio"), 75 | .product(name: "Logging", package: "swift-log"), 76 | .product(name: "_NIOFileSystem", package: "swift-nio"), 77 | ], 78 | path: "Tests/AsyncProcessTests" 79 | ), 80 | .target( 81 | name: "TSFCProcessSpawnSync", 82 | path: "Sources/CProcessSpawnSync", 83 | cSettings: [ 84 | .define("_GNU_SOURCE") 85 | ] 86 | ), 87 | .target( 88 | name: "TSFProcessSpawnSync", 89 | dependencies: [ 90 | "TSFCProcessSpawnSync", 91 | .product(name: "Atomics", package: "swift-atomics"), 92 | .product(name: "NIOConcurrencyHelpers", package: "swift-nio"), 93 | ], 94 | path: "Sources/ProcessSpawnSync" 95 | ), 96 | 97 | .target( 98 | name: "TSFFutures", 99 | dependencies: [ 100 | .product(name: "NIO", package: "swift-nio"), 101 | .product(name: "NIOFoundationCompat", package: "swift-nio"), 102 | .product(name: "SwiftToolsSupport-auto", package: "swift-tools-support-core"), 103 | ] 104 | ), 105 | .testTarget( 106 | name: "TSFFuturesTests", 107 | dependencies: [ 108 | "TSFFutures" 109 | ] 110 | ), 111 | .target( 112 | name: "TSFUtility", 113 | dependencies: [ 114 | "TSFFutures", 115 | .product(name: "NIO", package: "swift-nio"), 116 | .product(name: "NIOConcurrencyHelpers", package: "swift-nio"), 117 | .product(name: "NIOFoundationCompat", package: "swift-nio"), 118 | ] 119 | ), 120 | 121 | .target( 122 | name: "TSFCAS", 123 | dependencies: [ 124 | "TSFFutures", "TSFUtility", "CBLAKE3", 125 | .product(name: "SwiftProtobuf", package: "swift-protobuf"), 126 | ] 127 | ), 128 | .target( 129 | name: "TSFCASUtilities", 130 | dependencies: [ 131 | "TSFCAS", "TSFCASFileTree", 132 | ] 133 | ), 134 | .testTarget( 135 | name: "TSFCASTests", 136 | dependencies: ["TSFCAS"] 137 | ), 138 | .testTarget( 139 | name: "TSFCASUtilitiesTests", 140 | dependencies: ["TSFCASUtilities"] 141 | ), 142 | .target( 143 | name: "TSFCASFileTree", 144 | dependencies: ["TSFCAS"] 145 | ), 146 | .testTarget( 147 | name: "TSFCASFileTreeTests", 148 | dependencies: ["TSFCASFileTree"] 149 | ), 150 | ] 151 | ) 152 | -------------------------------------------------------------------------------- /Protos/CASFileTreeProtocol/file_tree.proto: -------------------------------------------------------------------------------- 1 | // This source file is part of the Swift.org open source project 2 | // 3 | // Copyright (c) 2020 Apple Inc. and the Swift project authors 4 | // Licensed under Apache License v2.0 with Runtime Library Exception 5 | // 6 | // See http://swift.org/LICENSE.txt for license information 7 | // See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 8 | 9 | syntax = "proto3"; 10 | 11 | option java_package = "com.apple.CASFileTreeProtocol"; 12 | 13 | enum LLBFileType { 14 | /// A plain file. 15 | PLAIN_FILE = 0x0; 16 | 17 | /// An executable file. 18 | EXECUTABLE = 0x1; 19 | 20 | /// A directory. 21 | DIRECTORY = 0x2; 22 | 23 | /// A symbolic link. 24 | SYMLINK = 0x3; 25 | } 26 | 27 | message LLBPosixFileDetails { 28 | /// The POSIX permissions (&0o7777). Masking is useful when storing entries 29 | /// with very restricted permissions (such as (perm & 0o0007) == 0). 30 | uint32 mode = 1; 31 | 32 | /// Owner user identifier. 33 | /// Semantically, absent owner == 0x0 ~= current uid. 34 | uint32 owner = 2; 35 | 36 | /// Owner group identifier. 37 | /// Semantically, absent owner == 0x0 ~= current gid. 38 | uint32 group = 3; 39 | } 40 | 41 | message LLBDirectoryEntry { 42 | /// The name of the directory entry. 43 | string name = 1; 44 | 45 | /// The type of the directory entry. 46 | LLBFileType type = 2; 47 | 48 | /// The (aggregate) size of the directory entry. 49 | uint64 size = 3; 50 | 51 | /// Mode and permissions. _Can_ optionally be present in the 52 | /// directory entry because the file can be just a direct blob reference. 53 | LLBPosixFileDetails posixDetails = 4; 54 | } 55 | 56 | /// The list of file names and associated information, of a directory. 57 | /// * The children are sorted by name. 58 | /// * FIXME: collation rules or UTF-8 normalization guarantees? 59 | message LLBDirectoryEntries { 60 | repeated LLBDirectoryEntry entries = 1; 61 | } 62 | 63 | enum LLBFileDataCompressionMethod { 64 | /// No compression is applied. 65 | NONE = 0x0; 66 | } 67 | 68 | message LLBFileInfo { 69 | /// The type of the CASTree entry. 70 | LLBFileType type = 1; 71 | 72 | /// The file data size or the aggregate directory data size (recursive). 73 | /// Whether directory data includes the size of the directory catalogs 74 | /// is unspecified. 75 | uint64 size = 2; 76 | 77 | /// OBSOLETE. Use posixDetails. 78 | /// The POSIX permissions (&0o777). Useful when storing entries 79 | /// with very restricted permissions (such as (perm & 0o007) == 0). 80 | uint32 posixPermissions = 3; 81 | 82 | /// Whether and what compression is applied to file data. 83 | /// * Compression ought not to be applied to symlinks. 84 | /// * Compression is applied after chunking, to retain seekability. 85 | LLBFileDataCompressionMethod compression = 4; 86 | 87 | /// Permission info useful for POSIX filesystems. 88 | LLBPosixFileDetails posixDetails = 5; 89 | 90 | oneof payload { 91 | /// Files and symlinks: 92 | /// * The file payload is contained in one or more 93 | /// fixed size references to [compressed] data. 94 | /// * The `fixedChunkSize` value helps to do O(1) seeking. 95 | uint64 fixedChunkSize = 11; 96 | 97 | /// Directories: 98 | /// * Directory entries are represented inline. 99 | LLBDirectoryEntries inlineChildren = 12; 100 | 101 | /// Directories: 102 | /// * Directory entries are represented as a reference to a B-tree. 103 | /// * The `compression` does have effect on the B-tree data. 104 | uint32 referencedChildrenTree = 13; 105 | } 106 | } 107 | -------------------------------------------------------------------------------- /Protos/CASProtocol/cas_object.proto: -------------------------------------------------------------------------------- 1 | // This source file is part of the Swift.org open source project 2 | // 3 | // Copyright (c) 2020 Apple Inc. and the Swift project authors 4 | // Licensed under Apache License v2.0 with Runtime Library Exception 5 | // 6 | // See http://swift.org/LICENSE.txt for license information 7 | // See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 8 | 9 | syntax = "proto3"; 10 | 11 | option java_package = "com.apple.CASProtocol"; 12 | 13 | import "CASProtocol/data_id.proto"; 14 | 15 | /// LLBPBCASObject represents the serialized from of CASObjects. It encodes the 16 | /// combination of the raw data of the object and its dependendent references. 17 | message LLBPBCASObject { 18 | repeated LLBDataID refs = 1; 19 | 20 | bytes data = 2; 21 | } 22 | -------------------------------------------------------------------------------- /Protos/CASProtocol/data_id.proto: -------------------------------------------------------------------------------- 1 | // This source file is part of the Swift.org open source project 2 | // 3 | // Copyright (c) 2020 Apple Inc. and the Swift project authors 4 | // Licensed under Apache License v2.0 with Runtime Library Exception 5 | // 6 | // See http://swift.org/LICENSE.txt for license information 7 | // See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 8 | 9 | syntax = "proto3"; 10 | 11 | option java_package = "com.apple.CASProtocol"; 12 | 13 | /// LLBDataID represents the digest of arbitrary data, and its purpose is to be a handler for interfacing with CAS 14 | /// systems. LLBDataID does not require the encoding of any particular hash function. Instead, it is expected that the 15 | /// CAS system itself that provides the digest. 16 | message LLBDataID { 17 | /// The bytes containing the digest of the contents store in the CAS. 18 | bytes bytes = 1; 19 | } 20 | -------------------------------------------------------------------------------- /Protos/module_map.asciipb: -------------------------------------------------------------------------------- 1 | mapping { 2 | module_name: "TSFCAS" 3 | proto_file_path: "CASProtocol/cas_object.proto" 4 | proto_file_path: "CASProtocol/data_id.proto" 5 | } 6 | mapping { 7 | module_name: "TSFCASFileTree" 8 | proto_file_path: "CASFileTreeProtocol/file_tree.proto" 9 | } 10 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # swift-tools-support-async 2 | 3 | Common infrastructural helpers on top of NIO for [llbuild2](https://github.com/apple/swift-llbuild2) and [swiftpm-on-llbuild2](https://github.com/apple/swiftpm-on-llbuild2) projects. This is **NOT** a general purpose package and is unlikely to ever become stable. 4 | 5 | ## License 6 | 7 | Copyright (c) 2020 Apple Inc. and the Swift project authors. 8 | Licensed under Apache License v2.0 with Runtime Library Exception. 9 | 10 | See https://swift.org/LICENSE.txt for license information. 11 | 12 | See https://swift.org/CONTRIBUTORS.txt for Swift project authors. 13 | -------------------------------------------------------------------------------- /Sources/AsyncProcess/ChunkSequence.swift: -------------------------------------------------------------------------------- 1 | //===----------------------------------------------------------------------===// 2 | // 3 | // This source file is part of the Swift open source project 4 | // 5 | // Copyright (c) 2022-2025 Apple Inc. and the Swift project authors 6 | // Licensed under Apache License v2.0 with Runtime Library Exception 7 | // 8 | // See https://swift.org/LICENSE.txt for license information 9 | // See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 10 | // 11 | //===----------------------------------------------------------------------===// 12 | 13 | // DO NOT EDIT - Make any changes in the upstream swift-async-process package, then re-run the vendoring script. 14 | 15 | import NIO 16 | 17 | #if os(Linux) || os(Android) || os(Windows) 18 | @preconcurrency import Foundation 19 | #else 20 | import Foundation 21 | #endif 22 | 23 | public struct IllegalStreamConsumptionError: Error { 24 | var description: String 25 | } 26 | 27 | public struct ChunkSequence: AsyncSequence & Sendable { 28 | private let contentStream: FileContentStream? 29 | 30 | public init( 31 | takingOwnershipOfFileHandle fileHandle: FileHandle, 32 | group: EventLoopGroup 33 | ) async throws { 34 | // This will close the fileHandle 35 | let contentStream = try await fileHandle.fileContentStream(eventLoop: group.any()) 36 | self.init(contentStream: contentStream) 37 | } 38 | 39 | internal func isSameAs(_ other: ChunkSequence) -> Bool { 40 | guard let myContentStream = self.contentStream else { 41 | return other.contentStream == nil 42 | } 43 | guard let otherContentStream = other.contentStream else { 44 | return self.contentStream == nil 45 | } 46 | return myContentStream.isSameAs(otherContentStream) 47 | } 48 | 49 | public func close() async throws { 50 | try await self.contentStream?.close() 51 | } 52 | 53 | private init(contentStream: FileContentStream?) { 54 | self.contentStream = contentStream 55 | } 56 | 57 | public static func makeEmptyStream() -> Self { 58 | return Self.init(contentStream: nil) 59 | } 60 | 61 | public func makeAsyncIterator() -> AsyncIterator { 62 | return AsyncIterator(self.contentStream) 63 | } 64 | 65 | public typealias Element = ByteBuffer 66 | public struct AsyncIterator: AsyncIteratorProtocol { 67 | public typealias Element = ByteBuffer 68 | internal typealias UnderlyingSequence = FileContentStream 69 | 70 | private var underlyingIterator: UnderlyingSequence.AsyncIterator? 71 | 72 | internal init(_ underlyingSequence: UnderlyingSequence?) { 73 | self.underlyingIterator = underlyingSequence?.makeAsyncIterator() 74 | } 75 | 76 | public mutating func next() async throws -> Element? { 77 | if self.underlyingIterator != nil { 78 | return try await self.underlyingIterator!.next() 79 | } else { 80 | throw IllegalStreamConsumptionError( 81 | description: """ 82 | Either `.discard`ed, `.inherit`ed or redirected this stream to a `.fileHandle`, 83 | cannot also consume it. To consume, please `.stream` it. 84 | """ 85 | ) 86 | } 87 | } 88 | } 89 | } 90 | -------------------------------------------------------------------------------- /Sources/AsyncProcess/EOFSequence.swift: -------------------------------------------------------------------------------- 1 | //===----------------------------------------------------------------------===// 2 | // 3 | // This source file is part of the Swift open source project 4 | // 5 | // Copyright (c) 2022-2025 Apple Inc. and the Swift project authors 6 | // Licensed under Apache License v2.0 with Runtime Library Exception 7 | // 8 | // See https://swift.org/LICENSE.txt for license information 9 | // See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 10 | // 11 | //===----------------------------------------------------------------------===// 12 | 13 | // DO NOT EDIT - Make any changes in the upstream swift-async-process package, then re-run the vendoring script. 14 | 15 | public struct EOFSequence: AsyncSequence & Sendable { 16 | public typealias Element = Element 17 | 18 | public struct AsyncIterator: AsyncIteratorProtocol { 19 | public mutating func next() async throws -> Element? { 20 | return nil 21 | } 22 | } 23 | 24 | public init(of type: Element.Type = Element.self) {} 25 | 26 | public func makeAsyncIterator() -> AsyncIterator { 27 | return AsyncIterator() 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /Sources/AsyncProcess/NIOAsyncPipeWriter.swift: -------------------------------------------------------------------------------- 1 | //===----------------------------------------------------------------------===// 2 | // 3 | // This source file is part of the Swift open source project 4 | // 5 | // Copyright (c) 2022-2025 Apple Inc. and the Swift project authors 6 | // Licensed under Apache License v2.0 with Runtime Library Exception 7 | // 8 | // See https://swift.org/LICENSE.txt for license information 9 | // See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 10 | // 11 | //===----------------------------------------------------------------------===// 12 | 13 | // DO NOT EDIT - Make any changes in the upstream swift-async-process package, then re-run the vendoring script. 14 | 15 | import Foundation 16 | import NIO 17 | 18 | struct NIOAsyncPipeWriter where Chunks.Element == ByteBuffer { 19 | static func sinkSequenceInto( 20 | _ chunks: Chunks, 21 | takingOwnershipOfFD fd: CInt, 22 | ignoreWriteErrors: Bool, 23 | eventLoop: EventLoop 24 | ) async throws { 25 | let channel = try await NIOPipeBootstrap(group: eventLoop) 26 | .channelOption(ChannelOptions.allowRemoteHalfClosure, value: true) 27 | .channelOption(ChannelOptions.autoRead, value: false) 28 | .takingOwnershipOfDescriptor( 29 | output: fd 30 | ).get() 31 | channel.close(mode: .input, promise: nil) 32 | return try await asyncDo { 33 | try await withTaskCancellationHandler { 34 | for try await chunk in chunks { 35 | do { 36 | try await channel.writeAndFlush(chunk).get() 37 | } catch { 38 | if !ignoreWriteErrors { 39 | throw error 40 | } 41 | break 42 | } 43 | } 44 | } onCancel: { 45 | channel.close(promise: nil) 46 | } 47 | } finally: { _ in 48 | do { 49 | try await channel.close() 50 | } catch ChannelError.alreadyClosed { 51 | // ok 52 | } 53 | } 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /Sources/AsyncProcess/ProcessExit.swift: -------------------------------------------------------------------------------- 1 | //===----------------------------------------------------------------------===// 2 | // 3 | // This source file is part of the Swift open source project 4 | // 5 | // Copyright (c) 2022-2025 Apple Inc. and the Swift project authors 6 | // Licensed under Apache License v2.0 with Runtime Library Exception 7 | // 8 | // See https://swift.org/LICENSE.txt for license information 9 | // See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 10 | // 11 | //===----------------------------------------------------------------------===// 12 | 13 | // DO NOT EDIT - Make any changes in the upstream swift-async-process package, then re-run the vendoring script. 14 | 15 | public struct ProcessExitExtendedInfo: Sendable { 16 | /// Reason the process exited. 17 | public var exitReason: ProcessExitReason 18 | 19 | /// Any errors that occurred whilst writing the provided `standardInput` sequence into the child process' standard input. 20 | public var standardInputWriteError: Optional 21 | } 22 | 23 | public enum ProcessExitReason: Hashable & Sendable { 24 | case exit(CInt) 25 | case signal(CInt) 26 | 27 | public func throwIfNonZero() throws { 28 | switch self { 29 | case .exit(0): 30 | return 31 | default: 32 | throw ProcessExecutionError(self) 33 | } 34 | } 35 | } 36 | 37 | extension ProcessExitReason { 38 | /// Turn into an integer like `$?` works in shells. 39 | /// 40 | /// Concretely, this means if the program exits normally with exit code `N`, `asShellExitCode == N`. But if the program exits because of a signal, then 41 | /// `asShellExitCode == N + 128`, so 128 gets added to the signal number. 42 | public var asShellExitCode: Int { 43 | switch self { 44 | case .exit(let code): 45 | return Int(code) 46 | case .signal(let code): 47 | return 128 + Int(code) 48 | } 49 | } 50 | 51 | /// Turn into an integer like Python's subprocess does. 52 | /// 53 | /// Concretely, this means if the program exits normally with exit code `N`, `asShellExitCode == N`. But if the program exits because of a signal, then 54 | /// `asShellExitCode == -N`, so the negative signal number gets returned. 55 | public var asPythonExitCode: Int { 56 | switch self { 57 | case .exit(let code): 58 | return Int(code) 59 | case .signal(let code): 60 | return -Int(code) 61 | } 62 | } 63 | } 64 | 65 | public struct ProcessExecutionError: Error & Hashable & Sendable { 66 | public var exitReason: ProcessExitReason 67 | 68 | public init(_ exitResult: ProcessExitReason) { 69 | self.exitReason = exitResult 70 | } 71 | } 72 | 73 | extension ProcessExecutionError: CustomStringConvertible { 74 | public var description: String { 75 | return "process exited non-zero: \(self.exitReason)" 76 | } 77 | } 78 | -------------------------------------------------------------------------------- /Sources/AsyncProcess/StructuredConcurrencyHelpers.swift: -------------------------------------------------------------------------------- 1 | //===----------------------------------------------------------------------===// 2 | // 3 | // This source file is part of the Swift open source project 4 | // 5 | // Copyright (c) 2022-2025 Apple Inc. and the Swift project authors 6 | // Licensed under Apache License v2.0 with Runtime Library Exception 7 | // 8 | // See https://swift.org/LICENSE.txt for license information 9 | // See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 10 | // 11 | //===----------------------------------------------------------------------===// 12 | 13 | // DO NOT EDIT - Make any changes in the upstream swift-async-process package, then re-run the vendoring script. 14 | 15 | //===----------------------------------------------------------------------===// 16 | // 17 | // This source file is part of the AsyncHTTPClient open source project 18 | // 19 | // Copyright (c) 2025 Apple Inc. and the AsyncHTTPClient project authors 20 | // Licensed under Apache License v2.0 21 | // 22 | // See LICENSE.txt for license information 23 | // See CONTRIBUTORS.txt for the list of AsyncHTTPClient project authors 24 | // 25 | // SPDX-License-Identifier: Apache-2.0 26 | // 27 | //===----------------------------------------------------------------------===// 28 | // swift-format-ignore 29 | // Note: Whitespace changes are used to workaround compiler bug 30 | // https://github.com/swiftlang/swift/issues/79285 31 | 32 | #if compiler(>=6.0) 33 | @inlinable 34 | @available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) 35 | internal func asyncDo( 36 | isolation: isolated (any Actor)? = #isolation, 37 | // DO NOT FIX THE WHITESPACE IN THE NEXT LINE UNTIL 5.10 IS UNSUPPORTED 38 | // https://github.com/swiftlang/swift/issues/79285 39 | _ body: () async throws -> sending R, finally: sending @escaping ((any Error)?) async throws -> Void) async throws -> sending R { 40 | let result: R 41 | do { 42 | result = try await body() 43 | } catch { 44 | // `body` failed, we need to invoke `finally` with the `error`. 45 | 46 | // This _looks_ unstructured but isn't really because we unconditionally always await the return. 47 | // We need to have an uncancelled task here to assure this is actually running in case we hit a 48 | // cancellation error. 49 | try await Task { 50 | try await finally(error) 51 | }.value 52 | throw error 53 | } 54 | 55 | // `body` succeeded, we need to invoke `finally` with `nil` (no error). 56 | 57 | // This _looks_ unstructured but isn't really because we unconditionally always await the return. 58 | // We need to have an uncancelled task here to assure this is actually running in case we hit a 59 | // cancellation error. 60 | try await Task { 61 | try await finally(nil) 62 | }.value 63 | return result 64 | } 65 | #else 66 | @inlinable 67 | @available(macOS 10.15, iOS 13.0, watchOS 6.0, tvOS 13.0, *) 68 | internal func asyncDo( 69 | _ body: () async throws -> R, 70 | finally: @escaping @Sendable ((any Error)?) async throws -> Void 71 | ) async throws -> R { 72 | let result: R 73 | do { 74 | result = try await body() 75 | } catch { 76 | // `body` failed, we need to invoke `finally` with the `error`. 77 | 78 | // This _looks_ unstructured but isn't really because we unconditionally always await the return. 79 | // We need to have an uncancelled task here to assure this is actually running in case we hit a 80 | // cancellation error. 81 | try await Task { 82 | try await finally(error) 83 | }.value 84 | throw error 85 | } 86 | 87 | // `body` succeeded, we need to invoke `finally` with `nil` (no error). 88 | 89 | // This _looks_ unstructured but isn't really because we unconditionally always await the return. 90 | // We need to have an uncancelled task here to assure this is actually running in case we hit a 91 | // cancellation error. 92 | try await Task { 93 | try await finally(nil) 94 | }.value 95 | return result 96 | } 97 | #endif 98 | -------------------------------------------------------------------------------- /Sources/CBLAKE3/blake3.c: -------------------------------------------------------------------------------- 1 | #if __x86_64__ 2 | 3 | #ifndef __SSE4_1__ 4 | #define BLAKE3_NO_SSE41 5 | #endif 6 | #ifndef __AVX2__ 7 | #define BLAKE3_NO_AVX2 8 | #endif 9 | #ifndef __AVX512__ 10 | #define BLAKE3_NO_AVX512 11 | #endif 12 | 13 | #include "../../ThirdParty/BLAKE3/c/blake3.c" 14 | #include "../../ThirdParty/BLAKE3/c/blake3_dispatch.c" 15 | #include "../../ThirdParty/BLAKE3/c/blake3_portable.c" 16 | #else 17 | #include "../../ThirdParty/BLAKE3/c/blake3.c" 18 | #include "../../ThirdParty/BLAKE3/c/blake3_dispatch.c" 19 | #include "../../ThirdParty/BLAKE3/c/blake3_portable.c" 20 | #endif 21 | -------------------------------------------------------------------------------- /Sources/CBLAKE3/blake3_avx2.c: -------------------------------------------------------------------------------- 1 | #if __AVX2__ 2 | #include "../../ThirdParty/BLAKE3/c/blake3_avx2.c" 3 | #endif 4 | -------------------------------------------------------------------------------- /Sources/CBLAKE3/blake3_avx512.c: -------------------------------------------------------------------------------- 1 | #if __AVX512__ 2 | #include "../../ThirdParty/BLAKE3/c/blake3_avx512.c" 3 | #endif 4 | -------------------------------------------------------------------------------- /Sources/CBLAKE3/blake3_impl.h: -------------------------------------------------------------------------------- 1 | ../../ThirdParty/BLAKE3/c/blake3_impl.h -------------------------------------------------------------------------------- /Sources/CBLAKE3/blake3_sse41.c: -------------------------------------------------------------------------------- 1 | #if __SSE4_1__ 2 | #include "../../ThirdParty/BLAKE3/c/blake3_sse41.c" 3 | #endif 4 | -------------------------------------------------------------------------------- /Sources/CBLAKE3/include/blake3.h: -------------------------------------------------------------------------------- 1 | ../../../ThirdParty/BLAKE3/c/blake3.h -------------------------------------------------------------------------------- /Sources/CProcessSpawnSync/include/CProcessSpawnSync.h: -------------------------------------------------------------------------------- 1 | //===----------------------------------------------------------------------===// 2 | // 3 | // This source file is part of the Swift open source project 4 | // 5 | // Copyright (c) 2022-2025 Apple Inc. and the Swift project authors 6 | // Licensed under Apache License v2.0 with Runtime Library Exception 7 | // 8 | // See https://swift.org/LICENSE.txt for license information 9 | // See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 10 | // 11 | //===----------------------------------------------------------------------===// 12 | 13 | // DO NOT EDIT - Make any changes in the upstream swift-async-process package, then re-run the vendoring script. 14 | 15 | #include "ps-api.h" 16 | -------------------------------------------------------------------------------- /Sources/CProcessSpawnSync/include/ps-api.h: -------------------------------------------------------------------------------- 1 | //===----------------------------------------------------------------------===// 2 | // 3 | // This source file is part of the Swift open source project 4 | // 5 | // Copyright (c) 2022-2025 Apple Inc. and the Swift project authors 6 | // Licensed under Apache License v2.0 with Runtime Library Exception 7 | // 8 | // See https://swift.org/LICENSE.txt for license information 9 | // See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 10 | // 11 | //===----------------------------------------------------------------------===// 12 | 13 | // DO NOT EDIT - Make any changes in the upstream swift-async-process package, then re-run the vendoring script. 14 | 15 | #ifndef PS_API_H 16 | #define PS_API_H 17 | 18 | #include 19 | #include 20 | 21 | typedef enum llb_ps_error_kind_s { 22 | PS_ERROR_KIND_EXECVE = 1, 23 | PS_ERROR_KIND_PIPE = 2, 24 | PS_ERROR_KIND_FCNTL = 3, 25 | PS_ERROR_KIND_SIGNAL = 4, 26 | PS_ERROR_KIND_SIGPROC_MASK = 5, 27 | PS_ERROR_KIND_CHDIR = 6, 28 | PS_ERROR_KIND_SETSID = 7, 29 | PS_ERROR_KIND_DUP2 = 8, 30 | PS_ERROR_KIND_READ_FROM_CHILD = 9, 31 | PS_ERROR_KIND_DUP = 10, 32 | PS_ERROR_KIND_SIGMASK_THREAD = 11, 33 | PS_ERROR_KIND_FAILED_CHILD_WAITPID = 12, 34 | } llb_ps_error_kind; 35 | 36 | typedef struct llb_ps_error_s { 37 | llb_ps_error_kind pse_kind; 38 | int pse_code; 39 | const char *pse_file; 40 | int pse_line; 41 | int pse_extra_info; 42 | } llb_ps_error; 43 | 44 | typedef enum llb_ps_fd_setup_kind_s { 45 | PS_MAP_FD = 1, 46 | PS_CLOSE_FD = 2, 47 | } llb_ps_fd_setup_kind; 48 | 49 | typedef struct llb_ps_fd_setup_s { 50 | llb_ps_fd_setup_kind psfd_kind; 51 | int psfd_parent_fd; 52 | } llb_ps_fd_setup; 53 | 54 | typedef struct llb_ps_process_configuration_s { 55 | const char *psc_path; 56 | 57 | // including argv[0] 58 | char **psc_argv; 59 | 60 | char **psc_env; 61 | 62 | const char *psc_cwd; 63 | 64 | int psc_fd_setup_count; 65 | const llb_ps_fd_setup *psc_fd_setup_instructions; 66 | 67 | bool psc_new_session; 68 | bool psc_close_other_fds; 69 | } llb_ps_process_configuration; 70 | 71 | pid_t llb_ps_spawn_process(llb_ps_process_configuration *config, llb_ps_error *out_error); 72 | 73 | void llb_ps_convert_exit_status(int in_status, bool *out_has_exited, bool *out_is_exit_code, int *out_code); 74 | 75 | #endif 76 | -------------------------------------------------------------------------------- /Sources/CProcessSpawnSync/internal-helpers.h: -------------------------------------------------------------------------------- 1 | //===----------------------------------------------------------------------===// 2 | // 3 | // This source file is part of the Swift open source project 4 | // 5 | // Copyright (c) 2022-2025 Apple Inc. and the Swift project authors 6 | // Licensed under Apache License v2.0 with Runtime Library Exception 7 | // 8 | // See https://swift.org/LICENSE.txt for license information 9 | // See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 10 | // 11 | //===----------------------------------------------------------------------===// 12 | 13 | // DO NOT EDIT - Make any changes in the upstream swift-async-process package, then re-run the vendoring script. 14 | 15 | #ifndef INTERNAL_HELPERS_H 16 | #define INTERNAL_HELPERS_H 17 | #include 18 | #include 19 | #include 20 | #include 21 | #if defined(__linux__) 22 | #include 23 | #endif 24 | #if defined(__APPLE__) 25 | #include 26 | 27 | ssize_t __getdirentries64(int fd, void *buf, size_t bufsize, off_t *basep); 28 | #endif 29 | 30 | static int positive_int_parse(const char *str) { 31 | int out = 0; 32 | char c = 0; 33 | 34 | while ((c = *str++) != 0) { 35 | out *= 10; 36 | if (c >= '0' && c <= '9') { 37 | out += c - '0'; 38 | } else { 39 | return -1; 40 | } 41 | } 42 | return out; 43 | } 44 | 45 | #if defined(__linux__) || defined(__APPLE__) 46 | // Platform-specific version that uses syscalls directly and doesn't allocate heap memory. 47 | // Safe to use after vfork() and before execve() 48 | static int highest_possibly_open_fd_dir_syscall(const char *fd_dir) { 49 | int highest_fd_so_far = 0; 50 | int dir_fd = open(fd_dir, O_RDONLY); 51 | if (dir_fd < 0) { 52 | // errno set by `open`. 53 | return -1; 54 | } 55 | 56 | // Buffer for directory entries - allocated on stack, no heap allocation 57 | char buffer[4096] = {0}; 58 | #if defined(__linux__) 59 | ssize_t bytes_read = -1; 60 | #elif defined(__APPLE__) 61 | ssize_t bytes_read = -1; 62 | off_t os_controlled_seek_pos = -1; 63 | #endif 64 | 65 | while (( 66 | #if defined(__linux__) 67 | # if defined(__GLIBC__) && __GLIBC__ == 2 && defined(__GLIBC_MINOR__) && __GLIBC_MINOR__ >= 30 68 | bytes_read = getdents64(dir_fd, (struct dirent64 *)buffer, sizeof(buffer)) 69 | # else 70 | bytes_read = syscall(SYS_getdents64, dir_fd, (struct dirent64 *)buffer, sizeof(buffer)) 71 | # endif 72 | #elif defined(__APPLE__) 73 | bytes_read = __getdirentries64(dir_fd, buffer, sizeof(buffer), &os_controlled_seek_pos) 74 | #endif 75 | ) > 0) { 76 | if (bytes_read < 0) { 77 | if (errno == EINTR) { 78 | continue; 79 | } else { 80 | // `errno` set by getdents64/getdirentries. 81 | highest_fd_so_far = -1; 82 | goto error; 83 | } 84 | } 85 | long offset = 0; 86 | while (offset < bytes_read) { 87 | #if defined(__linux__) 88 | struct dirent64 *entry = (struct dirent64 *)(buffer + offset); 89 | #elif defined(__APPLE__) 90 | struct dirent *entry = (struct dirent *)(buffer + offset); 91 | #endif 92 | 93 | // Skip "." and ".." entries 94 | if (entry->d_name[0] != '.') { 95 | int number = positive_int_parse(entry->d_name); 96 | if (number > highest_fd_so_far) { 97 | highest_fd_so_far = number; 98 | } 99 | } 100 | 101 | offset += entry->d_reclen; 102 | } 103 | } 104 | 105 | error: 106 | close(dir_fd); 107 | return highest_fd_so_far; 108 | } 109 | #endif 110 | 111 | static int highest_possibly_open_fd(void) { 112 | #if defined(__APPLE__) 113 | int hi = highest_possibly_open_fd_dir_syscall("/dev/fd"); 114 | if (hi < 0) { 115 | hi = getdtablesize(); 116 | } 117 | #elif defined(__linux__) 118 | int hi = highest_possibly_open_fd_dir_syscall("/proc/self/fd"); 119 | if (hi < 0) { 120 | hi = getdtablesize(); 121 | } 122 | #else 123 | int hi = 1024; 124 | #endif 125 | 126 | return hi; 127 | } 128 | 129 | static int block_everything_but_something_went_seriously_wrong_signals(sigset_t *old_mask) { 130 | sigset_t mask; 131 | int r = 0; 132 | r |= sigfillset(&mask); 133 | r |= sigdelset(&mask, SIGABRT); 134 | r |= sigdelset(&mask, SIGBUS); 135 | r |= sigdelset(&mask, SIGFPE); 136 | r |= sigdelset(&mask, SIGILL); 137 | r |= sigdelset(&mask, SIGKILL); 138 | r |= sigdelset(&mask, SIGSEGV); 139 | r |= sigdelset(&mask, SIGSTOP); 140 | r |= sigdelset(&mask, SIGSYS); 141 | r |= sigdelset(&mask, SIGTRAP); 142 | 143 | r |= pthread_sigmask(SIG_BLOCK, &mask, old_mask); 144 | return r; 145 | } 146 | #endif 147 | -------------------------------------------------------------------------------- /Sources/TSFCAS/DataID.swift: -------------------------------------------------------------------------------- 1 | // This source file is part of the Swift.org open source project 2 | // 3 | // Copyright (c) 2020-2021 Apple Inc. and the Swift project authors 4 | // Licensed under Apache License v2.0 with Runtime Library Exception 5 | // 6 | // See http://swift.org/LICENSE.txt for license information 7 | // See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 8 | 9 | import Foundation 10 | import NIOCore 11 | import TSCBasic 12 | import TSFUtility 13 | 14 | // MARK:- DataID Extensions - 15 | 16 | private enum DataIDKind: UInt8 { 17 | /// An id that is directly calculated based on a hash of the data. 18 | case directHash = 0 19 | case shareableHash = 4 20 | 21 | init?(from bytes: Data) { 22 | guard let first = bytes.first, 23 | let kind = DataIDKind(rawValue: first) 24 | else { 25 | return nil 26 | } 27 | self = kind 28 | } 29 | 30 | init?(from substring: Substring) { 31 | guard let first = substring.utf8.first, 32 | first >= UInt8(ascii: "0") 33 | else { 34 | return nil 35 | } 36 | self.init(rawValue: first - UInt8(ascii: "0")) 37 | } 38 | } 39 | 40 | extension LLBDataID: Hashable, CustomDebugStringConvertible { 41 | 42 | /// Represent DataID as string to encode it in messages. 43 | /// Properties of the string: the first character represents the kind, 44 | /// then '~', then the Base64 encoding follows. 45 | public var debugDescription: String { 46 | return ArraySlice(bytes.dropFirst()).base64URL(prepending: [ 47 | (bytes.first ?? 15) + UInt8(ascii: "0"), UInt8(ascii: "~"), 48 | ]) 49 | } 50 | 51 | public init?(bytes: [UInt8]) { 52 | let data = Data(bytes) 53 | guard DataIDKind(from: data) != nil else { 54 | return nil 55 | } 56 | self.bytes = data 57 | } 58 | 59 | public init(directHash bytes: [UInt8]) { 60 | self.bytes = Data([DataIDKind.directHash.rawValue] + bytes) 61 | } 62 | 63 | /// Initialize from the string form. 64 | public init?(string: String) { 65 | self.init(string: Substring(string)) 66 | } 67 | 68 | public init?(string: Substring) { 69 | // Test for the kind in the first position. 70 | guard let kind = DataIDKind(from: string) else { return nil } 71 | 72 | // Test for "~" in the second position. 73 | guard string.count >= 2 else { return nil } 74 | let tilde = string.utf8[string.utf8.index(string.startIndex, offsetBy: 1)] 75 | guard tilde == UInt8(ascii: "~") else { return nil } 76 | 77 | let b64substring = string.dropFirst(2) 78 | guard let completeBytes = [UInt8](base64URL: b64substring, prepending: [kind.rawValue]) 79 | else { 80 | return nil 81 | } 82 | 83 | self.bytes = Data(completeBytes) 84 | } 85 | } 86 | 87 | extension LLBDataID: Comparable { 88 | /// Compare DataID according to stable but arbitrary rules 89 | /// (not necessarily alphanumeric). 90 | public static func < (lhs: LLBDataID, rhs: LLBDataID) -> Bool { 91 | let a = lhs.bytes 92 | let b = rhs.bytes 93 | if a.count == b.count { 94 | for n in (0..) throws { 136 | guard let dataId = LLBDataID(bytes: Array(rawBytes)) else { 137 | throw LLBDataIDSliceError.decoding("from slice of size \(rawBytes.count)") 138 | } 139 | self = dataId 140 | } 141 | 142 | @inlinable 143 | public init(from rawBytes: LLBByteBuffer) throws { 144 | guard let dataId = LLBDataID(bytes: Array(buffer: rawBytes)) else { 145 | throw LLBDataIDSliceError.decoding("from slice of size \(rawBytes.readableBytes)") 146 | } 147 | self = dataId 148 | } 149 | 150 | @inlinable 151 | public func toBytes() -> ArraySlice { 152 | return ArraySlice(bytes) 153 | } 154 | 155 | @inlinable 156 | public func toBytes(into array: inout [UInt8]) { 157 | array += bytes 158 | } 159 | 160 | @inlinable 161 | public func toBytes(into buffer: inout LLBByteBuffer) throws { 162 | buffer.writeBytes(bytes) 163 | } 164 | 165 | @inlinable 166 | public var sliceSizeEstimate: Int { 167 | return bytes.count 168 | } 169 | } 170 | -------------------------------------------------------------------------------- /Sources/TSFCAS/DatabaseSpec.swift: -------------------------------------------------------------------------------- 1 | // This source file is part of the Swift.org open source project 2 | // 3 | // Copyright (c) 2020 Apple Inc. and the Swift project authors 4 | // Licensed under Apache License v2.0 with Runtime Library Exception 5 | // 6 | // See http://swift.org/LICENSE.txt for license information 7 | // See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 8 | 9 | import Foundation 10 | import NIOCore 11 | import TSFFutures 12 | 13 | /// A scheme for specifying a CASDatabase. 14 | public protocol LLBCASDatabaseScheme { 15 | /// The name of the scheme. 16 | static var scheme: String { get } 17 | 18 | /// Check if a URL is valid for this scheme. 19 | static func isValid(host: String?, port: Int?, path: String, query: String?) -> Bool 20 | 21 | /// Open a content store in this scheme. 22 | static func open(group: LLBFuturesDispatchGroup, url: URL) throws -> LLBCASDatabase 23 | } 24 | 25 | /// A specification for a CAS database location. 26 | /// 27 | /// Specifications are written using a URL scheme, for example: 28 | /// 29 | /// mem:// 30 | public struct LLBCASDatabaseSpec { 31 | /// The map of registered schemes. 32 | private static var registeredSchemes: [String: LLBCASDatabaseScheme.Type] = [ 33 | "mem": LLBInMemoryCASDatabaseScheme.self, 34 | "file": LLBFileBackedCASDatabaseScheme.self, 35 | ] 36 | 37 | /// Register a content store scheme type. 38 | /// 39 | /// This method is *not* thread safe. 40 | public static func register(schemeType: LLBCASDatabaseScheme.Type) { 41 | precondition(registeredSchemes[schemeType.scheme] == nil) 42 | registeredSchemes[schemeType.scheme] = schemeType 43 | } 44 | 45 | /// The underlying URL. 46 | public let url: URL 47 | 48 | /// The scheme definition. 49 | public let schemeType: LLBCASDatabaseScheme.Type 50 | 51 | public enum Error: Swift.Error { 52 | case noScheme 53 | case urlError(String) 54 | } 55 | 56 | /// Create a new spec for the given URL string. 57 | public init(_ string: String) throws { 58 | guard let url = URL(string: string) else { 59 | throw Error.urlError("URL parse error for \(string) for a CAS Database") 60 | } 61 | try self.init(url) 62 | } 63 | 64 | /// Create a new spec for the given URL. 65 | public init(_ url: URL) throws { 66 | guard let scheme = url.scheme else { 67 | throw Error.noScheme 68 | } 69 | 70 | // If the scheme isn't known, this isn't a valid spec. 71 | guard let schemeType = LLBCASDatabaseSpec.registeredSchemes[scheme] else { 72 | throw Error.urlError("Unknown URL scheme \"\(scheme)\" for a CAS Database at \(url)") 73 | } 74 | 75 | // Validate the URL with the scheme. 76 | if !schemeType.isValid(host: url.host, port: url.port, path: url.path, query: url.query) { 77 | throw Error.urlError("Invalid URL \(url) for a CAS Database") 78 | } 79 | 80 | self.url = url 81 | self.schemeType = schemeType 82 | } 83 | 84 | /// Open the specified store. 85 | public func open(group: LLBFuturesDispatchGroup) throws -> LLBCASDatabase { 86 | return try schemeType.open(group: group, url: url) 87 | } 88 | } 89 | 90 | extension LLBCASDatabaseSpec: Equatable { 91 | public static func == (lhs: LLBCASDatabaseSpec, rhs: LLBCASDatabaseSpec) -> Bool { 92 | return lhs.url == rhs.url 93 | } 94 | } 95 | -------------------------------------------------------------------------------- /Sources/TSFCAS/Generated/CASProtocol/cas_object.pb.swift: -------------------------------------------------------------------------------- 1 | // DO NOT EDIT. 2 | // swift-format-ignore-file 3 | // 4 | // Generated by the Swift generator plugin for the protocol buffer compiler. 5 | // Source: CASProtocol/cas_object.proto 6 | // 7 | // For information on using the generated types, please see the documentation: 8 | // https://github.com/apple/swift-protobuf/ 9 | 10 | // This source file is part of the Swift.org open source project 11 | // 12 | // Copyright (c) 2020 Apple Inc. and the Swift project authors 13 | // Licensed under Apache License v2.0 with Runtime Library Exception 14 | // 15 | // See http://swift.org/LICENSE.txt for license information 16 | // See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 17 | 18 | import Foundation 19 | import SwiftProtobuf 20 | 21 | // If the compiler emits an error on this type, it is because this file 22 | // was generated by a version of the `protoc` Swift plug-in that is 23 | // incompatible with the version of SwiftProtobuf to which you are linking. 24 | // Please ensure that you are building against the same version of the API 25 | // that was used to generate this file. 26 | fileprivate struct _GeneratedWithProtocGenSwiftVersion: SwiftProtobuf.ProtobufAPIVersionCheck { 27 | struct _2: SwiftProtobuf.ProtobufAPIVersion_2 {} 28 | typealias Version = _2 29 | } 30 | 31 | //// LLBPBCASObject represents the serialized from of CASObjects. It encodes the 32 | //// combination of the raw data of the object and its dependendent references. 33 | public struct LLBPBCASObject { 34 | // SwiftProtobuf.Message conformance is added in an extension below. See the 35 | // `Message` and `Message+*Additions` files in the SwiftProtobuf library for 36 | // methods supported on all messages. 37 | 38 | public var refs: [LLBDataID] = [] 39 | 40 | public var data: Data = Data() 41 | 42 | public var unknownFields = SwiftProtobuf.UnknownStorage() 43 | 44 | public init() {} 45 | } 46 | 47 | #if swift(>=5.5) && canImport(_Concurrency) 48 | extension LLBPBCASObject: @unchecked Sendable {} 49 | #endif // swift(>=5.5) && canImport(_Concurrency) 50 | 51 | // MARK: - Code below here is support for the SwiftProtobuf runtime. 52 | 53 | extension LLBPBCASObject: SwiftProtobuf.Message, SwiftProtobuf._MessageImplementationBase, SwiftProtobuf._ProtoNameProviding { 54 | public static let protoMessageName: String = "LLBPBCASObject" 55 | public static let _protobuf_nameMap: SwiftProtobuf._NameMap = [ 56 | 1: .same(proto: "refs"), 57 | 2: .same(proto: "data"), 58 | ] 59 | 60 | public mutating func decodeMessage(decoder: inout D) throws { 61 | while let fieldNumber = try decoder.nextFieldNumber() { 62 | // The use of inline closures is to circumvent an issue where the compiler 63 | // allocates stack space for every case branch when no optimizations are 64 | // enabled. https://github.com/apple/swift-protobuf/issues/1034 65 | switch fieldNumber { 66 | case 1: try { try decoder.decodeRepeatedMessageField(value: &self.refs) }() 67 | case 2: try { try decoder.decodeSingularBytesField(value: &self.data) }() 68 | default: break 69 | } 70 | } 71 | } 72 | 73 | public func traverse(visitor: inout V) throws { 74 | if !self.refs.isEmpty { 75 | try visitor.visitRepeatedMessageField(value: self.refs, fieldNumber: 1) 76 | } 77 | if !self.data.isEmpty { 78 | try visitor.visitSingularBytesField(value: self.data, fieldNumber: 2) 79 | } 80 | try unknownFields.traverse(visitor: &visitor) 81 | } 82 | 83 | public static func ==(lhs: LLBPBCASObject, rhs: LLBPBCASObject) -> Bool { 84 | if lhs.refs != rhs.refs {return false} 85 | if lhs.data != rhs.data {return false} 86 | if lhs.unknownFields != rhs.unknownFields {return false} 87 | return true 88 | } 89 | } 90 | -------------------------------------------------------------------------------- /Sources/TSFCAS/Generated/CASProtocol/data_id.pb.swift: -------------------------------------------------------------------------------- 1 | // DO NOT EDIT. 2 | // swift-format-ignore-file 3 | // 4 | // Generated by the Swift generator plugin for the protocol buffer compiler. 5 | // Source: CASProtocol/data_id.proto 6 | // 7 | // For information on using the generated types, please see the documentation: 8 | // https://github.com/apple/swift-protobuf/ 9 | 10 | // This source file is part of the Swift.org open source project 11 | // 12 | // Copyright (c) 2020 Apple Inc. and the Swift project authors 13 | // Licensed under Apache License v2.0 with Runtime Library Exception 14 | // 15 | // See http://swift.org/LICENSE.txt for license information 16 | // See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 17 | 18 | import Foundation 19 | import SwiftProtobuf 20 | 21 | // If the compiler emits an error on this type, it is because this file 22 | // was generated by a version of the `protoc` Swift plug-in that is 23 | // incompatible with the version of SwiftProtobuf to which you are linking. 24 | // Please ensure that you are building against the same version of the API 25 | // that was used to generate this file. 26 | fileprivate struct _GeneratedWithProtocGenSwiftVersion: SwiftProtobuf.ProtobufAPIVersionCheck { 27 | struct _2: SwiftProtobuf.ProtobufAPIVersion_2 {} 28 | typealias Version = _2 29 | } 30 | 31 | //// LLBDataID represents the digest of arbitrary data, and its purpose is to be a handler for interfacing with CAS 32 | //// systems. LLBDataID does not require the encoding of any particular hash function. Instead, it is expected that the 33 | //// CAS system itself that provides the digest. 34 | public struct LLBDataID { 35 | // SwiftProtobuf.Message conformance is added in an extension below. See the 36 | // `Message` and `Message+*Additions` files in the SwiftProtobuf library for 37 | // methods supported on all messages. 38 | 39 | //// The bytes containing the digest of the contents store in the CAS. 40 | public var bytes: Data = Data() 41 | 42 | public var unknownFields = SwiftProtobuf.UnknownStorage() 43 | 44 | public init() {} 45 | } 46 | 47 | #if swift(>=5.5) && canImport(_Concurrency) 48 | extension LLBDataID: @unchecked Sendable {} 49 | #endif // swift(>=5.5) && canImport(_Concurrency) 50 | 51 | // MARK: - Code below here is support for the SwiftProtobuf runtime. 52 | 53 | extension LLBDataID: SwiftProtobuf.Message, SwiftProtobuf._MessageImplementationBase, SwiftProtobuf._ProtoNameProviding { 54 | public static let protoMessageName: String = "LLBDataID" 55 | public static let _protobuf_nameMap: SwiftProtobuf._NameMap = [ 56 | 1: .same(proto: "bytes"), 57 | ] 58 | 59 | public mutating func decodeMessage(decoder: inout D) throws { 60 | while let fieldNumber = try decoder.nextFieldNumber() { 61 | // The use of inline closures is to circumvent an issue where the compiler 62 | // allocates stack space for every case branch when no optimizations are 63 | // enabled. https://github.com/apple/swift-protobuf/issues/1034 64 | switch fieldNumber { 65 | case 1: try { try decoder.decodeSingularBytesField(value: &self.bytes) }() 66 | default: break 67 | } 68 | } 69 | } 70 | 71 | public func traverse(visitor: inout V) throws { 72 | if !self.bytes.isEmpty { 73 | try visitor.visitSingularBytesField(value: self.bytes, fieldNumber: 1) 74 | } 75 | try unknownFields.traverse(visitor: &visitor) 76 | } 77 | 78 | public static func ==(lhs: LLBDataID, rhs: LLBDataID) -> Bool { 79 | if lhs.bytes != rhs.bytes {return false} 80 | if lhs.unknownFields != rhs.unknownFields {return false} 81 | return true 82 | } 83 | } 84 | -------------------------------------------------------------------------------- /Sources/TSFCAS/Implementations/Blake3DataID.swift: -------------------------------------------------------------------------------- 1 | // This source file is part of the Swift.org open source project 2 | // 3 | // Copyright (c) 2020 Apple Inc. and the Swift project authors 4 | // Licensed under Apache License v2.0 with Runtime Library Exception 5 | // 6 | // See http://swift.org/LICENSE.txt for license information 7 | // See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 8 | 9 | import CBLAKE3 10 | import NIOCore 11 | import TSFUtility 12 | 13 | extension LLBDataID { 14 | public init(blake3hash buffer: LLBByteBuffer, refs: [LLBDataID] = []) { 15 | var hasher = blake3_hasher() 16 | blake3_hasher_init(&hasher) 17 | 18 | for ref in refs { 19 | ref.bytes.withUnsafeBytes { content in 20 | blake3_hasher_update(&hasher, content.baseAddress, content.count) 21 | } 22 | } 23 | buffer.withUnsafeReadableBytes { data in 24 | blake3_hasher_update(&hasher, data.baseAddress, data.count) 25 | } 26 | 27 | let hash = [UInt8](unsafeUninitializedCapacity: Int(BLAKE3_OUT_LEN)) { (hash, len) in 28 | len = Int(BLAKE3_OUT_LEN) 29 | blake3_hasher_finalize(&hasher, hash.baseAddress, len) 30 | } 31 | 32 | self.init(directHash: hash) 33 | } 34 | 35 | public init(blake3hash data: [UInt8], refs: [LLBDataID] = []) { 36 | self.init(blake3hash: ArraySlice(data)) 37 | } 38 | public init(blake3hash string: String, refs: [LLBDataID] = []) { 39 | self.init(blake3hash: ArraySlice(string.utf8)) 40 | } 41 | 42 | public init(blake3hash slice: ArraySlice, refs: [LLBDataID] = []) { 43 | var hasher = blake3_hasher() 44 | blake3_hasher_init(&hasher) 45 | 46 | for ref in refs { 47 | ref.bytes.withUnsafeBytes { content in 48 | blake3_hasher_update(&hasher, content.baseAddress, content.count) 49 | } 50 | } 51 | slice.withUnsafeBytes { data in 52 | blake3_hasher_update(&hasher, data.baseAddress, data.count) 53 | } 54 | 55 | let hash = [UInt8](unsafeUninitializedCapacity: Int(BLAKE3_OUT_LEN)) { (hash, len) in 56 | len = Int(BLAKE3_OUT_LEN) 57 | blake3_hasher_finalize(&hasher, hash.baseAddress, len) 58 | } 59 | 60 | self.init(directHash: hash) 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /Sources/TSFCAS/Implementations/FileBackedCASDatabase.swift: -------------------------------------------------------------------------------- 1 | // This source file is part of the Swift.org open source project 2 | // 3 | // Copyright (c) 2020 Apple Inc. and the Swift project authors 4 | // Licensed under Apache License v2.0 with Runtime Library Exception 5 | // 6 | // See http://swift.org/LICENSE.txt for license information 7 | // See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 8 | 9 | import Foundation 10 | import NIO 11 | import TSCBasic 12 | import TSCLibc 13 | import TSCUtility 14 | import TSFFutures 15 | import TSFUtility 16 | 17 | public final class LLBFileBackedCASDatabase: LLBCASDatabase { 18 | /// Prefix for files written to disk. 19 | enum FileNamePrefix: String { 20 | case refs = "refs." 21 | case data = "data." 22 | } 23 | 24 | /// The content root path. 25 | public let path: AbsolutePath 26 | 27 | /// Threads capable of running futures. 28 | public let group: LLBFuturesDispatchGroup 29 | 30 | let threadPool: NIOThreadPool 31 | let fileIO: NonBlockingFileIO 32 | 33 | public init( 34 | group: LLBFuturesDispatchGroup, 35 | path: AbsolutePath 36 | ) { 37 | self.threadPool = NIOThreadPool(numberOfThreads: 6) 38 | threadPool.start() 39 | self.fileIO = NonBlockingFileIO(threadPool: threadPool) 40 | self.group = group 41 | self.path = path 42 | try? localFileSystem.createDirectory(path, recursive: true) 43 | } 44 | 45 | deinit { 46 | try? threadPool.syncShutdownGracefully() 47 | } 48 | 49 | private func fileName(for id: LLBDataID, prefix: FileNamePrefix) -> AbsolutePath { 50 | return path.appending(component: prefix.rawValue + id.debugDescription) 51 | } 52 | 53 | public func supportedFeatures() -> LLBFuture { 54 | group.next().makeSucceededFuture(LLBCASFeatures(preservesIDs: true)) 55 | } 56 | 57 | public func contains(_ id: LLBDataID, _ ctx: Context) -> LLBFuture { 58 | let refsFile = fileName(for: id, prefix: .refs) 59 | let dataFile = fileName(for: id, prefix: .data) 60 | let contains = localFileSystem.exists(refsFile) && localFileSystem.exists(dataFile) 61 | return group.next().makeSucceededFuture(contains) 62 | } 63 | 64 | func readFile(file: AbsolutePath) -> LLBFuture { 65 | let handleAndRegion = fileIO.openFile( 66 | path: file.pathString, eventLoop: group.next() 67 | ) 68 | 69 | let data: LLBFuture = handleAndRegion.flatMap { (handle, region) in 70 | let allocator = ByteBufferAllocator() 71 | return self.fileIO.read( 72 | fileRegion: region, 73 | allocator: allocator, 74 | eventLoop: self.group.next() 75 | ) 76 | } 77 | 78 | return handleAndRegion.and(data).flatMapThrowing { (handle, data) in 79 | try handle.0.close() 80 | return data 81 | } 82 | } 83 | 84 | public func get(_ id: LLBDataID, _ ctx: Context) -> LLBFuture { 85 | let refsFile = fileName(for: id, prefix: .refs) 86 | let dataFile = fileName(for: id, prefix: .data) 87 | 88 | let refsBytes: LLBFuture<[UInt8]> = readFile(file: refsFile).map { refsData in 89 | return Array(buffer: refsData) 90 | } 91 | 92 | let refs = refsBytes.flatMapThrowing { 93 | try JSONDecoder().decode([LLBDataID].self, from: Data($0)) 94 | } 95 | 96 | let data = readFile(file: dataFile) 97 | 98 | return refs.and(data).map { 99 | LLBCASObject(refs: $0.0, data: $0.1) 100 | } 101 | } 102 | 103 | var fs: FileSystem { localFileSystem } 104 | 105 | public func identify( 106 | refs: [LLBDataID] = [], 107 | data: LLBByteBuffer, 108 | _ ctx: Context 109 | ) -> LLBFuture { 110 | return group.next().makeSucceededFuture(LLBDataID(blake3hash: data, refs: refs)) 111 | } 112 | 113 | public func put( 114 | refs: [LLBDataID] = [], 115 | data: LLBByteBuffer, 116 | _ ctx: Context 117 | ) -> LLBFuture { 118 | let id = LLBDataID(blake3hash: data, refs: refs) 119 | return put(knownID: id, refs: refs, data: data, ctx) 120 | } 121 | 122 | public func put( 123 | knownID id: LLBDataID, 124 | refs: [LLBDataID] = [], 125 | data: LLBByteBuffer, 126 | _ ctx: Context 127 | ) -> LLBFuture { 128 | let dataFile = fileName(for: id, prefix: .data) 129 | let dataFuture = writeIfNeeded(data: data, path: dataFile) 130 | 131 | let refsFile = fileName(for: id, prefix: .refs) 132 | let refData = try! JSONEncoder().encode(refs) 133 | let refBytes = LLBByteBuffer.withBytes(refData) 134 | let refFuture = writeIfNeeded(data: refBytes, path: refsFile) 135 | 136 | return dataFuture.and(refFuture).map { _ in id } 137 | } 138 | 139 | /// Write the given data to the path if the size of data 140 | /// differs from the size at path. 141 | private func writeIfNeeded( 142 | data: LLBByteBuffer, 143 | path: AbsolutePath 144 | ) -> LLBFuture { 145 | let handle = fileIO.openFile( 146 | path: path.pathString, 147 | mode: .write, 148 | flags: .allowFileCreation(), 149 | eventLoop: group.next() 150 | ) 151 | 152 | let size = handle.flatMap { handle in 153 | self.fileIO.readFileSize( 154 | fileHandle: handle, 155 | eventLoop: self.group.next() 156 | ) 157 | } 158 | 159 | let result = size.and(handle).flatMap { (size, handle) -> LLBFuture in 160 | if size == data.readableBytes { 161 | return self.group.next().makeSucceededFuture(()) 162 | } 163 | 164 | return self.fileIO.write( 165 | fileHandle: handle, 166 | buffer: data, 167 | eventLoop: self.group.next() 168 | ) 169 | } 170 | 171 | return handle.and(result).flatMapThrowing { (handle, _) in 172 | try handle.close() 173 | } 174 | } 175 | 176 | } 177 | 178 | public struct LLBFileBackedCASDatabaseScheme: LLBCASDatabaseScheme { 179 | public static let scheme = "file" 180 | 181 | public static func isValid(host: String?, port: Int?, path: String, query: String?) -> Bool { 182 | return host == nil && port == nil && path != "" && query == nil 183 | } 184 | 185 | public static func open(group: LLBFuturesDispatchGroup, url: Foundation.URL) throws 186 | -> LLBCASDatabase 187 | { 188 | return try LLBFileBackedCASDatabase(group: group, path: AbsolutePath(validating: url.path)) 189 | } 190 | } 191 | -------------------------------------------------------------------------------- /Sources/TSFCAS/Implementations/InMemoryCASDatabase.swift: -------------------------------------------------------------------------------- 1 | // This source file is part of the Swift.org open source project 2 | // 3 | // Copyright (c) 2020 Apple Inc. and the Swift project authors 4 | // Licensed under Apache License v2.0 with Runtime Library Exception 5 | // 6 | // See http://swift.org/LICENSE.txt for license information 7 | // See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 8 | 9 | import Foundation 10 | import NIOConcurrencyHelpers 11 | import NIOCore 12 | import TSCUtility 13 | import TSFFutures 14 | import TSFUtility 15 | 16 | /// A simple in-memory implementation of the `LLBCASDatabase` protocol. 17 | public final class LLBInMemoryCASDatabase: Sendable { 18 | struct State: Sendable { 19 | /// The content. 20 | var content = [LLBDataID: LLBCASObject]() 21 | 22 | var totalDataBytes: Int = 0 23 | } 24 | 25 | private let state: NIOLockedValueBox = NIOLockedValueBox(State()) 26 | 27 | /// Threads capable of running futures. 28 | public let group: LLBFuturesDispatchGroup 29 | 30 | /// The total number of data bytes in the database (this does not include the size of refs). 31 | public var totalDataBytes: Int { 32 | return self.state.withLockedValue { state in 33 | return state.totalDataBytes 34 | } 35 | } 36 | 37 | /// Create an in-memory database. 38 | public init(group: LLBFuturesDispatchGroup) { 39 | self.group = group 40 | } 41 | 42 | /// Delete the data in the database. 43 | /// Intentionally not exposed via the CASDatabase protocol. 44 | public func delete(_ id: LLBDataID, recursive: Bool) -> LLBFuture { 45 | self.state.withLockedValue { state in 46 | unsafeDelete(state: &state, id, recursive: recursive) 47 | } 48 | return group.next().makeSucceededFuture(()) 49 | } 50 | private func unsafeDelete(state: inout State, _ id: LLBDataID, recursive: Bool) { 51 | guard let object = state.content[id] else { 52 | return 53 | } 54 | state.totalDataBytes -= object.data.readableBytes 55 | 56 | guard recursive else { 57 | return 58 | } 59 | 60 | for ref in object.refs { 61 | unsafeDelete(state: &state, ref, recursive: recursive) 62 | } 63 | } 64 | } 65 | 66 | extension LLBInMemoryCASDatabase: LLBCASDatabase { 67 | public func supportedFeatures() -> LLBFuture { 68 | return group.next().makeSucceededFuture(LLBCASFeatures(preservesIDs: true)) 69 | } 70 | 71 | public func contains(_ id: LLBDataID, _ ctx: Context) -> LLBFuture { 72 | let result = self.state.withLockedValue { state in 73 | state.content.index(forKey: id) != nil 74 | } 75 | return group.next().makeSucceededFuture(result) 76 | } 77 | 78 | public func get(_ id: LLBDataID, _ ctx: Context) -> LLBFuture { 79 | let result = self.state.withLockedValue { state in state.content[id] } 80 | return group.next().makeSucceededFuture(result) 81 | } 82 | 83 | public func identify(refs: [LLBDataID] = [], data: LLBByteBuffer, _ ctx: Context) -> LLBFuture< 84 | LLBDataID 85 | > { 86 | return group.next().makeSucceededFuture(LLBDataID(blake3hash: data, refs: refs)) 87 | } 88 | 89 | public func put(refs: [LLBDataID] = [], data: LLBByteBuffer, _ ctx: Context) -> LLBFuture< 90 | LLBDataID 91 | > { 92 | return put(knownID: LLBDataID(blake3hash: data, refs: refs), refs: refs, data: data, ctx) 93 | } 94 | 95 | public func put( 96 | knownID id: LLBDataID, refs: [LLBDataID] = [], data: LLBByteBuffer, _ ctx: Context 97 | ) -> LLBFuture { 98 | self.state.withLockedValue { state in 99 | guard state.content[id] == nil else { 100 | assert(state.content[id]?.data == data, "put data for id doesn't match") 101 | return 102 | } 103 | state.totalDataBytes += data.readableBytes 104 | state.content[id] = LLBCASObject(refs: refs, data: data) 105 | } 106 | return group.next().makeSucceededFuture(id) 107 | } 108 | } 109 | 110 | public struct LLBInMemoryCASDatabaseScheme: LLBCASDatabaseScheme { 111 | public static let scheme = "mem" 112 | 113 | public static func isValid(host: String?, port: Int?, path: String, query: String?) -> Bool { 114 | return host == nil && port == nil && path == "" && query == nil 115 | } 116 | 117 | public static func open(group: LLBFuturesDispatchGroup, url: Foundation.URL) throws 118 | -> LLBCASDatabase 119 | { 120 | return LLBInMemoryCASDatabase(group: group) 121 | } 122 | } 123 | -------------------------------------------------------------------------------- /Sources/TSFCAS/Object.swift: -------------------------------------------------------------------------------- 1 | // This source file is part of the Swift.org open source project 2 | // 3 | // Copyright (c) 2020 Apple Inc. and the Swift project authors 4 | // Licensed under Apache License v2.0 with Runtime Library Exception 5 | // 6 | // See http://swift.org/LICENSE.txt for license information 7 | // See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 8 | 9 | import Foundation 10 | import NIOCore 11 | import NIOFoundationCompat 12 | import TSFUtility 13 | 14 | // MARK:- CASObject Definition - 15 | 16 | public struct LLBCASObject: Equatable, Sendable { 17 | /// The list of references. 18 | public let refs: [LLBDataID] 19 | 20 | /// The object data. 21 | public let data: LLBByteBuffer 22 | 23 | public init(refs: [LLBDataID], data: LLBByteBuffer) { 24 | self.refs = refs 25 | self.data = data 26 | } 27 | } 28 | 29 | extension LLBCASObject { 30 | public init(refs: [LLBDataID], data: LLBByteBufferView) { 31 | self.init(refs: refs, data: LLBByteBuffer(data)) 32 | } 33 | } 34 | 35 | extension LLBCASObject { 36 | /// The size of the object data. 37 | public var size: Int { 38 | return data.readableBytes 39 | } 40 | } 41 | 42 | // MARK:- CASObjectRepresentable - 43 | 44 | public protocol LLBCASObjectRepresentable { 45 | func asCASObject() throws -> LLBCASObject 46 | } 47 | public protocol LLBCASObjectConstructable { 48 | init(from casObject: LLBCASObject) throws 49 | } 50 | 51 | // MARK:- CASObject Serializeable - 52 | 53 | extension LLBCASObject { 54 | public init(rawBytes: Data) throws { 55 | let pb = try LLBPBCASObject(serializedBytes: rawBytes) 56 | var data = LLBByteBufferAllocator().buffer(capacity: pb.data.count) 57 | data.writeBytes(pb.data) 58 | self.init(refs: pb.refs, data: data) 59 | } 60 | 61 | public func toData() throws -> Data { 62 | var pb = LLBPBCASObject() 63 | pb.refs = self.refs 64 | pb.data = Data(buffer: self.data) 65 | return try pb.serializedData() 66 | } 67 | } 68 | 69 | extension LLBCASObject: LLBSerializable { 70 | public init(from rawBytes: LLBByteBuffer) throws { 71 | let pb = try rawBytes.withUnsafeReadableBytes { 72 | try LLBPBCASObject( 73 | serializedBytes: Data( 74 | bytesNoCopy: UnsafeMutableRawPointer(mutating: $0.baseAddress!), 75 | count: $0.count, deallocator: .none)) 76 | } 77 | let refs = pb.refs 78 | var data = LLBByteBufferAllocator().buffer(capacity: pb.data.count) 79 | data.writeBytes(pb.data) 80 | self.init(refs: refs, data: data) 81 | } 82 | 83 | public func toBytes(into buffer: inout LLBByteBuffer) throws { 84 | buffer.writeBytes(try self.toData()) 85 | } 86 | 87 | } 88 | -------------------------------------------------------------------------------- /Sources/TSFCASFileTree/BinarySearch.swift: -------------------------------------------------------------------------------- 1 | // This source file is part of the Swift.org open source project 2 | // 3 | // Copyright (c) 2020 Apple Inc. and the Swift project authors 4 | // Licensed under Apache License v2.0 with Runtime Library Exception 5 | // 6 | // See http://swift.org/LICENSE.txt for license information 7 | // See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 8 | 9 | extension LLBCASFileTree { 10 | 11 | /// Search the index of a matching element consulting a given comparator. 12 | public static func binarySearch( 13 | _ elements: C, _ compare: (C.Element) -> Int 14 | ) -> C.Index? { 15 | var lo: C.Index = elements.startIndex 16 | var hi: C.Index = elements.index(before: elements.endIndex) 17 | 18 | while true { 19 | let distance = elements.distance(from: lo, to: hi) 20 | guard distance >= 0 else { break } 21 | 22 | // Compute the middle index of this iteration's search range. 23 | let mid = elements.index(lo, offsetBy: distance / 2) 24 | assert(elements.distance(from: elements.startIndex, to: mid) >= 0) 25 | assert(elements.distance(from: mid, to: elements.endIndex) > 0) 26 | 27 | // If there is a match, return the result. 28 | let cmp = compare(elements[mid]) 29 | if cmp == 0 { 30 | return mid 31 | } 32 | 33 | // Otherwise, continue to search. 34 | if cmp < 0 { 35 | hi = elements.index(before: mid) 36 | } else { 37 | lo = elements.index(after: mid) 38 | } 39 | } 40 | 41 | // Check exit conditions of the binary search. 42 | assert(elements.distance(from: elements.startIndex, to: lo) >= 0) 43 | assert(elements.distance(from: lo, to: elements.endIndex) >= 0) 44 | 45 | return nil 46 | } 47 | 48 | } 49 | -------------------------------------------------------------------------------- /Sources/TSFCASFileTree/CASFSClient.swift: -------------------------------------------------------------------------------- 1 | // This source file is part of the Swift.org open source project 2 | // 3 | // Copyright (c) 2020 Apple Inc. and the Swift project authors 4 | // Licensed under Apache License v2.0 with Runtime Library Exception 5 | // 6 | // See http://swift.org/LICENSE.txt for license information 7 | // See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 8 | 9 | import Foundation 10 | import NIOCore 11 | import TSCBasic 12 | import TSCUtility 13 | import TSFCAS 14 | 15 | /// A main API struct 16 | public struct LLBCASFSClient: Sendable { 17 | public let db: LLBCASDatabase 18 | 19 | /// Errors produced by CASClient 20 | public enum Error: Swift.Error { 21 | case noEntry(LLBDataID) 22 | case notSupportedYet 23 | case invalidUse 24 | case unexpectedNode 25 | } 26 | 27 | /// Remembers db 28 | public init(_ db: LLBCASDatabase) { 29 | self.db = db 30 | } 31 | 32 | /// Check that DataID exists in CAS 33 | public func exists(_ id: LLBDataID, _ ctx: Context) -> LLBFuture { 34 | return db.contains(id, ctx) 35 | } 36 | 37 | /// Load CASFSNode from CAS 38 | /// If object doesn't exist future fails with noEntry 39 | public func load(_ id: LLBDataID, type hint: LLBFileType? = nil, _ ctx: Context) -> LLBFuture< 40 | LLBCASFSNode 41 | > { 42 | return db.get(id, ctx).flatMapThrowing { objectOpt in 43 | guard let object = objectOpt else { 44 | throw Error.noEntry(id) 45 | } 46 | 47 | switch hint { 48 | case .directory?: 49 | let tree = try LLBCASFileTree(id: id, object: object) 50 | return LLBCASFSNode(tree: tree, db: self.db) 51 | case .plainFile?, .executable?: 52 | let blob = try LLBCASBlob(db: self.db, id: id, type: hint!, object: object, ctx) 53 | return LLBCASFSNode(blob: blob, db: self.db) 54 | case .symlink?, .UNRECOGNIZED?: 55 | // We don't support symlinks yet 56 | throw Error.notSupportedYet 57 | case nil: 58 | if let tree = try? LLBCASFileTree(id: id, object: object) { 59 | return LLBCASFSNode(tree: tree, db: self.db) 60 | } else if let blob = try? LLBCASBlob(db: self.db, id: id, object: object, ctx) { 61 | return LLBCASFSNode(blob: blob, db: self.db) 62 | } else { 63 | // We don't support symlinks yet 64 | throw Error.notSupportedYet 65 | } 66 | } 67 | } 68 | } 69 | 70 | /// Save ByteBuffer to CAS 71 | public func store(_ data: LLBByteBuffer, type: LLBFileType = .plainFile, _ ctx: Context) 72 | -> LLBFuture 73 | { 74 | LLBCASBlob.import(data: data, isExecutable: type == .executable, in: db, ctx).map { 75 | LLBCASFSNode(blob: $0, db: self.db) 76 | } 77 | } 78 | 79 | /// Save ArraySlice to CAS 80 | public func store(_ data: ArraySlice, type: LLBFileType = .plainFile, _ ctx: Context) 81 | -> LLBFuture 82 | { 83 | LLBCASBlob.import( 84 | data: LLBByteBuffer.withBytes(data), isExecutable: type == .executable, in: db, ctx 85 | ).map { LLBCASFSNode(blob: $0, db: self.db) } 86 | } 87 | 88 | /// Save Data to CAS 89 | public func store(_ data: Data, type: LLBFileType = .plainFile, _ ctx: Context) -> LLBFuture< 90 | LLBCASFSNode 91 | > { 92 | LLBCASBlob.import( 93 | data: LLBByteBuffer.withBytes(data), isExecutable: type == .executable, in: db, ctx 94 | ).map { LLBCASFSNode(blob: $0, db: self.db) } 95 | } 96 | 97 | } 98 | 99 | extension LLBCASFSClient { 100 | public func store(_ data: LLBByteBuffer, type: LLBFileType = .plainFile, _ ctx: Context) 101 | -> LLBFuture 102 | { 103 | LLBCASBlob.import(data: data, isExecutable: type == .executable, in: db, ctx).flatMap { 104 | $0.export(ctx) 105 | } 106 | } 107 | 108 | public func store(_ data: ArraySlice, type: LLBFileType = .plainFile, _ ctx: Context) 109 | -> LLBFuture 110 | { 111 | LLBCASBlob.import( 112 | data: LLBByteBuffer.withBytes(data), isExecutable: type == .executable, in: db, ctx 113 | ).flatMap { $0.export(ctx) } 114 | } 115 | 116 | public func store(_ data: Data, type: LLBFileType = .plainFile, _ ctx: Context) -> LLBFuture< 117 | LLBDataID 118 | > { 119 | LLBCASBlob.import( 120 | data: LLBByteBuffer.withBytes(data), isExecutable: type == .executable, in: db, ctx 121 | ).flatMap { $0.export(ctx) } 122 | } 123 | } 124 | 125 | extension LLBCASFSClient { 126 | /// Creates a new LLBCASFileTree node by prepending the tree with the given graph. For example, if the given id 127 | /// contains a reference to a CASFileTree containing [a.txt, b.txt], and path was 'some/path', the resulting 128 | /// CASFileTree would contain [some/path/a.txt, some/path/b.txt] (where both `some` and `path` represent 129 | /// CASFileTrees). 130 | public func wrap(_ id: LLBDataID, path: String, _ ctx: Context) -> LLBFuture { 131 | let absolutePath: AbsolutePath 132 | do { 133 | absolutePath = try AbsolutePath(validating: path, relativeTo: .root) 134 | } catch { 135 | return ctx.group.any().makeFailedFuture(error) 136 | } 137 | return self.load(id, ctx).flatMap { node in 138 | return absolutePath 139 | .components 140 | .dropFirst() 141 | .reversed() 142 | .reduce(self.db.group.next().makeSucceededFuture(node)) { future, pathComponent in 143 | future.flatMap { node in 144 | let entry = node.asDirectoryEntry(filename: pathComponent) 145 | return LLBCASFileTree.create(files: [entry], in: self.db, ctx).map { 146 | return LLBCASFSNode(tree: $0, db: self.db) 147 | } 148 | } 149 | } 150 | }.flatMapThrowing { 151 | guard let tree = $0.tree else { 152 | throw Error.unexpectedNode 153 | } 154 | return tree 155 | } 156 | } 157 | } 158 | -------------------------------------------------------------------------------- /Sources/TSFCASFileTree/CASFSNode.swift: -------------------------------------------------------------------------------- 1 | // This source file is part of the Swift.org open source project 2 | // 3 | // Copyright (c) 2020 Apple Inc. and the Swift project authors 4 | // Licensed under Apache License v2.0 with Runtime Library Exception 5 | // 6 | // See http://swift.org/LICENSE.txt for license information 7 | // See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 8 | 9 | import Foundation 10 | import TSFCAS 11 | 12 | /// A CAS object (can be tree or blob) 13 | public struct LLBCASFSNode { 14 | public enum Error: Swift.Error { 15 | case notApplicable 16 | } 17 | 18 | public enum NodeContent: Sendable { 19 | case tree(LLBCASFileTree) 20 | case blob(LLBCASBlob) 21 | } 22 | 23 | public let db: LLBCASDatabase 24 | public let value: NodeContent 25 | 26 | public init(tree: LLBCASFileTree, db: LLBCASDatabase) { 27 | self.db = db 28 | self.value = NodeContent.tree(tree) 29 | } 30 | 31 | public init(blob: LLBCASBlob, db: LLBCASDatabase) { 32 | self.db = db 33 | self.value = NodeContent.blob(blob) 34 | } 35 | 36 | /// Returns aggregated (for trees) or regular size of the Entry 37 | public func size() -> Int { 38 | switch value { 39 | case .tree(let tree): 40 | return tree.aggregateSize 41 | case .blob(let blob): 42 | return blob.size 43 | } 44 | } 45 | 46 | /// Gives CASFSNode type (meaningful for files) 47 | public func type() -> LLBFileType { 48 | switch value { 49 | case .tree(_): 50 | return .directory 51 | case .blob(let blob): 52 | return blob.type 53 | } 54 | } 55 | 56 | /// Optionally chainable tree access 57 | public var tree: LLBCASFileTree? { 58 | guard case .tree(let tree) = value else { 59 | return nil 60 | } 61 | return tree 62 | } 63 | 64 | /// Optionally chainable blob access 65 | public var blob: LLBCASBlob? { 66 | guard case .blob(let blob) = value else { 67 | return nil 68 | } 69 | return blob 70 | } 71 | 72 | public func asDirectoryEntry(filename: String) -> LLBDirectoryEntryID { 73 | switch value { 74 | case .tree(let tree): 75 | return tree.asDirectoryEntry(filename: filename) 76 | case .blob(let blob): 77 | return blob.asDirectoryEntry(filename: filename) 78 | } 79 | } 80 | } 81 | 82 | #if swift(>=5.5) && canImport(_Concurrency) 83 | extension LLBCASFSNode: Sendable {} 84 | #endif // swift(>=5.5) && canImport(_Concurrency) 85 | -------------------------------------------------------------------------------- /Sources/TSFCASFileTree/Context.swift: -------------------------------------------------------------------------------- 1 | // This source file is part of the Swift.org open source project 2 | // 3 | // Copyright (c) 2021 Apple Inc. and the Swift project authors 4 | // Licensed under Apache License v2.0 with Runtime Library Exception 5 | // 6 | // See http://swift.org/LICENSE.txt for license information 7 | // See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 8 | 9 | import TSCUtility 10 | 11 | private final class ContextKey {} 12 | 13 | /// Support storing and retrieving file tree import options from a context 14 | extension Context { 15 | public static func with(_ options: LLBCASFileTree.ImportOptions) -> Context { 16 | return Context( 17 | dictionaryLiteral: (ObjectIdentifier(LLBCASFileTree.ImportOptions.self), options as Any) 18 | ) 19 | } 20 | 21 | public var fileTreeImportOptions: LLBCASFileTree.ImportOptions? { 22 | get { 23 | guard 24 | let options = self[ 25 | ObjectIdentifier(LLBCASFileTree.ImportOptions.self), 26 | as: LLBCASFileTree.ImportOptions.self] 27 | else { 28 | return nil 29 | } 30 | 31 | return options 32 | } 33 | set { 34 | self[ObjectIdentifier(LLBCASFileTree.ImportOptions.self)] = newValue 35 | } 36 | } 37 | } 38 | 39 | /// Support storing and retrieving file tree export storage batcher from a context 40 | extension Context { 41 | private static let fileTreeExportStorageBatcherKey = ContextKey() 42 | 43 | public var fileTreeExportStorageBatcher: LLBBatchingFutureOperationQueue? { 44 | get { 45 | guard 46 | let options = self[ 47 | ObjectIdentifier(Self.fileTreeExportStorageBatcherKey), 48 | as: LLBBatchingFutureOperationQueue.self] 49 | else { 50 | return nil 51 | } 52 | 53 | return options 54 | } 55 | set { 56 | self[ObjectIdentifier(Self.fileTreeExportStorageBatcherKey)] = newValue 57 | } 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /Sources/TSFCASFileTree/DeclFileTree.swift: -------------------------------------------------------------------------------- 1 | // This source file is part of the Swift.org open source project 2 | // 3 | // Copyright (c) 2020 Apple Inc. and the Swift project authors 4 | // Licensed under Apache License v2.0 with Runtime Library Exception 5 | // 6 | // See http://swift.org/LICENSE.txt for license information 7 | // See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 8 | 9 | import NIOCore 10 | import TSCUtility 11 | import TSFCAS 12 | 13 | /// A declarative way to create a CASFileTree with content known upfront 14 | /// Usage example: 15 | /// let tree: LLBDeclFileTree = .dir(["dir1": .dir([]), "file1": .file()]) 16 | /// let casTreeFuture = tree.toTree(db: db) 17 | public indirect enum LLBDeclFileTree { 18 | case directory(files: [String: LLBDeclFileTree]) 19 | case file(contents: [UInt8]) 20 | 21 | public static func dir(_ files: [String: LLBDeclFileTree]) -> LLBDeclFileTree { 22 | return .directory(files: files) 23 | } 24 | 25 | public static func file(_ contents: [UInt8]) -> LLBDeclFileTree { 26 | return .file(contents: contents) 27 | } 28 | 29 | public static func file(_ contents: String) -> LLBDeclFileTree { 30 | return .file(contents: Array(contents.utf8)) 31 | } 32 | } 33 | 34 | extension LLBDeclFileTree: CustomDebugStringConvertible { 35 | public var debugDescription: String { 36 | switch self { 37 | case .directory(let files): 38 | return ".directory(\(files))" 39 | case .file(let contents): 40 | return ".file(\(contents.count))" 41 | } 42 | } 43 | } 44 | 45 | extension LLBCASFSClient { 46 | /// Save LLBDeclFileTree to CAS 47 | public func store(_ declTree: LLBDeclFileTree, _ ctx: Context) -> LLBFuture { 48 | switch declTree { 49 | case .directory: 50 | return storeDir(declTree, ctx).map { LLBCASFSNode(tree: $0, db: self.db) } 51 | case .file: 52 | return storeFile(declTree, ctx).map { LLBCASFSNode(blob: $0, db: self.db) } 53 | } 54 | } 55 | 56 | public func storeDir(_ declTree: LLBDeclFileTree, _ ctx: Context) -> LLBFuture { 57 | let loop = db.group.next() 58 | guard case .directory(files: let files) = declTree else { 59 | return loop.makeFailedFuture(Error.invalidUse) 60 | } 61 | let infosFutures: [LLBFuture] = files.map { arg in 62 | let (key, value) = arg 63 | switch value { 64 | case .directory: 65 | let treeFuture = storeDir(value, ctx) 66 | return treeFuture.map { tree in 67 | LLBDirectoryEntryID( 68 | info: .init(name: key, type: .directory, size: tree.aggregateSize), 69 | id: tree.id) 70 | } 71 | case .file(_): 72 | return storeFile(value, ctx).map { blob in 73 | blob.asDirectoryEntry(filename: key) 74 | } 75 | } 76 | } 77 | return LLBFuture.whenAllSucceed(infosFutures, on: loop).flatMap { infos in 78 | return LLBCASFileTree.create(files: infos, in: self.db, ctx) 79 | } 80 | } 81 | 82 | public func storeFile(_ declTree: LLBDeclFileTree, _ ctx: Context) -> LLBFuture { 83 | let loop = db.group.next() 84 | guard case .file(contents: let contents) = declTree else { 85 | return loop.makeFailedFuture(Error.invalidUse) 86 | } 87 | return LLBCASBlob.import( 88 | data: LLBByteBuffer.withBytes(contents), isExecutable: false, in: db, ctx) 89 | } 90 | 91 | } 92 | 93 | extension LLBCASFSClient { 94 | public func store(_ declTree: LLBDeclFileTree, _ ctx: Context) -> LLBFuture { 95 | switch declTree { 96 | case .directory: 97 | return storeDir(declTree, ctx).map { $0.id } 98 | case .file: 99 | return storeFile(declTree, ctx).flatMap { casBlob in 100 | casBlob.export(ctx) 101 | } 102 | } 103 | } 104 | } 105 | -------------------------------------------------------------------------------- /Sources/TSFCASFileTree/DirectoryEntry.swift: -------------------------------------------------------------------------------- 1 | // This source file is part of the Swift.org open source project 2 | // 3 | // Copyright (c) 2020-2021 Apple Inc. and the Swift project authors 4 | // Licensed under Apache License v2.0 with Runtime Library Exception 5 | // 6 | // See http://swift.org/LICENSE.txt for license information 7 | // See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 8 | 9 | import Foundation 10 | 11 | extension LLBDirectoryEntry { 12 | public init( 13 | name: String, type: LLBFileType, size: Int, posixDetails: LLBPosixFileDetails? = nil 14 | ) { 15 | self.init(name: name, type: type, size: UInt64(clamping: size), posixDetails: posixDetails) 16 | } 17 | 18 | public init( 19 | name: String, type: LLBFileType, size: UInt64, posixDetails: LLBPosixFileDetails? = nil 20 | ) { 21 | self.name = name 22 | self.type = type 23 | self.size = size 24 | if let pd = posixDetails, pd != LLBPosixFileDetails() { 25 | self.posixDetails = pd 26 | } 27 | } 28 | } 29 | 30 | extension LLBFileType { 31 | public var expectedPosixMode: mode_t { 32 | switch self { 33 | case .plainFile: 34 | return 0o644 35 | case .executable, .directory, .symlink: 36 | return 0o755 37 | case .UNRECOGNIZED: 38 | return 0 39 | } 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /Sources/TSFCASFileTree/Errors.swift: -------------------------------------------------------------------------------- 1 | // This source file is part of the Swift.org open source project 2 | // 3 | // Copyright (c) 2020 Apple Inc. and the Swift project authors 4 | // Licensed under Apache License v2.0 with Runtime Library Exception 5 | // 6 | // See http://swift.org/LICENSE.txt for license information 7 | // See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 8 | 9 | import TSCBasic 10 | import TSFCAS 11 | 12 | public enum LLBCASFileTreeFormatError: Error { 13 | /// The given id was referenced as a directory, but the object encoding didn't match expectations. 14 | case unexpectedDirectoryData(LLBDataID) 15 | 16 | /// The given id was referenced as a file, but the object encoding didn't match expectations. 17 | case unexpectedFileData(LLBDataID) 18 | 19 | /// The given id was referenced as a symlink, but the object encoding didn't match expectations. 20 | case unexpectedSymlinkData(LLBDataID) 21 | 22 | /// An unexpected error was thrown while communicating with the database. 23 | case unexpectedDatabaseError(Error) 24 | 25 | /// Formatting/protocol error. 26 | case formatError(reason: String) 27 | 28 | /// File size exceeds internal limits 29 | case fileTooLarge(path: AbsolutePath) 30 | 31 | /// Decompression failed 32 | case decompressFailed(String) 33 | } 34 | -------------------------------------------------------------------------------- /Sources/TSFCASFileTree/FileInfo.swift: -------------------------------------------------------------------------------- 1 | // This source file is part of the Swift.org open source project 2 | // 3 | // Copyright (c) 2020 Apple Inc. and the Swift project authors 4 | // Licensed under Apache License v2.0 with Runtime Library Exception 5 | // 6 | // See http://swift.org/LICENSE.txt for license information 7 | // See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 8 | 9 | import Foundation 10 | import NIOCore 11 | import SwiftProtobuf 12 | 13 | extension LLBFileInfo: LLBSerializable { 14 | /// Decode the given block back into a message. 15 | @inlinable 16 | public init(from rawBytes: LLBByteBuffer) throws { 17 | self = try Self.deserialize(from: rawBytes) 18 | } 19 | 20 | /// Produce an encoded blob that fully defines the structure contents. 21 | @inlinable 22 | public func toBytes(into buffer: inout LLBByteBuffer) throws { 23 | buffer.writeBytes(try serializedData()) 24 | } 25 | } 26 | 27 | extension LLBFileInfo { 28 | @inlinable 29 | public static func deserialize(from array: [UInt8]) throws -> Self { 30 | return try array.withUnsafeBufferPointer { try deserialize(from: $0) } 31 | } 32 | 33 | @inlinable 34 | public static func deserialize(from bytes: ArraySlice) throws -> Self { 35 | return try bytes.withUnsafeBufferPointer { try deserialize(from: $0) } 36 | } 37 | 38 | @inlinable 39 | public static func deserialize(from buffer: LLBByteBuffer) throws -> Self { 40 | return try buffer.withUnsafeReadableBytesWithStorageManagement { (buffer, mgr) in 41 | _ = mgr.retain() 42 | return try Self.init( 43 | serializedBytes: Data( 44 | bytesNoCopy: UnsafeMutableRawPointer(mutating: buffer.baseAddress!), 45 | count: buffer.count, 46 | deallocator: .custom({ _, _ in mgr.release() } 47 | ))) 48 | } 49 | } 50 | 51 | @inlinable 52 | public static func deserialize(from buffer: UnsafeBufferPointer) throws -> Self { 53 | return try Self.init( 54 | serializedBytes: Data( 55 | // NOTE: This doesn't actually mutate, which is why this is safe. 56 | bytesNoCopy: UnsafeMutableRawPointer(mutating: buffer.baseAddress!), 57 | count: buffer.count, deallocator: .none)) 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /Sources/TSFCASFileTree/TSCCASFileSystem.swift: -------------------------------------------------------------------------------- 1 | // This source file is part of the Swift.org open source project 2 | // 3 | // Copyright (c) 2020 Apple Inc. and the Swift project authors 4 | // Licensed under Apache License v2.0 with Runtime Library Exception 5 | // 6 | // See http://swift.org/LICENSE.txt for license information 7 | // See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 8 | 9 | import TSCBasic 10 | import TSCUtility 11 | import TSFCAS 12 | 13 | /// CAS backed FileSystem implementation rooted at the given CASTree. 14 | /// 15 | /// NOTE:- This class should *NOT* be used inside the db's NIO event loop. 16 | public final class TSCCASFileSystem: FileSystem { 17 | 18 | let rootTree: LLBCASFileTree 19 | let db: LLBCASDatabase 20 | let client: LLBCASFSClient 21 | let ctx: Context 22 | 23 | public init( 24 | db: LLBCASDatabase, 25 | rootTree: LLBCASFileTree, 26 | _ ctx: Context 27 | ) { 28 | self.db = db 29 | self.client = LLBCASFSClient(db) 30 | self.rootTree = rootTree 31 | self.ctx = ctx 32 | } 33 | 34 | public func exists(_ path: AbsolutePath, followSymlink: Bool) -> Bool { 35 | if path.isRoot { 36 | return true 37 | } 38 | let result = try? self.rootTree.lookup(path: path, in: self.db, Context()).wait() 39 | return result != nil 40 | } 41 | 42 | public func getDirectoryContents(_ path: AbsolutePath) throws -> [String] { 43 | if path.isRoot { 44 | return rootTree.files.map { $0.name } 45 | } 46 | 47 | let _result = try self.rootTree.lookup(path: path, in: self.db, ctx).wait() 48 | guard let result = _result else { throw FileSystemError(.noEntry, path) } 49 | 50 | // HACK: If this is a symlink, check if it points to a directory. 51 | // Move this to LLBCASFileTree.lookup() 52 | if result.info.type == .symlink, isDirectory(path) { 53 | let symlinkContents = try readFileContents(path).cString 54 | return try getDirectoryContents( 55 | path.parentDirectory.appending(RelativePath(symlinkContents))) 56 | } 57 | 58 | let entry = try self.client.load(result.id, ctx).wait() 59 | guard let tree = entry.tree else { throw FileSystemError(.notDirectory, path) } 60 | return tree.files.map { $0.name } 61 | } 62 | 63 | public func isDirectory(_ path: AbsolutePath) -> Bool { 64 | let fileType = self.fileType(of: path) 65 | if fileType == .directory { 66 | return true 67 | } 68 | 69 | // HACK: If this is a symlink, check if it points to a directory. 70 | // Move this to LLBCASFileTree.lookup() 71 | if fileType == .symlink { 72 | guard let symlinkContents = try? readFileContents(path).cString else { 73 | return false 74 | } 75 | return isDirectory(path.parentDirectory.appending(RelativePath(symlinkContents))) 76 | } 77 | 78 | return false 79 | } 80 | 81 | private func fileType(of path: AbsolutePath) -> LLBFileType? { 82 | if path.isRoot { return .directory } 83 | let fileType = try? self.rootTree.lookup(path: path, in: self.db, ctx).wait() 84 | return fileType?.info.type 85 | } 86 | 87 | public func isFile(_ path: AbsolutePath) -> Bool { 88 | let fileType = self.fileType(of: path) 89 | return fileType == .plainFile || fileType == .executable 90 | } 91 | 92 | public func isExecutableFile(_ path: AbsolutePath) -> Bool { 93 | fileType(of: path) == .executable 94 | } 95 | 96 | public func isSymlink(_ path: AbsolutePath) -> Bool { 97 | fileType(of: path) == .symlink 98 | } 99 | 100 | public func readFileContents(_ path: AbsolutePath) throws -> ByteString { 101 | if path.isRoot { 102 | throw FileSystemError(.ioError(code: 0), path) 103 | } 104 | 105 | let result = try rootTree.lookup(path: path, in: db, ctx).wait() 106 | guard let id = result?.id else { throw FileSystemError(.noEntry, path) } 107 | 108 | let entry = try client.load(id, ctx).wait() 109 | guard let blob = entry.blob else { throw FileSystemError(.ioError(code: 0), path) } 110 | let bytes = try blob.read(ctx).wait() 111 | return ByteString(bytes) 112 | } 113 | 114 | public func chmod(_ mode: FileMode, path: AbsolutePath, options: Set) throws { 115 | throw FileSystemError(.unsupported) 116 | } 117 | 118 | public func writeFileContents(_ path: AbsolutePath, bytes: ByteString) throws { 119 | throw FileSystemError(.unsupported) 120 | } 121 | 122 | public func createDirectory(_ path: AbsolutePath, recursive: Bool) throws { 123 | throw FileSystemError(.unsupported) 124 | } 125 | 126 | public func removeFileTree(_ path: AbsolutePath) throws { 127 | throw FileSystemError(.unsupported) 128 | } 129 | 130 | public func copy(from sourcePath: AbsolutePath, to destinationPath: AbsolutePath) throws { 131 | throw FileSystemError(.unsupported) 132 | } 133 | 134 | public func move(from sourcePath: AbsolutePath, to destinationPath: AbsolutePath) throws { 135 | throw FileSystemError(.unsupported) 136 | } 137 | 138 | public var cachesDirectory: AbsolutePath? { nil } 139 | 140 | public func createSymbolicLink( 141 | _ path: AbsolutePath, pointingAt destination: AbsolutePath, relative: Bool 142 | ) throws { 143 | throw FileSystemError(.unsupported) 144 | } 145 | 146 | public var currentWorkingDirectory: AbsolutePath? { nil } 147 | 148 | public func changeCurrentWorkingDirectory(to path: AbsolutePath) throws { 149 | throw FileSystemError(.unsupported) 150 | } 151 | 152 | public var homeDirectory: AbsolutePath { .root } 153 | 154 | public func isReadable(_ path: AbsolutePath) -> Bool { 155 | return !path.isRoot 156 | } 157 | 158 | public func isWritable(_ path: AbsolutePath) -> Bool { 159 | return false 160 | } 161 | 162 | public var tempDirectory: AbsolutePath { 163 | return (try? determineTempDirectory(nil)) ?? AbsolutePath.root.appending(component: "tmp") 164 | } 165 | } 166 | -------------------------------------------------------------------------------- /Sources/TSFCASUtilities/BufferedStreamWriter.swift: -------------------------------------------------------------------------------- 1 | import Foundation 2 | import NIOConcurrencyHelpers 3 | import NIOCore 4 | import TSCUtility 5 | import TSFCAS 6 | 7 | extension LLBByteBuffer { 8 | fileprivate var availableCapacity: Int { capacity - readableBytes } 9 | } 10 | 11 | /// Stream writer that buffers data before ingesting it into the CAS database. 12 | public class LLBBufferedStreamWriter { 13 | private let bufferSize: Int 14 | private let lock = NIOConcurrencyHelpers.NIOLock() 15 | private var outputWriter: LLBLinkedListStreamWriter 16 | private var currentBuffer: LLBByteBuffer 17 | private var currentBufferedChannel: UInt8? = nil 18 | 19 | public var latestID: LLBFuture? { 20 | return lock.withLock { outputWriter.latestID } 21 | } 22 | 23 | /// Creates a new buffered writer, with a default buffer size of 512kb to optimize for roundtrip read time. 24 | public init(_ db: LLBCASDatabase, bufferSize: Int = 1 << 19) { 25 | self.outputWriter = LLBLinkedListStreamWriter(db) 26 | self.bufferSize = bufferSize 27 | self.currentBuffer = LLBByteBufferAllocator.init().buffer(capacity: bufferSize) 28 | } 29 | 30 | public func rebase(onto newBase: LLBDataID, _ ctx: Context) { 31 | lock.withLock { 32 | outputWriter.rebase(onto: newBase, ctx) 33 | } 34 | } 35 | 36 | /// Writes a chunk of data into the stream. Flushes if the current buffer would overflow, or if the data to write 37 | /// is larger than the buffer size. 38 | public func write(data: LLBByteBuffer, channel: UInt8, _ ctx: Context = .init()) { 39 | lock.withLock { 40 | if channel != currentBufferedChannel 41 | || data.readableBytes > currentBuffer.availableCapacity 42 | { 43 | _flush(ctx) 44 | } 45 | 46 | currentBufferedChannel = channel 47 | 48 | // If data is larger or equal than buffer size, send as is. 49 | if data.readableBytes >= bufferSize { 50 | outputWriter.append(data: data, channel: channel, ctx) 51 | } else { 52 | // data is smaller than max chunk, and if we were going to overpass the chunk size, the above check 53 | // would have already flushed the data, so at this point we know there's space in the buffer. 54 | assert(data.readableBytes <= currentBuffer.availableCapacity) 55 | currentBuffer.writeImmutableBuffer(data) 56 | } 57 | 58 | // If we filled the buffer, send it out. 59 | if currentBuffer.availableCapacity == 0 { 60 | _flush(ctx) 61 | } 62 | } 63 | } 64 | 65 | /// Flushes the buffer into the stream writer. 66 | public func flush(_ ctx: Context = .init()) { 67 | lock.withLock { 68 | _flush(ctx) 69 | } 70 | } 71 | 72 | /// Private implementation of flush, must be called within the lock. 73 | private func _flush(_ ctx: Context) { 74 | if currentBuffer.readableBytes > 0, let currentBufferedChannel = currentBufferedChannel { 75 | outputWriter.append( 76 | data: currentBuffer, 77 | channel: currentBufferedChannel, ctx 78 | ) 79 | currentBuffer.clear() 80 | } 81 | } 82 | } 83 | -------------------------------------------------------------------------------- /Sources/TSFCASUtilities/LinkedListStream.swift: -------------------------------------------------------------------------------- 1 | // This source file is part of the Swift.org open source project 2 | // 3 | // Copyright (c) 2020 Apple Inc. and the Swift project authors 4 | // Licensed under Apache License v2.0 with Runtime Library Exception 5 | // 6 | // See http://swift.org/LICENSE.txt for license information 7 | // See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 8 | 9 | import NIOCore 10 | import TSCUtility 11 | import TSFCAS 12 | import TSFCASFileTree 13 | 14 | extension String { 15 | fileprivate func prepending(_ prefix: String) -> String { 16 | return prefix + self 17 | } 18 | } 19 | 20 | /// Basic writer implementation that resembles a linked list where each node contains control data (like the channel) 21 | /// and refs[0] always points to the dataID of the data chunk and refs[1] has the data ID for the next node in the 22 | /// chain, if it's not the last node. This implementation is not thread safe. 23 | public struct LLBLinkedListStreamWriter { 24 | private let db: LLBCASDatabase 25 | private let ext: String 26 | 27 | private var latestData: LLBFuture<(id: LLBDataID, aggregateSize: Int)>? 28 | 29 | public var latestID: LLBFuture? { 30 | latestData?.map { $0.id } 31 | } 32 | 33 | public init(_ db: LLBCASDatabase, ext: String? = nil) { 34 | self.db = db 35 | self.ext = ext?.prepending(".") ?? "" 36 | } 37 | 38 | // This rebases the current logs onto a new data ID, potentially losing all the previous uploads if not saved 39 | // previously. The newBase should be another dataID produced by a LLBLinkedListStreamWriter. 40 | public mutating func rebase(onto newBase: LLBDataID, _ ctx: Context) { 41 | self.latestData = LLBCASFSClient(db).load(newBase, ctx).map { 42 | $0.tree 43 | }.tsf_unwrapOptional(orStringError: "Expected an LLBCASTree").map { tree in 44 | (id: tree.id, aggregateSize: tree.aggregateSize) 45 | } 46 | } 47 | 48 | @discardableResult 49 | public mutating func append(data: LLBByteBuffer, channel: UInt8, _ ctx: Context) -> LLBFuture< 50 | LLBDataID 51 | > { 52 | let latestData = ( 53 | // Append on the previously cached node, or use nil as sentinel if this is the first write. 54 | self.latestData?.map { $0 } ?? db.group.next().makeSucceededFuture(nil)).flatMap { 55 | [db, ext] (previousData: (id: LLBDataID, aggregateSize: Int)?) -> LLBFuture< 56 | (id: LLBDataID, aggregateSize: Int) 57 | > in 58 | db.put(data: data, ctx).flatMap { [db, ext] contentID in 59 | 60 | var entries = [ 61 | LLBDirectoryEntryID( 62 | info: .init( 63 | name: "\(channel)\(ext)", type: .plainFile, size: data.readableBytes 64 | ), 65 | id: contentID 66 | ) 67 | ] 68 | 69 | let aggregateSize: Int 70 | if let (prevID, prevSize) = previousData { 71 | entries.append( 72 | LLBDirectoryEntryID( 73 | info: .init(name: "prev", type: .directory, size: prevSize), 74 | id: prevID 75 | ) 76 | ) 77 | aggregateSize = prevSize + data.readableBytes 78 | } else { 79 | aggregateSize = data.readableBytes 80 | } 81 | 82 | return LLBCASFileTree.create(files: entries, in: db, ctx).map { 83 | (id: $0.id, aggregateSize: aggregateSize) 84 | } 85 | } 86 | } 87 | 88 | self.latestData = latestData 89 | return latestData.map { $0.id } 90 | } 91 | } 92 | 93 | extension LLBLinkedListStreamWriter { 94 | @discardableResult 95 | @inlinable 96 | public mutating func append(data: LLBByteBuffer, _ ctx: Context) -> LLBFuture { 97 | return append(data: data, channel: 0, ctx) 98 | } 99 | } 100 | -------------------------------------------------------------------------------- /Sources/TSFCASUtilities/StreamReader.swift: -------------------------------------------------------------------------------- 1 | // This source file is part of the Swift.org open source project 2 | // 3 | // Copyright (c) 2020 Apple Inc. and the Swift project authors 4 | // Licensed under Apache License v2.0 with Runtime Library Exception 5 | // 6 | // See http://swift.org/LICENSE.txt for license information 7 | // See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 8 | 9 | import NIOCore 10 | import TSCUtility 11 | import TSFCAS 12 | import TSFCASFileTree 13 | 14 | /// Implements the reading logic to read any kind of streaming data storage implemented. Currently it's hardcoded to 15 | /// read the LinkedList stream contents, but could get extended in the future when new storage structures are 16 | /// implemented. This should be the unified API to read streaming content, so that readers do not need to understand 17 | /// which writer was used to store the data. 18 | public struct LLBCASStreamReader { 19 | private let db: LLBCASDatabase 20 | 21 | public init(_ db: LLBCASDatabase) { 22 | self.db = db 23 | } 24 | 25 | public func read( 26 | id: LLBDataID, 27 | channels: [UInt8]?, 28 | lastReadID: LLBDataID?, 29 | _ ctx: Context, 30 | readerBlock: @escaping (UInt8, LLBByteBufferView) throws -> Bool 31 | ) -> LLBFuture { 32 | return innerRead( 33 | id: id, 34 | channels: channels, 35 | lastReadID: lastReadID, 36 | ctx, 37 | readerBlock: readerBlock 38 | ).map { _ in () } 39 | } 40 | 41 | private func innerRead( 42 | id: LLBDataID, 43 | channels: [UInt8]? = nil, 44 | lastReadID: LLBDataID? = nil, 45 | _ ctx: Context, 46 | readerBlock: @escaping (UInt8, LLBByteBufferView) throws -> Bool 47 | ) -> LLBFuture { 48 | if id == lastReadID { 49 | return db.group.next().makeSucceededFuture(true) 50 | } 51 | 52 | return LLBCASFSClient(db).load(id, ctx).flatMap { node in 53 | guard let tree = node.tree else { 54 | return self.db.group.next().makeFailedFuture(LLBCASStreamError.invalid) 55 | } 56 | 57 | let readChainFuture: LLBFuture 58 | 59 | // If there is a "prev" directory, treat this as a linked list implementation. This is an implementation 60 | // detail that could be cleaned up later, but the idea is that there's a single unified reader entity. 61 | if let (id, _) = tree.lookup("prev") { 62 | readChainFuture = self.innerRead( 63 | id: id, 64 | channels: channels, 65 | lastReadID: lastReadID, 66 | ctx, 67 | readerBlock: readerBlock 68 | ) 69 | } else { 70 | // If this is the last node, schedule a sentinel read that returns to keep on reading. 71 | readChainFuture = self.db.group.next().makeSucceededFuture(true) 72 | } 73 | 74 | return readChainFuture.flatMap { shouldContinue -> LLBFuture in 75 | // If we don't want to continue reading, or if the channel is not requested, close the current chain 76 | // and propagate the desire to keep on reading. 77 | guard shouldContinue else { 78 | return self.db.group.next().makeSucceededFuture(shouldContinue) 79 | } 80 | 81 | let files = tree.files.filter { 82 | $0.type == .plainFile 83 | } 84 | 85 | guard files.count == 1, let (contentID, _) = tree.lookup(files[0].name) else { 86 | return self.db.group.next().makeFailedFuture(LLBCASStreamError.invalid) 87 | } 88 | 89 | let channelOpt = files.first.flatMap { $0.name.split(separator: ".").first }.flatMap 90 | { UInt8($0) } 91 | 92 | guard let channel = channelOpt, 93 | channels?.contains(channel) ?? true 94 | else { 95 | return self.db.group.next().makeSucceededFuture(true) 96 | } 97 | 98 | return LLBCASFSClient(self.db).load(contentID, ctx).flatMap { node in 99 | guard let blob = node.blob else { 100 | return self.db.group.next().makeFailedFuture(LLBCASStreamError.missing) 101 | } 102 | 103 | return blob.read(ctx).flatMapThrowing { byteBufferView in 104 | return try readerBlock(channel, byteBufferView) 105 | } 106 | } 107 | } 108 | } 109 | } 110 | } 111 | 112 | // Convenience extension for default parameters. 113 | extension LLBCASStreamReader { 114 | @inlinable 115 | public func read( 116 | id: LLBDataID, 117 | _ ctx: Context, 118 | readerBlock: @escaping (UInt8, LLBByteBufferView) throws -> Bool 119 | ) -> LLBFuture { 120 | return read(id: id, channels: nil, lastReadID: nil, ctx, readerBlock: readerBlock) 121 | } 122 | 123 | @inlinable 124 | public func read( 125 | id: LLBDataID, 126 | channels: [UInt8], 127 | _ ctx: Context, 128 | readerBlock: @escaping (UInt8, LLBByteBufferView) throws -> Bool 129 | ) -> LLBFuture { 130 | return read(id: id, channels: channels, lastReadID: nil, ctx, readerBlock: readerBlock) 131 | } 132 | 133 | @inlinable 134 | public func read( 135 | id: LLBDataID, 136 | lastReadID: LLBDataID, 137 | _ ctx: Context, 138 | readerBlock: @escaping (UInt8, LLBByteBufferView) throws -> Bool 139 | ) -> LLBFuture { 140 | return read(id: id, channels: nil, lastReadID: lastReadID, ctx, readerBlock: readerBlock) 141 | } 142 | } 143 | 144 | /// Common error types for stream protocol implementations. 145 | public enum LLBCASStreamError: Error { 146 | case invalid 147 | case missing 148 | } 149 | -------------------------------------------------------------------------------- /Sources/TSFFutures/BatchingFutureOperationQueue.swift: -------------------------------------------------------------------------------- 1 | // This source file is part of the Swift.org open source project 2 | // 3 | // Copyright (c) 2020 Apple Inc. and the Swift project authors 4 | // Licensed under Apache License v2.0 with Runtime Library Exception 5 | // 6 | // See http://swift.org/LICENSE.txt for license information 7 | // See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 8 | 9 | import Foundation 10 | import NIOCore 11 | import TSCUtility 12 | 13 | /// Run the given computations on a given array in batches, exercising 14 | /// a specified amount of parallelism. 15 | /// 16 | /// - Discussion: 17 | /// For some blocking operations (such as file system accesses) executing 18 | /// them on the NIO loops is very expensive since it blocks the event 19 | /// processing machinery. Here we use extra threads for such operations. 20 | public struct LLBBatchingFutureOperationQueue: Sendable { 21 | 22 | /// Threads capable of running futures. 23 | public let group: LLBFuturesDispatchGroup 24 | 25 | /// Queue of outstanding operations. 26 | @usableFromInline 27 | let operationQueue: OperationQueue 28 | 29 | /// Because `LLBBatchingFutureOperationQueue` is a struct, the compiler 30 | /// will claim that `maxOpCount`'s setter is `mutating`, even though 31 | /// `OperationQueue` is a threadsafe class. 32 | /// This method exists as a workaround to adjust the underlying concurrency 33 | /// of the operation queue without unnecessary synchronization. 34 | public func setMaxOpCount(_ maxOpCount: Int) { 35 | operationQueue.maxConcurrentOperationCount = maxOpCount 36 | } 37 | 38 | /// Maximum number of operations executed concurrently. 39 | public var maxOpCount: Int { 40 | get { operationQueue.maxConcurrentOperationCount } 41 | set { self.setMaxOpCount(newValue) } 42 | } 43 | 44 | /// Return the number of operations currently queued. 45 | @inlinable 46 | public var opCount: Int { 47 | return operationQueue.operationCount 48 | } 49 | 50 | /// Whether the queue is suspended. 51 | @inlinable 52 | public var isSuspended: Bool { 53 | return operationQueue.isSuspended 54 | } 55 | 56 | /// 57 | /// - Parameters: 58 | /// - name: Unique string label, for logging. 59 | /// - group: Threads capable of running futures. 60 | /// - maxConcurrentOperationCount: 61 | /// Operations to execute in parallel. 62 | @inlinable 63 | public init( 64 | name: String, group: LLBFuturesDispatchGroup, maxConcurrentOperationCount maxOpCount: Int, 65 | qualityOfService: QualityOfService = .default 66 | ) { 67 | self.group = group 68 | self.operationQueue = OperationQueue( 69 | tsf_withName: name, maxConcurrentOperationCount: maxOpCount) 70 | self.operationQueue.qualityOfService = qualityOfService 71 | } 72 | 73 | @inlinable 74 | public func execute(_ body: @escaping () throws -> T) -> LLBFuture { 75 | let promise = group.next().makePromise(of: T.self) 76 | operationQueue.addOperation { 77 | promise.fulfill(body) 78 | } 79 | return promise.futureResult 80 | } 81 | 82 | @inlinable 83 | public func execute(_ body: @escaping () -> LLBFuture) -> LLBFuture { 84 | let promise = group.next().makePromise(of: T.self) 85 | operationQueue.addOperation { 86 | let f = body() 87 | f.cascade(to: promise) 88 | 89 | // Wait for completion, to ensure we maintain at most N concurrent operations. 90 | _ = try? f.wait() 91 | } 92 | return promise.futureResult 93 | } 94 | 95 | /// Order-preserving parallel execution. Wait for everything to complete. 96 | @inlinable 97 | public func execute( 98 | _ args: [A], minStride: Int = 1, _ body: @escaping (ArraySlice) throws -> [T] 99 | ) -> LLBFuture<[T]> { 100 | let futures: [LLBFuture<[T]>] = executeNoWait(args, minStride: minStride, body) 101 | let loop = futures.first?.eventLoop ?? group.next() 102 | return LLBFuture<[T]>.whenAllSucceed(futures, on: loop).map { $0.flatMap { $0 } } 103 | } 104 | 105 | /// Order-preserving parallel execution. 106 | /// Do not wait for all executions to complete, returning individual futures. 107 | @inlinable 108 | public func executeNoWait( 109 | _ args: [A], minStride: Int = 1, maxStride: Int = Int.max, 110 | _ body: @escaping (ArraySlice) throws -> [T] 111 | ) -> [LLBFuture<[T]>] { 112 | let batches: [ArraySlice] = args.tsc_sliceBy( 113 | maxStride: max(minStride, min(maxStride, args.count / maxOpCount))) 114 | return batches.map { arg in execute { try body(arg) } } 115 | } 116 | 117 | } 118 | -------------------------------------------------------------------------------- /Sources/TSFFutures/CancellableFuture.swift: -------------------------------------------------------------------------------- 1 | // This source file is part of the Swift.org open source project 2 | // 3 | // Copyright (c) 2020 Apple Inc. and the Swift project authors 4 | // Licensed under Apache License v2.0 with Runtime Library Exception 5 | // 6 | // See http://swift.org/LICENSE.txt for license information 7 | // See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 8 | 9 | import NIOCore 10 | 11 | /// A construct which expresses operations which can be asynchronously 12 | /// cancelled. The cancellation is not guaranteed and no ordering guarantees 13 | /// are provided with respect to the order of future's callbacks and the 14 | /// cancel operation returning. 15 | public struct LLBCancellableFuture: LLBCancelProtocol { 16 | /// The underlying future. 17 | public let future: LLBFuture 18 | 19 | /// The way to asynchronously cancel the operation backing up the future. 20 | public let canceller: LLBCanceller 21 | 22 | /// Initialize the future with a given canceller. 23 | public init(_ future: LLBFuture, canceller specificCanceller: LLBCanceller? = nil) { 24 | self.future = future 25 | let canceller = specificCanceller ?? LLBCanceller() 26 | self.canceller = canceller 27 | self.future.whenComplete { _ in 28 | // Do not invoke the cancel handler if the future 29 | // has already terminated. This is a bit opportunistic 30 | // and can miss some cancellation invocations, but 31 | // we expect the cancellation handlers to be no-op 32 | // when cancelling something that's not there. 33 | canceller.abandon() 34 | } 35 | } 36 | 37 | /// Initialize with a given handler which can be 38 | /// subsequently invoked through self.canceller.cancel() 39 | public init(_ future: LLBFuture, handler: LLBCancelProtocol) { 40 | self = LLBCancellableFuture(future, canceller: LLBCanceller(handler)) 41 | } 42 | 43 | /// Conformance to the `CancelProtocol`. 44 | public func cancel(reason: String?) { 45 | canceller.cancel(reason: reason) 46 | } 47 | } 48 | 49 | /// Some surface compatibility with EventLoopFuture to minimize 50 | /// the amount of code change in tests and other places. 51 | extension LLBCancellableFuture { 52 | #if swift(>=5.7) 53 | @available( 54 | *, noasync, message: "wait() can block indefinitely, prefer get()", renamed: "get()" 55 | ) 56 | @inlinable 57 | public func wait() throws -> T { 58 | try future.wait() 59 | } 60 | #else 61 | @inlinable 62 | public func wait() throws -> T { 63 | try future.wait() 64 | } 65 | #endif 66 | 67 | @available(macOS 10.15, iOS 13.0, tvOS 13.0, watchOS 6.0, *) 68 | @inlinable 69 | public func get() async throws -> T { 70 | try await future.get() 71 | } 72 | } 73 | -------------------------------------------------------------------------------- /Sources/TSFFutures/CancellablePromise.swift: -------------------------------------------------------------------------------- 1 | // This source file is part of the Swift.org open source project 2 | // 3 | // Copyright (c) 2020 Apple Inc. and the Swift project authors 4 | // Licensed under Apache License v2.0 with Runtime Library Exception 5 | // 6 | // See http://swift.org/LICENSE.txt for license information 7 | // See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 8 | 9 | import Atomics 10 | import NIOConcurrencyHelpers 11 | import NIOCore 12 | 13 | public enum LLBCancellablePromiseError: Error { 14 | case promiseFulfilled 15 | case promiseCancelled 16 | case promiseLeaked 17 | } 18 | 19 | /// A promise that can be cancelled prematurely. 20 | /// The CancellablePromise supports two types of API access: 21 | /// - The writer access that fulfills the promise or cancels it, getting 22 | /// back indication of whether or not that operation was successful. 23 | /// - The reader access that checks if the promise has been fulfilled. 24 | open class LLBCancellablePromise: @unchecked /* because inheritance... */ Sendable { 25 | /// Underlying promise. Private to avoid messing with out outside 26 | /// of CancellablePromise lifecycle protection. 27 | private let promise: LLBPromise 28 | 29 | /// The current state of the promise. 30 | /// - inProgress: The promise is waiting to be fulfilled or cancelled. 31 | /// - fulfilled: The promise has been fulfilled with a value or error. 32 | /// - cancelled: The promise has been cancelled via cancel(_:) 33 | public enum State: Int { 34 | case inProgress 35 | case fulfilled 36 | case cancelled 37 | } 38 | 39 | /// A state maintaining the lifecycle of the promise. 40 | @usableFromInline 41 | let state_: ManagedAtomic 42 | 43 | @inlinable 44 | public var state: State { 45 | return State(rawValue: state_.load(ordering: .relaxed))! 46 | } 47 | 48 | /// The eventual result of the promise. 49 | public var futureResult: LLBFuture { 50 | return promise.futureResult 51 | } 52 | 53 | /// Whether the promise was fulfilled or cancelled. 54 | @inlinable 55 | public var isCompleted: Bool { 56 | return state != .inProgress 57 | } 58 | 59 | /// Whether the promise was cancelled. 60 | @inlinable 61 | public var isCancelled: Bool { 62 | return state == .cancelled 63 | } 64 | 65 | /// Initialize a new promise off the given event loop. 66 | public convenience init(on loop: LLBFuturesDispatchLoop) { 67 | self.init(promise: loop.makePromise()) 68 | } 69 | 70 | /// Initialize a promise directly. Less safe because the promise 71 | /// could be accidentally fulfilled outside of CancellablePromise lifecycle. 72 | public init(promise: LLBPromise) { 73 | self.promise = promise 74 | self.state_ = .init(State.inProgress.rawValue) 75 | } 76 | 77 | /// Returns `true` if the state has been modified from .inProgress. 78 | private func modifyState(_ newState: State) -> Bool { 79 | assert(newState != .inProgress) 80 | return state_.compareExchange( 81 | expected: State.inProgress.rawValue, desired: newState.rawValue, 82 | ordering: .sequentiallyConsistent 83 | ).0 84 | } 85 | 86 | /// Fulfill the promise and return `true` if the promise was been fulfilled 87 | /// by this call, as opposed to having aready been fulfilled. 88 | open func fail(_ error: Swift.Error) -> Bool { 89 | let justModified = modifyState(State.fulfilled) 90 | if justModified { 91 | promise.fail(error) 92 | } 93 | return justModified 94 | } 95 | 96 | /// Cancel the promise and return `true` if the promise was been fulfilled 97 | /// by this call, as opposed to having aready been fulfilled. 98 | open func cancel(_ error: Swift.Error) -> Bool { 99 | let justModified = modifyState(State.cancelled) 100 | if justModified { 101 | promise.fail(error) 102 | } 103 | return justModified 104 | } 105 | 106 | /// Fulfill the promise and return `true` if the promise was been fulfilled 107 | /// by this call, as opposed to having aready been fulfilled. 108 | open func succeed(_ value: T) -> Bool { 109 | let justModified = modifyState(State.fulfilled) 110 | if justModified { 111 | promise.succeed(value) 112 | } 113 | return justModified 114 | } 115 | 116 | deinit { 117 | _ = cancel(LLBCancellablePromiseError.promiseLeaked) 118 | } 119 | } 120 | 121 | extension LLBFuture { 122 | 123 | /// Execute the given operation if a specified promise is not complete. 124 | /// Otherwise encode a `CancellablePromiseError`. 125 | @inlinable 126 | public func ifNotCompleteThen( 127 | check promise: LLBCancellablePromise

, _ operation: @escaping (Value) -> LLBFuture 128 | ) -> LLBFuture { 129 | flatMap { value in 130 | switch promise.state { 131 | case .inProgress: 132 | return operation(value) 133 | case .fulfilled: 134 | return self.eventLoop.makeFailedFuture(LLBCancellablePromiseError.promiseFulfilled) 135 | case .cancelled: 136 | return self.eventLoop.makeFailedFuture(LLBCancellablePromiseError.promiseCancelled) 137 | } 138 | } 139 | } 140 | 141 | /// Execute the given operation if a specified promise is not complete. 142 | /// Otherwise encode a `CancellablePromiseError`. 143 | @inlinable 144 | public func ifNotCompleteMap( 145 | check promise: LLBCancellablePromise

, _ operation: @escaping (Value) -> O 146 | ) -> LLBFuture { 147 | flatMapThrowing { value in 148 | switch promise.state { 149 | case .inProgress: 150 | return operation(value) 151 | case .fulfilled: 152 | throw LLBCancellablePromiseError.promiseFulfilled 153 | case .cancelled: 154 | throw LLBCancellablePromiseError.promiseCancelled 155 | } 156 | } 157 | } 158 | 159 | /// Post the result of a future onto the cancellable promise. 160 | @inlinable 161 | public func cascade(to promise: LLBCancellablePromise) { 162 | guard promise.isCompleted == false else { return } 163 | whenComplete { result in 164 | switch result { 165 | case .success(let value): _ = promise.succeed(value) 166 | case .failure(let error): _ = promise.fail(error) 167 | } 168 | } 169 | } 170 | } 171 | -------------------------------------------------------------------------------- /Sources/TSFFutures/Canceller.swift: -------------------------------------------------------------------------------- 1 | // This source file is part of the Swift.org open source project 2 | // 3 | // Copyright (c) 2020 Apple Inc. and the Swift project authors 4 | // Licensed under Apache License v2.0 with Runtime Library Exception 5 | // 6 | // See http://swift.org/LICENSE.txt for license information 7 | // See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 8 | 9 | import NIOConcurrencyHelpers 10 | 11 | public protocol LLBCancelProtocol { 12 | func cancel(reason: String?) 13 | } 14 | 15 | /// An object serving as a cancellation handler which can 16 | /// be supplied a cancellation procedure much later in its lifetime. 17 | public final class LLBCanceller { 18 | private let mutex_ = NIOConcurrencyHelpers.NIOLock() 19 | 20 | /// Whether and why it was cancelled. 21 | private var finalReason_: FinalReason? = nil 22 | 23 | /// A handler to when the cancellation is requested. 24 | private var handler_: LLBCancelProtocol? 25 | 26 | /// A reason for reaching the final state. 27 | private enum FinalReason { 28 | /// Cancelled with a specified reason. 29 | case cancelled(reason: String) 30 | /// Cancellation won't be needed. 31 | case abandoned 32 | } 33 | 34 | public init(_ cancelHandler: LLBCancelProtocol? = nil) { 35 | handler_ = cancelHandler 36 | } 37 | 38 | /// Checks whether the object has been cancelled. 39 | public var isCancelled: Bool { 40 | mutex_.lock() 41 | guard case .cancelled? = finalReason_ else { 42 | mutex_.unlock() 43 | return false 44 | } 45 | mutex_.unlock() 46 | return true 47 | } 48 | 49 | /// Return the reason for cancelling. 50 | public var cancelReason: String? { 51 | mutex_.lock() 52 | guard case .cancelled(let reason)? = finalReason_ else { 53 | mutex_.unlock() 54 | return nil 55 | } 56 | mutex_.unlock() 57 | return reason 58 | } 59 | 60 | /// Atomically replace the cancellation handler. 61 | public func set(handler newHandler: LLBCancelProtocol?) { 62 | mutex_.lock() 63 | let oldHandler = handler_ 64 | handler_ = newHandler 65 | if case .cancelled(let reason) = finalReason_ { 66 | oldHandler?.cancel(reason: reason) 67 | newHandler?.cancel(reason: reason) 68 | } 69 | mutex_.unlock() 70 | } 71 | 72 | /// Do not cancel anything even if requested. 73 | public func abandon() { 74 | mutex_.lock() 75 | finalReason_ = .abandoned 76 | handler_ = nil 77 | mutex_.unlock() 78 | } 79 | 80 | /// Cancel an outstanding operation. 81 | public func cancel(reason specifiedReason: String? = nil) { 82 | mutex_.lock() 83 | 84 | guard finalReason_ == nil else { 85 | // Already cancelled or abandoned. 86 | mutex_.unlock() 87 | return 88 | } 89 | 90 | let reason = specifiedReason ?? "no reason given" 91 | finalReason_ = .cancelled(reason: reason) 92 | let handler = handler_ 93 | 94 | mutex_.unlock() 95 | 96 | handler?.cancel(reason: reason) 97 | } 98 | 99 | deinit { 100 | mutex_.lock() 101 | guard case .cancelled(let reason) = finalReason_, let handler = handler_ else { 102 | mutex_.unlock() 103 | return 104 | } 105 | mutex_.unlock() 106 | handler.cancel(reason: reason + " (in deinit)") 107 | } 108 | } 109 | 110 | // Allow Canceller serve as a cancellation handler. 111 | extension LLBCanceller: LLBCancelProtocol {} 112 | 113 | /// Create a chain of single-purpose handlers. 114 | public final class LLBCancelHandlersChain: LLBCancelProtocol { 115 | private let lock = NIOConcurrencyHelpers.NIOLock() 116 | private var head: LLBCancelProtocol? 117 | private var tail: LLBCancelProtocol? 118 | 119 | public init(_ first: LLBCancelProtocol? = nil, _ second: LLBCancelProtocol? = nil) { 120 | self.head = first 121 | self.tail = second 122 | } 123 | 124 | /// Add another handler to the chain. 125 | public func add(handler: LLBCancelProtocol, for canceller: LLBCanceller) { 126 | lock.withLockVoid { 127 | guard let head = self.head else { 128 | self.head = handler 129 | return 130 | } 131 | guard let tail = self.tail else { 132 | self.tail = handler 133 | return 134 | } 135 | self.head = handler 136 | self.tail = LLBCancelHandlersChain(head, tail) 137 | } 138 | 139 | if let reason = canceller.cancelReason { 140 | cancel(reason: reason) 141 | } 142 | } 143 | 144 | /// Cancel the operations in the handlers chain. 145 | public func cancel(reason: String?) { 146 | let (h, t): (LLBCancelProtocol?, LLBCancelProtocol?) = lock.withLock { 147 | let pair = (self.head, self.tail) 148 | self.head = nil 149 | self.tail = nil 150 | return pair 151 | } 152 | h?.cancel(reason: reason) 153 | t?.cancel(reason: reason) 154 | } 155 | } 156 | -------------------------------------------------------------------------------- /Sources/TSFFutures/EventualResultsCache.swift: -------------------------------------------------------------------------------- 1 | // This source file is part of the Swift.org open source project 2 | // 3 | // Copyright (c) 2020 Apple Inc. and the Swift project authors 4 | // Licensed under Apache License v2.0 with Runtime Library Exception 5 | // 6 | // See http://swift.org/LICENSE.txt for license information 7 | // See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 8 | 9 | import NIOConcurrencyHelpers 10 | import NIOCore 11 | 12 | /// Cache of keys to eventually obtainable values. 13 | /// 14 | /// This cache coalesces requests and avoids re-obtaining values multiple times. 15 | public final class LLBEventualResultsCache: LLBFutureDeduplicator 16 | { 17 | /// The already cached keys. 18 | @usableFromInline 19 | internal var storage = [Key: LLBFuture]() 20 | 21 | /// Return the number of entries in the cache. 22 | @inlinable 23 | public var count: Int { 24 | return lock.withLock { storage.count } 25 | } 26 | 27 | @inlinable 28 | override internal func lockedCacheGet(key: Key) -> LLBFuture? { 29 | return storage[key] 30 | } 31 | 32 | @inlinable 33 | override internal func lockedCacheSet(_ key: Key, _ future: LLBFuture) { 34 | storage[key] = future 35 | } 36 | 37 | @inlinable 38 | public override subscript(_ key: Key) -> LLBFuture? { 39 | get { return super[key] } 40 | set { lock.withLockVoid { storage[key] = newValue } } 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /Sources/TSFFutures/FutureOperationQueue.swift: -------------------------------------------------------------------------------- 1 | // This source file is part of the Swift.org open source project 2 | // 3 | // Copyright (c) 2020-2021 Apple Inc. and the Swift project authors 4 | // Licensed under Apache License v2.0 with Runtime Library Exception 5 | // 6 | // See http://swift.org/LICENSE.txt for license information 7 | // See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 8 | 9 | import Foundation 10 | import NIO 11 | import NIOConcurrencyHelpers 12 | 13 | /// A queue for future-producing operations, which limits how many can run 14 | /// concurrently. 15 | public final class LLBFutureOperationQueue: Sendable { 16 | struct State: Sendable { 17 | /// Maximum allowed number of work items concurrently executing. 18 | var maxConcurrentOperations: Int 19 | 20 | /// The number of executing futures. 21 | var numExecuting = 0 22 | 23 | /// The user-specified "shares" that are currently being processed. 24 | var numSharesInFLight = 0 25 | 26 | /// The queue of operations to run. 27 | var workQueue = NIO.CircularBuffer() 28 | } 29 | 30 | struct WorkItem { 31 | let loop: LLBFuturesDispatchLoop 32 | let share: Int 33 | let notifyWhenScheduled: LLBPromise? 34 | let run: () -> Void 35 | } 36 | 37 | private let state: NIOLockedValueBox 38 | 39 | /// Maximum allowed number of shares concurrently executing. 40 | /// This option independently sets a cap on concurrency. 41 | private let maxConcurrentShares: Int 42 | 43 | public var maxConcurrentOperations: Int { 44 | get { 45 | return self.state.withLockedValue { state in 46 | return state.maxConcurrentOperations 47 | } 48 | } 49 | set { 50 | self.scheduleMoreTasks { state in 51 | state.maxConcurrentOperations = max(1, newValue) 52 | } 53 | } 54 | } 55 | 56 | /// Return the number of operations currently queued. 57 | public var opCount: Int { 58 | return self.state.withLockedValue { state in 59 | return state.numExecuting + state.workQueue.count 60 | } 61 | } 62 | 63 | /// Create a new limiter which will only initiate `maxConcurrentOperations` 64 | /// operations simultaneously. 65 | public init(maxConcurrentOperations: Int, maxConcurrentShares: Int = .max) { 66 | self.state = NIOLockedValueBox( 67 | State(maxConcurrentOperations: max(1, maxConcurrentOperations))) 68 | self.maxConcurrentShares = max(1, maxConcurrentShares) 69 | } 70 | 71 | /// NB: calls wait() on a current thread, beware. 72 | @available( 73 | *, noasync, 74 | message: "This method blocks indefinitely, don't use from 'async' or SwiftNIO EventLoops" 75 | ) 76 | @available(*, deprecated, message: "This method blocks indefinitely and returns a future") 77 | public func enqueueWithBackpressure( 78 | on loop: LLBFuturesDispatchLoop, share: Int = 1, body: @escaping () -> LLBFuture 79 | ) -> LLBFuture { 80 | let scheduled = loop.makePromise(of: Void.self) 81 | 82 | let future: LLBFuture = enqueue( 83 | on: loop, share: share, notifyWhenScheduled: scheduled, body: body) 84 | 85 | try! scheduled.futureResult.wait() 86 | 87 | return future 88 | } 89 | 90 | /// Add an operation into the queue, which can run immediately 91 | /// or at some unspecified time in the future, as permitted by 92 | /// the `maxConcurrentOperations` setting. 93 | /// The `share` option independently controls maximum allowed concurrency. 94 | /// The queue can support low number of high-share loads, or high number of 95 | /// low-share loads. Useful to model queue size in bytes. 96 | /// For such use cases, set share to the payload size in bytes. 97 | public func enqueue( 98 | on loop: LLBFuturesDispatchLoop, share: Int = 1, 99 | notifyWhenScheduled: LLBPromise? = nil, body: @escaping () -> LLBFuture 100 | ) -> LLBFuture { 101 | let promise = loop.makePromise(of: T.self) 102 | 103 | func runBody() { 104 | let f = body() 105 | f.whenComplete { _ in 106 | self.scheduleMoreTasks { state in 107 | assert(state.numExecuting >= 1) 108 | assert(state.numSharesInFLight >= share) 109 | state.numExecuting -= 1 110 | state.numSharesInFLight -= share 111 | } 112 | } 113 | f.cascade(to: promise) 114 | } 115 | 116 | let workItem = WorkItem( 117 | loop: loop, share: share, notifyWhenScheduled: notifyWhenScheduled, run: runBody) 118 | 119 | self.scheduleMoreTasks { state in 120 | state.workQueue.append(workItem) 121 | } 122 | 123 | return promise.futureResult 124 | } 125 | 126 | private func scheduleMoreTasks(performUnderLock: (inout State) -> Void) { 127 | // Decrement our counter, and get a new item to run if available. 128 | typealias Item = (loop: LLBFuturesDispatchLoop, notify: LLBPromise?, run: () -> Void) 129 | let toExecute: [Item] = self.state.withLockedValue { state in 130 | performUnderLock(&state) 131 | 132 | var scheduleItems: [Item] = [] 133 | 134 | // If we have room to execute the operation, 135 | // do so immediately (outside the lock). 136 | while state.numExecuting < state.maxConcurrentOperations, 137 | state.numSharesInFLight < self.maxConcurrentShares 138 | { 139 | 140 | // Schedule a new operation, if available. 141 | guard let op = state.workQueue.popFirst() else { 142 | break 143 | } 144 | 145 | state.numExecuting += 1 146 | state.numSharesInFLight += op.share 147 | scheduleItems.append((op.loop, op.notifyWhenScheduled, op.run)) 148 | } 149 | 150 | return scheduleItems 151 | } 152 | 153 | for (loop, notify, run) in toExecute { 154 | loop.execute { 155 | notify?.succeed(()) 156 | run() 157 | } 158 | } 159 | } 160 | } 161 | -------------------------------------------------------------------------------- /Sources/TSFFutures/Futures.swift: -------------------------------------------------------------------------------- 1 | // This source file is part of the Swift.org open source project 2 | // 3 | // Copyright (c) 2020 Apple Inc. and the Swift project authors 4 | // Licensed under Apache License v2.0 with Runtime Library Exception 5 | // 6 | // See http://swift.org/LICENSE.txt for license information 7 | // See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 8 | 9 | import NIO 10 | import TSCBasic 11 | import TSCUtility 12 | 13 | public typealias LLBFuture = NIO.EventLoopFuture 14 | public typealias LLBPromise = NIO.EventLoopPromise 15 | public typealias LLBFuturesDispatchGroup = NIO.EventLoopGroup 16 | public typealias LLBFuturesDispatchLoop = NIO.EventLoop 17 | 18 | public func LLBMakeDefaultDispatchGroup() -> LLBFuturesDispatchGroup { 19 | return MultiThreadedEventLoopGroup(numberOfThreads: System.coreCount) 20 | } 21 | 22 | extension LLBPromise { 23 | /// Fulfill the promise from a returned value, or fail the promise if throws. 24 | @inlinable 25 | public func fulfill(_ body: () throws -> Value) { 26 | do { 27 | try succeed(body()) 28 | } catch { 29 | fail(error) 30 | } 31 | } 32 | } 33 | 34 | /// Support storing and retrieving dispatch group from a context 35 | extension Context { 36 | public static func with(_ group: LLBFuturesDispatchGroup) -> Context { 37 | return Context( 38 | dictionaryLiteral: (ObjectIdentifier(LLBFuturesDispatchGroup.self), group as Any)) 39 | } 40 | 41 | public var group: LLBFuturesDispatchGroup { 42 | get { 43 | guard 44 | let group = self[ 45 | ObjectIdentifier(LLBFuturesDispatchGroup.self), as: LLBFuturesDispatchGroup.self 46 | ] 47 | else { 48 | fatalError("no futures dispatch group") 49 | } 50 | return group 51 | } 52 | set { 53 | self[ObjectIdentifier(LLBFuturesDispatchGroup.self)] = newValue 54 | } 55 | } 56 | } 57 | 58 | extension LLBFuture { 59 | public func tsf_unwrapOptional( 60 | orError error: Swift.Error 61 | ) -> EventLoopFuture where Value == T? { 62 | self.flatMapThrowing { value in 63 | guard let value = value else { 64 | throw error 65 | } 66 | return value 67 | } 68 | } 69 | 70 | public func tsf_unwrapOptional( 71 | orStringError error: String 72 | ) -> EventLoopFuture where Value == T? { 73 | tsf_unwrapOptional(orError: StringError(error)) 74 | } 75 | } 76 | -------------------------------------------------------------------------------- /Sources/TSFFutures/OperationQueue+Extensions.swift: -------------------------------------------------------------------------------- 1 | // This source file is part of the Swift.org open source project 2 | // 3 | // Copyright (c) 2020 Apple Inc. and the Swift project authors 4 | // Licensed under Apache License v2.0 with Runtime Library Exception 5 | // 6 | // See http://swift.org/LICENSE.txt for license information 7 | // See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 8 | 9 | import Foundation 10 | 11 | extension OperationQueue { 12 | public convenience init(tsf_withName: String, maxConcurrentOperationCount: Int) { 13 | self.init() 14 | self.name = name 15 | self.maxConcurrentOperationCount = maxConcurrentOperationCount 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /Sources/TSFFutures/OrderManager.swift: -------------------------------------------------------------------------------- 1 | // This source file is part of the Swift.org open source project 2 | // 3 | // Copyright (c) 2020 Apple Inc. and the Swift project authors 4 | // Licensed under Apache License v2.0 with Runtime Library Exception 5 | // 6 | // See http://swift.org/LICENSE.txt for license information 7 | // See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 8 | 9 | import Dispatch 10 | import NIO 11 | import NIOConcurrencyHelpers 12 | 13 | /// The `OrderManager` allows explicitly specify dependencies between 14 | /// various callbacks. This is necessary to avoid or induce race conditions 15 | /// in otherwise timing-dependent code, making such code deterministic. 16 | /// The semantics is as follows: the `OrderManager` invokes callbacks 17 | /// specified as arguments to the `order(_:_)` functions starting with 1. 18 | /// The callbacks of order `n` are guaranteed to run to completion before 19 | /// callbacks of order `n+1` are run. If there's a gap in the callbacks order 20 | /// sequence, the callbacks are suspended until the missing callback is 21 | /// registered, `reset()` is called, or global timeout occurs. 22 | /// In addition to that, `reset()` restarts the global timeout. 23 | /// 24 | /// Example: 25 | /// 26 | /// let manager = OrderManager(on: ...) 27 | /// manager.order(3, { print("3") }) 28 | /// manager.order(1, { print("1") }) 29 | /// manager.order(2, { print("2") }) 30 | /// try manager.order(4).wait() 31 | /// 32 | /// The following will be printed out: 33 | /// 34 | /// 1 35 | /// 2 36 | /// 3 37 | /// 38 | 39 | public class LLBOrderManager { 40 | 41 | // A safety timer, not to be exceeded. 42 | private let cancelTimer = DispatchSource.makeTimerSource() 43 | private let timeout: DispatchTimeInterval 44 | 45 | private typealias WaitListElement = ( 46 | order: Int, promise: LLBPromise, file: String, line: Int 47 | ) 48 | private let lock = NIOConcurrencyHelpers.NIOLock() 49 | private var waitlist = [WaitListElement]() 50 | private var nextToRun = 1 51 | 52 | private var eventLoop: EventLoop { 53 | lock.withLock { 54 | switch groupDesignator { 55 | case .managedGroup(let group): 56 | return group.next() 57 | case .externallySuppliedGroup(let group): 58 | return group.next() 59 | } 60 | } 61 | } 62 | 63 | private enum GroupDesignator { 64 | case managedGroup(LLBFuturesDispatchGroup) 65 | case externallySuppliedGroup(LLBFuturesDispatchGroup) 66 | } 67 | private var groupDesignator: GroupDesignator 68 | 69 | public enum Error: Swift.Error { 70 | case orderManagerReset(file: String, line: Int) 71 | } 72 | 73 | public init(on loop: EventLoop, timeout: DispatchTimeInterval = .seconds(60)) { 74 | self.groupDesignator = GroupDesignator.externallySuppliedGroup(loop) 75 | self.timeout = timeout 76 | restartInactivityTimer() 77 | cancelTimer.setEventHandler { [weak self] in 78 | guard let self = self else { return } 79 | _ = self.reset() 80 | self.cancelTimer.cancel() 81 | } 82 | cancelTimer.resume() 83 | } 84 | 85 | private func restartInactivityTimer() { 86 | cancelTimer.schedule(deadline: DispatchTime.now() + timeout, repeating: .never) 87 | 88 | } 89 | 90 | /// Run a specified callback in a particular order. 91 | @discardableResult 92 | public func order( 93 | _ n: Int, file: String = #file, line: Int = #line, _ callback: @escaping () throws -> T 94 | ) -> EventLoopFuture { 95 | let promise = eventLoop.makePromise(of: Void.self) 96 | 97 | lock.withLockVoid { 98 | waitlist.append((order: n, promise: promise, file: file, line: line)) 99 | } 100 | 101 | let future = promise.futureResult.flatMapThrowing { 102 | try callback() 103 | } 104 | 105 | future.whenComplete { _ in 106 | self.lock.withLockVoid { 107 | if n == self.nextToRun { 108 | self.nextToRun += 1 109 | } 110 | } 111 | self.unblockWaiters() 112 | } 113 | 114 | unblockWaiters() 115 | return future 116 | } 117 | 118 | @discardableResult 119 | public func order(_ n: Int, file: String = #file, line: Int = #line) -> EventLoopFuture { 120 | return order(n, file: file, line: line, {}) 121 | } 122 | 123 | private func unblockWaiters() { 124 | let wakeup: [EventLoopPromise] = lock.withLock { 125 | let wakeupPromises = 126 | waitlist 127 | .filter({ $0.order <= nextToRun }) 128 | .map({ $0.promise }) 129 | waitlist = waitlist.filter({ $0.order > nextToRun }) 130 | return wakeupPromises 131 | } 132 | wakeup.forEach { $0.succeed(()) } 133 | } 134 | 135 | /// Fail all ordered callbacks. Not calling the callback functions 136 | /// specified as argument to order(_:_), but failing the outcome. 137 | public func reset(file: String = #file, line: Int = #line) -> EventLoopFuture { 138 | restartInactivityTimer() 139 | let lock = self.lock 140 | 141 | let futures = failPromises(file: file, line: line) 142 | 143 | return EventLoopFuture.whenAllSucceed(futures, on: eventLoop).map { [weak self] _ in 144 | guard let self = self else { return } 145 | lock.withLockVoid { 146 | assert(self.waitlist.isEmpty) 147 | self.nextToRun = 1 148 | } 149 | } 150 | } 151 | 152 | @discardableResult 153 | private func failPromises(file: String = #file, line: Int = #line) -> [EventLoopFuture] { 154 | let toCancel: [WaitListElement] = lock.withLock { 155 | let cancelList = waitlist 156 | waitlist = [] 157 | nextToRun = Int.max 158 | return cancelList 159 | } 160 | let error = Error.orderManagerReset(file: file, line: line) 161 | return toCancel.sorted(by: { $0.order < $1.order }).map { 162 | $0.promise.fail(error) 163 | return $0.promise.futureResult.flatMapErrorThrowing { _ in () } 164 | } 165 | } 166 | 167 | deinit { 168 | cancelTimer.setEventHandler {} 169 | cancelTimer.cancel() 170 | 171 | failPromises() 172 | 173 | guard case .managedGroup(let group) = groupDesignator else { 174 | return 175 | } 176 | 177 | let q = DispatchQueue(label: "tsf.OrderManager") 178 | q.async { try! group.syncShutdownGracefully() } 179 | } 180 | } 181 | -------------------------------------------------------------------------------- /Sources/TSFUtility/ByteBuffer.swift: -------------------------------------------------------------------------------- 1 | // This source file is part of the Swift.org open source project 2 | // 3 | // Copyright (c) 2020 Apple Inc. and the Swift project authors 4 | // Licensed under Apache License v2.0 with Runtime Library Exception 5 | // 6 | // See http://swift.org/LICENSE.txt for license information 7 | // See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 8 | 9 | import Foundation 10 | import NIO 11 | import NIOFoundationCompat 12 | 13 | public typealias LLBByteBuffer = NIO.ByteBuffer 14 | public typealias LLBByteBufferAllocator = NIO.ByteBufferAllocator 15 | public typealias LLBByteBufferView = NIO.ByteBufferView 16 | 17 | extension LLBByteBuffer { 18 | public static func withBytes(_ data: ArraySlice) -> LLBByteBuffer { 19 | return LLBByteBuffer(bytes: data) 20 | } 21 | 22 | public static func withBytes(_ data: Data) -> LLBByteBuffer { 23 | return LLBByteBuffer(data: data) 24 | } 25 | 26 | public static func withBytes(_ data: [UInt8]) -> LLBByteBuffer { 27 | return LLBByteBuffer(bytes: data) 28 | } 29 | } 30 | 31 | extension LLBByteBuffer { 32 | public mutating func reserveWriteCapacity(_ count: Int) { 33 | self.reserveCapacity(self.writerIndex + count) 34 | } 35 | 36 | public mutating func unsafeWrite( 37 | _ writeCallback: (UnsafeMutableRawBufferPointer) -> (wrote: Int, R) 38 | ) -> R { 39 | var returnValue: R? = nil 40 | self.writeWithUnsafeMutableBytes(minimumWritableBytes: 0) { ptr -> Int in 41 | let (wrote, ret) = writeCallback(ptr) 42 | returnValue = ret 43 | return wrote 44 | } 45 | return returnValue! 46 | } 47 | 48 | } 49 | -------------------------------------------------------------------------------- /Sources/TSFUtility/FastData.swift: -------------------------------------------------------------------------------- 1 | // This source file is part of the Swift.org open source project 2 | // 3 | // Copyright (c) 2020 Apple Inc. and the Swift project authors 4 | // Licensed under Apache License v2.0 with Runtime Library Exception 5 | // 6 | // See http://swift.org/LICENSE.txt for license information 7 | // See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 8 | 9 | import Foundation 10 | import NIOCore 11 | 12 | /// Something that exposes working withContiguousStorage 13 | public enum LLBFastData { 14 | case slice(ArraySlice) 15 | case view(LLBByteBuffer) 16 | case data(Data) 17 | case pointer(UnsafeRawBufferPointer, deallocator: (UnsafeRawBufferPointer) -> Void) 18 | 19 | public init(_ data: [UInt8]) { self = .slice(ArraySlice(data)) } 20 | public init(_ data: ArraySlice) { self = .slice(data) } 21 | public init(_ data: LLBByteBuffer) { self = .view(data) } 22 | public init(_ data: Data) { 23 | precondition(data.regions.count == 1) 24 | self = .data(data) 25 | } 26 | public init( 27 | _ pointer: UnsafeRawBufferPointer, deallocator: @escaping (UnsafeRawBufferPointer) -> Void 28 | ) { 29 | self = .pointer(pointer, deallocator: deallocator) 30 | } 31 | 32 | public var count: Int { 33 | switch self { 34 | case .slice(let data): 35 | return data.count 36 | case .view(let data): 37 | return data.readableBytes 38 | case .data(let data): 39 | return data.count 40 | case .pointer(let ptr, _): 41 | return ptr.count 42 | } 43 | } 44 | 45 | public func withContiguousStorage(_ cb: (UnsafeBufferPointer) throws -> R) rethrows 46 | -> R 47 | { 48 | switch self { 49 | case .slice(let data): 50 | return try data.withContiguousStorageIfAvailable(cb)! 51 | case .view(let data): 52 | return try data.readableBytesView.withContiguousStorageIfAvailable(cb)! 53 | case .data(let data): 54 | precondition(data.regions.count == 1) 55 | return try data.withUnsafeBytes { rawPtr in 56 | let ptr = UnsafeRawBufferPointer(rawPtr).bindMemory(to: UInt8.self) 57 | return try cb(ptr) 58 | } 59 | case .pointer(let rawPtr, _): 60 | let ptr = UnsafeRawBufferPointer(rawPtr).bindMemory(to: UInt8.self) 61 | return try cb(ptr) 62 | } 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /Sources/TSFUtility/Serializable.swift: -------------------------------------------------------------------------------- 1 | // This source file is part of the Swift.org open source project 2 | // 3 | // Copyright (c) 2020 Apple Inc. and the Swift project authors 4 | // Licensed under Apache License v2.0 with Runtime Library Exception 5 | // 6 | // See http://swift.org/LICENSE.txt for license information 7 | // See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 8 | 9 | import Foundation 10 | import NIOCore 11 | 12 | /// Serializable protocol describes a structure that can be serialized 13 | /// into a buffer of bytes, and deserialized back from the buffer of bytes. 14 | public typealias LLBSerializable = LLBSerializableIn & LLBSerializableOut 15 | 16 | public enum LLBSerializableError: Error { 17 | case unknownError(String) 18 | } 19 | 20 | public protocol LLBSerializableIn { 21 | /// Decode the given block back into a message. 22 | init(from rawBytes: LLBByteBuffer) throws 23 | } 24 | 25 | public protocol LLBSerializableOut { 26 | /// Produce an encoded blob that fully defines the structure contents. 27 | func toBytes(into buffer: inout LLBByteBuffer) throws 28 | } 29 | 30 | extension LLBSerializableOut { 31 | public func toBytes() throws -> LLBByteBuffer { 32 | var buffer = LLBByteBufferAllocator().buffer(capacity: 0) 33 | try toBytes(into: &buffer) 34 | return buffer 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /Tests/AsyncProcessTests/AsyncByteBufferLineSequenceTests.swift: -------------------------------------------------------------------------------- 1 | //===----------------------------------------------------------------------===// 2 | // 3 | // This source file is part of the Swift open source project 4 | // 5 | // Copyright (c) 2022-2025 Apple Inc. and the Swift project authors 6 | // Licensed under Apache License v2.0 with Runtime Library Exception 7 | // 8 | // See https://swift.org/LICENSE.txt for license information 9 | // See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 10 | // 11 | //===----------------------------------------------------------------------===// 12 | 13 | // DO NOT EDIT - Make any changes in the upstream swift-async-process package, then re-run the vendoring script. 14 | 15 | import Foundation 16 | import NIO 17 | import TSFAsyncProcess 18 | import XCTest 19 | 20 | final class AsyncByteBufferLineSequenceTests: XCTestCase { 21 | func testJustManyNewlines() async throws { 22 | for n in 0..<100 { 23 | let inputs: [ByteBuffer] = [ByteBuffer(repeating: UInt8(ascii: "\n"), count: n)] 24 | let lines = try await Array(inputs.async.splitIntoLines().strings) 25 | XCTAssertEqual(Array(repeating: "", count: n), lines) 26 | } 27 | } 28 | 29 | func testJustOneNewlineAtATime() async throws { 30 | for n in 0..<100 { 31 | let inputs: [ByteBuffer] = Array( 32 | repeating: ByteBuffer(integer: UInt8(ascii: "\n")), count: n) 33 | let lines = try await Array(inputs.async.splitIntoLines().strings) 34 | XCTAssertEqual(Array(repeating: "", count: n), lines) 35 | } 36 | } 37 | 38 | func testManyChunksNoNewlineDeliveringLastChunk() async throws { 39 | for n in 1..<100 { 40 | let inputs: [ByteBuffer] = [ByteBuffer(repeating: 0, count: n)] 41 | let lines = try await Array(inputs.async.splitIntoLines().strings) 42 | XCTAssertEqual([String(repeating: "\0", count: n)], lines) 43 | } 44 | } 45 | 46 | func testManyChunksNoNewlineNotDeliveringLastChunk() async throws { 47 | for n in 0..<100 { 48 | let inputs: [ByteBuffer] = [ByteBuffer(repeating: 0, count: n)] 49 | let lines = try await Array( 50 | inputs.async.splitIntoLines(dropLastChunkIfNoNewline: true).strings) 51 | XCTAssertEqual([], lines) 52 | } 53 | } 54 | 55 | func testOverlyLongLineIsSplit() async throws { 56 | var inputs = Array(repeating: ByteBuffer(integer: UInt8(0)), count: 10) 57 | inputs.append(ByteBuffer(integer: UInt8(ascii: "\n"))) 58 | let lines = try await Array( 59 | inputs.async.splitIntoLines( 60 | maximumAllowableBufferSize: 3, 61 | dropLastChunkIfNoNewline: true 62 | ).strings) 63 | XCTAssertEqual(["\0\0\0\0", "\0\0\0\0", "\0\0"], lines) 64 | } 65 | 66 | func testOverlyLongLineIsSplitByDefault() async throws { 67 | var inputs = [ByteBuffer(repeating: UInt8(0), count: 1024 * 1024 - 2)] // almost at the limit 68 | inputs.append(ByteBuffer(integer: UInt8(ascii: "\0"))) 69 | inputs.append(ByteBuffer(integer: UInt8(ascii: "\0"))) // hitting the limit 70 | inputs.append(ByteBuffer(integer: UInt8(ascii: "\0"))) // over the limit 71 | inputs.append(ByteBuffer(integer: UInt8(ascii: "\n"))) // too late 72 | let lines = try await Array( 73 | inputs.async.splitIntoLines( 74 | dropTerminator: false, 75 | dropLastChunkIfNoNewline: true 76 | ).strings) 77 | XCTAssertEqual([String(repeating: "\0", count: 1024 * 1024 + 1), "\n"], lines) 78 | } 79 | } 80 | -------------------------------------------------------------------------------- /Tests/AsyncProcessTests/Helpers+LogRecorderHandler.swift: -------------------------------------------------------------------------------- 1 | //===----------------------------------------------------------------------===// 2 | // 3 | // This source file is part of the Swift open source project 4 | // 5 | // Copyright (c) 2022-2025 Apple Inc. and the Swift project authors 6 | // Licensed under Apache License v2.0 with Runtime Library Exception 7 | // 8 | // See https://swift.org/LICENSE.txt for license information 9 | // See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 10 | // 11 | //===----------------------------------------------------------------------===// 12 | 13 | // DO NOT EDIT - Make any changes in the upstream swift-async-process package, then re-run the vendoring script. 14 | 15 | import Logging 16 | import NIOConcurrencyHelpers 17 | 18 | final internal class LogRecorderHandler: LogHandler { 19 | internal let state = NIOLockedValueBox(State()) 20 | 21 | struct FullLogMessage: Equatable { 22 | var level: Logger.Level 23 | var message: Logger.Message 24 | var metadata: Logger.Metadata 25 | } 26 | 27 | struct State { 28 | var metadata: [String: Logger.Metadata.Value] = [:] 29 | var messages: [FullLogMessage] = [] 30 | var logLevel: Logger.Level = .trace 31 | } 32 | 33 | func makeLogger() -> Logger { 34 | return Logger(label: "LogRecorder for tests", factory: { _ in self }) 35 | } 36 | 37 | func log( 38 | level: Logger.Level, 39 | message: Logger.Message, 40 | metadata: Logger.Metadata?, 41 | source: String, 42 | file: String, 43 | function: String, 44 | line: UInt 45 | ) { 46 | let fullMessage = FullLogMessage( 47 | level: level, 48 | message: message, 49 | metadata: self.metadata.merging(metadata ?? [:]) { l, r in r } 50 | ) 51 | self.state.withLockedValue { state in 52 | state.messages.append(fullMessage) 53 | } 54 | } 55 | 56 | var recordedMessages: [FullLogMessage] { 57 | return self.state.withLockedValue { $0.messages } 58 | } 59 | 60 | subscript(metadataKey key: String) -> Logging.Logger.Metadata.Value? { 61 | get { 62 | return self.state.withLockedValue { 63 | $0.metadata[key] 64 | } 65 | } 66 | set { 67 | self.state.withLockedValue { 68 | $0.metadata[key] = newValue 69 | } 70 | } 71 | } 72 | 73 | var metadata: Logging.Logger.Metadata { 74 | get { 75 | return self.state.withLockedValue { 76 | $0.metadata 77 | } 78 | } 79 | 80 | set { 81 | return self.state.withLockedValue { 82 | $0.metadata = newValue 83 | } 84 | } 85 | } 86 | 87 | var logLevel: Logging.Logger.Level { 88 | get { 89 | return self.state.withLockedValue { 90 | $0.logLevel 91 | } 92 | } 93 | 94 | set { 95 | return self.state.withLockedValue { 96 | $0.logLevel = newValue 97 | } 98 | } 99 | } 100 | } 101 | -------------------------------------------------------------------------------- /Tests/TSFCASFileTreeTests/CASBlobTests.swift: -------------------------------------------------------------------------------- 1 | // This source file is part of the Swift.org open source project 2 | // 3 | // Copyright (c) 2020 Apple Inc. and the Swift project authors 4 | // Licensed under Apache License v2.0 with Runtime Library Exception 5 | // 6 | // See http://swift.org/LICENSE.txt for license information 7 | // See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 8 | 9 | import TSCBasic 10 | import TSCUtility 11 | import TSFCASFileTree 12 | import XCTest 13 | 14 | class CASBlobTests: XCTestCase { 15 | var group: LLBFuturesDispatchGroup! 16 | 17 | override func setUp() { 18 | group = LLBMakeDefaultDispatchGroup() 19 | } 20 | 21 | override func tearDown() { 22 | try! group.syncShutdownGracefully() 23 | group = nil 24 | } 25 | 26 | func testBasics() throws { 27 | try withTemporaryFile(suffix: ".dat") { tmp in 28 | // Check several chunk sizes, to probe boundary conditions. 29 | try checkOneBlob(tmp, chunkSize: 16, [UInt8](repeating: 1, count: 512)) 30 | try checkOneBlob(tmp, chunkSize: 1024, [UInt8](repeating: 1, count: 512)) 31 | 32 | // Compression only works with larger objects, due to hard coded constants in importer. 33 | try checkOneBlob(tmp, chunkSize: 1024, [UInt8](repeating: 1, count: 2048)) 34 | } 35 | } 36 | 37 | func checkOneBlob(_ tmp: TemporaryFile, chunkSize: Int, _ contents: [UInt8]) throws { 38 | try localFileSystem.writeFileContents(tmp.path, bytes: ByteString(contents)) 39 | 40 | let db = LLBInMemoryCASDatabase(group: group) 41 | let ctx = Context() 42 | let id = try LLBCASFileTree.import( 43 | path: tmp.path, to: db, 44 | options: LLBCASFileTree.ImportOptions(fileChunkSize: chunkSize), stats: nil, ctx 45 | ).wait() 46 | 47 | let blob = try LLBCASBlob.parse(id: id, in: db, ctx).wait() 48 | XCTAssertEqual(blob.size, contents.count) 49 | 50 | // Check various read patterns. 51 | for testRange in [0..<0, 0..<1, 0.. LLBByteBuffer { 92 | var buffer = allocator.buffer(capacity: objectSize) 93 | for j in 0.. LLBByteBuffer { 57 | var buffer = allocator.buffer(capacity: objectSize) 58 | for j in 0.. Bool in 46 | print(data.count) 47 | readBuffer.writeBytes(data) 48 | timesCalled += 1 49 | return true 50 | }.wait() 51 | 52 | XCTAssertEqual(timesCalled, 1) 53 | XCTAssertEqual( 54 | Data(readBuffer.readableBytesView), Data(LLBByteBufferView(repeating: 65, count: 32))) 55 | } 56 | 57 | func testDifferentChannels() throws { 58 | let db = LLBInMemoryCASDatabase(group: group) 59 | let ctx = Context() 60 | let allocator = LLBByteBufferAllocator() 61 | 62 | let writer = LLBBufferedStreamWriter(db, bufferSize: 32) 63 | 64 | var buffer = allocator.buffer(capacity: 128) 65 | buffer.writeRepeatingByte(65, count: 16) 66 | 67 | writer.write(data: buffer, channel: 0) 68 | 69 | // Nil because it hasn't buffered out yet. 70 | XCTAssertNil(writer.latestID) 71 | 72 | buffer.clear() 73 | buffer.writeRepeatingByte(65, count: 16) 74 | writer.write(data: buffer, channel: 1) 75 | 76 | // Flush to send the remaining data. 77 | writer.flush() 78 | 79 | // Now there should be an ID because we've reached the buffer size 80 | let dataID = try XCTUnwrap(writer.latestID).wait() 81 | 82 | let reader = LLBCASStreamReader(db) 83 | 84 | var readBuffer = allocator.buffer(capacity: 32) 85 | 86 | var timesCalled = 0 87 | var channelsRead = Set() 88 | try reader.read(id: dataID, ctx) { (channel, data) -> Bool in 89 | readBuffer.writeBytes(data) 90 | channelsRead.insert(channel) 91 | timesCalled += 1 92 | return true 93 | }.wait() 94 | 95 | XCTAssertEqual(timesCalled, 2) 96 | XCTAssertEqual( 97 | Data(readBuffer.readableBytesView), Data(LLBByteBufferView(repeating: 65, count: 32))) 98 | XCTAssertEqual(channelsRead, [0, 1]) 99 | } 100 | } 101 | -------------------------------------------------------------------------------- /Tests/TSFCASUtilitiesTests/LinkedListStreamTests.swift: -------------------------------------------------------------------------------- 1 | // This source file is part of the Swift.org open source project 2 | // 3 | // Copyright (c) 2020 Apple Inc. and the Swift project authors 4 | // Licensed under Apache License v2.0 with Runtime Library Exception 5 | // 6 | // See http://swift.org/LICENSE.txt for license information 7 | // See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 8 | 9 | import TSCBasic 10 | import TSCUtility 11 | import TSFCAS 12 | import TSFCASFileTree 13 | import TSFCASUtilities 14 | import XCTest 15 | 16 | class LinkedListStreamTests: XCTestCase { 17 | let group = LLBMakeDefaultDispatchGroup() 18 | 19 | func testSingleLine() throws { 20 | let db = LLBInMemoryCASDatabase(group: group) 21 | let ctx = Context() 22 | 23 | var writer = LLBLinkedListStreamWriter(db) 24 | 25 | writer.append(data: LLBByteBuffer(string: "hello, world!"), ctx) 26 | 27 | let reader = LLBCASStreamReader(db) 28 | 29 | let latestID = try writer.latestID!.wait() 30 | 31 | var contentRead = false 32 | try reader.read(id: latestID, ctx) { (channel, data) -> Bool in 33 | let stringData = String(decoding: Data(data), as: UTF8.self) 34 | XCTAssertEqual(stringData, "hello, world!") 35 | contentRead = true 36 | return true 37 | }.wait() 38 | 39 | XCTAssertTrue(contentRead) 40 | } 41 | 42 | func testStreamSingleChannel() throws { 43 | let db = LLBInMemoryCASDatabase(group: group) 44 | let ctx = Context() 45 | 46 | let writeStream = Array(0...20).map { "Stream line \($0)" } 47 | 48 | var writer = LLBLinkedListStreamWriter(db) 49 | 50 | for block in writeStream { 51 | writer.append(data: LLBByteBuffer(string: block), ctx) 52 | } 53 | 54 | let reader = LLBCASStreamReader(db) 55 | 56 | let latestID = try writer.latestID!.wait() 57 | 58 | var readStream = [String]() 59 | 60 | try reader.read(id: latestID, ctx) { (channel, data) -> Bool in 61 | let block = String(decoding: Data(data), as: UTF8.self) 62 | readStream.append(block) 63 | return true 64 | }.wait() 65 | 66 | XCTAssertEqual(readStream, writeStream) 67 | } 68 | 69 | func testStreamMultiChannel() throws { 70 | let db = LLBInMemoryCASDatabase(group: group) 71 | let ctx = Context() 72 | 73 | let writeStream: [(UInt8, String)] = Array(0...20).map { ($0 % 4, "Stream line \($0)") } 74 | 75 | var writer = LLBLinkedListStreamWriter(db) 76 | 77 | for (channel, block) in writeStream { 78 | writer.append(data: LLBByteBuffer(string: block), channel: channel, ctx) 79 | } 80 | 81 | let reader = LLBCASStreamReader(db) 82 | 83 | let latestID = try writer.latestID!.wait() 84 | 85 | var readStream = [String]() 86 | 87 | try reader.read(id: latestID, channels: [0, 1], ctx) { (channel, data) -> Bool in 88 | let block = String(decoding: Data(data), as: UTF8.self) 89 | readStream.append(block) 90 | return true 91 | }.wait() 92 | 93 | let filteredWriteStream = writeStream.filter { $0.0 <= 1 }.map { $0.1 } 94 | 95 | XCTAssertEqual(readStream, filteredWriteStream) 96 | } 97 | 98 | func testStreamAggregateSize() throws { 99 | let db = LLBInMemoryCASDatabase(group: group) 100 | let ctx = Context() 101 | 102 | let writeStream: [(UInt8, String)] = Array(0...1).map { ($0 % 2, "Stream line \($0)") } 103 | 104 | var writer = LLBLinkedListStreamWriter(db) 105 | 106 | for (channel, block) in writeStream { 107 | writer.append(data: LLBByteBuffer(string: block), channel: channel, ctx) 108 | } 109 | 110 | let reader = LLBCASStreamReader(db) 111 | 112 | let latestID = try writer.latestID!.wait() 113 | 114 | let node = try LLBCASFSClient(db).load(latestID, ctx).wait() 115 | 116 | var readLength: Int = 0 117 | try reader.read(id: latestID, ctx) { (channel, data) -> Bool in 118 | readLength += data.count 119 | return true 120 | }.wait() 121 | 122 | XCTAssertEqual(node.size(), readLength) 123 | } 124 | 125 | func testStreamReadLimit() throws { 126 | let db = LLBInMemoryCASDatabase(group: group) 127 | let ctx = Context() 128 | 129 | let writeStream = Array(0...20).map { "Stream line \($0)" } 130 | 131 | var writer = LLBLinkedListStreamWriter(db) 132 | 133 | for block in writeStream { 134 | writer.append(data: LLBByteBuffer(string: block), ctx) 135 | } 136 | 137 | let reader = LLBCASStreamReader(db) 138 | 139 | let latestID = try writer.latestID!.wait() 140 | 141 | var readStream = [String]() 142 | 143 | var stopped = false 144 | try reader.read(id: latestID, ctx) { (channel, data) -> Bool in 145 | // Read only 5 elements 146 | if readStream.count == 5 { 147 | stopped = true 148 | return false 149 | } 150 | guard !stopped else { 151 | XCTFail("Requested to stop but kept receiving data") 152 | return false 153 | } 154 | 155 | let block = String(decoding: Data(data), as: UTF8.self) 156 | readStream.append(block) 157 | return true 158 | }.wait() 159 | 160 | XCTAssertEqual(readStream, Array(writeStream.prefix(5))) 161 | } 162 | 163 | func testStreamFromPreviousState() throws { 164 | let db = LLBInMemoryCASDatabase(group: group) 165 | let ctx = Context() 166 | 167 | let writeStream = Array(0...20).map { "Stream line \($0)" } 168 | 169 | var writer = LLBLinkedListStreamWriter(db) 170 | 171 | for block in writeStream { 172 | writer.append(data: LLBByteBuffer(string: block), ctx) 173 | } 174 | 175 | let startMarker = try writer.latestID!.wait() 176 | 177 | let writeStream2 = Array(21...40).map { "Stream line \($0)" } 178 | 179 | for block in writeStream2 { 180 | writer.append(data: LLBByteBuffer(string: block), ctx) 181 | } 182 | 183 | let reader = LLBCASStreamReader(db) 184 | 185 | let latestID = try writer.latestID!.wait() 186 | 187 | var readStream = [String]() 188 | 189 | try reader.read(id: latestID, lastReadID: startMarker, ctx) { (channel, data) -> Bool in 190 | let block = String(decoding: Data(data), as: UTF8.self) 191 | readStream.append(block) 192 | return true 193 | }.wait() 194 | 195 | XCTAssertEqual(readStream, writeStream2) 196 | } 197 | } 198 | -------------------------------------------------------------------------------- /Tests/TSFFuturesTests/BatchingFutureOperationQueue.swift: -------------------------------------------------------------------------------- 1 | // 2 | // Copyright © 2019-2021 Apple, Inc. All rights reserved. 3 | // 4 | 5 | import Atomics 6 | import NIO 7 | import NIOConcurrencyHelpers 8 | import TSFFutures 9 | import XCTest 10 | 11 | class BatchingFutureOperationQueueTests: XCTestCase { 12 | 13 | // Test dynamic capacity increase. 14 | func testDynamic() throws { 15 | let group = MultiThreadedEventLoopGroup(numberOfThreads: 1) 16 | defer { try! group.syncShutdownGracefully() } 17 | 18 | let manager = LLBOrderManager(on: group.next(), timeout: .seconds(5)) 19 | 20 | var q = LLBBatchingFutureOperationQueue( 21 | name: "foo", group: group, maxConcurrentOperationCount: 1) 22 | 23 | let opsInFlight = ManagedAtomic(0) 24 | 25 | let future1: LLBFuture = q.execute { () -> LLBFuture in 26 | opsInFlight.wrappingIncrement(ordering: .relaxed) 27 | return manager.order(1).flatMap { 28 | manager.order(6) { 29 | opsInFlight.wrappingDecrement(ordering: .relaxed) 30 | } 31 | } 32 | } 33 | 34 | let future2: LLBFuture = q.execute { () -> LLBFuture in 35 | opsInFlight.wrappingIncrement(ordering: .relaxed) 36 | return manager.order(3).flatMap { 37 | manager.order(6) { 38 | opsInFlight.wrappingDecrement(ordering: .relaxed) 39 | } 40 | } 41 | } 42 | 43 | // Wait until future1 adss to opsInFlight. 44 | try manager.order(2).wait() 45 | XCTAssertEqual(opsInFlight.load(ordering: .relaxed), 1) 46 | 47 | // The test breaks without this line. 48 | q.maxOpCount += 1 49 | 50 | try manager.order(4).wait() 51 | XCTAssertEqual(opsInFlight.load(ordering: .relaxed), 2) 52 | try manager.order(5).wait() 53 | 54 | try manager.order(7).wait() 55 | XCTAssertEqual(opsInFlight.load(ordering: .relaxed), 0) 56 | 57 | try future2.wait() 58 | try future1.wait() 59 | 60 | } 61 | 62 | // Test setMaxOpCount on immutable queue. 63 | func testSetMaxConcurrency() throws { 64 | let group = MultiThreadedEventLoopGroup(numberOfThreads: 1) 65 | defer { try! group.syncShutdownGracefully() } 66 | 67 | let q = LLBBatchingFutureOperationQueue( 68 | name: "foo", group: group, maxConcurrentOperationCount: 1) 69 | q.setMaxOpCount(q.maxOpCount + 1) 70 | XCTAssertEqual(q.maxOpCount, 2) 71 | } 72 | 73 | } 74 | -------------------------------------------------------------------------------- /Tests/TSFFuturesTests/CancellableFutureTests.swift: -------------------------------------------------------------------------------- 1 | // 2 | // Copyright © 2019-2020 Apple, Inc. All rights reserved. 3 | // 4 | 5 | import Atomics 6 | import NIO 7 | import NIOConcurrencyHelpers 8 | import TSFFutures 9 | import XCTest 10 | 11 | class CancellableFutureTests: XCTestCase { 12 | 13 | var group: EventLoopGroup! 14 | 15 | override func setUp() { 16 | super.setUp() 17 | 18 | group = MultiThreadedEventLoopGroup(numberOfThreads: 1) 19 | } 20 | 21 | override func tearDown() { 22 | super.tearDown() 23 | 24 | try! group.syncShutdownGracefully() 25 | group = nil 26 | } 27 | 28 | /// This is a mock for some function that is able to cancel 29 | /// future's underlying operation. 30 | struct Handler: LLBCancelProtocol { 31 | private let called = ManagedAtomic(0) 32 | 33 | var wasCalled: Bool { 34 | return timesCalled > 0 35 | } 36 | 37 | var timesCalled: Int { 38 | return called.load(ordering: .relaxed) 39 | } 40 | 41 | func cancel(reason: String?) { 42 | called.wrappingIncrement(ordering: .relaxed) 43 | } 44 | } 45 | 46 | public enum GenericError: Swift.Error { 47 | case error 48 | } 49 | 50 | func testBasicSuccess() throws { 51 | let promise = group.next().makePromise(of: Void.self) 52 | let handler = Handler() 53 | let cf = LLBCancellableFuture(promise.futureResult, canceller: .init(handler)) 54 | promise.succeed(()) 55 | XCTAssertNoThrow(try promise.futureResult.wait()) 56 | XCTAssertFalse(handler.wasCalled) 57 | cf.canceller.cancel(reason: #function) 58 | // Canceller won't be invoked if the future was already fired. 59 | XCTAssertFalse(handler.wasCalled) 60 | } 61 | 62 | func testBasicFailure() throws { 63 | let promise = group.next().makePromise(of: Void.self) 64 | let handler = Handler() 65 | let cf = LLBCancellableFuture(promise.futureResult, canceller: .init(handler)) 66 | promise.fail(GenericError.error) 67 | XCTAssertThrowsError(try promise.futureResult.wait()) { error in 68 | XCTAssert(error is GenericError) 69 | } 70 | XCTAssertFalse(handler.wasCalled) 71 | cf.canceller.cancel(reason: #function) 72 | // Canceller won't be invoked if the future was already fired. 73 | XCTAssertFalse(handler.wasCalled) 74 | } 75 | 76 | func testBasicCancellation() throws { 77 | let promise = group.next().makePromise(of: Void.self) 78 | let handler = Handler() 79 | let cf = LLBCancellableFuture(promise.futureResult, canceller: .init(handler)) 80 | cf.cancel(reason: #function) 81 | promise.succeed(()) 82 | XCTAssertNoThrow(try promise.futureResult.wait()) 83 | XCTAssertTrue(handler.wasCalled) 84 | } 85 | 86 | func testLateCancellation() throws { 87 | let promise = group.next().makePromise(of: Void.self) 88 | let handler = Handler() 89 | let cf = LLBCancellableFuture(promise.futureResult, canceller: .init(handler)) 90 | promise.succeed(()) 91 | XCTAssertNoThrow(try promise.futureResult.wait()) 92 | cf.cancel(reason: #function) 93 | XCTAssertFalse(handler.wasCalled) 94 | } 95 | 96 | func testDoubleCancellation() throws { 97 | let promise = group.next().makePromise(of: Void.self) 98 | let handler = Handler() 99 | let cf = LLBCancellableFuture(promise.futureResult, canceller: .init(handler)) 100 | cf.cancel(reason: #function) 101 | promise.succeed(()) 102 | XCTAssertNoThrow(try promise.futureResult.wait()) 103 | XCTAssertTrue(handler.wasCalled) 104 | cf.canceller.cancel(reason: #function) 105 | XCTAssertEqual(handler.timesCalled, 1) 106 | } 107 | 108 | func testLateInitialization() throws { 109 | let promise = group.next().makePromise(of: Void.self) 110 | let handler = Handler() 111 | let cf = LLBCancellableFuture(promise.futureResult) 112 | cf.cancel(reason: #function) 113 | // Setting the handler after cancelling. 114 | cf.canceller.set(handler: handler) 115 | promise.succeed(()) 116 | XCTAssertNoThrow(try promise.futureResult.wait()) 117 | XCTAssertTrue(handler.wasCalled) 118 | cf.canceller.cancel(reason: #function) 119 | XCTAssertEqual(handler.timesCalled, 1) 120 | } 121 | 122 | func testLateInitializationAndCancellation() throws { 123 | let promise = group.next().makePromise(of: Void.self) 124 | let handler = Handler() 125 | let cf = LLBCancellableFuture(promise.futureResult) 126 | promise.succeed(()) 127 | XCTAssertNoThrow(try promise.futureResult.wait()) 128 | // Setting the handler after cancelling. 129 | cf.canceller.set(handler: handler) 130 | cf.cancel(reason: #function) 131 | XCTAssertFalse(handler.wasCalled) 132 | } 133 | 134 | } 135 | -------------------------------------------------------------------------------- /Tests/TSFFuturesTests/CancellablePromiseTests.swift: -------------------------------------------------------------------------------- 1 | // 2 | // Copyright © 2019-2020 Apple, Inc. All rights reserved. 3 | // 4 | 5 | import NIO 6 | import NIOConcurrencyHelpers 7 | import TSFFutures 8 | import XCTest 9 | 10 | class CancellablePromiseTests: XCTestCase { 11 | 12 | var group: EventLoopGroup! 13 | 14 | override func setUp() { 15 | super.setUp() 16 | 17 | group = MultiThreadedEventLoopGroup(numberOfThreads: 1) 18 | } 19 | 20 | override func tearDown() { 21 | super.tearDown() 22 | 23 | try! group.syncShutdownGracefully() 24 | group = nil 25 | } 26 | 27 | public enum GenericError: Swift.Error { 28 | case error 29 | case error1 30 | case error2 31 | } 32 | 33 | func testBasicSuccess() throws { 34 | let promise = LLBCancellablePromise(on: group.next()) 35 | XCTAssertTrue(promise.succeed(())) 36 | XCTAssertNoThrow(try promise.futureResult.wait()) 37 | } 38 | 39 | func testBasicFailure() throws { 40 | let promise = LLBCancellablePromise(on: group.next()) 41 | XCTAssertTrue(promise.fail(GenericError.error)) 42 | XCTAssertThrowsError(try promise.futureResult.wait()) 43 | } 44 | 45 | func testCancel() throws { 46 | let promise = LLBCancellablePromise(on: group.next()) 47 | XCTAssertTrue(promise.cancel(GenericError.error)) 48 | XCTAssertThrowsError(try promise.futureResult.wait()) { error in 49 | XCTAssert(error is GenericError) 50 | } 51 | XCTAssertFalse(promise.succeed(())) 52 | } 53 | 54 | func testDoubleCancel() throws { 55 | let promise = LLBCancellablePromise(on: group.next()) 56 | XCTAssertTrue(promise.cancel(GenericError.error)) 57 | XCTAssertFalse(promise.cancel(GenericError.error1)) 58 | XCTAssertThrowsError(try promise.futureResult.wait()) { error in 59 | guard case .error? = error as? GenericError else { 60 | XCTFail("Unexpected throw \(error)") 61 | return 62 | } 63 | } 64 | XCTAssertFalse(promise.fail(GenericError.error2)) 65 | } 66 | 67 | func testLeakIsOK() throws { 68 | let _ = LLBCancellablePromise(on: group.next()) 69 | } 70 | 71 | } 72 | -------------------------------------------------------------------------------- /Tests/TSFFuturesTests/CancellerTests.swift: -------------------------------------------------------------------------------- 1 | // 2 | // Copyright © 2019-2020 Apple, Inc. All rights reserved. 3 | // 4 | 5 | import Atomics 6 | import NIO 7 | import NIOConcurrencyHelpers 8 | import TSFFutures 9 | import XCTest 10 | 11 | class CancellerTests: XCTestCase { 12 | 13 | var group: EventLoopGroup! 14 | 15 | override func setUp() { 16 | super.setUp() 17 | 18 | group = MultiThreadedEventLoopGroup(numberOfThreads: 1) 19 | } 20 | 21 | override func tearDown() { 22 | super.tearDown() 23 | 24 | try! group.syncShutdownGracefully() 25 | group = nil 26 | } 27 | 28 | /// This is a mock for some function that is able to cancel 29 | /// future's underlying operation. 30 | struct Handler: LLBCancelProtocol { 31 | private let called = ManagedAtomic(0) 32 | 33 | var wasCalled: Bool { 34 | return timesCalled > 0 35 | } 36 | 37 | var timesCalled: Int { 38 | return called.load(ordering: .relaxed) 39 | } 40 | 41 | func cancel(reason: String?) { 42 | called.wrappingIncrement(ordering: .relaxed) 43 | } 44 | } 45 | 46 | func testCancel() throws { 47 | let handler = Handler() 48 | let canceller = LLBCanceller(handler) 49 | XCTAssertFalse(handler.wasCalled) 50 | canceller.cancel(reason: #function) 51 | XCTAssertTrue(handler.wasCalled) 52 | } 53 | 54 | func testDoubleCancellation() throws { 55 | let handler = Handler() 56 | let canceller = LLBCanceller(handler) 57 | canceller.cancel(reason: #function) 58 | XCTAssertTrue(handler.wasCalled) 59 | canceller.cancel(reason: #function) 60 | XCTAssertEqual(handler.timesCalled, 1) 61 | } 62 | 63 | func testLateInitialization() throws { 64 | let handler = Handler() 65 | let canceller = LLBCanceller() 66 | canceller.cancel(reason: #function) 67 | // Setting the handler after cancelling. 68 | canceller.set(handler: handler) 69 | XCTAssertTrue(handler.wasCalled) 70 | canceller.cancel(reason: #function) 71 | XCTAssertEqual(handler.timesCalled, 1) 72 | } 73 | 74 | func testAbandonFirst() throws { 75 | let handler = Handler() 76 | let canceller = LLBCanceller() 77 | canceller.abandon() 78 | canceller.cancel(reason: #function) 79 | canceller.set(handler: handler) 80 | XCTAssertFalse(handler.wasCalled) 81 | } 82 | 83 | func testAbandonLast() throws { 84 | let handler = Handler() 85 | let canceller = LLBCanceller() 86 | canceller.cancel(reason: #function) 87 | canceller.abandon() 88 | canceller.set(handler: handler) 89 | XCTAssertFalse(handler.wasCalled) 90 | } 91 | 92 | } 93 | -------------------------------------------------------------------------------- /Tests/TSFFuturesTests/EventualResultsCacheTests.swift: -------------------------------------------------------------------------------- 1 | // 2 | // Copyright © 2019 Apple, Inc. All rights reserved. 3 | // 4 | 5 | import NIO 6 | import TSFFutures 7 | import XCTest 8 | 9 | class EventualResultsCacheTests: XCTestCase { 10 | 11 | var group: LLBFuturesDispatchGroup! 12 | 13 | override func setUp() { 14 | super.setUp() 15 | 16 | group = LLBMakeDefaultDispatchGroup() 17 | } 18 | 19 | override func tearDown() { 20 | super.tearDown() 21 | 22 | try! group.syncShutdownGracefully() 23 | group = nil 24 | } 25 | 26 | /// Test that we don't re-resolve the cached value 27 | func testSerialCoalescing() throws { 28 | let cache = LLBEventualResultsCache(group: group) 29 | var hits = 0 30 | 31 | let v1Future = cache.value(for: 1) { key in 32 | hits += 1 33 | return group.next().makeSucceededFuture("\(hits)") 34 | } 35 | 36 | let v1 = try v1Future.wait() 37 | 38 | let v2Future = cache.value(for: 1) { key in 39 | XCTFail("Unexpected resolver invocation") 40 | hits += 1 41 | return group.next().makeSucceededFuture("\(hits)") 42 | } 43 | 44 | let v2 = try v2Future.wait() 45 | 46 | XCTAssertEqual(v1, v2) 47 | XCTAssertEqual(hits, 1) 48 | } 49 | 50 | /// Test that we don't re-resolve even if resolution takes time. 51 | func testParallelCoalescing() throws { 52 | let cache = LLBEventualResultsCache(group: group) 53 | var hits = 0 54 | 55 | func resolver(_ key: Int) -> LLBFuture { 56 | hits += 1 57 | let promise = group.next().makePromise(of: String.self) 58 | _ = group.next().scheduleTask(in: TimeAmount.milliseconds(100)) { 59 | promise.succeed("\(hits)") 60 | } 61 | return promise.futureResult 62 | } 63 | 64 | let v1Future = cache.value(for: 1, with: resolver) 65 | let v2Future = cache.value(for: 1, with: resolver) 66 | 67 | XCTAssertEqual(try v1Future.wait(), try v2Future.wait()) 68 | XCTAssertEqual(hits, 1) 69 | } 70 | 71 | /// Test that we don't re-resolve an in-flight value when requesting multiple 72 | func testMultipleValueResolution() throws { 73 | let cache = LLBEventualResultsCache(group: group) 74 | 75 | // Immediate resolution. 76 | _ = try cache.value(for: 0) { key in 77 | return group.next().makeSucceededFuture(0) 78 | }.wait() 79 | 80 | // Delayed resolution. 81 | _ = cache.value(for: 1) { key in 82 | let promise = group.next().makePromise(of: Int.self) 83 | _ = group.next().scheduleTask(in: TimeAmount.milliseconds(100)) { 84 | promise.succeed(key) 85 | } 86 | return promise.futureResult 87 | } 88 | 89 | let futures = cache.values(for: [0, 1, 2, 3]) { keys in 90 | XCTAssertFalse(keys.contains(0), "Unexpected resolver invocation") 91 | XCTAssertFalse(keys.contains(1), "Unexpected resolver invocation") 92 | XCTAssertTrue(keys.contains(2), "Unexpected resolver invocation") 93 | XCTAssertTrue(keys.contains(3), "Unexpected resolver invocation") 94 | return group.next().makeSucceededFuture(keys) 95 | } 96 | 97 | let results = try LLBFuture.whenAllSucceed(futures, on: group.next()).wait() 98 | 99 | XCTAssertEqual([0, 1, 2, 3], results) 100 | } 101 | 102 | } 103 | -------------------------------------------------------------------------------- /Tests/TSFFuturesTests/FutureDeduplicatorTests.swift: -------------------------------------------------------------------------------- 1 | // 2 | // Copyright © 2019 Apple, Inc. All rights reserved. 3 | // 4 | 5 | import NIO 6 | import TSFFutures 7 | import XCTest 8 | 9 | class FutureDeduplicatorTests: XCTestCase { 10 | 11 | var group: LLBFuturesDispatchGroup! 12 | 13 | override func setUp() { 14 | super.setUp() 15 | 16 | group = LLBMakeDefaultDispatchGroup() 17 | } 18 | 19 | override func tearDown() { 20 | super.tearDown() 21 | 22 | try! group.syncShutdownGracefully() 23 | group = nil 24 | } 25 | 26 | /// Test that we don't re-resolve the cached value 27 | func testSerialCoalescing() throws { 28 | let cache = LLBFutureDeduplicator(group: group) 29 | var hits = 0 30 | 31 | let v1Future = cache.value(for: 1) { key in 32 | hits += 1 33 | return group.next().makeSucceededFuture("\(hits)") 34 | } 35 | 36 | let v1 = try v1Future.wait() 37 | 38 | let v2Future = cache.value(for: 1) { key in 39 | hits += 1 40 | return group.next().makeSucceededFuture("\(hits)") 41 | } 42 | 43 | let v2 = try v2Future.wait() 44 | 45 | XCTAssertEqual(hits, 2) 46 | XCTAssertNotEqual(v1, v2) 47 | } 48 | 49 | /// Test that we don't re-resolve even if resolution takes time. 50 | func testParallelCoalescing() throws { 51 | let cache = LLBFutureDeduplicator(group: group) 52 | var hits = 0 53 | 54 | func resolver(_ key: Int) -> LLBFuture { 55 | hits += 1 56 | let promise = group.next().makePromise(of: String.self) 57 | _ = group.next().scheduleTask(in: TimeAmount.milliseconds(100)) { 58 | promise.succeed("\(hits)") 59 | } 60 | return promise.futureResult 61 | } 62 | 63 | let v1Future = cache.value(for: 1, with: resolver) 64 | let v2Future = cache.value(for: 1, with: resolver) 65 | 66 | XCTAssertEqual(try v1Future.wait(), try v2Future.wait()) 67 | XCTAssertEqual(hits, 1) 68 | } 69 | 70 | /// Test that we don't re-resolve an in-flight value when requesting multiple 71 | func testMultipleValueResolution() throws { 72 | let cache = LLBFutureDeduplicator(group: group) 73 | 74 | // Immediate resolution. 75 | _ = try cache.value(for: 0) { key in 76 | return group.next().makeSucceededFuture(0) 77 | }.wait() 78 | 79 | // Delayed resolution. 80 | _ = cache.value(for: 1) { key in 81 | let promise = group.next().makePromise(of: Int.self) 82 | _ = group.next().scheduleTask(in: TimeAmount.milliseconds(100)) { 83 | promise.succeed(key) 84 | } 85 | return promise.futureResult 86 | } 87 | 88 | let futures = cache.values(for: [0, 1, 2, 3]) { keys in 89 | // This has already been resolved once, so we are expected 90 | // to resolve it anew in the `FutureDeduplicator` abstraction. 91 | // See `EventualResultsCache` for a different behavior. 92 | XCTAssertTrue(keys.contains(0), "Unexpected resolver invocation") 93 | XCTAssertFalse(keys.contains(1), "Unexpected resolver invocation") 94 | XCTAssertTrue(keys.contains(2), "Unexpected resolver invocation") 95 | XCTAssertTrue(keys.contains(3), "Unexpected resolver invocation") 96 | return group.next().makeSucceededFuture(keys) 97 | } 98 | 99 | let results = try LLBFuture.whenAllSucceed(futures, on: group.next()).wait() 100 | 101 | XCTAssertEqual([0, 1, 2, 3], results) 102 | } 103 | 104 | } 105 | -------------------------------------------------------------------------------- /Tests/TSFFuturesTests/FutureOperationQueueTests.swift: -------------------------------------------------------------------------------- 1 | // 2 | // Copyright © 2019-2021 Apple, Inc. All rights reserved. 3 | // 4 | 5 | import Atomics 6 | import NIO 7 | import NIOConcurrencyHelpers 8 | import TSFFutures 9 | import XCTest 10 | 11 | class FutureOperationQueueTests: XCTestCase { 12 | func testBasics() throws { 13 | let group = LLBMakeDefaultDispatchGroup() 14 | defer { try! group.syncShutdownGracefully() } 15 | 16 | let loop = group.next() 17 | let p1 = loop.makePromise(of: Bool.self) 18 | let p2 = loop.makePromise(of: Bool.self) 19 | let p3 = loop.makePromise(of: Bool.self) 20 | var p1Started = false 21 | var p2Started = false 22 | var p3Started = false 23 | 24 | let manager = LLBOrderManager(on: group.next()) 25 | 26 | let q = LLBFutureOperationQueue(maxConcurrentOperations: 2) 27 | 28 | // Start the first two operations, they should run immediately. 29 | _ = q.enqueue(on: loop) { () -> LLBFuture in 30 | p1Started = true 31 | return manager.order(2).flatMap { 32 | p1.futureResult 33 | } 34 | } 35 | _ = q.enqueue(on: loop) { () -> LLBFuture in 36 | p2Started = true 37 | return manager.order(1).flatMap { 38 | p2.futureResult 39 | } 40 | } 41 | 42 | // Start the third, it should queue. 43 | _ = q.enqueue(on: loop) { () -> LLBFuture in 44 | p3Started = true 45 | return manager.order(4).flatMap { 46 | p3.futureResult 47 | } 48 | } 49 | 50 | try manager.order(3).wait() 51 | XCTAssertEqual(p1Started, true) 52 | XCTAssertEqual(p2Started, true) 53 | XCTAssertEqual(p3Started, false) 54 | 55 | // Complete the first. 56 | p1.succeed(true) 57 | try manager.order(5).wait() 58 | 59 | // Now p3 should have started. 60 | XCTAssertEqual(p3Started, true) 61 | p2.succeed(true) 62 | p3.succeed(true) 63 | 64 | _ = try! p1.futureResult.wait() 65 | _ = try! p2.futureResult.wait() 66 | _ = try! p3.futureResult.wait() 67 | } 68 | 69 | // Stress test. 70 | func testStress() throws { 71 | let group = MultiThreadedEventLoopGroup(numberOfThreads: 2) 72 | defer { try! group.syncShutdownGracefully() } 73 | 74 | let q = LLBFutureOperationQueue(maxConcurrentOperations: 2) 75 | 76 | let atomic = ManagedAtomic(0) 77 | var futures: [LLBFuture] = [] 78 | let lock = NIOConcurrencyHelpers.NIOLock() 79 | DispatchQueue.concurrentPerform(iterations: 1_000) { i in 80 | let result = q.enqueue(on: group.next()) { () -> LLBFuture in 81 | // Check that we aren't executing more operations than we would want. 82 | let p = group.next().makePromise(of: Bool.self) 83 | let prior = atomic.loadThenWrappingIncrement(ordering: .relaxed) 84 | XCTAssert(prior >= 0 && prior < 2, "saw \(prior + 1) concurrent tasks at start") 85 | p.futureResult.whenComplete { _ in 86 | let prior = atomic.loadThenWrappingDecrement(ordering: .relaxed) 87 | XCTAssert(prior > 0 && prior <= 2, "saw \(prior) concurrent tasks at end") 88 | } 89 | 90 | // Complete the future at some point 91 | group.next().execute { 92 | p.succeed(true) 93 | } 94 | 95 | return p.futureResult 96 | } 97 | 98 | lock.withLockVoid { 99 | futures.append(result) 100 | } 101 | } 102 | 103 | lock.withLockVoid { 104 | for future in futures { 105 | _ = try! future.wait() 106 | } 107 | } 108 | } 109 | 110 | // Test dynamic capacity increase. 111 | func testDynamic() throws { 112 | let group = MultiThreadedEventLoopGroup(numberOfThreads: 1) 113 | defer { try! group.syncShutdownGracefully() } 114 | 115 | let manager = LLBOrderManager(on: group.next(), timeout: .seconds(5)) 116 | 117 | let q = LLBFutureOperationQueue(maxConcurrentOperations: 1) 118 | 119 | let opsInFlight = ManagedAtomic(0) 120 | 121 | let future1: LLBFuture = q.enqueue(on: group.next()) { 122 | opsInFlight.wrappingIncrement(ordering: .relaxed) 123 | return manager.order(1).flatMap { 124 | manager.order(6) { 125 | opsInFlight.wrappingDecrement(ordering: .relaxed) 126 | } 127 | } 128 | } 129 | 130 | let future2: LLBFuture = q.enqueue(on: group.next()) { 131 | opsInFlight.wrappingIncrement(ordering: .relaxed) 132 | return manager.order(3).flatMap { 133 | manager.order(6) { 134 | opsInFlight.wrappingDecrement(ordering: .relaxed) 135 | } 136 | } 137 | } 138 | 139 | // Wait until future1 adss to opsInFlight. 140 | try manager.order(2).wait() 141 | XCTAssertEqual(opsInFlight.load(ordering: .relaxed), 1) 142 | 143 | // The test breaks without this line. 144 | q.maxConcurrentOperations += 1 145 | 146 | try manager.order(4).wait() 147 | XCTAssertEqual(opsInFlight.load(ordering: .relaxed), 2) 148 | try manager.order(5).wait() 149 | 150 | try manager.order(7).wait() 151 | XCTAssertEqual(opsInFlight.load(ordering: .relaxed), 0) 152 | 153 | try future2.wait() 154 | try future1.wait() 155 | } 156 | 157 | } 158 | -------------------------------------------------------------------------------- /Tests/TSFFuturesTests/OrderManagerTests.swift: -------------------------------------------------------------------------------- 1 | // 2 | // Copyright © 2020 Apple, Inc. All rights reserved. 3 | // 4 | 5 | import NIO 6 | import TSFFutures 7 | import XCTest 8 | 9 | class OrderManagerTests: XCTestCase { 10 | 11 | func testOrderManagerWithLoop() throws { 12 | let group = MultiThreadedEventLoopGroup(numberOfThreads: 1) 13 | defer { 14 | try! group.syncShutdownGracefully() 15 | } 16 | do { 17 | let manager = LLBOrderManager(on: group.next()) 18 | try manager.reset().wait() 19 | } 20 | } 21 | 22 | } 23 | -------------------------------------------------------------------------------- /Utilities/build_proto_toolchain.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -eu 2 | # 3 | # This source file is part of the Swift.org open source project 4 | # 5 | # Copyright (c) 2020 Apple Inc. and the Swift project authors 6 | # Licensed under Apache License v2.0 with Runtime Library Exception 7 | # 8 | # See http://swift.org/LICENSE.txt for license information 9 | # See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 10 | 11 | PROTOC_ZIP=protoc-21.10-osx-universal_binary.zip 12 | PROTOC_URL="https://github.com/protocolbuffers/protobuf/releases/download/v21.10/$PROTOC_ZIP" 13 | 14 | UTILITIES_DIR="$(dirname "$0")" 15 | TOOLS_DIR="$UTILITIES_DIR/tools" 16 | 17 | mkdir -p "$TOOLS_DIR" 18 | 19 | if [[ ! -f "$UTILITIES_DIR/tools/$PROTOC_ZIP" ]]; then 20 | curl -L "$PROTOC_URL" --output "$TOOLS_DIR/$PROTOC_ZIP" 21 | unzip -o "$TOOLS_DIR/$PROTOC_ZIP" -d "$TOOLS_DIR" 22 | fi 23 | 24 | # Use swift build instead of cloning the repo to make sure that the generated code matches the SwiftProtobuf library 25 | # being used as a dependency in the build. This might be a bit slower, but it's correct. 26 | swift build -c release --product protoc-gen-swift --package-path "$UTILITIES_DIR/.." 27 | 28 | cp "$UTILITIES_DIR"/../.build/release/protoc-gen-swift "$TOOLS_DIR/bin" 29 | --------------------------------------------------------------------------------