├── .gitignore ├── .gitmodules ├── CODEOWNERS ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── LICENSE.txt ├── Makefile ├── Package.resolved ├── Package.swift ├── Protos ├── CASFileTreeProtocol │ └── file_tree.proto ├── CASProtocol │ ├── cas_object.proto │ └── data_id.proto └── module_map.asciipb ├── README.md ├── Sources ├── CBLAKE3 │ ├── blake3.c │ ├── blake3_avx2.c │ ├── blake3_avx512.c │ ├── blake3_impl.h │ ├── blake3_sse41.c │ └── include │ │ └── blake3.h ├── TSFCAS │ ├── DataID.swift │ ├── Database.swift │ ├── DatabaseSpec.swift │ ├── Generated │ │ └── CASProtocol │ │ │ ├── cas_object.pb.swift │ │ │ └── data_id.pb.swift │ ├── Implementations │ │ ├── Blake3DataID.swift │ │ ├── FileBackedCASDatabase.swift │ │ └── InMemoryCASDatabase.swift │ └── Object.swift ├── TSFCASFileTree │ ├── BinarySearch.swift │ ├── CASBlob.swift │ ├── CASFSClient.swift │ ├── CASFSNode.swift │ ├── ConcurrentFileTreeWalker.swift │ ├── Context.swift │ ├── DeclFileTree.swift │ ├── DirectoryEntry.swift │ ├── Errors.swift │ ├── FileInfo.swift │ ├── FileTree.swift │ ├── FileTreeExport.swift │ ├── FileTreeImport.swift │ ├── FilesystemObject.swift │ ├── Generated │ │ └── CASFileTreeProtocol │ │ │ └── file_tree.pb.swift │ ├── Internal │ │ ├── ConcurrentFilesystemScanner.swift │ │ ├── FileSegmenter.swift │ │ └── FileTreeParser.swift │ └── TSCCASFileSystem.swift ├── TSFCASUtilities │ ├── BufferedStreamWriter.swift │ ├── LinkedListStream.swift │ └── StreamReader.swift ├── TSFFutures │ ├── BatchingFutureOperationQueue.swift │ ├── CancellableFuture.swift │ ├── CancellablePromise.swift │ ├── Canceller.swift │ ├── EventualResultsCache.swift │ ├── FutureDeduplicator.swift │ ├── FutureOperationQueue.swift │ ├── Futures.swift │ ├── OperationQueue+Extensions.swift │ └── OrderManager.swift └── TSFUtility │ ├── ByteBuffer.swift │ ├── FastData.swift │ ├── FutureFileSystem.swift │ └── Serializable.swift ├── Tests ├── TSFCASFileTreeTests │ ├── CASBlobTests.swift │ ├── FileTreeImportExportTests.swift │ └── FileTreeTests.swift ├── TSFCASTests │ ├── DataIDTests.swift │ ├── FileBackedCASDatabaseTests.swift │ └── InMemoryCASDatabaseTests.swift ├── TSFCASUtilitiesTests │ ├── BufferedStreamWriterTests.swift │ └── LinkedListStreamTests.swift └── TSFFuturesTests │ ├── BatchingFutureOperationQueue.swift │ ├── CancellableFutureTests.swift │ ├── CancellablePromiseTests.swift │ ├── CancellerTests.swift │ ├── EventualResultsCacheTests.swift │ ├── FutureDeduplicatorTests.swift │ ├── FutureOperationQueueTests.swift │ └── OrderManagerTests.swift └── Utilities └── build_proto_toolchain.sh /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | /.build 3 | /Packages 4 | /*.xcodeproj 5 | xcuserdata/ 6 | DerivedData/ 7 | .swiftpm/xcode/package.xcworkspace/contents.xcworkspacedata 8 | /Utilities/tools 9 | 10 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "ThirdParty/BLAKE3"] 2 | path = ThirdParty/BLAKE3 3 | url = https://github.com/BLAKE3-team/BLAKE3.git 4 | -------------------------------------------------------------------------------- /CODEOWNERS: -------------------------------------------------------------------------------- 1 | # This file is a list of the people responsible for ensuring that patches for a 2 | # particular part of Swift are reviewed, either by themself or by someone else. 3 | # They are also the gatekeepers for their part of Swift, with the final word on 4 | # what goes in or not. 5 | 6 | # The list is sorted by surname and formatted to allow easy grepping and 7 | # beautification by scripts. The fields are: name (N), email (E), web-address 8 | # (W), PGP key ID and fingerprint (P), description (D), and snail-mail address 9 | # (S). 10 | 11 | # N: David M. Bryson 12 | # E: dmbryson@apple.com 13 | # D: Everything in tools-support-async not covered by someone else 14 | 15 | ### 16 | 17 | # The following lines are used by GitHub to automatically recommend reviewers. 18 | 19 | * @dmbryson 20 | 21 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Code of Conduct 2 | To be a truly great community, Swift.org needs to welcome developers from all walks of life, 3 | with different backgrounds, and with a wide range of experience. A diverse and friendly 4 | community will have more great ideas, more unique perspectives, and produce more great 5 | code. We will work diligently to make the Swift community welcoming to everyone. 6 | 7 | To give clarity of what is expected of our members, Swift.org has adopted the code of conduct 8 | defined by [contributor-covenant.org](https://www.contributor-covenant.org). This document is used across many open source 9 | communities, and we think it articulates our values well. The full text is copied below: 10 | 11 | ### Contributor Code of Conduct v1.3 12 | As contributors and maintainers of this project, and in the interest of fostering an open and 13 | welcoming community, we pledge to respect all people who contribute through reporting 14 | issues, posting feature requests, updating documentation, submitting pull requests or patches, 15 | and other activities. 16 | 17 | We are committed to making participation in this project a harassment-free experience for 18 | everyone, regardless of level of experience, gender, gender identity and expression, sexual 19 | orientation, disability, personal appearance, body size, race, ethnicity, age, religion, or 20 | nationality. 21 | 22 | Examples of unacceptable behavior by participants include: 23 | - The use of sexualized language or imagery 24 | - Personal attacks 25 | - Trolling or insulting/derogatory comments 26 | - Public or private harassment 27 | - Publishing other’s private information, such as physical or electronic addresses, without explicit permission 28 | - Other unethical or unprofessional conduct 29 | 30 | Project maintainers have the right and responsibility to remove, edit, or reject comments, 31 | commits, code, wiki edits, issues, and other contributions that are not aligned to this Code of 32 | Conduct, or to ban temporarily or permanently any contributor for other behaviors that they 33 | deem inappropriate, threatening, offensive, or harmful. 34 | 35 | By adopting this Code of Conduct, project maintainers commit themselves to fairly and 36 | consistently applying these principles to every aspect of managing this project. Project 37 | maintainers who do not follow or enforce the Code of Conduct may be permanently removed 38 | from the project team. 39 | 40 | This code of conduct applies both within project spaces and in public spaces when an 41 | individual is representing the project or its community. 42 | 43 | Instances of abusive, harassing, or otherwise unacceptable behavior may be reported by 44 | contacting a project maintainer at [conduct@swift.org](mailto:conduct@swift.org). All complaints will be reviewed and 45 | investigated and will result in a response that is deemed necessary and appropriate to the 46 | circumstances. Maintainers are obligated to maintain confidentiality with regard to the reporter 47 | of an incident. 48 | 49 | *This policy is adapted from the Contributor Code of Conduct [version 1.3.0](http://contributor-covenant.org/version/1/3/0/).* 50 | 51 | ### Reporting 52 | A working group of community members is committed to promptly addressing any [reported 53 | issues](mailto:conduct@swift.org). Working group members are volunteers appointed by the project lead, with a 54 | preference for individuals with varied backgrounds and perspectives. Membership is expected 55 | to change regularly, and may grow or shrink. 56 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | By submitting a pull request, you represent that you have the right to license 2 | your contribution to Apple and the community, and agree by submitting the patch 3 | that your contributions are licensed under the [Swift 4 | license](https://swift.org/LICENSE.txt). 5 | 6 | --- 7 | 8 | Before submitting the pull request, please make sure you have tested your 9 | changes and that they follow the Swift project [guidelines for contributing 10 | code](https://swift.org/contributing/#contributing-code). 11 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | # This source file is part of the Swift.org open source project 2 | # 3 | # Copyright (c) 2020 Apple Inc. and the Swift project authors 4 | # Licensed under Apache License v2.0 with Runtime Library Exception 5 | # 6 | # See http://swift.org/LICENSE.txt for license information 7 | # See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 8 | 9 | .PHONY: 10 | generate: clean generate-protos 11 | 12 | # These command should be executed any time the proto definitions change. It is 13 | # not required to be generated as part of a regular `swift build` since we're 14 | # checking in the generated sources. 15 | .PHONY: 16 | generate-protos: proto-toolchain 17 | mkdir -p Sources/TSFCAS/Generated 18 | Utilities/tools/bin/protoc \ 19 | -I=Protos \ 20 | --plugin=Utilities/tools/bin/protoc-gen-swift \ 21 | --swift_out=Sources/TSFCAS/Generated \ 22 | --swift_opt=Visibility=Public \ 23 | --swift_opt=ProtoPathModuleMappings=Protos/module_map.asciipb \ 24 | $$(find Protos/CASProtocol -name \*.proto) 25 | mkdir -p Sources/TSFCASFileTree/Generated 26 | Utilities/tools/bin/protoc \ 27 | -I=Protos \ 28 | --plugin=Utilities/tools/bin/protoc-gen-swift \ 29 | --swift_out=Sources/TSFCASFileTree/Generated \ 30 | --swift_opt=Visibility=Public \ 31 | --swift_opt=ProtoPathModuleMappings=Protos/module_map.asciipb \ 32 | $$(find Protos/CASFileTreeProtocol -name \*.proto) 33 | 34 | .PHONY: 35 | proto-toolchain: 36 | Utilities/build_proto_toolchain.sh 37 | 38 | .PHONY: 39 | clean: 40 | rm -rf Sources/TSFCAS/Generated 41 | rm -rf Sources/TSFCASFileTree/Generated 42 | -------------------------------------------------------------------------------- /Package.resolved: -------------------------------------------------------------------------------- 1 | { 2 | "object": { 3 | "pins": [ 4 | { 5 | "package": "swift-atomics", 6 | "repositoryURL": "https://github.com/apple/swift-atomics.git", 7 | "state": { 8 | "branch": null, 9 | "revision": "cd142fd2f64be2100422d658e7411e39489da985", 10 | "version": "1.2.0" 11 | } 12 | }, 13 | { 14 | "package": "swift-collections", 15 | "repositoryURL": "https://github.com/apple/swift-collections.git", 16 | "state": { 17 | "branch": null, 18 | "revision": "671108c96644956dddcd89dd59c203dcdb36cec7", 19 | "version": "1.1.4" 20 | } 21 | }, 22 | { 23 | "package": "swift-nio", 24 | "repositoryURL": "https://github.com/apple/swift-nio.git", 25 | "state": { 26 | "branch": null, 27 | "revision": "f7dc3f527576c398709b017584392fb58592e7f5", 28 | "version": "2.75.0" 29 | } 30 | }, 31 | { 32 | "package": "SwiftProtobuf", 33 | "repositoryURL": "https://github.com/apple/swift-protobuf.git", 34 | "state": { 35 | "branch": null, 36 | "revision": "ebc7251dd5b37f627c93698e4374084d98409633", 37 | "version": "1.28.2" 38 | } 39 | }, 40 | { 41 | "package": "swift-system", 42 | "repositoryURL": "https://github.com/apple/swift-system.git", 43 | "state": { 44 | "branch": null, 45 | "revision": "c8a44d836fe7913603e246acab7c528c2e780168", 46 | "version": "1.4.0" 47 | } 48 | }, 49 | { 50 | "package": "swift-tools-support-core", 51 | "repositoryURL": "https://github.com/apple/swift-tools-support-core.git", 52 | "state": { 53 | "branch": null, 54 | "revision": "ad2fc22a00b898c7af5d8c74a555666f67a06720", 55 | "version": "0.7.0" 56 | } 57 | } 58 | ] 59 | }, 60 | "version": 1 61 | } 62 | -------------------------------------------------------------------------------- /Package.swift: -------------------------------------------------------------------------------- 1 | // swift-tools-version:5.5 2 | import PackageDescription 3 | import class Foundation.ProcessInfo 4 | 5 | let macOSPlatform: SupportedPlatform 6 | let iOSPlatform: SupportedPlatform 7 | if let deploymentTarget = ProcessInfo.processInfo.environment["SWIFTTSC_MACOS_DEPLOYMENT_TARGET"] { 8 | macOSPlatform = .macOS(deploymentTarget) 9 | } else { 10 | macOSPlatform = .macOS(.v10_15) 11 | } 12 | if let deploymentTarget = ProcessInfo.processInfo.environment["SWIFTTSC_IOS_DEPLOYMENT_TARGET"] { 13 | iOSPlatform = .iOS(deploymentTarget) 14 | } else { 15 | iOSPlatform = .iOS(.v13) 16 | } 17 | 18 | let package = Package( 19 | name: "swift-tools-support-async", 20 | platforms: [ 21 | macOSPlatform, 22 | iOSPlatform 23 | ], 24 | products: [ 25 | .library( 26 | name: "SwiftToolsSupportAsync", 27 | targets: ["TSFFutures", "TSFUtility"]), 28 | .library( 29 | name: "SwiftToolsSupportCAS", 30 | targets: ["TSFCAS", "TSFCASFileTree", "TSFCASUtilities"]), 31 | ], 32 | dependencies: [ 33 | .package(url: "https://github.com/apple/swift-nio.git", from: "2.68.0"), 34 | .package(url: "https://github.com/apple/swift-protobuf.git", from: "1.28.0"), 35 | .package(url: "https://github.com/apple/swift-tools-support-core.git", "0.5.8" ..< "0.8.0"), 36 | ], 37 | targets: [ 38 | // BLAKE3 hash support 39 | .target( 40 | name: "CBLAKE3", 41 | dependencies: [], 42 | cSettings: [ 43 | .headerSearchPath("./"), 44 | ] 45 | ), 46 | 47 | .target( 48 | name: "TSFFutures", 49 | dependencies: [ 50 | .product(name: "NIO", package: "swift-nio"), 51 | .product(name: "NIOFoundationCompat", package: "swift-nio"), 52 | .product(name: "SwiftToolsSupport-auto", package: "swift-tools-support-core") 53 | ] 54 | ), 55 | .testTarget( 56 | name: "TSFFuturesTests", 57 | dependencies: [ 58 | "TSFFutures", 59 | ] 60 | ), 61 | .target( 62 | name: "TSFUtility", 63 | dependencies: [ 64 | "TSFFutures", 65 | .product(name: "NIO", package: "swift-nio"), 66 | .product(name: "NIOConcurrencyHelpers", package: "swift-nio"), 67 | .product(name: "NIOFoundationCompat", package: "swift-nio"), 68 | ] 69 | ), 70 | 71 | .target( 72 | name: "TSFCAS", 73 | dependencies: [ 74 | "TSFFutures", "TSFUtility", "CBLAKE3", 75 | .product(name: "SwiftProtobuf", package: "swift-protobuf") 76 | ] 77 | ), 78 | .target( 79 | name: "TSFCASUtilities", 80 | dependencies: [ 81 | "TSFCAS", "TSFCASFileTree", 82 | ] 83 | ), 84 | .testTarget( 85 | name: "TSFCASTests", 86 | dependencies: ["TSFCAS"] 87 | ), 88 | .testTarget( 89 | name: "TSFCASUtilitiesTests", 90 | dependencies: ["TSFCASUtilities"] 91 | ), 92 | .target( 93 | name: "TSFCASFileTree", 94 | dependencies: ["TSFCAS"] 95 | ), 96 | .testTarget( 97 | name: "TSFCASFileTreeTests", 98 | dependencies: ["TSFCASFileTree"] 99 | ), 100 | ] 101 | ) 102 | -------------------------------------------------------------------------------- /Protos/CASFileTreeProtocol/file_tree.proto: -------------------------------------------------------------------------------- 1 | // This source file is part of the Swift.org open source project 2 | // 3 | // Copyright (c) 2020 Apple Inc. and the Swift project authors 4 | // Licensed under Apache License v2.0 with Runtime Library Exception 5 | // 6 | // See http://swift.org/LICENSE.txt for license information 7 | // See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 8 | 9 | syntax = "proto3"; 10 | 11 | option java_package = "com.apple.CASFileTreeProtocol"; 12 | 13 | enum LLBFileType { 14 | /// A plain file. 15 | PLAIN_FILE = 0x0; 16 | 17 | /// An executable file. 18 | EXECUTABLE = 0x1; 19 | 20 | /// A directory. 21 | DIRECTORY = 0x2; 22 | 23 | /// A symbolic link. 24 | SYMLINK = 0x3; 25 | } 26 | 27 | message LLBPosixFileDetails { 28 | /// The POSIX permissions (&0o7777). Masking is useful when storing entries 29 | /// with very restricted permissions (such as (perm & 0o0007) == 0). 30 | uint32 mode = 1; 31 | 32 | /// Owner user identifier. 33 | /// Semantically, absent owner == 0x0 ~= current uid. 34 | uint32 owner = 2; 35 | 36 | /// Owner group identifier. 37 | /// Semantically, absent owner == 0x0 ~= current gid. 38 | uint32 group = 3; 39 | } 40 | 41 | message LLBDirectoryEntry { 42 | /// The name of the directory entry. 43 | string name = 1; 44 | 45 | /// The type of the directory entry. 46 | LLBFileType type = 2; 47 | 48 | /// The (aggregate) size of the directory entry. 49 | uint64 size = 3; 50 | 51 | /// Mode and permissions. _Can_ optionally be present in the 52 | /// directory entry because the file can be just a direct blob reference. 53 | LLBPosixFileDetails posixDetails = 4; 54 | } 55 | 56 | /// The list of file names and associated information, of a directory. 57 | /// * The children are sorted by name. 58 | /// * FIXME: collation rules or UTF-8 normalization guarantees? 59 | message LLBDirectoryEntries { 60 | repeated LLBDirectoryEntry entries = 1; 61 | } 62 | 63 | enum LLBFileDataCompressionMethod { 64 | /// No compression is applied. 65 | NONE = 0x0; 66 | } 67 | 68 | message LLBFileInfo { 69 | /// The type of the CASTree entry. 70 | LLBFileType type = 1; 71 | 72 | /// The file data size or the aggregate directory data size (recursive). 73 | /// Whether directory data includes the size of the directory catalogs 74 | /// is unspecified. 75 | uint64 size = 2; 76 | 77 | /// OBSOLETE. Use posixDetails. 78 | /// The POSIX permissions (&0o777). Useful when storing entries 79 | /// with very restricted permissions (such as (perm & 0o007) == 0). 80 | uint32 posixPermissions = 3; 81 | 82 | /// Whether and what compression is applied to file data. 83 | /// * Compression ought not to be applied to symlinks. 84 | /// * Compression is applied after chunking, to retain seekability. 85 | LLBFileDataCompressionMethod compression = 4; 86 | 87 | /// Permission info useful for POSIX filesystems. 88 | LLBPosixFileDetails posixDetails = 5; 89 | 90 | oneof payload { 91 | /// Files and symlinks: 92 | /// * The file payload is contained in one or more 93 | /// fixed size references to [compressed] data. 94 | /// * The `fixedChunkSize` value helps to do O(1) seeking. 95 | uint64 fixedChunkSize = 11; 96 | 97 | /// Directories: 98 | /// * Directory entries are represented inline. 99 | LLBDirectoryEntries inlineChildren = 12; 100 | 101 | /// Directories: 102 | /// * Directory entries are represented as a reference to a B-tree. 103 | /// * The `compression` does have effect on the B-tree data. 104 | uint32 referencedChildrenTree = 13; 105 | } 106 | } 107 | -------------------------------------------------------------------------------- /Protos/CASProtocol/cas_object.proto: -------------------------------------------------------------------------------- 1 | // This source file is part of the Swift.org open source project 2 | // 3 | // Copyright (c) 2020 Apple Inc. and the Swift project authors 4 | // Licensed under Apache License v2.0 with Runtime Library Exception 5 | // 6 | // See http://swift.org/LICENSE.txt for license information 7 | // See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 8 | 9 | syntax = "proto3"; 10 | 11 | option java_package = "com.apple.CASProtocol"; 12 | 13 | import "CASProtocol/data_id.proto"; 14 | 15 | /// LLBPBCASObject represents the serialized from of CASObjects. It encodes the 16 | /// combination of the raw data of the object and its dependendent references. 17 | message LLBPBCASObject { 18 | repeated LLBDataID refs = 1; 19 | 20 | bytes data = 2; 21 | } 22 | -------------------------------------------------------------------------------- /Protos/CASProtocol/data_id.proto: -------------------------------------------------------------------------------- 1 | // This source file is part of the Swift.org open source project 2 | // 3 | // Copyright (c) 2020 Apple Inc. and the Swift project authors 4 | // Licensed under Apache License v2.0 with Runtime Library Exception 5 | // 6 | // See http://swift.org/LICENSE.txt for license information 7 | // See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 8 | 9 | syntax = "proto3"; 10 | 11 | option java_package = "com.apple.CASProtocol"; 12 | 13 | /// LLBDataID represents the digest of arbitrary data, and its purpose is to be a handler for interfacing with CAS 14 | /// systems. LLBDataID does not require the encoding of any particular hash function. Instead, it is expected that the 15 | /// CAS system itself that provides the digest. 16 | message LLBDataID { 17 | /// The bytes containing the digest of the contents store in the CAS. 18 | bytes bytes = 1; 19 | } 20 | -------------------------------------------------------------------------------- /Protos/module_map.asciipb: -------------------------------------------------------------------------------- 1 | mapping { 2 | module_name: "TSFCAS" 3 | proto_file_path: "CASProtocol/cas_object.proto" 4 | proto_file_path: "CASProtocol/data_id.proto" 5 | } 6 | mapping { 7 | module_name: "TSFCASFileTree" 8 | proto_file_path: "CASFileTreeProtocol/file_tree.proto" 9 | } 10 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # swift-tools-support-async 2 | 3 | Common infrastructural helpers on top of NIO for [llbuild2](https://github.com/apple/swift-llbuild2) and [swiftpm-on-llbuild2](https://github.com/apple/swiftpm-on-llbuild2) projects. This is **NOT** a general purpose package and is unlikely to ever become stable. 4 | 5 | ## License 6 | 7 | Copyright (c) 2020 Apple Inc. and the Swift project authors. 8 | Licensed under Apache License v2.0 with Runtime Library Exception. 9 | 10 | See https://swift.org/LICENSE.txt for license information. 11 | 12 | See https://swift.org/CONTRIBUTORS.txt for Swift project authors. 13 | -------------------------------------------------------------------------------- /Sources/CBLAKE3/blake3.c: -------------------------------------------------------------------------------- 1 | #if __x86_64__ 2 | 3 | #ifndef __SSE4_1__ 4 | #define BLAKE3_NO_SSE41 5 | #endif 6 | #ifndef __AVX2__ 7 | #define BLAKE3_NO_AVX2 8 | #endif 9 | #ifndef __AVX512__ 10 | #define BLAKE3_NO_AVX512 11 | #endif 12 | 13 | #include "../../ThirdParty/BLAKE3/c/blake3.c" 14 | #include "../../ThirdParty/BLAKE3/c/blake3_dispatch.c" 15 | #include "../../ThirdParty/BLAKE3/c/blake3_portable.c" 16 | #else 17 | #include "../../ThirdParty/BLAKE3/c/blake3.c" 18 | #include "../../ThirdParty/BLAKE3/c/blake3_dispatch.c" 19 | #include "../../ThirdParty/BLAKE3/c/blake3_portable.c" 20 | #endif 21 | -------------------------------------------------------------------------------- /Sources/CBLAKE3/blake3_avx2.c: -------------------------------------------------------------------------------- 1 | #if __AVX2__ 2 | #include "../../ThirdParty/BLAKE3/c/blake3_avx2.c" 3 | #endif 4 | -------------------------------------------------------------------------------- /Sources/CBLAKE3/blake3_avx512.c: -------------------------------------------------------------------------------- 1 | #if __AVX512__ 2 | #include "../../ThirdParty/BLAKE3/c/blake3_avx512.c" 3 | #endif 4 | -------------------------------------------------------------------------------- /Sources/CBLAKE3/blake3_impl.h: -------------------------------------------------------------------------------- 1 | ../../ThirdParty/BLAKE3/c/blake3_impl.h -------------------------------------------------------------------------------- /Sources/CBLAKE3/blake3_sse41.c: -------------------------------------------------------------------------------- 1 | #if __SSE4_1__ 2 | #include "../../ThirdParty/BLAKE3/c/blake3_sse41.c" 3 | #endif 4 | -------------------------------------------------------------------------------- /Sources/CBLAKE3/include/blake3.h: -------------------------------------------------------------------------------- 1 | ../../../ThirdParty/BLAKE3/c/blake3.h -------------------------------------------------------------------------------- /Sources/TSFCAS/DataID.swift: -------------------------------------------------------------------------------- 1 | // This source file is part of the Swift.org open source project 2 | // 3 | // Copyright (c) 2020-2021 Apple Inc. and the Swift project authors 4 | // Licensed under Apache License v2.0 with Runtime Library Exception 5 | // 6 | // See http://swift.org/LICENSE.txt for license information 7 | // See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 8 | 9 | import Foundation 10 | 11 | import NIOCore 12 | import TSCBasic 13 | import TSFUtility 14 | 15 | 16 | // MARK:- DataID Extensions - 17 | 18 | fileprivate enum DataIDKind: UInt8 { 19 | /// An id that is directly calculated based on a hash of the data. 20 | case directHash = 0 21 | case shareableHash = 4 22 | 23 | init?(from bytes: Data) { 24 | guard let first = bytes.first, 25 | let kind = DataIDKind(rawValue: first) else { 26 | return nil 27 | } 28 | self = kind 29 | } 30 | 31 | init?(from substring: Substring) { 32 | guard let first = substring.utf8.first, 33 | first >= UInt8(ascii: "0") else { 34 | return nil 35 | } 36 | self.init(rawValue: first - UInt8(ascii: "0")) 37 | } 38 | } 39 | 40 | 41 | extension LLBDataID: Hashable, CustomDebugStringConvertible { 42 | 43 | /// Represent DataID as string to encode it in messages. 44 | /// Properties of the string: the first character represents the kind, 45 | /// then '~', then the Base64 encoding follows. 46 | public var debugDescription: String { 47 | return ArraySlice(bytes.dropFirst()).base64URL(prepending: 48 | [(bytes.first ?? 15) + UInt8(ascii: "0"), UInt8(ascii: "~")]) 49 | } 50 | 51 | public init?(bytes: [UInt8]) { 52 | let data = Data(bytes) 53 | guard DataIDKind(from: data) != nil else { 54 | return nil 55 | } 56 | self.bytes = data 57 | } 58 | 59 | public init(directHash bytes: [UInt8]) { 60 | self.bytes = Data([DataIDKind.directHash.rawValue] + bytes) 61 | } 62 | 63 | /// Initialize from the string form. 64 | public init?(string: String) { 65 | self.init(string: Substring(string)) 66 | } 67 | 68 | public init?(string: Substring) { 69 | // Test for the kind in the first position. 70 | guard let kind = DataIDKind(from: string) else { return nil } 71 | 72 | // Test for "~" in the second position. 73 | guard string.count >= 2 else { return nil } 74 | let tilde = string.utf8[string.utf8.index(string.startIndex, offsetBy: 1)] 75 | guard tilde == UInt8(ascii: "~") else { return nil } 76 | 77 | let b64substring = string.dropFirst(2) 78 | guard let completeBytes = [UInt8](base64URL: b64substring, prepending: [kind.rawValue]) else { 79 | return nil 80 | } 81 | 82 | self.bytes = Data(completeBytes) 83 | } 84 | } 85 | 86 | 87 | extension LLBDataID: Comparable { 88 | /// Compare DataID according to stable but arbitrary rules 89 | /// (not necessarily alphanumeric). 90 | public static func < (lhs: LLBDataID, rhs: LLBDataID) -> Bool { 91 | let a = lhs.bytes 92 | let b = rhs.bytes 93 | if a.count == b.count { 94 | for n in (0..) throws { 138 | guard let dataId = LLBDataID(bytes: Array(rawBytes)) else { 139 | throw LLBDataIDSliceError.decoding("from slice of size \(rawBytes.count)") 140 | } 141 | self = dataId 142 | } 143 | 144 | @inlinable 145 | public init(from rawBytes: LLBByteBuffer) throws { 146 | guard let dataId = LLBDataID(bytes: Array(buffer: rawBytes)) else { 147 | throw LLBDataIDSliceError.decoding("from slice of size \(rawBytes.readableBytes)") 148 | } 149 | self = dataId 150 | } 151 | 152 | @inlinable 153 | public func toBytes() -> ArraySlice { 154 | return ArraySlice(bytes) 155 | } 156 | 157 | @inlinable 158 | public func toBytes(into array: inout [UInt8]) { 159 | array += bytes 160 | } 161 | 162 | @inlinable 163 | public func toBytes(into buffer: inout LLBByteBuffer) throws { 164 | buffer.writeBytes(bytes) 165 | } 166 | 167 | @inlinable 168 | public var sliceSizeEstimate: Int { 169 | return bytes.count 170 | } 171 | } 172 | 173 | -------------------------------------------------------------------------------- /Sources/TSFCAS/Database.swift: -------------------------------------------------------------------------------- 1 | // This source file is part of the Swift.org open source project 2 | // 3 | // Copyright (c) 2020 Apple Inc. and the Swift project authors 4 | // Licensed under Apache License v2.0 with Runtime Library Exception 5 | // 6 | // See http://swift.org/LICENSE.txt for license information 7 | // See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 8 | 9 | import Foundation 10 | 11 | import NIOCore 12 | import TSCUtility 13 | 14 | @_exported import TSFFutures 15 | @_exported import TSFUtility 16 | 17 | 18 | /// Error wrappers that implementations may use to communicate desired higher 19 | /// level responses. 20 | public enum LLBCASDatabaseError: Error { 21 | /// The database encountered a network related error that may resolve if the 22 | /// operation is tried again (with some delay). 23 | case retryableNetworkError(Error) 24 | 25 | /// The database encountered a network related error that is not recoverable. 26 | case terminalNetworkError(Error) 27 | } 28 | 29 | /// Features supported by a CAS Database 30 | public struct LLBCASFeatures: Codable { 31 | 32 | /// Whether a database is "ID preserving" 33 | /// 34 | /// An ID preserving database will *always* honor the id passed to a 35 | /// `put(knownID: ...)` request. i.e. on success the returned 36 | /// DataID will match. 37 | public let preservesIDs: Bool 38 | 39 | public init(preservesIDs: Bool = true) { 40 | self.preservesIDs = preservesIDs 41 | } 42 | } 43 | 44 | /// A content-addressable database protocol 45 | /// 46 | /// THREAD-SAFETY: The database is expected to be thread-safe. 47 | public protocol LLBCASDatabase: AnyObject & Sendable { 48 | var group: LLBFuturesDispatchGroup { get } 49 | 50 | /// Get the supported features of this database implementation 51 | func supportedFeatures() -> LLBFuture 52 | 53 | /// Check if the database contains the given `id`. 54 | func contains(_ id: LLBDataID, _ ctx: Context) -> LLBFuture 55 | 56 | /// Get the object corresponding to the given `id`. 57 | /// 58 | /// - Parameters: 59 | /// - id: The id of the object to look up 60 | /// - Returns: The object, or nil if not present in the database. 61 | func get(_ id: LLBDataID, _ ctx: Context) -> LLBFuture 62 | 63 | /// Calculate the DataID for the given CAS object. 64 | /// 65 | /// The implementation *MUST* return a valid content-address, such 66 | /// that a subsequent call to `put(knownID:...` will return an identical 67 | /// `id`. This method should be implemented as efficiently as possible, 68 | /// ideally locally. 69 | /// 70 | /// NOTE: The implementations *MAY* store the content, as if it were `put`. 71 | /// Clients *MAY NOT* assume the data has been written. 72 | /// 73 | /// 74 | /// - Parameters: 75 | /// - refs: The list of objects references. 76 | /// - data: The object contents. 77 | /// - Returns: The id representing the combination of contents and refs. 78 | func identify(refs: [LLBDataID], data: LLBByteBuffer, _ ctx: Context) -> LLBFuture 79 | 80 | /// Store an object. 81 | /// 82 | /// - Parameters: 83 | /// - refs: The list of objects references. 84 | /// - data: The object contents. 85 | /// - Returns: The id representing the combination of contents and refs. 86 | func put(refs: [LLBDataID], data: LLBByteBuffer, _ ctx: Context) -> LLBFuture 87 | 88 | /// Store an object with a known id. 89 | /// 90 | /// In such situations, the `id` *MUST* be a valid content-address for the 91 | /// object, such that there *MUST NOT* be any other combination of refs 92 | /// and data which could yield the same `id`. The `id`, however, *MAY* 93 | /// be different from the id the database would otherwise have assigned given 94 | /// the content without a known ID. 95 | /// 96 | /// NOTE: The implementation *MAY* choose to reject the known ID, and store 97 | /// the data under its own. The client *MUST* respect the provided result ID, 98 | /// and *MAY NOT* assume that a successful write allows access under the 99 | /// provided `id`. 100 | /// 101 | /// - Parameters: 102 | /// - id: The id of the object, if known. 103 | /// - refs: The list of object references. 104 | /// - data: The object contents. 105 | func put(knownID id: LLBDataID, refs: [LLBDataID], data: LLBByteBuffer, _ ctx: Context) -> LLBFuture 106 | } 107 | 108 | public extension LLBCASDatabase { 109 | @inlinable 110 | func identify(_ object: LLBCASObject, _ ctx: Context) -> LLBFuture { 111 | return identify(refs: object.refs, data: object.data, ctx) 112 | } 113 | 114 | @inlinable 115 | func put(_ object: LLBCASObject, _ ctx: Context) -> LLBFuture { 116 | return put(refs: object.refs, data: object.data, ctx) 117 | } 118 | 119 | @inlinable 120 | func put(knownID id: LLBDataID, object: LLBCASObject, _ ctx: Context) -> LLBFuture { 121 | return put(knownID: id, refs: object.refs, data: object.data, ctx) 122 | } 123 | } 124 | 125 | public extension LLBCASDatabase { 126 | @inlinable 127 | func identify(refs: [LLBDataID], data: LLBByteBufferView, _ ctx: Context) -> LLBFuture { 128 | return identify(refs: refs, data: LLBByteBuffer(data), ctx) 129 | } 130 | 131 | @inlinable 132 | func put(refs: [LLBDataID], data: LLBByteBufferView, _ ctx: Context) -> LLBFuture { 133 | return put(refs: refs, data: LLBByteBuffer(data), ctx) 134 | } 135 | 136 | @inlinable 137 | func put(data: LLBByteBuffer, _ ctx: Context) -> LLBFuture { 138 | return self.put(refs: [], data: data, ctx) 139 | } 140 | 141 | @inlinable 142 | func put(knownID id: LLBDataID, refs: [LLBDataID], data: LLBByteBufferView, _ ctx: Context) -> LLBFuture { 143 | return put(knownID: id, refs: refs, data: LLBByteBuffer(data), ctx) 144 | } 145 | } 146 | 147 | /// Support storing and retrieving a CAS database from a context 148 | public extension Context { 149 | static func with(_ db: LLBCASDatabase) -> Context { 150 | return Context(dictionaryLiteral: (ObjectIdentifier(LLBCASDatabase.self), db as Any)) 151 | } 152 | 153 | var db: LLBCASDatabase { 154 | get { 155 | guard let db = self[ObjectIdentifier(LLBCASDatabase.self), as: LLBCASDatabase.self] else { 156 | fatalError("no CAS database") 157 | } 158 | return db 159 | } 160 | set { 161 | self[ObjectIdentifier(LLBCASDatabase.self)] = newValue 162 | } 163 | } 164 | } 165 | -------------------------------------------------------------------------------- /Sources/TSFCAS/DatabaseSpec.swift: -------------------------------------------------------------------------------- 1 | // This source file is part of the Swift.org open source project 2 | // 3 | // Copyright (c) 2020 Apple Inc. and the Swift project authors 4 | // Licensed under Apache License v2.0 with Runtime Library Exception 5 | // 6 | // See http://swift.org/LICENSE.txt for license information 7 | // See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 8 | 9 | import Foundation 10 | 11 | import NIOCore 12 | import TSFFutures 13 | 14 | 15 | /// A scheme for specifying a CASDatabase. 16 | public protocol LLBCASDatabaseScheme { 17 | /// The name of the scheme. 18 | static var scheme: String { get } 19 | 20 | /// Check if a URL is valid for this scheme. 21 | static func isValid(host: String?, port: Int?, path: String, query: String?) -> Bool 22 | 23 | /// Open a content store in this scheme. 24 | static func open(group: LLBFuturesDispatchGroup, url: URL) throws -> LLBCASDatabase 25 | } 26 | 27 | 28 | /// A specification for a CAS database location. 29 | /// 30 | /// Specifications are written using a URL scheme, for example: 31 | /// 32 | /// mem:// 33 | public struct LLBCASDatabaseSpec { 34 | /// The map of registered schemes. 35 | private static var registeredSchemes: [String: LLBCASDatabaseScheme.Type] = [ 36 | "mem": LLBInMemoryCASDatabaseScheme.self, 37 | "file": LLBFileBackedCASDatabaseScheme.self, 38 | ] 39 | 40 | /// Register a content store scheme type. 41 | /// 42 | /// This method is *not* thread safe. 43 | public static func register(schemeType: LLBCASDatabaseScheme.Type) { 44 | precondition(registeredSchemes[schemeType.scheme] == nil) 45 | registeredSchemes[schemeType.scheme] = schemeType 46 | } 47 | 48 | /// The underlying URL. 49 | public let url: URL 50 | 51 | /// The scheme definition. 52 | public let schemeType: LLBCASDatabaseScheme.Type 53 | 54 | public enum Error: Swift.Error { 55 | case noScheme 56 | case urlError(String) 57 | } 58 | 59 | /// Create a new spec for the given URL string. 60 | public init(_ string: String) throws { 61 | guard let url = URL(string: string) else { 62 | throw Error.urlError("URL parse error for \(string) for a CAS Database") 63 | } 64 | try self.init(url) 65 | } 66 | 67 | /// Create a new spec for the given URL. 68 | public init(_ url: URL) throws { 69 | guard let scheme = url.scheme else { 70 | throw Error.noScheme 71 | } 72 | 73 | // If the scheme isn't known, this isn't a valid spec. 74 | guard let schemeType = LLBCASDatabaseSpec.registeredSchemes[scheme] else { 75 | throw Error.urlError("Unknown URL scheme \"\(scheme)\" for a CAS Database at \(url)") 76 | } 77 | 78 | // Validate the URL with the scheme. 79 | if !schemeType.isValid(host: url.host, port: url.port, path: url.path, query: url.query) { 80 | throw Error.urlError("Invalid URL \(url) for a CAS Database") 81 | } 82 | 83 | self.url = url 84 | self.schemeType = schemeType 85 | } 86 | 87 | /// Open the specified store. 88 | public func open(group: LLBFuturesDispatchGroup) throws -> LLBCASDatabase { 89 | return try schemeType.open(group: group, url: url) 90 | } 91 | } 92 | 93 | extension LLBCASDatabaseSpec: Equatable { 94 | public static func ==(lhs: LLBCASDatabaseSpec, rhs: LLBCASDatabaseSpec) -> Bool { 95 | return lhs.url == rhs.url 96 | } 97 | } 98 | -------------------------------------------------------------------------------- /Sources/TSFCAS/Generated/CASProtocol/cas_object.pb.swift: -------------------------------------------------------------------------------- 1 | // DO NOT EDIT. 2 | // swift-format-ignore-file 3 | // 4 | // Generated by the Swift generator plugin for the protocol buffer compiler. 5 | // Source: CASProtocol/cas_object.proto 6 | // 7 | // For information on using the generated types, please see the documentation: 8 | // https://github.com/apple/swift-protobuf/ 9 | 10 | // This source file is part of the Swift.org open source project 11 | // 12 | // Copyright (c) 2020 Apple Inc. and the Swift project authors 13 | // Licensed under Apache License v2.0 with Runtime Library Exception 14 | // 15 | // See http://swift.org/LICENSE.txt for license information 16 | // See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 17 | 18 | import Foundation 19 | import SwiftProtobuf 20 | 21 | // If the compiler emits an error on this type, it is because this file 22 | // was generated by a version of the `protoc` Swift plug-in that is 23 | // incompatible with the version of SwiftProtobuf to which you are linking. 24 | // Please ensure that you are building against the same version of the API 25 | // that was used to generate this file. 26 | fileprivate struct _GeneratedWithProtocGenSwiftVersion: SwiftProtobuf.ProtobufAPIVersionCheck { 27 | struct _2: SwiftProtobuf.ProtobufAPIVersion_2 {} 28 | typealias Version = _2 29 | } 30 | 31 | //// LLBPBCASObject represents the serialized from of CASObjects. It encodes the 32 | //// combination of the raw data of the object and its dependendent references. 33 | public struct LLBPBCASObject { 34 | // SwiftProtobuf.Message conformance is added in an extension below. See the 35 | // `Message` and `Message+*Additions` files in the SwiftProtobuf library for 36 | // methods supported on all messages. 37 | 38 | public var refs: [LLBDataID] = [] 39 | 40 | public var data: Data = Data() 41 | 42 | public var unknownFields = SwiftProtobuf.UnknownStorage() 43 | 44 | public init() {} 45 | } 46 | 47 | #if swift(>=5.5) && canImport(_Concurrency) 48 | extension LLBPBCASObject: @unchecked Sendable {} 49 | #endif // swift(>=5.5) && canImport(_Concurrency) 50 | 51 | // MARK: - Code below here is support for the SwiftProtobuf runtime. 52 | 53 | extension LLBPBCASObject: SwiftProtobuf.Message, SwiftProtobuf._MessageImplementationBase, SwiftProtobuf._ProtoNameProviding { 54 | public static let protoMessageName: String = "LLBPBCASObject" 55 | public static let _protobuf_nameMap: SwiftProtobuf._NameMap = [ 56 | 1: .same(proto: "refs"), 57 | 2: .same(proto: "data"), 58 | ] 59 | 60 | public mutating func decodeMessage(decoder: inout D) throws { 61 | while let fieldNumber = try decoder.nextFieldNumber() { 62 | // The use of inline closures is to circumvent an issue where the compiler 63 | // allocates stack space for every case branch when no optimizations are 64 | // enabled. https://github.com/apple/swift-protobuf/issues/1034 65 | switch fieldNumber { 66 | case 1: try { try decoder.decodeRepeatedMessageField(value: &self.refs) }() 67 | case 2: try { try decoder.decodeSingularBytesField(value: &self.data) }() 68 | default: break 69 | } 70 | } 71 | } 72 | 73 | public func traverse(visitor: inout V) throws { 74 | if !self.refs.isEmpty { 75 | try visitor.visitRepeatedMessageField(value: self.refs, fieldNumber: 1) 76 | } 77 | if !self.data.isEmpty { 78 | try visitor.visitSingularBytesField(value: self.data, fieldNumber: 2) 79 | } 80 | try unknownFields.traverse(visitor: &visitor) 81 | } 82 | 83 | public static func ==(lhs: LLBPBCASObject, rhs: LLBPBCASObject) -> Bool { 84 | if lhs.refs != rhs.refs {return false} 85 | if lhs.data != rhs.data {return false} 86 | if lhs.unknownFields != rhs.unknownFields {return false} 87 | return true 88 | } 89 | } 90 | -------------------------------------------------------------------------------- /Sources/TSFCAS/Generated/CASProtocol/data_id.pb.swift: -------------------------------------------------------------------------------- 1 | // DO NOT EDIT. 2 | // swift-format-ignore-file 3 | // 4 | // Generated by the Swift generator plugin for the protocol buffer compiler. 5 | // Source: CASProtocol/data_id.proto 6 | // 7 | // For information on using the generated types, please see the documentation: 8 | // https://github.com/apple/swift-protobuf/ 9 | 10 | // This source file is part of the Swift.org open source project 11 | // 12 | // Copyright (c) 2020 Apple Inc. and the Swift project authors 13 | // Licensed under Apache License v2.0 with Runtime Library Exception 14 | // 15 | // See http://swift.org/LICENSE.txt for license information 16 | // See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 17 | 18 | import Foundation 19 | import SwiftProtobuf 20 | 21 | // If the compiler emits an error on this type, it is because this file 22 | // was generated by a version of the `protoc` Swift plug-in that is 23 | // incompatible with the version of SwiftProtobuf to which you are linking. 24 | // Please ensure that you are building against the same version of the API 25 | // that was used to generate this file. 26 | fileprivate struct _GeneratedWithProtocGenSwiftVersion: SwiftProtobuf.ProtobufAPIVersionCheck { 27 | struct _2: SwiftProtobuf.ProtobufAPIVersion_2 {} 28 | typealias Version = _2 29 | } 30 | 31 | //// LLBDataID represents the digest of arbitrary data, and its purpose is to be a handler for interfacing with CAS 32 | //// systems. LLBDataID does not require the encoding of any particular hash function. Instead, it is expected that the 33 | //// CAS system itself that provides the digest. 34 | public struct LLBDataID { 35 | // SwiftProtobuf.Message conformance is added in an extension below. See the 36 | // `Message` and `Message+*Additions` files in the SwiftProtobuf library for 37 | // methods supported on all messages. 38 | 39 | //// The bytes containing the digest of the contents store in the CAS. 40 | public var bytes: Data = Data() 41 | 42 | public var unknownFields = SwiftProtobuf.UnknownStorage() 43 | 44 | public init() {} 45 | } 46 | 47 | #if swift(>=5.5) && canImport(_Concurrency) 48 | extension LLBDataID: @unchecked Sendable {} 49 | #endif // swift(>=5.5) && canImport(_Concurrency) 50 | 51 | // MARK: - Code below here is support for the SwiftProtobuf runtime. 52 | 53 | extension LLBDataID: SwiftProtobuf.Message, SwiftProtobuf._MessageImplementationBase, SwiftProtobuf._ProtoNameProviding { 54 | public static let protoMessageName: String = "LLBDataID" 55 | public static let _protobuf_nameMap: SwiftProtobuf._NameMap = [ 56 | 1: .same(proto: "bytes"), 57 | ] 58 | 59 | public mutating func decodeMessage(decoder: inout D) throws { 60 | while let fieldNumber = try decoder.nextFieldNumber() { 61 | // The use of inline closures is to circumvent an issue where the compiler 62 | // allocates stack space for every case branch when no optimizations are 63 | // enabled. https://github.com/apple/swift-protobuf/issues/1034 64 | switch fieldNumber { 65 | case 1: try { try decoder.decodeSingularBytesField(value: &self.bytes) }() 66 | default: break 67 | } 68 | } 69 | } 70 | 71 | public func traverse(visitor: inout V) throws { 72 | if !self.bytes.isEmpty { 73 | try visitor.visitSingularBytesField(value: self.bytes, fieldNumber: 1) 74 | } 75 | try unknownFields.traverse(visitor: &visitor) 76 | } 77 | 78 | public static func ==(lhs: LLBDataID, rhs: LLBDataID) -> Bool { 79 | if lhs.bytes != rhs.bytes {return false} 80 | if lhs.unknownFields != rhs.unknownFields {return false} 81 | return true 82 | } 83 | } 84 | -------------------------------------------------------------------------------- /Sources/TSFCAS/Implementations/Blake3DataID.swift: -------------------------------------------------------------------------------- 1 | // This source file is part of the Swift.org open source project 2 | // 3 | // Copyright (c) 2020 Apple Inc. and the Swift project authors 4 | // Licensed under Apache License v2.0 with Runtime Library Exception 5 | // 6 | // See http://swift.org/LICENSE.txt for license information 7 | // See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 8 | 9 | 10 | import CBLAKE3 11 | import NIOCore 12 | import TSFUtility 13 | 14 | 15 | public extension LLBDataID { 16 | init(blake3hash buffer: LLBByteBuffer, refs: [LLBDataID] = []) { 17 | var hasher = blake3_hasher() 18 | blake3_hasher_init(&hasher) 19 | 20 | for ref in refs { 21 | ref.bytes.withUnsafeBytes { content in 22 | blake3_hasher_update(&hasher, content.baseAddress, content.count) 23 | } 24 | } 25 | buffer.withUnsafeReadableBytes { data in 26 | blake3_hasher_update(&hasher, data.baseAddress, data.count) 27 | } 28 | 29 | let hash = [UInt8](unsafeUninitializedCapacity: Int(BLAKE3_OUT_LEN)) { (hash, len) in 30 | len = Int(BLAKE3_OUT_LEN) 31 | blake3_hasher_finalize(&hasher, hash.baseAddress, len) 32 | } 33 | 34 | self.init(directHash: hash) 35 | } 36 | 37 | init(blake3hash data: [UInt8], refs: [LLBDataID] = []) { 38 | self.init(blake3hash: ArraySlice(data)) 39 | } 40 | init(blake3hash string: String, refs: [LLBDataID] = []) { 41 | self.init(blake3hash: ArraySlice(string.utf8)) 42 | } 43 | 44 | init(blake3hash slice: ArraySlice, refs: [LLBDataID] = []) { 45 | var hasher = blake3_hasher() 46 | blake3_hasher_init(&hasher) 47 | 48 | for ref in refs { 49 | ref.bytes.withUnsafeBytes { content in 50 | blake3_hasher_update(&hasher, content.baseAddress, content.count) 51 | } 52 | } 53 | slice.withUnsafeBytes { data in 54 | blake3_hasher_update(&hasher, data.baseAddress, data.count) 55 | } 56 | 57 | let hash = [UInt8](unsafeUninitializedCapacity: Int(BLAKE3_OUT_LEN)) { (hash, len) in 58 | len = Int(BLAKE3_OUT_LEN) 59 | blake3_hasher_finalize(&hasher, hash.baseAddress, len) 60 | } 61 | 62 | self.init(directHash: hash) 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /Sources/TSFCAS/Implementations/FileBackedCASDatabase.swift: -------------------------------------------------------------------------------- 1 | // This source file is part of the Swift.org open source project 2 | // 3 | // Copyright (c) 2020 Apple Inc. and the Swift project authors 4 | // Licensed under Apache License v2.0 with Runtime Library Exception 5 | // 6 | // See http://swift.org/LICENSE.txt for license information 7 | // See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 8 | 9 | import Foundation 10 | 11 | import NIO 12 | import TSCBasic 13 | import TSCLibc 14 | import TSCUtility 15 | import TSFFutures 16 | import TSFUtility 17 | 18 | 19 | public final class LLBFileBackedCASDatabase: LLBCASDatabase { 20 | /// Prefix for files written to disk. 21 | enum FileNamePrefix: String { 22 | case refs = "refs." 23 | case data = "data." 24 | } 25 | 26 | /// The content root path. 27 | public let path: AbsolutePath 28 | 29 | /// Threads capable of running futures. 30 | public let group: LLBFuturesDispatchGroup 31 | 32 | let threadPool: NIOThreadPool 33 | let fileIO: NonBlockingFileIO 34 | 35 | public init( 36 | group: LLBFuturesDispatchGroup, 37 | path: AbsolutePath 38 | ) { 39 | self.threadPool = NIOThreadPool(numberOfThreads: 6) 40 | threadPool.start() 41 | self.fileIO = NonBlockingFileIO(threadPool: threadPool) 42 | self.group = group 43 | self.path = path 44 | try? localFileSystem.createDirectory(path, recursive: true) 45 | } 46 | 47 | deinit { 48 | try? threadPool.syncShutdownGracefully() 49 | } 50 | 51 | private func fileName(for id: LLBDataID, prefix: FileNamePrefix) -> AbsolutePath { 52 | return path.appending(component: prefix.rawValue + id.debugDescription) 53 | } 54 | 55 | public func supportedFeatures() -> LLBFuture { 56 | group.next().makeSucceededFuture(LLBCASFeatures(preservesIDs: true)) 57 | } 58 | 59 | public func contains(_ id: LLBDataID, _ ctx: Context) -> LLBFuture { 60 | let refsFile = fileName(for: id, prefix: .refs) 61 | let dataFile = fileName(for: id, prefix: .data) 62 | let contains = localFileSystem.exists(refsFile) && localFileSystem.exists(dataFile) 63 | return group.next().makeSucceededFuture(contains) 64 | } 65 | 66 | func readFile(file: AbsolutePath) -> LLBFuture { 67 | let handleAndRegion = fileIO.openFile( 68 | path: file.pathString, eventLoop: group.next() 69 | ) 70 | 71 | let data: LLBFuture = handleAndRegion.flatMap { (handle, region) in 72 | let allocator = ByteBufferAllocator() 73 | return self.fileIO.read( 74 | fileRegion: region, 75 | allocator: allocator, 76 | eventLoop: self.group.next() 77 | ) 78 | } 79 | 80 | return handleAndRegion.and(data).flatMapThrowing { (handle, data) in 81 | try handle.0.close() 82 | return data 83 | } 84 | } 85 | 86 | public func get(_ id: LLBDataID, _ ctx: Context) -> LLBFuture { 87 | let refsFile = fileName(for: id, prefix: .refs) 88 | let dataFile = fileName(for: id, prefix: .data) 89 | 90 | let refsBytes: LLBFuture<[UInt8]> = readFile(file: refsFile).map { refsData in 91 | return Array(buffer: refsData) 92 | } 93 | 94 | let refs = refsBytes.flatMapThrowing { 95 | try JSONDecoder().decode([LLBDataID].self, from: Data($0)) 96 | } 97 | 98 | let data = readFile(file: dataFile) 99 | 100 | return refs.and(data).map { 101 | LLBCASObject(refs: $0.0, data: $0.1) 102 | } 103 | } 104 | 105 | var fs: FileSystem { localFileSystem } 106 | 107 | public func identify( 108 | refs: [LLBDataID] = [], 109 | data: LLBByteBuffer, 110 | _ ctx: Context 111 | ) -> LLBFuture { 112 | return group.next().makeSucceededFuture(LLBDataID(blake3hash: data, refs: refs)) 113 | } 114 | 115 | public func put( 116 | refs: [LLBDataID] = [], 117 | data: LLBByteBuffer, 118 | _ ctx: Context 119 | ) -> LLBFuture { 120 | let id = LLBDataID(blake3hash: data, refs: refs) 121 | return put(knownID: id, refs: refs, data: data, ctx) 122 | } 123 | 124 | public func put( 125 | knownID id: LLBDataID, 126 | refs: [LLBDataID] = [], 127 | data: LLBByteBuffer, 128 | _ ctx: Context 129 | ) -> LLBFuture { 130 | let dataFile = fileName(for: id, prefix: .data) 131 | let dataFuture = writeIfNeeded(data: data, path: dataFile) 132 | 133 | let refsFile = fileName(for: id, prefix: .refs) 134 | let refData = try! JSONEncoder().encode(refs) 135 | let refBytes = LLBByteBuffer.withBytes(refData) 136 | let refFuture = writeIfNeeded(data: refBytes, path: refsFile) 137 | 138 | return dataFuture.and(refFuture).map { _ in id } 139 | } 140 | 141 | /// Write the given data to the path if the size of data 142 | /// differs from the size at path. 143 | private func writeIfNeeded( 144 | data: LLBByteBuffer, 145 | path: AbsolutePath 146 | ) -> LLBFuture { 147 | let handle = fileIO.openFile( 148 | path: path.pathString, 149 | mode: .write, 150 | flags: .allowFileCreation(), 151 | eventLoop: group.next() 152 | ) 153 | 154 | let size = handle.flatMap { handle in 155 | self.fileIO.readFileSize( 156 | fileHandle: handle, 157 | eventLoop: self.group.next() 158 | ) 159 | } 160 | 161 | let result = size.and(handle).flatMap { (size, handle) -> LLBFuture in 162 | if size == data.readableBytes { 163 | return self.group.next().makeSucceededFuture(()) 164 | } 165 | 166 | return self.fileIO.write( 167 | fileHandle: handle, 168 | buffer: data, 169 | eventLoop: self.group.next() 170 | ) 171 | } 172 | 173 | return handle.and(result).flatMapThrowing { (handle, _) in 174 | try handle.close() 175 | } 176 | } 177 | 178 | } 179 | 180 | public struct LLBFileBackedCASDatabaseScheme: LLBCASDatabaseScheme { 181 | public static let scheme = "file" 182 | 183 | public static func isValid(host: String?, port: Int?, path: String, query: String?) -> Bool { 184 | return host == nil && port == nil && path != "" && query == nil 185 | } 186 | 187 | public static func open(group: LLBFuturesDispatchGroup, url: Foundation.URL) throws -> LLBCASDatabase { 188 | return try LLBFileBackedCASDatabase(group: group, path: AbsolutePath(validating: url.path)) 189 | } 190 | } 191 | 192 | -------------------------------------------------------------------------------- /Sources/TSFCAS/Implementations/InMemoryCASDatabase.swift: -------------------------------------------------------------------------------- 1 | // This source file is part of the Swift.org open source project 2 | // 3 | // Copyright (c) 2020 Apple Inc. and the Swift project authors 4 | // Licensed under Apache License v2.0 with Runtime Library Exception 5 | // 6 | // See http://swift.org/LICENSE.txt for license information 7 | // See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 8 | 9 | import Foundation 10 | 11 | import NIOCore 12 | import NIOConcurrencyHelpers 13 | 14 | import TSCUtility 15 | import TSFFutures 16 | import TSFUtility 17 | 18 | 19 | /// A simple in-memory implementation of the `LLBCASDatabase` protocol. 20 | public final class LLBInMemoryCASDatabase: Sendable { 21 | struct State: Sendable { 22 | /// The content. 23 | var content = [LLBDataID: LLBCASObject]() 24 | 25 | var totalDataBytes: Int = 0 26 | } 27 | 28 | private let state: NIOLockedValueBox = NIOLockedValueBox(State()) 29 | 30 | /// Threads capable of running futures. 31 | public let group: LLBFuturesDispatchGroup 32 | 33 | /// The total number of data bytes in the database (this does not include the size of refs). 34 | public var totalDataBytes: Int { 35 | return self.state.withLockedValue { state in 36 | return state.totalDataBytes 37 | } 38 | } 39 | 40 | /// Create an in-memory database. 41 | public init(group: LLBFuturesDispatchGroup) { 42 | self.group = group 43 | } 44 | 45 | /// Delete the data in the database. 46 | /// Intentionally not exposed via the CASDatabase protocol. 47 | public func delete(_ id: LLBDataID, recursive: Bool) -> LLBFuture { 48 | self.state.withLockedValue { state in 49 | unsafeDelete(state: &state, id, recursive: recursive) 50 | } 51 | return group.next().makeSucceededFuture(()) 52 | } 53 | private func unsafeDelete(state: inout State, _ id: LLBDataID, recursive: Bool) { 54 | guard let object = state.content[id] else { 55 | return 56 | } 57 | state.totalDataBytes -= object.data.readableBytes 58 | 59 | guard recursive else { 60 | return 61 | } 62 | 63 | for ref in object.refs { 64 | unsafeDelete(state: &state, ref, recursive: recursive) 65 | } 66 | } 67 | } 68 | 69 | extension LLBInMemoryCASDatabase: LLBCASDatabase { 70 | public func supportedFeatures() -> LLBFuture { 71 | return group.next().makeSucceededFuture(LLBCASFeatures(preservesIDs: true)) 72 | } 73 | 74 | public func contains(_ id: LLBDataID, _ ctx: Context) -> LLBFuture { 75 | let result = self.state.withLockedValue { state in 76 | state.content.index(forKey: id) != nil 77 | } 78 | return group.next().makeSucceededFuture(result) 79 | } 80 | 81 | public func get(_ id: LLBDataID, _ ctx: Context) -> LLBFuture { 82 | let result = self.state.withLockedValue { state in state.content[id] } 83 | return group.next().makeSucceededFuture(result) 84 | } 85 | 86 | public func identify(refs: [LLBDataID] = [], data: LLBByteBuffer, _ ctx: Context) -> LLBFuture { 87 | return group.next().makeSucceededFuture(LLBDataID(blake3hash: data, refs: refs)) 88 | } 89 | 90 | public func put(refs: [LLBDataID] = [], data: LLBByteBuffer, _ ctx: Context) -> LLBFuture { 91 | return put(knownID: LLBDataID(blake3hash: data, refs: refs), refs: refs, data: data, ctx) 92 | } 93 | 94 | public func put(knownID id: LLBDataID, refs: [LLBDataID] = [], data: LLBByteBuffer, _ ctx: Context) -> LLBFuture { 95 | self.state.withLockedValue { state in 96 | guard state.content[id] == nil else { 97 | assert(state.content[id]?.data == data, "put data for id doesn't match") 98 | return 99 | } 100 | state.totalDataBytes += data.readableBytes 101 | state.content[id] = LLBCASObject(refs: refs, data: data) 102 | } 103 | return group.next().makeSucceededFuture(id) 104 | } 105 | } 106 | 107 | public struct LLBInMemoryCASDatabaseScheme: LLBCASDatabaseScheme { 108 | public static let scheme = "mem" 109 | 110 | public static func isValid(host: String?, port: Int?, path: String, query: String?) -> Bool { 111 | return host == nil && port == nil && path == "" && query == nil 112 | } 113 | 114 | public static func open(group: LLBFuturesDispatchGroup, url: Foundation.URL) throws -> LLBCASDatabase { 115 | return LLBInMemoryCASDatabase(group: group) 116 | } 117 | } 118 | -------------------------------------------------------------------------------- /Sources/TSFCAS/Object.swift: -------------------------------------------------------------------------------- 1 | // This source file is part of the Swift.org open source project 2 | // 3 | // Copyright (c) 2020 Apple Inc. and the Swift project authors 4 | // Licensed under Apache License v2.0 with Runtime Library Exception 5 | // 6 | // See http://swift.org/LICENSE.txt for license information 7 | // See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 8 | 9 | import Foundation 10 | 11 | import TSFUtility 12 | import NIOCore 13 | import NIOFoundationCompat 14 | 15 | 16 | // MARK:- CASObject Definition - 17 | 18 | public struct LLBCASObject: Equatable, Sendable { 19 | /// The list of references. 20 | public let refs: [LLBDataID] 21 | 22 | /// The object data. 23 | public let data: LLBByteBuffer 24 | 25 | public init(refs: [LLBDataID], data: LLBByteBuffer) { 26 | self.refs = refs 27 | self.data = data 28 | } 29 | } 30 | 31 | public extension LLBCASObject { 32 | init(refs: [LLBDataID], data: LLBByteBufferView) { 33 | self.init(refs: refs, data: LLBByteBuffer(data)) 34 | } 35 | } 36 | 37 | public extension LLBCASObject { 38 | /// The size of the object data. 39 | var size: Int { 40 | return data.readableBytes 41 | } 42 | } 43 | 44 | // MARK:- CASObjectRepresentable - 45 | 46 | public protocol LLBCASObjectRepresentable { 47 | func asCASObject() throws -> LLBCASObject 48 | } 49 | public protocol LLBCASObjectConstructable { 50 | init(from casObject: LLBCASObject) throws 51 | } 52 | 53 | // MARK:- CASObject Serializeable - 54 | 55 | public extension LLBCASObject { 56 | init(rawBytes: Data) throws { 57 | let pb = try LLBPBCASObject(serializedBytes: rawBytes) 58 | var data = LLBByteBufferAllocator().buffer(capacity: pb.data.count) 59 | data.writeBytes(pb.data) 60 | self.init(refs: pb.refs, data: data) 61 | } 62 | 63 | func toData() throws -> Data { 64 | var pb = LLBPBCASObject() 65 | pb.refs = self.refs 66 | pb.data = Data(buffer: self.data) 67 | return try pb.serializedData() 68 | } 69 | } 70 | 71 | extension LLBCASObject: LLBSerializable { 72 | public init(from rawBytes: LLBByteBuffer) throws { 73 | let pb = try rawBytes.withUnsafeReadableBytes { 74 | try LLBPBCASObject(serializedBytes: Data(bytesNoCopy: UnsafeMutableRawPointer(mutating: $0.baseAddress!), count: $0.count, deallocator: .none)) 75 | } 76 | let refs = pb.refs 77 | var data = LLBByteBufferAllocator().buffer(capacity: pb.data.count) 78 | data.writeBytes(pb.data) 79 | self.init(refs: refs, data: data) 80 | } 81 | 82 | public func toBytes(into buffer: inout LLBByteBuffer) throws { 83 | buffer.writeBytes(try self.toData()) 84 | } 85 | 86 | } 87 | 88 | -------------------------------------------------------------------------------- /Sources/TSFCASFileTree/BinarySearch.swift: -------------------------------------------------------------------------------- 1 | // This source file is part of the Swift.org open source project 2 | // 3 | // Copyright (c) 2020 Apple Inc. and the Swift project authors 4 | // Licensed under Apache License v2.0 with Runtime Library Exception 5 | // 6 | // See http://swift.org/LICENSE.txt for license information 7 | // See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 8 | 9 | 10 | extension LLBCASFileTree { 11 | 12 | /// Search the index of a matching element consulting a given comparator. 13 | public static func binarySearch(_ elements: C, _ compare: (C.Element) -> Int) -> C.Index? { 14 | var lo: C.Index = elements.startIndex 15 | var hi: C.Index = elements.index(before: elements.endIndex) 16 | 17 | while true { 18 | let distance = elements.distance(from: lo, to: hi) 19 | guard distance >= 0 else { break } 20 | 21 | // Compute the middle index of this iteration's search range. 22 | let mid = elements.index(lo, offsetBy: distance/2) 23 | assert(elements.distance(from: elements.startIndex, to: mid) >= 0) 24 | assert(elements.distance(from: mid, to: elements.endIndex) > 0) 25 | 26 | // If there is a match, return the result. 27 | let cmp = compare(elements[mid]) 28 | if cmp == 0 { 29 | return mid 30 | } 31 | 32 | // Otherwise, continue to search. 33 | if cmp < 0 { 34 | hi = elements.index(before: mid) 35 | } else { 36 | lo = elements.index(after: mid) 37 | } 38 | } 39 | 40 | // Check exit conditions of the binary search. 41 | assert(elements.distance(from: elements.startIndex, to: lo) >= 0) 42 | assert(elements.distance(from: lo, to: elements.endIndex) >= 0) 43 | 44 | return nil 45 | } 46 | 47 | } 48 | -------------------------------------------------------------------------------- /Sources/TSFCASFileTree/CASFSClient.swift: -------------------------------------------------------------------------------- 1 | // This source file is part of the Swift.org open source project 2 | // 3 | // Copyright (c) 2020 Apple Inc. and the Swift project authors 4 | // Licensed under Apache License v2.0 with Runtime Library Exception 5 | // 6 | // See http://swift.org/LICENSE.txt for license information 7 | // See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 8 | 9 | import Foundation 10 | 11 | import NIOCore 12 | import TSFCAS 13 | import TSCBasic 14 | import TSCUtility 15 | 16 | 17 | /// A main API struct 18 | public struct LLBCASFSClient { 19 | public let db: LLBCASDatabase 20 | 21 | /// Errors produced by CASClient 22 | public enum Error: Swift.Error { 23 | case noEntry(LLBDataID) 24 | case notSupportedYet 25 | case invalidUse 26 | case unexpectedNode 27 | } 28 | 29 | /// Remembers db 30 | public init(_ db: LLBCASDatabase) { 31 | self.db = db 32 | } 33 | 34 | /// Check that DataID exists in CAS 35 | public func exists(_ id: LLBDataID, _ ctx: Context) -> LLBFuture { 36 | return db.contains(id, ctx) 37 | } 38 | 39 | /// Load CASFSNode from CAS 40 | /// If object doesn't exist future fails with noEntry 41 | public func load(_ id: LLBDataID, type hint: LLBFileType? = nil, _ ctx: Context) -> LLBFuture { 42 | return db.get(id, ctx).flatMapThrowing { objectOpt in 43 | guard let object = objectOpt else { 44 | throw Error.noEntry(id) 45 | } 46 | 47 | switch hint { 48 | case .directory?: 49 | let tree = try LLBCASFileTree(id: id, object: object) 50 | return LLBCASFSNode(tree: tree, db: self.db) 51 | case .plainFile?, .executable?: 52 | let blob = try LLBCASBlob(db: self.db, id: id, type: hint!, object: object, ctx) 53 | return LLBCASFSNode(blob: blob, db: self.db) 54 | case .symlink?, .UNRECOGNIZED?: 55 | // We don't support symlinks yet 56 | throw Error.notSupportedYet 57 | case nil: 58 | if let tree = try? LLBCASFileTree(id: id, object: object) { 59 | return LLBCASFSNode(tree: tree, db: self.db) 60 | } else if let blob = try? LLBCASBlob(db: self.db, id: id, object: object, ctx) { 61 | return LLBCASFSNode(blob: blob, db: self.db) 62 | } else { 63 | // We don't support symlinks yet 64 | throw Error.notSupportedYet 65 | } 66 | } 67 | } 68 | } 69 | 70 | /// Save ByteBuffer to CAS 71 | public func store(_ data: LLBByteBuffer, type: LLBFileType = .plainFile, _ ctx: Context) -> LLBFuture { 72 | LLBCASBlob.import(data: data, isExecutable: type == .executable, in: db, ctx).map { LLBCASFSNode(blob: $0, db: self.db) } 73 | } 74 | 75 | /// Save ArraySlice to CAS 76 | public func store(_ data: ArraySlice, type: LLBFileType = .plainFile, _ ctx: Context) -> LLBFuture { 77 | LLBCASBlob.import(data: LLBByteBuffer.withBytes(data), isExecutable: type == .executable, in: db, ctx).map { LLBCASFSNode(blob: $0, db: self.db) } 78 | } 79 | 80 | /// Save Data to CAS 81 | public func store(_ data: Data, type: LLBFileType = .plainFile, _ ctx: Context) -> LLBFuture { 82 | LLBCASBlob.import(data: LLBByteBuffer.withBytes(data), isExecutable: type == .executable, in: db, ctx).map { LLBCASFSNode(blob: $0, db: self.db) } 83 | } 84 | 85 | } 86 | 87 | extension LLBCASFSClient { 88 | public func store(_ data: LLBByteBuffer, type: LLBFileType = .plainFile, _ ctx: Context) -> LLBFuture { 89 | LLBCASBlob.import(data: data, isExecutable: type == .executable, in: db, ctx).flatMap { $0.export(ctx) } 90 | } 91 | 92 | public func store(_ data: ArraySlice, type: LLBFileType = .plainFile, _ ctx: Context) -> LLBFuture { 93 | LLBCASBlob.import(data: LLBByteBuffer.withBytes(data), isExecutable: type == .executable, in: db, ctx).flatMap { $0.export(ctx) } 94 | } 95 | 96 | public func store(_ data: Data, type: LLBFileType = .plainFile, _ ctx: Context) -> LLBFuture { 97 | LLBCASBlob.import(data: LLBByteBuffer.withBytes(data), isExecutable: type == .executable, in: db, ctx).flatMap { $0.export(ctx) } 98 | } 99 | } 100 | 101 | extension LLBCASFSClient { 102 | /// Creates a new LLBCASFileTree node by prepending the tree with the given graph. For example, if the given id 103 | /// contains a reference to a CASFileTree containing [a.txt, b.txt], and path was 'some/path', the resulting 104 | /// CASFileTree would contain [some/path/a.txt, some/path/b.txt] (where both `some` and `path` represent 105 | /// CASFileTrees). 106 | public func wrap(_ id: LLBDataID, path: String, _ ctx: Context) -> LLBFuture { 107 | let absolutePath: AbsolutePath 108 | do { 109 | absolutePath = try AbsolutePath(validating: path, relativeTo: .root) 110 | } catch { 111 | return ctx.group.any().makeFailedFuture(error) 112 | } 113 | return self.load(id, ctx).flatMap { node in 114 | return absolutePath 115 | .components 116 | .dropFirst() 117 | .reversed() 118 | .reduce(self.db.group.next().makeSucceededFuture(node)) { future, pathComponent in 119 | future.flatMap { node in 120 | let entry = node.asDirectoryEntry(filename: pathComponent) 121 | return LLBCASFileTree.create(files: [entry], in: self.db, ctx).map { 122 | return LLBCASFSNode(tree: $0, db: self.db) 123 | } 124 | } 125 | } 126 | }.flatMapThrowing { 127 | guard let tree = $0.tree else { 128 | throw Error.unexpectedNode 129 | } 130 | return tree 131 | } 132 | } 133 | } 134 | -------------------------------------------------------------------------------- /Sources/TSFCASFileTree/CASFSNode.swift: -------------------------------------------------------------------------------- 1 | // This source file is part of the Swift.org open source project 2 | // 3 | // Copyright (c) 2020 Apple Inc. and the Swift project authors 4 | // Licensed under Apache License v2.0 with Runtime Library Exception 5 | // 6 | // See http://swift.org/LICENSE.txt for license information 7 | // See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 8 | 9 | import Foundation 10 | 11 | import TSFCAS 12 | 13 | 14 | /// A CAS object (can be tree or blob) 15 | public struct LLBCASFSNode { 16 | public enum Error: Swift.Error { 17 | case notApplicable 18 | } 19 | 20 | public enum NodeContent { 21 | case tree(LLBCASFileTree) 22 | case blob(LLBCASBlob) 23 | } 24 | 25 | public let db: LLBCASDatabase 26 | public let value: NodeContent 27 | 28 | public init(tree: LLBCASFileTree, db: LLBCASDatabase) { 29 | self.db = db 30 | self.value = NodeContent.tree(tree) 31 | } 32 | 33 | public init(blob: LLBCASBlob, db: LLBCASDatabase) { 34 | self.db = db 35 | self.value = NodeContent.blob(blob) 36 | } 37 | 38 | /// Returns aggregated (for trees) or regular size of the Entry 39 | public func size() -> Int { 40 | switch value { 41 | case .tree(let tree): 42 | return tree.aggregateSize 43 | case .blob(let blob): 44 | return blob.size 45 | } 46 | } 47 | 48 | /// Gives CASFSNode type (meaningful for files) 49 | public func type() -> LLBFileType { 50 | switch value { 51 | case .tree(_): 52 | return .directory 53 | case .blob(let blob): 54 | return blob.type 55 | } 56 | } 57 | 58 | /// Optionally chainable tree access 59 | public var tree: LLBCASFileTree? { 60 | guard case .tree(let tree) = value else { 61 | return nil 62 | } 63 | return tree 64 | } 65 | 66 | /// Optionally chainable blob access 67 | public var blob: LLBCASBlob? { 68 | guard case .blob(let blob) = value else { 69 | return nil 70 | } 71 | return blob 72 | } 73 | 74 | public func asDirectoryEntry(filename: String) -> LLBDirectoryEntryID { 75 | switch value { 76 | case let .tree(tree): 77 | return tree.asDirectoryEntry(filename: filename) 78 | case let .blob(blob): 79 | return blob.asDirectoryEntry(filename: filename) 80 | } 81 | } 82 | } 83 | -------------------------------------------------------------------------------- /Sources/TSFCASFileTree/ConcurrentFileTreeWalker.swift: -------------------------------------------------------------------------------- 1 | // This source file is part of the Swift.org open source project 2 | // 3 | // Copyright (c) 2020 Apple Inc. and the Swift project authors 4 | // Licensed under Apache License v2.0 with Runtime Library Exception 5 | // 6 | // See http://swift.org/LICENSE.txt for license information 7 | // See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 8 | 9 | import Foundation 10 | 11 | import NIOCore 12 | import NIOConcurrencyHelpers 13 | import TSCBasic 14 | import TSCUtility 15 | 16 | import TSFCAS 17 | 18 | 19 | protocol RetrieveChildrenProtocol: AnyObject { 20 | associatedtype Item 21 | 22 | /// Get the item's children based on the item. 23 | func children(of: Item, _ ctx: Context) -> LLBFuture<[Item]> 24 | } 25 | 26 | /// Walk the hierarchy with bounded concurrency. 27 | final class ConcurrentHierarchyWalker { 28 | 29 | private let group: LLBFuturesDispatchGroup 30 | private let futureOpQueue: LLBFutureOperationQueue 31 | private let getChildren: (_ of: Item, _ ctx: Context) -> LLBFuture<[Item]> 32 | 33 | public init(group: LLBFuturesDispatchGroup, delegate: Delegate, maxConcurrentOperations: Int = 100) where Delegate.Item == Item { 34 | self.group = group 35 | self.getChildren = { (item, ctx) in 36 | delegate.children(of: item, ctx) 37 | } 38 | self.futureOpQueue = .init(maxConcurrentOperations: maxConcurrentOperations) 39 | } 40 | 41 | public func walk(_ item: Item, _ ctx: Context) -> LLBFuture { 42 | return futureOpQueue.enqueue(on: group.next()) { 43 | self.getChildren(item, ctx) 44 | }.flatMap { more in 45 | let futures = more.map { self.walk($0, ctx) } 46 | return LLBFuture.whenAllSucceed(futures, on: self.group.next()).map { _ in () } 47 | } 48 | } 49 | } 50 | 51 | public class LLBConcurrentFileTreeWalker: RetrieveChildrenProtocol { 52 | let db: LLBCASDatabase 53 | let client: LLBCASFSClient 54 | let filterCallback: (FilterArgument) -> Bool 55 | 56 | public struct FilterArgument: CustomDebugStringConvertible { 57 | public let path: AbsolutePath? 58 | public let type: LLBFileType 59 | public let size: Int 60 | } 61 | 62 | public struct Item { 63 | /// The description of the current CAS filesystem entry. 64 | let arg: FilterArgument 65 | 66 | /// Used to explode the CAS filesystem further. 67 | let id: LLBDataID 68 | 69 | /// A reference to a scan result to make scan() function 70 | /// concurrency-safe (and reentrant, not that we need it). 71 | let scanResult: ScanResult 72 | } 73 | 74 | public class ScanResult { 75 | let lock = NIOConcurrencyHelpers.NIOLock() 76 | var collectedArguments = [FilterArgument]() 77 | 78 | public func reapResult() -> [FilterArgument] { 79 | lock.withLock { 80 | let result = collectedArguments 81 | collectedArguments = [] 82 | return result 83 | } 84 | } 85 | } 86 | 87 | public init(db: LLBCASDatabase, _ filter: @escaping (FilterArgument) -> Bool) { 88 | self.db = db 89 | self.client = LLBCASFSClient(db) 90 | self.filterCallback = filter 91 | } 92 | 93 | /// Concurrently scan the filesystem in CAS, returning the unsorted 94 | /// list of entries that the filter has accepted. 95 | /// Scanning a single file will result in a single entry with no name. 96 | public func scan(root: LLBDataID, _ ctx: Context) -> LLBFuture<[FilterArgument]> { 97 | let root = Item(arg: FilterArgument(path: .root, type: .UNRECOGNIZED(.min), size: 0), id: root, scanResult: ScanResult()) 98 | let walker = ConcurrentHierarchyWalker(group: db.group, delegate: self) 99 | return walker.walk(root, ctx).map { () in 100 | root.scanResult.reapResult() 101 | } 102 | } 103 | 104 | /// Get the children of a (directory) item. 105 | public func children(of item: Item, _ ctx: Context) -> LLBFuture<[Item]> { 106 | let typeHint: LLBFileType? 107 | switch item.arg.type { 108 | case .UNRECOGNIZED(.min): 109 | typeHint = nil 110 | case let type: 111 | typeHint = type 112 | } 113 | 114 | return client.load(item.id, type: typeHint, ctx).map { node in 115 | if typeHint == nil, item.arg.path == .root, item.arg.size == 0 { 116 | // This is our root. Check if we're allowed to go past it. 117 | let dirEntry = node.asDirectoryEntry(filename: "-") 118 | let rootItem = Item(arg: FilterArgument(path: node.tree != nil ? .root : nil, type: dirEntry.info.type, size: Int(clamping: dirEntry.info.size)), id: dirEntry.id, scanResult: item.scanResult) 119 | guard self.filter(rootItem) else { 120 | return [] 121 | } 122 | } 123 | 124 | switch node.value { 125 | case let .tree(tree): 126 | var directories = [Item]() 127 | for (index, entry) in tree.files.enumerated() { 128 | let entryPath = item.arg.path!.appending(component: entry.name) 129 | let entryItem = Item(arg: FilterArgument(path: entryPath, type: entry.type, size: Int(clamping: entry.size)), id: tree.object.refs[index], scanResult: item.scanResult) 130 | guard self.filter(entryItem) else { 131 | continue 132 | } 133 | 134 | if case .directory = entry.type { 135 | directories.append(entryItem) 136 | } 137 | } 138 | return directories 139 | case let .blob(blob): 140 | let entryItem = Item(arg: FilterArgument(path: nil, type: blob.type, size: blob.size), id: item.id, scanResult: item.scanResult) 141 | _ = self.filter(entryItem) 142 | return [] 143 | } 144 | } 145 | } 146 | 147 | private func filter(_ item: Item) -> Bool { 148 | 149 | if case .UNRECOGNIZED(.min) = item.arg.type { 150 | // We don't expect this to come from the outside of this file. 151 | // But it is technically possible, so just ignore. 152 | return false 153 | } 154 | 155 | guard filterCallback(item.arg) else { 156 | return false 157 | } 158 | 159 | item.scanResult.lock.withLock { 160 | item.scanResult.collectedArguments.append(item.arg) 161 | } 162 | 163 | return true 164 | } 165 | } 166 | 167 | extension LLBConcurrentFileTreeWalker.FilterArgument { 168 | public var debugDescription: String { 169 | let path = self.path?.pathString ?? "" 170 | switch type { 171 | case .directory where self.path == .root: 172 | return "\(path)\(size == 0 ? "" : " \(sizeString)")" 173 | case .directory: 174 | return "\(path)/\(size == 0 ? "" : " \(sizeString)")" 175 | case .plainFile: 176 | return "\(path) \(sizeString)" 177 | case .executable: 178 | return "\(path)* \(sizeString)" 179 | case .symlink: 180 | return "\(path)@" 181 | case .UNRECOGNIZED(let code): 182 | return "\(path)?(\(code))" 183 | } 184 | } 185 | 186 | private var sizeString: String { 187 | if size < 100_000 { 188 | return "\(size) bytes" 189 | } else if size < 100_000_000 { 190 | return String(format: "%.1f MB", Double(size) / 1_000_000) 191 | } else { 192 | return String(format: "%.1f GB", Double(size) / 1_000_000_000) 193 | } 194 | } 195 | } 196 | -------------------------------------------------------------------------------- /Sources/TSFCASFileTree/Context.swift: -------------------------------------------------------------------------------- 1 | // This source file is part of the Swift.org open source project 2 | // 3 | // Copyright (c) 2021 Apple Inc. and the Swift project authors 4 | // Licensed under Apache License v2.0 with Runtime Library Exception 5 | // 6 | // See http://swift.org/LICENSE.txt for license information 7 | // See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 8 | 9 | import TSCUtility 10 | 11 | private final class ContextKey {} 12 | 13 | /// Support storing and retrieving file tree import options from a context 14 | public extension Context { 15 | static func with(_ options: LLBCASFileTree.ImportOptions) -> Context { 16 | return Context(dictionaryLiteral: (ObjectIdentifier(LLBCASFileTree.ImportOptions.self), options as Any)) 17 | } 18 | 19 | var fileTreeImportOptions: LLBCASFileTree.ImportOptions? { 20 | get { 21 | guard let options = self[ObjectIdentifier(LLBCASFileTree.ImportOptions.self), as: LLBCASFileTree.ImportOptions.self] else { 22 | return nil 23 | } 24 | 25 | return options 26 | } 27 | set { 28 | self[ObjectIdentifier(LLBCASFileTree.ImportOptions.self)] = newValue 29 | } 30 | } 31 | } 32 | 33 | /// Support storing and retrieving file tree export storage batcher from a context 34 | public extension Context { 35 | private static let fileTreeExportStorageBatcherKey = ContextKey() 36 | 37 | var fileTreeExportStorageBatcher: LLBBatchingFutureOperationQueue? { 38 | get { 39 | guard let options = self[ObjectIdentifier(Self.fileTreeExportStorageBatcherKey), as: LLBBatchingFutureOperationQueue.self] else { 40 | return nil 41 | } 42 | 43 | return options 44 | } 45 | set { 46 | self[ObjectIdentifier(Self.fileTreeExportStorageBatcherKey)] = newValue 47 | } 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /Sources/TSFCASFileTree/DeclFileTree.swift: -------------------------------------------------------------------------------- 1 | // This source file is part of the Swift.org open source project 2 | // 3 | // Copyright (c) 2020 Apple Inc. and the Swift project authors 4 | // Licensed under Apache License v2.0 with Runtime Library Exception 5 | // 6 | // See http://swift.org/LICENSE.txt for license information 7 | // See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 8 | 9 | import NIOCore 10 | import TSCUtility 11 | import TSFCAS 12 | 13 | 14 | /// A declarative way to create a CASFileTree with content known upfront 15 | /// Usage example: 16 | /// let tree: LLBDeclFileTree = .dir(["dir1": .dir([]), "file1": .file()]) 17 | /// let casTreeFuture = tree.toTree(db: db) 18 | public indirect enum LLBDeclFileTree { 19 | case directory(files: [String: LLBDeclFileTree]) 20 | case file(contents: [UInt8]) 21 | 22 | public static func dir(_ files: [String: LLBDeclFileTree]) -> LLBDeclFileTree { 23 | return .directory(files: files) 24 | } 25 | 26 | public static func file(_ contents: [UInt8]) -> LLBDeclFileTree { 27 | return .file(contents: contents) 28 | } 29 | 30 | public static func file(_ contents: String) -> LLBDeclFileTree { 31 | return .file(contents: Array(contents.utf8)) 32 | } 33 | } 34 | 35 | extension LLBDeclFileTree: CustomDebugStringConvertible { 36 | public var debugDescription: String { 37 | switch self { 38 | case let .directory(files): 39 | return ".directory(\(files))" 40 | case let .file(contents): 41 | return ".file(\(contents.count))" 42 | } 43 | } 44 | } 45 | 46 | 47 | extension LLBCASFSClient { 48 | /// Save LLBDeclFileTree to CAS 49 | public func store(_ declTree: LLBDeclFileTree, _ ctx: Context) -> LLBFuture { 50 | switch declTree { 51 | case .directory: 52 | return storeDir(declTree, ctx).map { LLBCASFSNode(tree: $0, db: self.db) } 53 | case .file: 54 | return storeFile(declTree, ctx).map { LLBCASFSNode(blob: $0, db: self.db) } 55 | } 56 | } 57 | 58 | public func storeDir(_ declTree: LLBDeclFileTree, _ ctx: Context) -> LLBFuture { 59 | let loop = db.group.next() 60 | guard case .directory(files: let files) = declTree else { 61 | return loop.makeFailedFuture(Error.invalidUse) 62 | } 63 | let infosFutures: [LLBFuture] = files.map { arg in 64 | let (key, value) = arg 65 | switch value { 66 | case .directory: 67 | let treeFuture = storeDir(value, ctx) 68 | return treeFuture.map { tree in 69 | LLBDirectoryEntryID(info: .init(name: key, type: .directory, size: tree.aggregateSize), 70 | id: tree.id)} 71 | case .file(_): 72 | return storeFile(value, ctx).map { blob in 73 | blob.asDirectoryEntry(filename: key) 74 | } 75 | } 76 | } 77 | return LLBFuture.whenAllSucceed(infosFutures, on: loop).flatMap { infos in 78 | return LLBCASFileTree.create(files: infos, in: self.db, ctx) 79 | } 80 | } 81 | 82 | public func storeFile(_ declTree: LLBDeclFileTree, _ ctx: Context) -> LLBFuture { 83 | let loop = db.group.next() 84 | guard case .file(contents: let contents) = declTree else { 85 | return loop.makeFailedFuture(Error.invalidUse) 86 | } 87 | return LLBCASBlob.import(data: LLBByteBuffer.withBytes(contents), isExecutable: false, in: db, ctx) 88 | } 89 | 90 | } 91 | 92 | extension LLBCASFSClient { 93 | public func store(_ declTree: LLBDeclFileTree, _ ctx: Context) -> LLBFuture { 94 | switch declTree { 95 | case .directory: 96 | return storeDir(declTree, ctx).map{ $0.id } 97 | case .file: 98 | return storeFile(declTree, ctx).flatMap { casBlob in 99 | casBlob.export(ctx) 100 | } 101 | } 102 | } 103 | } 104 | -------------------------------------------------------------------------------- /Sources/TSFCASFileTree/DirectoryEntry.swift: -------------------------------------------------------------------------------- 1 | // This source file is part of the Swift.org open source project 2 | // 3 | // Copyright (c) 2020-2021 Apple Inc. and the Swift project authors 4 | // Licensed under Apache License v2.0 with Runtime Library Exception 5 | // 6 | // See http://swift.org/LICENSE.txt for license information 7 | // See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 8 | 9 | import Foundation 10 | 11 | extension LLBDirectoryEntry { 12 | public init(name: String, type: LLBFileType, size: Int, posixDetails: LLBPosixFileDetails? = nil) { 13 | self.init(name: name, type: type, size: UInt64(clamping: size), posixDetails: posixDetails) 14 | } 15 | 16 | public init(name: String, type: LLBFileType, size: UInt64, posixDetails: LLBPosixFileDetails? = nil) { 17 | self.name = name 18 | self.type = type 19 | self.size = size 20 | if let pd = posixDetails, pd != LLBPosixFileDetails() { 21 | self.posixDetails = pd 22 | } 23 | } 24 | } 25 | 26 | extension LLBFileType { 27 | public var expectedPosixMode: mode_t { 28 | switch self { 29 | case .plainFile: 30 | return 0o644 31 | case .executable, .directory, .symlink: 32 | return 0o755 33 | case .UNRECOGNIZED: 34 | return 0 35 | } 36 | } 37 | } 38 | 39 | -------------------------------------------------------------------------------- /Sources/TSFCASFileTree/Errors.swift: -------------------------------------------------------------------------------- 1 | // This source file is part of the Swift.org open source project 2 | // 3 | // Copyright (c) 2020 Apple Inc. and the Swift project authors 4 | // Licensed under Apache License v2.0 with Runtime Library Exception 5 | // 6 | // See http://swift.org/LICENSE.txt for license information 7 | // See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 8 | 9 | import TSCBasic 10 | 11 | import TSFCAS 12 | 13 | 14 | public enum LLBCASFileTreeFormatError: Error { 15 | /// The given id was referenced as a directory, but the object encoding didn't match expectations. 16 | case unexpectedDirectoryData(LLBDataID) 17 | 18 | /// The given id was referenced as a file, but the object encoding didn't match expectations. 19 | case unexpectedFileData(LLBDataID) 20 | 21 | /// The given id was referenced as a symlink, but the object encoding didn't match expectations. 22 | case unexpectedSymlinkData(LLBDataID) 23 | 24 | /// An unexpected error was thrown while communicating with the database. 25 | case unexpectedDatabaseError(Error) 26 | 27 | /// Formatting/protocol error. 28 | case formatError(reason: String) 29 | 30 | /// File size exceeds internal limits 31 | case fileTooLarge(path: AbsolutePath) 32 | 33 | /// Decompression failed 34 | case decompressFailed(String) 35 | } 36 | 37 | -------------------------------------------------------------------------------- /Sources/TSFCASFileTree/FileInfo.swift: -------------------------------------------------------------------------------- 1 | // This source file is part of the Swift.org open source project 2 | // 3 | // Copyright (c) 2020 Apple Inc. and the Swift project authors 4 | // Licensed under Apache License v2.0 with Runtime Library Exception 5 | // 6 | // See http://swift.org/LICENSE.txt for license information 7 | // See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 8 | 9 | import Foundation 10 | 11 | import NIOCore 12 | import SwiftProtobuf 13 | 14 | extension LLBFileInfo: LLBSerializable { 15 | /// Decode the given block back into a message. 16 | @inlinable 17 | public init(from rawBytes: LLBByteBuffer) throws { 18 | self = try Self.deserialize(from: rawBytes) 19 | } 20 | 21 | /// Produce an encoded blob that fully defines the structure contents. 22 | @inlinable 23 | public func toBytes(into buffer: inout LLBByteBuffer) throws { 24 | buffer.writeBytes(try serializedData()) 25 | } 26 | } 27 | 28 | 29 | extension LLBFileInfo { 30 | @inlinable 31 | public static func deserialize(from array: [UInt8]) throws -> Self { 32 | return try array.withUnsafeBufferPointer{ try deserialize(from: $0) } 33 | } 34 | 35 | @inlinable 36 | public static func deserialize(from bytes: ArraySlice) throws -> Self { 37 | return try bytes.withUnsafeBufferPointer{ try deserialize(from: $0) } 38 | } 39 | 40 | @inlinable 41 | public static func deserialize(from buffer: LLBByteBuffer) throws -> Self { 42 | return try buffer.withUnsafeReadableBytesWithStorageManagement { (buffer, mgr) in 43 | _ = mgr.retain() 44 | return try Self.init(serializedBytes: Data( 45 | bytesNoCopy: UnsafeMutableRawPointer(mutating: buffer.baseAddress!), 46 | count: buffer.count, 47 | deallocator: .custom({ _,_ in mgr.release() } 48 | ))) 49 | } 50 | } 51 | 52 | @inlinable 53 | public static func deserialize(from buffer: UnsafeBufferPointer) throws -> Self { 54 | return try Self.init(serializedBytes: Data( 55 | // NOTE: This doesn't actually mutate, which is why this is safe. 56 | bytesNoCopy: UnsafeMutableRawPointer(mutating: buffer.baseAddress!), 57 | count: buffer.count, deallocator: .none)) 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /Sources/TSFCASFileTree/TSCCASFileSystem.swift: -------------------------------------------------------------------------------- 1 | // This source file is part of the Swift.org open source project 2 | // 3 | // Copyright (c) 2020 Apple Inc. and the Swift project authors 4 | // Licensed under Apache License v2.0 with Runtime Library Exception 5 | // 6 | // See http://swift.org/LICENSE.txt for license information 7 | // See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 8 | 9 | import TSCBasic 10 | import TSCUtility 11 | 12 | import TSFCAS 13 | 14 | /// CAS backed FileSystem implementation rooted at the given CASTree. 15 | /// 16 | /// NOTE:- This class should *NOT* be used inside the db's NIO event loop. 17 | public final class TSCCASFileSystem: FileSystem { 18 | 19 | let rootTree: LLBCASFileTree 20 | let db: LLBCASDatabase 21 | let client: LLBCASFSClient 22 | let ctx: Context 23 | 24 | public init( 25 | db: LLBCASDatabase, 26 | rootTree: LLBCASFileTree, 27 | _ ctx: Context 28 | ) { 29 | self.db = db 30 | self.client = LLBCASFSClient(db) 31 | self.rootTree = rootTree 32 | self.ctx = ctx 33 | } 34 | 35 | public func exists(_ path: AbsolutePath, followSymlink: Bool) -> Bool { 36 | if path.isRoot { 37 | return true 38 | } 39 | let result = try? self.rootTree.lookup(path: path, in: self.db, Context()).wait() 40 | return result != nil 41 | } 42 | 43 | public func getDirectoryContents(_ path: AbsolutePath) throws -> [String] { 44 | if path.isRoot { 45 | return rootTree.files.map { $0.name } 46 | } 47 | 48 | let _result = try self.rootTree.lookup(path: path, in: self.db, ctx).wait() 49 | guard let result = _result else { throw FileSystemError(.noEntry, path) } 50 | 51 | // HACK: If this is a symlink, check if it points to a directory. 52 | // Move this to LLBCASFileTree.lookup() 53 | if result.info.type == .symlink, isDirectory(path) { 54 | let symlinkContents = try readFileContents(path).cString 55 | return try getDirectoryContents(path.parentDirectory.appending(RelativePath(symlinkContents))) 56 | } 57 | 58 | let entry = try self.client.load(result.id, ctx).wait() 59 | guard let tree = entry.tree else { throw FileSystemError(.notDirectory, path) } 60 | return tree.files.map{ $0.name } 61 | } 62 | 63 | public func isDirectory(_ path: AbsolutePath) -> Bool { 64 | let fileType = self.fileType(of: path) 65 | if fileType == .directory { 66 | return true 67 | } 68 | 69 | // HACK: If this is a symlink, check if it points to a directory. 70 | // Move this to LLBCASFileTree.lookup() 71 | if fileType == .symlink { 72 | guard let symlinkContents = try? readFileContents(path).cString else { 73 | return false 74 | } 75 | return isDirectory(path.parentDirectory.appending(RelativePath(symlinkContents))) 76 | } 77 | 78 | return false 79 | } 80 | 81 | private func fileType(of path: AbsolutePath) -> LLBFileType? { 82 | if path.isRoot { return .directory } 83 | let fileType = try? self.rootTree.lookup(path: path, in: self.db, ctx).wait() 84 | return fileType?.info.type 85 | } 86 | 87 | public func isFile(_ path: AbsolutePath) -> Bool { 88 | let fileType = self.fileType(of: path) 89 | return fileType == .plainFile || fileType == .executable 90 | } 91 | 92 | public func isExecutableFile(_ path: AbsolutePath) -> Bool { 93 | fileType(of: path) == .executable 94 | } 95 | 96 | public func isSymlink(_ path: AbsolutePath) -> Bool { 97 | fileType(of: path) == .symlink 98 | } 99 | 100 | public func readFileContents(_ path: AbsolutePath) throws -> ByteString { 101 | if path.isRoot { 102 | throw FileSystemError(.ioError(code: 0), path) 103 | } 104 | 105 | let result = try rootTree.lookup(path: path, in: db, ctx).wait() 106 | guard let id = result?.id else { throw FileSystemError(.noEntry, path) } 107 | 108 | let entry = try client.load(id, ctx).wait() 109 | guard let blob = entry.blob else { throw FileSystemError(.ioError(code: 0), path) } 110 | let bytes = try blob.read(ctx).wait() 111 | return ByteString(bytes) 112 | } 113 | 114 | public func chmod(_ mode: FileMode, path: AbsolutePath, options: Set) throws { 115 | throw FileSystemError(.unsupported) 116 | } 117 | 118 | public func writeFileContents(_ path: AbsolutePath, bytes: ByteString) throws { 119 | throw FileSystemError(.unsupported) 120 | } 121 | 122 | public func createDirectory(_ path: AbsolutePath, recursive: Bool) throws { 123 | throw FileSystemError(.unsupported) 124 | } 125 | 126 | public func removeFileTree(_ path: AbsolutePath) throws { 127 | throw FileSystemError(.unsupported) 128 | } 129 | 130 | public func copy(from sourcePath: AbsolutePath, to destinationPath: AbsolutePath) throws { 131 | throw FileSystemError(.unsupported) 132 | } 133 | 134 | public func move(from sourcePath: AbsolutePath, to destinationPath: AbsolutePath) throws { 135 | throw FileSystemError(.unsupported) 136 | } 137 | 138 | public var cachesDirectory: AbsolutePath? { nil } 139 | 140 | public func createSymbolicLink(_ path: AbsolutePath, pointingAt destination: AbsolutePath, relative: Bool) throws { 141 | throw FileSystemError(.unsupported) 142 | } 143 | 144 | public var currentWorkingDirectory: AbsolutePath? { nil } 145 | 146 | public func changeCurrentWorkingDirectory(to path: AbsolutePath) throws { 147 | throw FileSystemError(.unsupported) 148 | } 149 | 150 | public var homeDirectory: AbsolutePath { .root } 151 | 152 | public func isReadable(_ path: AbsolutePath) -> Bool { 153 | return !path.isRoot 154 | } 155 | 156 | public func isWritable(_ path: AbsolutePath) -> Bool { 157 | return false 158 | } 159 | 160 | public var tempDirectory: AbsolutePath { 161 | return (try? determineTempDirectory(nil)) ?? AbsolutePath.root.appending(component: "tmp") 162 | } 163 | } 164 | -------------------------------------------------------------------------------- /Sources/TSFCASUtilities/BufferedStreamWriter.swift: -------------------------------------------------------------------------------- 1 | import Foundation 2 | import NIOCore 3 | import NIOConcurrencyHelpers 4 | import TSCUtility 5 | import TSFCAS 6 | 7 | fileprivate extension LLBByteBuffer { 8 | var availableCapacity: Int { capacity - readableBytes } 9 | } 10 | 11 | /// Stream writer that buffers data before ingesting it into the CAS database. 12 | public class LLBBufferedStreamWriter { 13 | private let bufferSize: Int 14 | private let lock = NIOConcurrencyHelpers.NIOLock() 15 | private var outputWriter: LLBLinkedListStreamWriter 16 | private var currentBuffer: LLBByteBuffer 17 | private var currentBufferedChannel: UInt8? = nil 18 | 19 | public var latestID: LLBFuture? { 20 | return lock.withLock { outputWriter.latestID } 21 | } 22 | 23 | /// Creates a new buffered writer, with a default buffer size of 512kb to optimize for roundtrip read time. 24 | public init(_ db: LLBCASDatabase, bufferSize: Int = 1 << 19) { 25 | self.outputWriter = LLBLinkedListStreamWriter(db) 26 | self.bufferSize = bufferSize 27 | self.currentBuffer = LLBByteBufferAllocator.init().buffer(capacity: bufferSize) 28 | } 29 | 30 | public func rebase(onto newBase: LLBDataID, _ ctx: Context) { 31 | lock.withLock { 32 | outputWriter.rebase(onto: newBase, ctx) 33 | } 34 | } 35 | 36 | /// Writes a chunk of data into the stream. Flushes if the current buffer would overflow, or if the data to write 37 | /// is larger than the buffer size. 38 | public func write(data: LLBByteBuffer, channel: UInt8, _ ctx: Context = .init()) { 39 | lock.withLock { 40 | if channel != currentBufferedChannel || data.readableBytes > currentBuffer.availableCapacity 41 | { 42 | _flush(ctx) 43 | } 44 | 45 | currentBufferedChannel = channel 46 | 47 | // If data is larger or equal than buffer size, send as is. 48 | if data.readableBytes >= bufferSize { 49 | outputWriter.append(data: data, channel: channel, ctx) 50 | } else { 51 | // data is smaller than max chunk, and if we were going to overpass the chunk size, the above check 52 | // would have already flushed the data, so at this point we know there's space in the buffer. 53 | assert(data.readableBytes <= currentBuffer.availableCapacity) 54 | currentBuffer.writeImmutableBuffer(data) 55 | } 56 | 57 | // If we filled the buffer, send it out. 58 | if currentBuffer.availableCapacity == 0 { 59 | _flush(ctx) 60 | } 61 | } 62 | } 63 | 64 | /// Flushes the buffer into the stream writer. 65 | public func flush(_ ctx: Context = .init()) { 66 | lock.withLock { 67 | _flush(ctx) 68 | } 69 | } 70 | 71 | /// Private implementation of flush, must be called within the lock. 72 | private func _flush(_ ctx: Context) { 73 | if currentBuffer.readableBytes > 0, let currentBufferedChannel = currentBufferedChannel { 74 | outputWriter.append( 75 | data: currentBuffer, 76 | channel: currentBufferedChannel, ctx 77 | ) 78 | currentBuffer.clear() 79 | } 80 | } 81 | } 82 | -------------------------------------------------------------------------------- /Sources/TSFCASUtilities/LinkedListStream.swift: -------------------------------------------------------------------------------- 1 | // This source file is part of the Swift.org open source project 2 | // 3 | // Copyright (c) 2020 Apple Inc. and the Swift project authors 4 | // Licensed under Apache License v2.0 with Runtime Library Exception 5 | // 6 | // See http://swift.org/LICENSE.txt for license information 7 | // See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 8 | 9 | import NIOCore 10 | import TSCUtility 11 | import TSFCAS 12 | import TSFCASFileTree 13 | 14 | private extension String { 15 | func prepending(_ prefix: String) -> String { 16 | return prefix + self 17 | } 18 | } 19 | 20 | /// Basic writer implementation that resembles a linked list where each node contains control data (like the channel) 21 | /// and refs[0] always points to the dataID of the data chunk and refs[1] has the data ID for the next node in the 22 | /// chain, if it's not the last node. This implementation is not thread safe. 23 | public struct LLBLinkedListStreamWriter { 24 | private let db: LLBCASDatabase 25 | private let ext: String 26 | 27 | private var latestData: LLBFuture<(id: LLBDataID, aggregateSize: Int)>? 28 | 29 | public var latestID: LLBFuture? { 30 | latestData?.map { $0.id } 31 | } 32 | 33 | public init(_ db: LLBCASDatabase, ext: String? = nil) { 34 | self.db = db 35 | self.ext = ext?.prepending(".") ?? "" 36 | } 37 | 38 | // This rebases the current logs onto a new data ID, potentially losing all the previous uploads if not saved 39 | // previously. The newBase should be another dataID produced by a LLBLinkedListStreamWriter. 40 | public mutating func rebase(onto newBase: LLBDataID, _ ctx: Context) { 41 | self.latestData = LLBCASFSClient(db).load(newBase, ctx).map{ 42 | $0.tree 43 | }.tsf_unwrapOptional(orStringError: "Expected an LLBCASTree").map { tree in 44 | (id: tree.id, aggregateSize: tree.aggregateSize) 45 | } 46 | } 47 | 48 | @discardableResult 49 | public mutating func append(data: LLBByteBuffer, channel: UInt8, _ ctx: Context) -> LLBFuture { 50 | let latestData = ( 51 | // Append on the previously cached node, or use nil as sentinel if this is the first write. 52 | self.latestData?.map { $0 } ?? db.group.next().makeSucceededFuture(nil) 53 | ).flatMap { [db, ext] (previousData: (id: LLBDataID, aggregateSize: Int)?) -> LLBFuture<(id: LLBDataID, aggregateSize: Int)> in 54 | db.put(data: data, ctx).flatMap { [db, ext] contentID in 55 | 56 | var entries = [ 57 | LLBDirectoryEntryID( 58 | info: .init(name: "\(channel)\(ext)", type: .plainFile, size: data.readableBytes), 59 | id: contentID 60 | ), 61 | ] 62 | 63 | let aggregateSize: Int 64 | if let (prevID, prevSize) = previousData { 65 | entries.append( 66 | LLBDirectoryEntryID( 67 | info: .init(name: "prev", type: .directory, size: prevSize), 68 | id: prevID 69 | ) 70 | ) 71 | aggregateSize = prevSize + data.readableBytes 72 | } else { 73 | aggregateSize = data.readableBytes 74 | } 75 | 76 | return LLBCASFileTree.create(files: entries, in: db, ctx).map { (id: $0.id, aggregateSize: aggregateSize) } 77 | } 78 | } 79 | 80 | self.latestData = latestData 81 | return latestData.map { $0.id } 82 | } 83 | } 84 | 85 | public extension LLBLinkedListStreamWriter { 86 | @discardableResult 87 | @inlinable 88 | mutating func append(data: LLBByteBuffer, _ ctx: Context) -> LLBFuture { 89 | return append(data: data, channel: 0, ctx) 90 | } 91 | } 92 | -------------------------------------------------------------------------------- /Sources/TSFCASUtilities/StreamReader.swift: -------------------------------------------------------------------------------- 1 | // This source file is part of the Swift.org open source project 2 | // 3 | // Copyright (c) 2020 Apple Inc. and the Swift project authors 4 | // Licensed under Apache License v2.0 with Runtime Library Exception 5 | // 6 | // See http://swift.org/LICENSE.txt for license information 7 | // See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 8 | 9 | import NIOCore 10 | import TSCUtility 11 | import TSFCAS 12 | import TSFCASFileTree 13 | 14 | /// Implements the reading logic to read any kind of streaming data storage implemented. Currently it's hardcoded to 15 | /// read the LinkedList stream contents, but could get extended in the future when new storage structures are 16 | /// implemented. This should be the unified API to read streaming content, so that readers do not need to understand 17 | /// which writer was used to store the data. 18 | public struct LLBCASStreamReader { 19 | private let db: LLBCASDatabase 20 | 21 | public init(_ db: LLBCASDatabase) { 22 | self.db = db 23 | } 24 | 25 | public func read( 26 | id: LLBDataID, 27 | channels: [UInt8]?, 28 | lastReadID: LLBDataID?, 29 | _ ctx: Context, 30 | readerBlock: @escaping (UInt8, LLBByteBufferView) throws -> Bool 31 | ) -> LLBFuture { 32 | return innerRead( 33 | id: id, 34 | channels: channels, 35 | lastReadID: lastReadID, 36 | ctx, 37 | readerBlock: readerBlock 38 | ).map { _ in () } 39 | } 40 | 41 | private func innerRead( 42 | id: LLBDataID, 43 | channels: [UInt8]? = nil, 44 | lastReadID: LLBDataID? = nil, 45 | _ ctx: Context, 46 | readerBlock: @escaping (UInt8, LLBByteBufferView) throws -> Bool 47 | ) -> LLBFuture { 48 | if id == lastReadID { 49 | return db.group.next().makeSucceededFuture(true) 50 | } 51 | 52 | return LLBCASFSClient(db).load(id, ctx).flatMap { node in 53 | guard let tree = node.tree else { 54 | return self.db.group.next().makeFailedFuture(LLBCASStreamError.invalid) 55 | } 56 | 57 | let readChainFuture: LLBFuture 58 | 59 | // If there is a "prev" directory, treat this as a linked list implementation. This is an implementation 60 | // detail that could be cleaned up later, but the idea is that there's a single unified reader entity. 61 | if let (id, _) = tree.lookup("prev") { 62 | readChainFuture = self.innerRead( 63 | id: id, 64 | channels: channels, 65 | lastReadID: lastReadID, 66 | ctx, 67 | readerBlock: readerBlock 68 | ) 69 | } else { 70 | // If this is the last node, schedule a sentinel read that returns to keep on reading. 71 | readChainFuture = self.db.group.next().makeSucceededFuture(true) 72 | } 73 | 74 | return readChainFuture.flatMap { shouldContinue -> LLBFuture in 75 | // If we don't want to continue reading, or if the channel is not requested, close the current chain 76 | // and propagate the desire to keep on reading. 77 | guard shouldContinue else { 78 | return self.db.group.next().makeSucceededFuture(shouldContinue) 79 | } 80 | 81 | let files = tree.files.filter { 82 | $0.type == .plainFile 83 | } 84 | 85 | guard files.count == 1, let (contentID, _) = tree.lookup(files[0].name) else { 86 | return self.db.group.next().makeFailedFuture(LLBCASStreamError.invalid) 87 | } 88 | 89 | let channelOpt = files.first.flatMap { $0.name.split(separator: ".").first }.flatMap { UInt8($0) } 90 | 91 | guard let channel = channelOpt, 92 | channels?.contains(channel) ?? true else { 93 | return self.db.group.next().makeSucceededFuture(true) 94 | } 95 | 96 | return LLBCASFSClient(self.db).load(contentID, ctx).flatMap { node in 97 | guard let blob = node.blob else { 98 | return self.db.group.next().makeFailedFuture(LLBCASStreamError.missing) 99 | } 100 | 101 | return blob.read(ctx).flatMapThrowing { byteBufferView in 102 | return try readerBlock(channel, byteBufferView) 103 | } 104 | } 105 | } 106 | } 107 | } 108 | } 109 | 110 | // Convenience extension for default parameters. 111 | public extension LLBCASStreamReader { 112 | @inlinable 113 | func read( 114 | id: LLBDataID, 115 | _ ctx: Context, 116 | readerBlock: @escaping (UInt8, LLBByteBufferView) throws -> Bool 117 | ) -> LLBFuture { 118 | return read(id: id, channels: nil, lastReadID: nil, ctx, readerBlock: readerBlock) 119 | } 120 | 121 | @inlinable 122 | func read( 123 | id: LLBDataID, 124 | channels: [UInt8], 125 | _ ctx: Context, 126 | readerBlock: @escaping (UInt8, LLBByteBufferView) throws -> Bool 127 | ) -> LLBFuture { 128 | return read(id: id, channels: channels, lastReadID: nil, ctx, readerBlock: readerBlock) 129 | } 130 | 131 | @inlinable 132 | func read( 133 | id: LLBDataID, 134 | lastReadID: LLBDataID, 135 | _ ctx: Context, 136 | readerBlock: @escaping (UInt8, LLBByteBufferView) throws -> Bool 137 | ) -> LLBFuture { 138 | return read(id: id, channels: nil, lastReadID: lastReadID, ctx, readerBlock: readerBlock) 139 | } 140 | } 141 | 142 | /// Common error types for stream protocol implementations. 143 | public enum LLBCASStreamError: Error { 144 | case invalid 145 | case missing 146 | } 147 | -------------------------------------------------------------------------------- /Sources/TSFFutures/BatchingFutureOperationQueue.swift: -------------------------------------------------------------------------------- 1 | // This source file is part of the Swift.org open source project 2 | // 3 | // Copyright (c) 2020 Apple Inc. and the Swift project authors 4 | // Licensed under Apache License v2.0 with Runtime Library Exception 5 | // 6 | // See http://swift.org/LICENSE.txt for license information 7 | // See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 8 | 9 | import Foundation 10 | 11 | import NIOCore 12 | import TSCUtility 13 | 14 | 15 | /// Run the given computations on a given array in batches, exercising 16 | /// a specified amount of parallelism. 17 | /// 18 | /// - Discussion: 19 | /// For some blocking operations (such as file system accesses) executing 20 | /// them on the NIO loops is very expensive since it blocks the event 21 | /// processing machinery. Here we use extra threads for such operations. 22 | public struct LLBBatchingFutureOperationQueue: Sendable { 23 | 24 | /// Threads capable of running futures. 25 | public let group: LLBFuturesDispatchGroup 26 | 27 | /// Queue of outstanding operations. 28 | @usableFromInline 29 | let operationQueue: OperationQueue 30 | 31 | /// Because `LLBBatchingFutureOperationQueue` is a struct, the compiler 32 | /// will claim that `maxOpCount`'s setter is `mutating`, even though 33 | /// `OperationQueue` is a threadsafe class. 34 | /// This method exists as a workaround to adjust the underlying concurrency 35 | /// of the operation queue without unnecessary synchronization. 36 | public func setMaxOpCount(_ maxOpCount: Int) { 37 | operationQueue.maxConcurrentOperationCount = maxOpCount 38 | } 39 | 40 | /// Maximum number of operations executed concurrently. 41 | public var maxOpCount: Int { 42 | get { operationQueue.maxConcurrentOperationCount } 43 | set { self.setMaxOpCount(newValue) } 44 | } 45 | 46 | /// Return the number of operations currently queued. 47 | @inlinable 48 | public var opCount: Int { 49 | return operationQueue.operationCount 50 | } 51 | 52 | /// Whether the queue is suspended. 53 | @inlinable 54 | public var isSuspended: Bool { 55 | return operationQueue.isSuspended 56 | } 57 | 58 | /// 59 | /// - Parameters: 60 | /// - name: Unique string label, for logging. 61 | /// - group: Threads capable of running futures. 62 | /// - maxConcurrentOperationCount: 63 | /// Operations to execute in parallel. 64 | @inlinable 65 | public init(name: String, group: LLBFuturesDispatchGroup, maxConcurrentOperationCount maxOpCount: Int, qualityOfService: QualityOfService = .default) { 66 | self.group = group 67 | self.operationQueue = OperationQueue(tsf_withName: name, maxConcurrentOperationCount: maxOpCount) 68 | self.operationQueue.qualityOfService = qualityOfService 69 | } 70 | 71 | @inlinable 72 | public func execute(_ body: @escaping () throws -> T) -> LLBFuture { 73 | let promise = group.next().makePromise(of: T.self) 74 | operationQueue.addOperation { 75 | promise.fulfill(body) 76 | } 77 | return promise.futureResult 78 | } 79 | 80 | @inlinable 81 | public func execute(_ body: @escaping () -> LLBFuture) -> LLBFuture { 82 | let promise = group.next().makePromise(of: T.self) 83 | operationQueue.addOperation { 84 | let f = body() 85 | f.cascade(to: promise) 86 | 87 | // Wait for completion, to ensure we maintain at most N concurrent operations. 88 | _ = try? f.wait() 89 | } 90 | return promise.futureResult 91 | } 92 | 93 | /// Order-preserving parallel execution. Wait for everything to complete. 94 | @inlinable 95 | public func execute(_ args: [A], minStride: Int = 1, _ body: @escaping (ArraySlice) throws -> [T]) -> LLBFuture<[T]> { 96 | let futures: [LLBFuture<[T]>] = executeNoWait(args, minStride: minStride, body) 97 | let loop = futures.first?.eventLoop ?? group.next() 98 | return LLBFuture<[T]>.whenAllSucceed(futures, on: loop).map{$0.flatMap{$0}} 99 | } 100 | 101 | /// Order-preserving parallel execution. 102 | /// Do not wait for all executions to complete, returning individual futures. 103 | @inlinable 104 | public func executeNoWait(_ args: [A], minStride: Int = 1, maxStride: Int = Int.max, _ body: @escaping (ArraySlice) throws -> [T]) -> [LLBFuture<[T]>] { 105 | let batches: [ArraySlice] = args.tsc_sliceBy(maxStride: max(minStride, min(maxStride, args.count / maxOpCount))) 106 | return batches.map{arg in execute{try body(arg)}} 107 | } 108 | 109 | } 110 | -------------------------------------------------------------------------------- /Sources/TSFFutures/CancellableFuture.swift: -------------------------------------------------------------------------------- 1 | // This source file is part of the Swift.org open source project 2 | // 3 | // Copyright (c) 2020 Apple Inc. and the Swift project authors 4 | // Licensed under Apache License v2.0 with Runtime Library Exception 5 | // 6 | // See http://swift.org/LICENSE.txt for license information 7 | // See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 8 | 9 | import NIOCore 10 | 11 | /// A construct which expresses operations which can be asynchronously 12 | /// cancelled. The cancellation is not guaranteed and no ordering guarantees 13 | /// are provided with respect to the order of future's callbacks and the 14 | /// cancel operation returning. 15 | public struct LLBCancellableFuture: LLBCancelProtocol { 16 | /// The underlying future. 17 | public let future: LLBFuture 18 | 19 | /// The way to asynchronously cancel the operation backing up the future. 20 | public let canceller: LLBCanceller 21 | 22 | /// Initialize the future with a given canceller. 23 | public init(_ future: LLBFuture, canceller specificCanceller: LLBCanceller? = nil) { 24 | self.future = future 25 | let canceller = specificCanceller ?? LLBCanceller() 26 | self.canceller = canceller 27 | self.future.whenComplete { _ in 28 | // Do not invoke the cancel handler if the future 29 | // has already terminated. This is a bit opportunistic 30 | // and can miss some cancellation invocations, but 31 | // we expect the cancellation handlers to be no-op 32 | // when cancelling something that's not there. 33 | canceller.abandon() 34 | } 35 | } 36 | 37 | /// Initialize with a given handler which can be 38 | /// subsequently invoked through self.canceller.cancel() 39 | public init(_ future: LLBFuture, handler: LLBCancelProtocol) { 40 | self = LLBCancellableFuture(future, canceller: LLBCanceller(handler)) 41 | } 42 | 43 | /// Conformance to the `CancelProtocol`. 44 | public func cancel(reason: String?) { 45 | canceller.cancel(reason: reason) 46 | } 47 | } 48 | 49 | 50 | /// Some surface compatibility with EventLoopFuture to minimize 51 | /// the amount of code change in tests and other places. 52 | extension LLBCancellableFuture { 53 | #if swift(>=5.7) 54 | @available(*, noasync, message: "wait() can block indefinitely, prefer get()", renamed: "get()") 55 | @inlinable 56 | public func wait() throws -> T { 57 | try future.wait() 58 | } 59 | #else 60 | @inlinable 61 | public func wait() throws -> T { 62 | try future.wait() 63 | } 64 | #endif 65 | 66 | @available(macOS 10.15, iOS 13.0, tvOS 13.0, watchOS 6.0, *) 67 | @inlinable 68 | public func get() async throws -> T { 69 | try await future.get() 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /Sources/TSFFutures/CancellablePromise.swift: -------------------------------------------------------------------------------- 1 | // This source file is part of the Swift.org open source project 2 | // 3 | // Copyright (c) 2020 Apple Inc. and the Swift project authors 4 | // Licensed under Apache License v2.0 with Runtime Library Exception 5 | // 6 | // See http://swift.org/LICENSE.txt for license information 7 | // See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 8 | 9 | import Atomics 10 | import NIOCore 11 | import NIOConcurrencyHelpers 12 | 13 | 14 | public enum LLBCancellablePromiseError: Error { 15 | case promiseFulfilled 16 | case promiseCancelled 17 | case promiseLeaked 18 | } 19 | 20 | /// A promise that can be cancelled prematurely. 21 | /// The CancellablePromise supports two types of API access: 22 | /// - The writer access that fulfills the promise or cancels it, getting 23 | /// back indication of whether or not that operation was successful. 24 | /// - The reader access that checks if the promise has been fulfilled. 25 | open class LLBCancellablePromise: @unchecked /* because inheritance... */ Sendable { 26 | /// Underlying promise. Private to avoid messing with out outside 27 | /// of CancellablePromise lifecycle protection. 28 | private let promise: LLBPromise 29 | 30 | /// The current state of the promise. 31 | /// - inProgress: The promise is waiting to be fulfilled or cancelled. 32 | /// - fulfilled: The promise has been fulfilled with a value or error. 33 | /// - cancelled: The promise has been cancelled via cancel(_:) 34 | public enum State: Int { 35 | case inProgress 36 | case fulfilled 37 | case cancelled 38 | } 39 | 40 | /// A state maintaining the lifecycle of the promise. 41 | @usableFromInline 42 | let state_: ManagedAtomic 43 | 44 | @inlinable 45 | public var state: State { 46 | return State(rawValue: state_.load(ordering: .relaxed))! 47 | } 48 | 49 | /// The eventual result of the promise. 50 | public var futureResult: LLBFuture { 51 | return promise.futureResult 52 | } 53 | 54 | /// Whether the promise was fulfilled or cancelled. 55 | @inlinable 56 | public var isCompleted: Bool { 57 | return state != .inProgress 58 | } 59 | 60 | /// Whether the promise was cancelled. 61 | @inlinable 62 | public var isCancelled: Bool { 63 | return state == .cancelled 64 | } 65 | 66 | /// Initialize a new promise off the given event loop. 67 | public convenience init(on loop: LLBFuturesDispatchLoop) { 68 | self.init(promise: loop.makePromise()) 69 | } 70 | 71 | /// Initialize a promise directly. Less safe because the promise 72 | /// could be accidentally fulfilled outside of CancellablePromise lifecycle. 73 | public init(promise: LLBPromise) { 74 | self.promise = promise 75 | self.state_ = .init(State.inProgress.rawValue) 76 | } 77 | 78 | /// Returns `true` if the state has been modified from .inProgress. 79 | private func modifyState(_ newState: State) -> Bool { 80 | assert(newState != .inProgress) 81 | return state_.compareExchange( 82 | expected: State.inProgress.rawValue, desired: newState.rawValue, ordering: .sequentiallyConsistent 83 | ).0 84 | } 85 | 86 | /// Fulfill the promise and return `true` if the promise was been fulfilled 87 | /// by this call, as opposed to having aready been fulfilled. 88 | open func fail(_ error: Swift.Error) -> Bool { 89 | let justModified = modifyState(State.fulfilled) 90 | if justModified { 91 | promise.fail(error) 92 | } 93 | return justModified 94 | } 95 | 96 | /// Cancel the promise and return `true` if the promise was been fulfilled 97 | /// by this call, as opposed to having aready been fulfilled. 98 | open func cancel(_ error: Swift.Error) -> Bool { 99 | let justModified = modifyState(State.cancelled) 100 | if justModified { 101 | promise.fail(error) 102 | } 103 | return justModified 104 | } 105 | 106 | /// Fulfill the promise and return `true` if the promise was been fulfilled 107 | /// by this call, as opposed to having aready been fulfilled. 108 | open func succeed(_ value: T) -> Bool { 109 | let justModified = modifyState(State.fulfilled) 110 | if justModified { 111 | promise.succeed(value) 112 | } 113 | return justModified 114 | } 115 | 116 | deinit { 117 | _ = cancel(LLBCancellablePromiseError.promiseLeaked) 118 | } 119 | } 120 | 121 | extension LLBFuture { 122 | 123 | /// Execute the given operation if a specified promise is not complete. 124 | /// Otherwise encode a `CancellablePromiseError`. 125 | @inlinable 126 | public func ifNotCompleteThen(check promise: LLBCancellablePromise

, _ operation: @escaping (Value) -> LLBFuture) -> LLBFuture { 127 | flatMap { value in 128 | switch promise.state { 129 | case .inProgress: 130 | return operation(value) 131 | case .fulfilled: 132 | return self.eventLoop.makeFailedFuture(LLBCancellablePromiseError.promiseFulfilled) 133 | case .cancelled: 134 | return self.eventLoop.makeFailedFuture(LLBCancellablePromiseError.promiseCancelled) 135 | } 136 | } 137 | } 138 | 139 | /// Execute the given operation if a specified promise is not complete. 140 | /// Otherwise encode a `CancellablePromiseError`. 141 | @inlinable 142 | public func ifNotCompleteMap(check promise: LLBCancellablePromise

, _ operation: @escaping (Value) -> O) -> LLBFuture { 143 | flatMapThrowing { value in 144 | switch promise.state { 145 | case .inProgress: 146 | return operation(value) 147 | case .fulfilled: 148 | throw LLBCancellablePromiseError.promiseFulfilled 149 | case .cancelled: 150 | throw LLBCancellablePromiseError.promiseCancelled 151 | } 152 | } 153 | } 154 | 155 | /// Post the result of a future onto the cancellable promise. 156 | @inlinable 157 | public func cascade(to promise: LLBCancellablePromise) { 158 | guard promise.isCompleted == false else { return } 159 | whenComplete { result in 160 | switch result { 161 | case let .success(value): _ = promise.succeed(value) 162 | case let .failure(error): _ = promise.fail(error) 163 | } 164 | } 165 | } 166 | } 167 | -------------------------------------------------------------------------------- /Sources/TSFFutures/Canceller.swift: -------------------------------------------------------------------------------- 1 | // This source file is part of the Swift.org open source project 2 | // 3 | // Copyright (c) 2020 Apple Inc. and the Swift project authors 4 | // Licensed under Apache License v2.0 with Runtime Library Exception 5 | // 6 | // See http://swift.org/LICENSE.txt for license information 7 | // See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 8 | 9 | import NIOConcurrencyHelpers 10 | 11 | public protocol LLBCancelProtocol { 12 | func cancel(reason: String?) 13 | } 14 | 15 | /// An object serving as a cancellation handler which can 16 | /// be supplied a cancellation procedure much later in its lifetime. 17 | public final class LLBCanceller { 18 | private let mutex_ = NIOConcurrencyHelpers.NIOLock() 19 | 20 | /// Whether and why it was cancelled. 21 | private var finalReason_: FinalReason? = nil 22 | 23 | /// A handler to when the cancellation is requested. 24 | private var handler_: LLBCancelProtocol? 25 | 26 | /// A reason for reaching the final state. 27 | private enum FinalReason { 28 | /// Cancelled with a specified reason. 29 | case cancelled(reason: String) 30 | /// Cancellation won't be needed. 31 | case abandoned 32 | } 33 | 34 | public init(_ cancelHandler: LLBCancelProtocol? = nil) { 35 | handler_ = cancelHandler 36 | } 37 | 38 | /// Checks whether the object has been cancelled. 39 | public var isCancelled: Bool { 40 | mutex_.lock() 41 | guard case .cancelled? = finalReason_ else { 42 | mutex_.unlock() 43 | return false 44 | } 45 | mutex_.unlock() 46 | return true 47 | } 48 | 49 | /// Return the reason for cancelling. 50 | public var cancelReason: String? { 51 | mutex_.lock() 52 | guard case let .cancelled(reason)? = finalReason_ else { 53 | mutex_.unlock() 54 | return nil 55 | } 56 | mutex_.unlock() 57 | return reason 58 | } 59 | 60 | /// Atomically replace the cancellation handler. 61 | public func set(handler newHandler: LLBCancelProtocol?) { 62 | mutex_.lock() 63 | let oldHandler = handler_ 64 | handler_ = newHandler 65 | if case .cancelled(let reason) = finalReason_ { 66 | oldHandler?.cancel(reason: reason) 67 | newHandler?.cancel(reason: reason) 68 | } 69 | mutex_.unlock() 70 | } 71 | 72 | /// Do not cancel anything even if requested. 73 | public func abandon() { 74 | mutex_.lock() 75 | finalReason_ = .abandoned 76 | handler_ = nil 77 | mutex_.unlock() 78 | } 79 | 80 | /// Cancel an outstanding operation. 81 | public func cancel(reason specifiedReason: String? = nil) { 82 | mutex_.lock() 83 | 84 | guard finalReason_ == nil else { 85 | // Already cancelled or abandoned. 86 | mutex_.unlock() 87 | return 88 | } 89 | 90 | let reason = specifiedReason ?? "no reason given" 91 | finalReason_ = .cancelled(reason: reason) 92 | let handler = handler_ 93 | 94 | mutex_.unlock() 95 | 96 | handler?.cancel(reason: reason) 97 | } 98 | 99 | deinit { 100 | mutex_.lock() 101 | guard case .cancelled(let reason) = finalReason_, let handler = handler_ else { 102 | mutex_.unlock() 103 | return 104 | } 105 | mutex_.unlock() 106 | handler.cancel(reason: reason + " (in deinit)") 107 | } 108 | } 109 | 110 | // Allow Canceller serve as a cancellation handler. 111 | extension LLBCanceller: LLBCancelProtocol { } 112 | 113 | /// Create a chain of single-purpose handlers. 114 | public final class LLBCancelHandlersChain: LLBCancelProtocol { 115 | private let lock = NIOConcurrencyHelpers.NIOLock() 116 | private var head: LLBCancelProtocol? 117 | private var tail: LLBCancelProtocol? 118 | 119 | public init(_ first: LLBCancelProtocol? = nil, _ second: LLBCancelProtocol? = nil) { 120 | self.head = first 121 | self.tail = second 122 | } 123 | 124 | /// Add another handler to the chain. 125 | public func add(handler: LLBCancelProtocol, for canceller: LLBCanceller) { 126 | lock.withLockVoid { 127 | guard let head = self.head else { 128 | self.head = handler 129 | return 130 | } 131 | guard let tail = self.tail else { 132 | self.tail = handler 133 | return 134 | } 135 | self.head = handler 136 | self.tail = LLBCancelHandlersChain(head, tail) 137 | } 138 | 139 | if let reason = canceller.cancelReason { 140 | cancel(reason: reason) 141 | } 142 | } 143 | 144 | /// Cancel the operations in the handlers chain. 145 | public func cancel(reason: String?) { 146 | let (h, t): (LLBCancelProtocol?, LLBCancelProtocol?) = lock.withLock { 147 | let pair = (self.head, self.tail) 148 | self.head = nil 149 | self.tail = nil 150 | return pair 151 | } 152 | h?.cancel(reason: reason) 153 | t?.cancel(reason: reason) 154 | } 155 | } 156 | -------------------------------------------------------------------------------- /Sources/TSFFutures/EventualResultsCache.swift: -------------------------------------------------------------------------------- 1 | // This source file is part of the Swift.org open source project 2 | // 3 | // Copyright (c) 2020 Apple Inc. and the Swift project authors 4 | // Licensed under Apache License v2.0 with Runtime Library Exception 5 | // 6 | // See http://swift.org/LICENSE.txt for license information 7 | // See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 8 | 9 | import NIOCore 10 | import NIOConcurrencyHelpers 11 | 12 | /// Cache of keys to eventually obtainable values. 13 | /// 14 | /// This cache coalesces requests and avoids re-obtaining values multiple times. 15 | public final class LLBEventualResultsCache: LLBFutureDeduplicator { 16 | /// The already cached keys. 17 | @usableFromInline 18 | internal var storage = [Key: LLBFuture]() 19 | 20 | /// Return the number of entries in the cache. 21 | @inlinable 22 | public var count: Int { 23 | get { return lock.withLock { storage.count } } 24 | } 25 | 26 | @inlinable 27 | override internal func lockedCacheGet(key: Key) -> LLBFuture? { 28 | return storage[key] 29 | } 30 | 31 | @inlinable 32 | override internal func lockedCacheSet(_ key: Key, _ future: LLBFuture) { 33 | storage[key] = future 34 | } 35 | 36 | @inlinable 37 | public override subscript(_ key: Key) -> LLBFuture? { 38 | get { return super[key] } 39 | set { lock.withLockVoid { storage[key] = newValue } } 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /Sources/TSFFutures/FutureOperationQueue.swift: -------------------------------------------------------------------------------- 1 | // This source file is part of the Swift.org open source project 2 | // 3 | // Copyright (c) 2020-2021 Apple Inc. and the Swift project authors 4 | // Licensed under Apache License v2.0 with Runtime Library Exception 5 | // 6 | // See http://swift.org/LICENSE.txt for license information 7 | // See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 8 | 9 | import Foundation 10 | 11 | import NIOConcurrencyHelpers 12 | import NIO 13 | 14 | 15 | /// A queue for future-producing operations, which limits how many can run 16 | /// concurrently. 17 | public final class LLBFutureOperationQueue: Sendable { 18 | struct State: Sendable { 19 | /// Maximum allowed number of work items concurrently executing. 20 | var maxConcurrentOperations: Int 21 | 22 | /// The number of executing futures. 23 | var numExecuting = 0 24 | 25 | /// The user-specified "shares" that are currently being processed. 26 | var numSharesInFLight = 0 27 | 28 | /// The queue of operations to run. 29 | var workQueue = NIO.CircularBuffer() 30 | } 31 | 32 | struct WorkItem { 33 | let loop: LLBFuturesDispatchLoop 34 | let share: Int 35 | let notifyWhenScheduled: LLBPromise? 36 | let run: () -> Void 37 | } 38 | 39 | private let state: NIOLockedValueBox 40 | 41 | /// Maximum allowed number of shares concurrently executing. 42 | /// This option independently sets a cap on concurrency. 43 | private let maxConcurrentShares: Int 44 | 45 | public var maxConcurrentOperations: Int { 46 | get { 47 | return self.state.withLockedValue { state in 48 | return state.maxConcurrentOperations 49 | } 50 | } 51 | set { 52 | self.scheduleMoreTasks { state in 53 | state.maxConcurrentOperations = max(1, newValue) 54 | } 55 | } 56 | } 57 | 58 | /// Return the number of operations currently queued. 59 | public var opCount: Int { 60 | return self.state.withLockedValue { state in 61 | return state.numExecuting + state.workQueue.count 62 | } 63 | } 64 | 65 | /// Create a new limiter which will only initiate `maxConcurrentOperations` 66 | /// operations simultaneously. 67 | public init(maxConcurrentOperations: Int, maxConcurrentShares: Int = .max) { 68 | self.state = NIOLockedValueBox(State(maxConcurrentOperations: max(1, maxConcurrentOperations))) 69 | self.maxConcurrentShares = max(1, maxConcurrentShares) 70 | } 71 | 72 | /// NB: calls wait() on a current thread, beware. 73 | @available(*, noasync, message: "This method blocks indefinitely, don't use from 'async' or SwiftNIO EventLoops") 74 | @available(*, deprecated, message: "This method blocks indefinitely and returns a future") 75 | public func enqueueWithBackpressure(on loop: LLBFuturesDispatchLoop, share: Int = 1, body: @escaping () -> LLBFuture) -> LLBFuture { 76 | let scheduled = loop.makePromise(of: Void.self) 77 | 78 | let future: LLBFuture = enqueue(on: loop, share: share, notifyWhenScheduled: scheduled, body: body) 79 | 80 | try! scheduled.futureResult.wait() 81 | 82 | return future 83 | } 84 | 85 | /// Add an operation into the queue, which can run immediately 86 | /// or at some unspecified time in the future, as permitted by 87 | /// the `maxConcurrentOperations` setting. 88 | /// The `share` option independently controls maximum allowed concurrency. 89 | /// The queue can support low number of high-share loads, or high number of 90 | /// low-share loads. Useful to model queue size in bytes. 91 | /// For such use cases, set share to the payload size in bytes. 92 | public func enqueue(on loop: LLBFuturesDispatchLoop, share: Int = 1, notifyWhenScheduled: LLBPromise? = nil, body: @escaping () -> LLBFuture) -> LLBFuture { 93 | let promise = loop.makePromise(of: T.self) 94 | 95 | func runBody() { 96 | let f = body() 97 | f.whenComplete { _ in 98 | self.scheduleMoreTasks { state in 99 | assert(state.numExecuting >= 1) 100 | assert(state.numSharesInFLight >= share) 101 | state.numExecuting -= 1 102 | state.numSharesInFLight -= share 103 | } 104 | } 105 | f.cascade(to: promise) 106 | } 107 | 108 | let workItem = WorkItem(loop: loop, share: share, notifyWhenScheduled: notifyWhenScheduled, run: runBody) 109 | 110 | self.scheduleMoreTasks { state in 111 | state.workQueue.append(workItem) 112 | } 113 | 114 | return promise.futureResult 115 | } 116 | 117 | private func scheduleMoreTasks(performUnderLock: (inout State) -> Void) { 118 | // Decrement our counter, and get a new item to run if available. 119 | typealias Item = (loop: LLBFuturesDispatchLoop, notify: LLBPromise?, run: () -> Void) 120 | let toExecute: [Item] = self.state.withLockedValue { state in 121 | performUnderLock(&state) 122 | 123 | var scheduleItems: [Item] = [] 124 | 125 | // If we have room to execute the operation, 126 | // do so immediately (outside the lock). 127 | while state.numExecuting < state.maxConcurrentOperations, 128 | state.numSharesInFLight < self.maxConcurrentShares { 129 | 130 | // Schedule a new operation, if available. 131 | guard let op = state.workQueue.popFirst() else { 132 | break 133 | } 134 | 135 | state.numExecuting += 1 136 | state.numSharesInFLight += op.share 137 | scheduleItems.append((op.loop, op.notifyWhenScheduled, op.run)) 138 | } 139 | 140 | return scheduleItems 141 | } 142 | 143 | for (loop, notify, run) in toExecute { 144 | loop.execute { 145 | notify?.succeed(()) 146 | run() 147 | } 148 | } 149 | } 150 | } 151 | -------------------------------------------------------------------------------- /Sources/TSFFutures/Futures.swift: -------------------------------------------------------------------------------- 1 | // This source file is part of the Swift.org open source project 2 | // 3 | // Copyright (c) 2020 Apple Inc. and the Swift project authors 4 | // Licensed under Apache License v2.0 with Runtime Library Exception 5 | // 6 | // See http://swift.org/LICENSE.txt for license information 7 | // See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 8 | 9 | import NIO 10 | 11 | import TSCBasic 12 | import TSCUtility 13 | 14 | public typealias LLBFuture = NIO.EventLoopFuture 15 | public typealias LLBPromise = NIO.EventLoopPromise 16 | public typealias LLBFuturesDispatchGroup = NIO.EventLoopGroup 17 | public typealias LLBFuturesDispatchLoop = NIO.EventLoop 18 | 19 | 20 | public func LLBMakeDefaultDispatchGroup() -> LLBFuturesDispatchGroup { 21 | return MultiThreadedEventLoopGroup(numberOfThreads: System.coreCount) 22 | } 23 | 24 | public extension LLBPromise { 25 | /// Fulfill the promise from a returned value, or fail the promise if throws. 26 | @inlinable 27 | func fulfill(_ body: () throws -> Value) { 28 | do { 29 | try succeed(body()) 30 | } catch { 31 | fail(error) 32 | } 33 | } 34 | } 35 | 36 | 37 | /// Support storing and retrieving dispatch group from a context 38 | public extension Context { 39 | static func with(_ group: LLBFuturesDispatchGroup) -> Context { 40 | return Context(dictionaryLiteral: (ObjectIdentifier(LLBFuturesDispatchGroup.self), group as Any)) 41 | } 42 | 43 | var group: LLBFuturesDispatchGroup { 44 | get { 45 | guard let group = self[ObjectIdentifier(LLBFuturesDispatchGroup.self), as: LLBFuturesDispatchGroup.self] else { 46 | fatalError("no futures dispatch group") 47 | } 48 | return group 49 | } 50 | set { 51 | self[ObjectIdentifier(LLBFuturesDispatchGroup.self)] = newValue 52 | } 53 | } 54 | } 55 | 56 | extension LLBFuture { 57 | public func tsf_unwrapOptional( 58 | orError error: Swift.Error 59 | ) -> EventLoopFuture where Value == T? { 60 | self.flatMapThrowing { value in 61 | guard let value = value else { 62 | throw error 63 | } 64 | return value 65 | } 66 | } 67 | 68 | public func tsf_unwrapOptional( 69 | orStringError error: String 70 | ) -> EventLoopFuture where Value == T? { 71 | tsf_unwrapOptional(orError: StringError(error)) 72 | } 73 | } 74 | -------------------------------------------------------------------------------- /Sources/TSFFutures/OperationQueue+Extensions.swift: -------------------------------------------------------------------------------- 1 | // This source file is part of the Swift.org open source project 2 | // 3 | // Copyright (c) 2020 Apple Inc. and the Swift project authors 4 | // Licensed under Apache License v2.0 with Runtime Library Exception 5 | // 6 | // See http://swift.org/LICENSE.txt for license information 7 | // See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 8 | 9 | import Foundation 10 | 11 | public extension OperationQueue { 12 | convenience init(tsf_withName: String, maxConcurrentOperationCount: Int) { 13 | self.init() 14 | self.name = name 15 | self.maxConcurrentOperationCount = maxConcurrentOperationCount 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /Sources/TSFFutures/OrderManager.swift: -------------------------------------------------------------------------------- 1 | // This source file is part of the Swift.org open source project 2 | // 3 | // Copyright (c) 2020 Apple Inc. and the Swift project authors 4 | // Licensed under Apache License v2.0 with Runtime Library Exception 5 | // 6 | // See http://swift.org/LICENSE.txt for license information 7 | // See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 8 | 9 | import Dispatch 10 | 11 | import NIO 12 | import NIOConcurrencyHelpers 13 | 14 | 15 | /// The `OrderManager` allows explicitly specify dependencies between 16 | /// various callbacks. This is necessary to avoid or induce race conditions 17 | /// in otherwise timing-dependent code, making such code deterministic. 18 | /// The semantics is as follows: the `OrderManager` invokes callbacks 19 | /// specified as arguments to the `order(_:_)` functions starting with 1. 20 | /// The callbacks of order `n` are guaranteed to run to completion before 21 | /// callbacks of order `n+1` are run. If there's a gap in the callbacks order 22 | /// sequence, the callbacks are suspended until the missing callback is 23 | /// registered, `reset()` is called, or global timeout occurs. 24 | /// In addition to that, `reset()` restarts the global timeout. 25 | /// 26 | /// Example: 27 | /// 28 | /// let manager = OrderManager(on: ...) 29 | /// manager.order(3, { print("3") }) 30 | /// manager.order(1, { print("1") }) 31 | /// manager.order(2, { print("2") }) 32 | /// try manager.order(4).wait() 33 | /// 34 | /// The following will be printed out: 35 | /// 36 | /// 1 37 | /// 2 38 | /// 3 39 | /// 40 | 41 | public class LLBOrderManager { 42 | 43 | // A safety timer, not to be exceeded. 44 | private let cancelTimer = DispatchSource.makeTimerSource() 45 | private let timeout: DispatchTimeInterval 46 | 47 | private typealias WaitListElement = (order: Int, promise: LLBPromise, file: String, line: Int) 48 | private let lock = NIOConcurrencyHelpers.NIOLock() 49 | private var waitlist = [WaitListElement]() 50 | private var nextToRun = 1 51 | 52 | private var eventLoop: EventLoop { 53 | lock.withLock { 54 | switch groupDesignator { 55 | case let .managedGroup(group): 56 | return group.next() 57 | case let .externallySuppliedGroup(group): 58 | return group.next() 59 | } 60 | } 61 | } 62 | 63 | private enum GroupDesignator { 64 | case managedGroup(LLBFuturesDispatchGroup) 65 | case externallySuppliedGroup(LLBFuturesDispatchGroup) 66 | } 67 | private var groupDesignator: GroupDesignator 68 | 69 | public enum Error: Swift.Error { 70 | case orderManagerReset(file: String, line: Int) 71 | } 72 | 73 | public init(on loop: EventLoop, timeout: DispatchTimeInterval = .seconds(60)) { 74 | self.groupDesignator = GroupDesignator.externallySuppliedGroup(loop) 75 | self.timeout = timeout 76 | restartInactivityTimer() 77 | cancelTimer.setEventHandler { [weak self] in 78 | guard let self = self else { return } 79 | _ = self.reset() 80 | self.cancelTimer.cancel() 81 | } 82 | cancelTimer.resume() 83 | } 84 | 85 | private func restartInactivityTimer() { 86 | cancelTimer.schedule(deadline: DispatchTime.now() + timeout, repeating: .never) 87 | 88 | } 89 | 90 | /// Run a specified callback in a particular order. 91 | @discardableResult 92 | public func order(_ n: Int, file: String = #file, line: Int = #line, _ callback: @escaping () throws -> T) -> EventLoopFuture { 93 | let promise = eventLoop.makePromise(of: Void.self) 94 | 95 | lock.withLockVoid { 96 | waitlist.append((order: n, promise: promise, file: file, line: line)) 97 | } 98 | 99 | let future = promise.futureResult.flatMapThrowing { 100 | try callback() 101 | } 102 | 103 | future.whenComplete { _ in 104 | self.lock.withLockVoid { 105 | if n == self.nextToRun { 106 | self.nextToRun += 1 107 | } 108 | } 109 | self.unblockWaiters() 110 | } 111 | 112 | unblockWaiters() 113 | return future 114 | } 115 | 116 | @discardableResult 117 | public func order(_ n: Int, file: String = #file, line: Int = #line) -> EventLoopFuture { 118 | return order(n, file: file, line: line, {}) 119 | } 120 | 121 | private func unblockWaiters() { 122 | let wakeup: [EventLoopPromise] = lock.withLock { 123 | let wakeupPromises = waitlist 124 | .filter({$0.order <= nextToRun}) 125 | .map({$0.promise}) 126 | waitlist = waitlist.filter({$0.order > nextToRun}) 127 | return wakeupPromises 128 | } 129 | wakeup.forEach { $0.succeed(()) } 130 | } 131 | 132 | /// Fail all ordered callbacks. Not calling the callback functions 133 | /// specified as argument to order(_:_), but failing the outcome. 134 | public func reset(file: String = #file, line: Int = #line) -> EventLoopFuture { 135 | restartInactivityTimer() 136 | let lock = self.lock 137 | 138 | let futures = failPromises(file: file, line: line) 139 | 140 | return EventLoopFuture.whenAllSucceed(futures, on: eventLoop).map { [weak self] _ in 141 | guard let self = self else { return } 142 | lock.withLockVoid { 143 | assert(self.waitlist.isEmpty) 144 | self.nextToRun = 1 145 | } 146 | } 147 | } 148 | 149 | @discardableResult 150 | private func failPromises(file: String = #file, line: Int = #line) -> [EventLoopFuture] { 151 | let toCancel: [WaitListElement] = lock.withLock { 152 | let cancelList = waitlist 153 | waitlist = [] 154 | nextToRun = Int.max 155 | return cancelList 156 | } 157 | let error = Error.orderManagerReset(file: file, line: line) 158 | return toCancel.sorted(by: {$0.order < $1.order}).map { 159 | $0.promise.fail(error) 160 | return $0.promise.futureResult.flatMapErrorThrowing { _ in () } 161 | } 162 | } 163 | 164 | deinit { 165 | cancelTimer.setEventHandler { } 166 | cancelTimer.cancel() 167 | 168 | failPromises() 169 | 170 | guard case let .managedGroup(group) = groupDesignator else { 171 | return 172 | } 173 | 174 | let q = DispatchQueue(label: "tsf.OrderManager") 175 | q.async { try! group.syncShutdownGracefully() } 176 | } 177 | } 178 | -------------------------------------------------------------------------------- /Sources/TSFUtility/ByteBuffer.swift: -------------------------------------------------------------------------------- 1 | // This source file is part of the Swift.org open source project 2 | // 3 | // Copyright (c) 2020 Apple Inc. and the Swift project authors 4 | // Licensed under Apache License v2.0 with Runtime Library Exception 5 | // 6 | // See http://swift.org/LICENSE.txt for license information 7 | // See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 8 | 9 | import NIO 10 | import NIOFoundationCompat 11 | import Foundation 12 | 13 | public typealias LLBByteBuffer = NIO.ByteBuffer 14 | public typealias LLBByteBufferAllocator = NIO.ByteBufferAllocator 15 | public typealias LLBByteBufferView = NIO.ByteBufferView 16 | 17 | 18 | public extension LLBByteBuffer { 19 | static func withBytes(_ data: ArraySlice) -> LLBByteBuffer { 20 | return LLBByteBuffer(bytes: data) 21 | } 22 | 23 | static func withBytes(_ data: Data) -> LLBByteBuffer { 24 | return LLBByteBuffer(data: data) 25 | } 26 | 27 | static func withBytes(_ data: Array) -> LLBByteBuffer { 28 | return LLBByteBuffer(bytes: data) 29 | } 30 | } 31 | 32 | extension LLBByteBuffer { 33 | public mutating func reserveWriteCapacity(_ count: Int) { 34 | self.reserveCapacity(self.writerIndex + count) 35 | } 36 | 37 | public mutating func unsafeWrite(_ writeCallback: (UnsafeMutableRawBufferPointer) -> (wrote: Int, R)) -> R { 38 | var returnValue: R? = nil 39 | self.writeWithUnsafeMutableBytes(minimumWritableBytes: 0) { ptr -> Int in 40 | let (wrote, ret) = writeCallback(ptr) 41 | returnValue = ret 42 | return wrote 43 | } 44 | return returnValue! 45 | } 46 | 47 | } 48 | -------------------------------------------------------------------------------- /Sources/TSFUtility/FastData.swift: -------------------------------------------------------------------------------- 1 | // This source file is part of the Swift.org open source project 2 | // 3 | // Copyright (c) 2020 Apple Inc. and the Swift project authors 4 | // Licensed under Apache License v2.0 with Runtime Library Exception 5 | // 6 | // See http://swift.org/LICENSE.txt for license information 7 | // See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 8 | 9 | import Foundation 10 | 11 | import NIOCore 12 | 13 | /// Something that exposes working withContiguousStorage 14 | public enum LLBFastData { 15 | case slice(ArraySlice) 16 | case view(LLBByteBuffer) 17 | case data(Data) 18 | case pointer(UnsafeRawBufferPointer, deallocator: (UnsafeRawBufferPointer) -> Void) 19 | 20 | public init(_ data: [UInt8]) { self = .slice(ArraySlice(data)) } 21 | public init(_ data: ArraySlice) { self = .slice(data) } 22 | public init(_ data: LLBByteBuffer) { self = .view(data) } 23 | public init(_ data: Data) { 24 | precondition(data.regions.count == 1) 25 | self = .data(data) 26 | } 27 | public init(_ pointer: UnsafeRawBufferPointer, deallocator: @escaping (UnsafeRawBufferPointer) -> Void) { 28 | self = .pointer(pointer, deallocator: deallocator) 29 | } 30 | 31 | public var count: Int { 32 | switch self { 33 | case let .slice(data): 34 | return data.count 35 | case let .view(data): 36 | return data.readableBytes 37 | case let .data(data): 38 | return data.count 39 | case let .pointer(ptr, _): 40 | return ptr.count 41 | } 42 | } 43 | 44 | public func withContiguousStorage(_ cb: (UnsafeBufferPointer) throws -> R) rethrows -> R { 45 | switch self { 46 | case let .slice(data): 47 | return try data.withContiguousStorageIfAvailable(cb)! 48 | case let .view(data): 49 | return try data.readableBytesView.withContiguousStorageIfAvailable(cb)! 50 | case let .data(data): 51 | precondition(data.regions.count == 1) 52 | return try data.withUnsafeBytes { rawPtr in 53 | let ptr = UnsafeRawBufferPointer(rawPtr).bindMemory(to: UInt8.self) 54 | return try cb(ptr) 55 | } 56 | case let .pointer(rawPtr, _): 57 | let ptr = UnsafeRawBufferPointer(rawPtr).bindMemory(to: UInt8.self) 58 | return try cb(ptr) 59 | } 60 | } 61 | } 62 | 63 | -------------------------------------------------------------------------------- /Sources/TSFUtility/FutureFileSystem.swift: -------------------------------------------------------------------------------- 1 | // This source file is part of the Swift.org open source project 2 | // 3 | // Copyright (c) 2020 Apple Inc. and the Swift project authors 4 | // Licensed under Apache License v2.0 with Runtime Library Exception 5 | // 6 | // See http://swift.org/LICENSE.txt for license information 7 | // See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 8 | 9 | import Foundation 10 | 11 | import NIOCore 12 | import TSCBasic 13 | import TSFFutures 14 | 15 | 16 | /// Asynchronous file system interface integrated with `Future`s. 17 | public struct LLBFutureFileSystem { 18 | 19 | public let batchingQueue: LLBBatchingFutureOperationQueue 20 | 21 | /// - Parameters: 22 | /// - group: Threads capable of running futures. 23 | /// - maxConcurrentOperationCount: 24 | /// Operations to execute in parallel. 25 | public init(group: LLBFuturesDispatchGroup) { 26 | let solidStateDriveParallelism = 8 27 | self.batchingQueue = LLBBatchingFutureOperationQueue(name: "llbuild2.futureFileSystem", group: group, maxConcurrentOperationCount: solidStateDriveParallelism) 28 | } 29 | 30 | public func readFileContents(_ path: AbsolutePath) -> LLBFuture> { 31 | let pathString = path.pathString 32 | return batchingQueue.execute { 33 | try ArraySlice(Self.syncRead(pathString)) 34 | } 35 | } 36 | 37 | public func readFileContentsWithStat(_ path: AbsolutePath) -> LLBFuture<(contents: ArraySlice, stat: stat)> { 38 | let pathString = path.pathString 39 | return batchingQueue.execute { 40 | try Self.syncReadWithStat(pathString) 41 | } 42 | } 43 | 44 | public func getFileInfo(_ path: AbsolutePath) -> LLBFuture { 45 | let pathString = path.pathString 46 | return batchingQueue.execute { 47 | var sb = stat() 48 | guard stat(pathString, &sb) != -1 else { 49 | throw FileSystemError(errno: errno, path) 50 | } 51 | return sb 52 | } 53 | } 54 | 55 | /// Read in the given file. 56 | public static func syncRead(_ path: String) throws -> [UInt8] { 57 | let fd = try Self.openImpl(path) 58 | defer { close(fd) } 59 | 60 | let expectedFileSize = 8192 // Greater than 78% of stdlib headers. 61 | let firstBuffer = try syncReadComplete(fd: fd, readSize: expectedFileSize) 62 | guard firstBuffer.count == expectedFileSize else { 63 | // A small file was read without hitting stat(). Good. 64 | return firstBuffer 65 | } 66 | 67 | // Fast path failed. Measure file size and try to swallow it whole. 68 | var sb = stat() 69 | guard fstat(fd, &sb) == 0 else { 70 | throw FileSystemError(.ioError(code: 0), try? AbsolutePath(validating: path)) 71 | } 72 | 73 | if expectedFileSize > sb.st_size { 74 | // File size is less than what was already read in. 75 | throw FileSystemError(.ioError(code: 0), try? AbsolutePath(validating: path)) 76 | } else if expectedFileSize == sb.st_size { 77 | // Avoid copying if the file is exactly 8kiB. 78 | return firstBuffer 79 | } 80 | 81 | return try [UInt8](unsafeUninitializedCapacity: Int(sb.st_size)) { ptr, initializedCount in 82 | var consumedSize = expectedFileSize 83 | defer { initializedCount = consumedSize } 84 | 85 | // Copy the already read bytes. 86 | firstBuffer.withUnsafeBytes { firstBufferBytes in 87 | let alreadyRead = UnsafeRawBufferPointer(start: firstBufferBytes.baseAddress!, count: expectedFileSize) 88 | UnsafeMutableRawBufferPointer(ptr).copyMemory(from: alreadyRead) 89 | } 90 | 91 | consumedSize += try unsafeReadCompleteImpl(fd: fd, ptr, bufferOffset: consumedSize, fileOffset: 0) 92 | } 93 | } 94 | 95 | /// Return the bytes and sometimes the stat information for the file. 96 | /// The stat information is a byproduct and can be used as an optimization. 97 | private static func syncReadWithStat(_ path: String) throws -> (contents: ArraySlice, stat: stat) { 98 | let fd = try Self.openImpl(path) 99 | defer { close(fd) } 100 | 101 | // Fast path failed. Measure file size and try to swallow it whole. 102 | var sb = stat() 103 | guard fstat(fd, &sb) == 0 else { 104 | throw FileSystemError(.ioError(code: errno), try? AbsolutePath(validating: path)) 105 | } 106 | 107 | let data = try syncReadComplete(fd: fd, readSize: Int(sb.st_size)) 108 | guard data.count == sb.st_size else { 109 | // File size is less than advertised. 110 | throw FileSystemError(.ioError(code: 0), try? AbsolutePath(validating: path)) 111 | } 112 | 113 | return (contents: ArraySlice(data), stat: sb) 114 | } 115 | 116 | /// Read until reaches the readSize or an EOF. 117 | /// The difference between hitting the buffer with or without EOF can not 118 | /// be inferred from the return value of this function. 119 | public static func syncReadComplete(fd: CInt, readSize: Int, fileOffset: Int = 0) throws -> [UInt8] { 120 | 121 | return try [UInt8](unsafeUninitializedCapacity: readSize) { ptr, initializedCount in 122 | initializedCount = try unsafeReadCompleteImpl(fd: fd, ptr, bufferOffset: 0, fileOffset: fileOffset) 123 | } 124 | } 125 | 126 | public static func openImpl(_ path: String, flags: CInt = O_RDONLY) throws -> CInt { 127 | let fd = open(path, flags | O_CLOEXEC) 128 | guard fd != -1 else { 129 | // FIXME: Need to fix FileSystemError to not require an AbsolutePath. 130 | throw FileSystemError(errno: errno, (try? AbsolutePath(validating: path)) ?? .root) 131 | } 132 | return fd 133 | } 134 | 135 | /// Read until the end of the given buffer or EOF. 136 | /// Returns the bytes read. 137 | private static func unsafeReadCompleteImpl(fd: CInt, _ ptr: UnsafeMutableBufferPointer, bufferOffset: Int, fileOffset: Int) throws -> Int { 138 | var offset = 0 139 | while bufferOffset + offset < ptr.count { 140 | let (off, overflow) = fileOffset.addingReportingOverflow(offset) 141 | guard !overflow else { 142 | // FIXME: Need to fix FileSystemError to allow ERANGE. 143 | throw FileSystemError(.unknownOSError) 144 | } 145 | 146 | let count = try unsafeReadImpl(fd: fd, ptr, bufferOffset: bufferOffset + offset, fileOffset: off) 147 | if count > 0 { 148 | offset += count 149 | } else if count == 0 { 150 | break 151 | } else { 152 | fatalError("read() returned \(count)") 153 | } 154 | } 155 | 156 | return offset 157 | } 158 | 159 | private static func unsafeReadImpl(fd: CInt, _ ptr: UnsafeMutableBufferPointer, bufferOffset: Int, fileOffset: Int) throws -> Int { 160 | assert(bufferOffset < ptr.count) 161 | 162 | while true { 163 | guard let off = off_t(exactly: fileOffset) else { 164 | // FIXME: Need to fix FileSystemError to allow ERANGE. 165 | throw FileSystemError(.unknownOSError) 166 | } 167 | 168 | let ret = pread(fd, ptr.baseAddress! + bufferOffset, ptr.count - bufferOffset, off) 169 | switch ret { 170 | case let count where count > 0: 171 | return count 172 | case 0: 173 | return 0 174 | case -1: 175 | guard errno == EINTR else { 176 | throw FileSystemError.init(.ioError(code: errno)) 177 | } 178 | continue 179 | default: 180 | fatalError("pread() returned \(ret)") 181 | } 182 | } 183 | } 184 | 185 | } 186 | -------------------------------------------------------------------------------- /Sources/TSFUtility/Serializable.swift: -------------------------------------------------------------------------------- 1 | // This source file is part of the Swift.org open source project 2 | // 3 | // Copyright (c) 2020 Apple Inc. and the Swift project authors 4 | // Licensed under Apache License v2.0 with Runtime Library Exception 5 | // 6 | // See http://swift.org/LICENSE.txt for license information 7 | // See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 8 | 9 | import Foundation 10 | 11 | import NIOCore 12 | 13 | /// Serializable protocol describes a structure that can be serialized 14 | /// into a buffer of bytes, and deserialized back from the buffer of bytes. 15 | public typealias LLBSerializable = LLBSerializableIn & LLBSerializableOut 16 | 17 | public enum LLBSerializableError: Error { 18 | case unknownError(String) 19 | } 20 | 21 | public protocol LLBSerializableIn { 22 | /// Decode the given block back into a message. 23 | init(from rawBytes: LLBByteBuffer) throws 24 | } 25 | 26 | public protocol LLBSerializableOut { 27 | /// Produce an encoded blob that fully defines the structure contents. 28 | func toBytes(into buffer: inout LLBByteBuffer) throws 29 | } 30 | 31 | extension LLBSerializableOut { 32 | public func toBytes() throws -> LLBByteBuffer { 33 | var buffer = LLBByteBufferAllocator().buffer(capacity: 0) 34 | try toBytes(into: &buffer) 35 | return buffer 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /Tests/TSFCASFileTreeTests/CASBlobTests.swift: -------------------------------------------------------------------------------- 1 | // This source file is part of the Swift.org open source project 2 | // 3 | // Copyright (c) 2020 Apple Inc. and the Swift project authors 4 | // Licensed under Apache License v2.0 with Runtime Library Exception 5 | // 6 | // See http://swift.org/LICENSE.txt for license information 7 | // See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 8 | 9 | import XCTest 10 | 11 | import TSCBasic 12 | import TSCUtility 13 | 14 | import TSFCASFileTree 15 | 16 | 17 | class CASBlobTests: XCTestCase { 18 | var group: LLBFuturesDispatchGroup! 19 | 20 | override func setUp() { 21 | group = LLBMakeDefaultDispatchGroup() 22 | } 23 | 24 | override func tearDown() { 25 | try! group.syncShutdownGracefully() 26 | group = nil 27 | } 28 | 29 | func testBasics() throws { 30 | try withTemporaryFile(suffix: ".dat") { tmp in 31 | // Check several chunk sizes, to probe boundary conditions. 32 | try checkOneBlob(tmp, chunkSize: 16, [UInt8](repeating: 1, count: 512)) 33 | try checkOneBlob(tmp, chunkSize: 1024, [UInt8](repeating: 1, count: 512)) 34 | 35 | // Compression only works with larger objects, due to hard coded constants in importer. 36 | try checkOneBlob(tmp, chunkSize: 1024, [UInt8](repeating: 1, count: 2048)) 37 | } 38 | } 39 | 40 | func checkOneBlob(_ tmp: TemporaryFile, chunkSize: Int, _ contents: [UInt8]) throws { 41 | try localFileSystem.writeFileContents(tmp.path, bytes: ByteString(contents)) 42 | 43 | let db = LLBInMemoryCASDatabase(group: group) 44 | let ctx = Context() 45 | let id = try LLBCASFileTree.import(path: tmp.path, to: db, 46 | options: LLBCASFileTree.ImportOptions(fileChunkSize: chunkSize), stats: nil, ctx).wait() 47 | 48 | let blob = try LLBCASBlob.parse(id: id, in: db, ctx).wait() 49 | XCTAssertEqual(blob.size, contents.count) 50 | 51 | // Check various read patterns. 52 | for testRange in [0 ..< 0, 0 ..< 1, 0 ..< contents.count, 10 ..< 20, 20 ..< 128, 128 ..< 512] { 53 | 54 | let blobRange = try blob.read(range: testRange, ctx).wait() 55 | let bytes: [UInt8] = Array(LLBByteBuffer(blobRange).readableBytesView.prefix(testRange.count)) 56 | XCTAssertEqual(ArraySlice(bytes), contents[testRange]) 57 | } 58 | } 59 | } 60 | -------------------------------------------------------------------------------- /Tests/TSFCASFileTreeTests/FileTreeImportExportTests.swift: -------------------------------------------------------------------------------- 1 | // This source file is part of the Swift.org open source project 2 | // 3 | // Copyright (c) 2020 Apple Inc. and the Swift project authors 4 | // Licensed under Apache License v2.0 with Runtime Library Exception 5 | // 6 | // See http://swift.org/LICENSE.txt for license information 7 | // See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 8 | 9 | 10 | import Dispatch 11 | import XCTest 12 | 13 | import TSCBasic 14 | import TSCUtility 15 | 16 | import TSFCASFileTree 17 | 18 | 19 | class ImportExportTests: XCTestCase { 20 | 21 | var testOptions: LLBCASFileTree.ImportOptions { 22 | var options = LLBCASFileTree.ImportOptions() 23 | // These settings are important for keeping tests small. 24 | options.fileChunkSize = 4096 25 | options.minMmapSize = 4096 26 | return options 27 | } 28 | 29 | func testBasicFilesystemExport() throws{ 30 | let group = LLBMakeDefaultDispatchGroup() 31 | let ctx = Context() 32 | 33 | try withTemporaryDirectory(prefix: #function, removeTreeOnDeinit: true) { dir in 34 | let tmpdir = dir.appending(component: "first") 35 | 36 | // Create sample file system content. 37 | let fs = TSCBasic.localFileSystem 38 | try fs.createDirectory(tmpdir) 39 | 40 | let db = LLBInMemoryCASDatabase(group: group) 41 | 42 | let inTree: LLBDeclFileTree = .dir(["a.txt": .file("hi"), 43 | "dir": .dir(["b.txt": .file("hello"), 44 | "c.txt": .file("world") 45 | ]) 46 | ]) 47 | let id = try LLBCASFSClient(db).store(inTree, ctx).wait().asDirectoryEntry(filename: "").id 48 | 49 | // Get the object. 50 | let tree: LLBCASFileTree 51 | do { 52 | let casObject = try db.get(id, ctx).wait() 53 | tree = try LLBCASFileTree(id: id, object: casObject!) 54 | } catch { 55 | XCTFail("Unexpected CASTree download error: \(errno)") 56 | throw error 57 | } 58 | 59 | // Check the result. 60 | XCTAssertEqual(tree.files, [ 61 | LLBDirectoryEntry(name: "a.txt", type: .plainFile, size: 2), 62 | LLBDirectoryEntry(name: "dir", type: .directory, size: 10)]) 63 | 64 | // Export the results. 65 | let tmpdir2 = dir.appending(component: "second") 66 | try fs.createDirectory(tmpdir2) 67 | try LLBCASFileTree.export( 68 | id, 69 | from: db, 70 | to: tmpdir2, 71 | stats: LLBCASFileTree.ExportProgressStatsInt64(), 72 | ctx 73 | ).wait() 74 | 75 | // Check the file was exported. 76 | XCTAssertEqual(try fs.readFileContents(tmpdir2.appending(component: "a.txt")), "hi") 77 | } 78 | 79 | XCTAssertNoThrow(try group.syncShutdownGracefully()) 80 | } 81 | 82 | func testBasicFilesystemImport() throws{ 83 | let group = LLBMakeDefaultDispatchGroup() 84 | let ctx = Context() 85 | 86 | for (wireFormat, expectedUploadSize) in [(LLBCASFileTree.WireFormat.binary, 68), (.compressed, 68)] { 87 | try withTemporaryDirectory(prefix: #function, removeTreeOnDeinit: true) { dir in 88 | let tmpdir = dir.appending(component: "first") 89 | 90 | // Create sample file system content. 91 | let fs = TSCBasic.localFileSystem 92 | try fs.createDirectory(tmpdir) 93 | try fs.writeFileContents(tmpdir.appending(component: "a.txt"), bytes: "hi") 94 | let subpath = tmpdir.appending(component: "dir") 95 | try fs.createDirectory(subpath, recursive: true) 96 | try fs.writeFileContents(subpath.appending(component: "b.txt"), bytes: "hello") 97 | try fs.writeFileContents(subpath.appending(component: "c.txt"), bytes: "world") 98 | 99 | let db = LLBInMemoryCASDatabase(group: group) 100 | let stats = LLBCASFileTree.ImportProgressStats() 101 | 102 | let id = try LLBCASFileTree.import(path: tmpdir, to: db, options: testOptions.with(wireFormat: wireFormat), stats: stats, ctx).wait() 103 | XCTAssertEqual(stats.uploadedBytes - stats.uploadedMetadataBytes, 12) 104 | XCTAssertEqual(stats.uploadedBytes, expectedUploadSize) 105 | XCTAssertEqual(stats.importedBytes, expectedUploadSize) 106 | XCTAssertEqual(stats.toImportBytes, expectedUploadSize) 107 | XCTAssertEqual(stats.phase, .ImportSucceeded) 108 | 109 | // Get the object. 110 | let tree: LLBCASFileTree 111 | do { 112 | let casObject = try db.get(id, ctx).wait() 113 | tree = try LLBCASFileTree(id: id, object: casObject!) 114 | } catch { 115 | XCTFail("Unexpected CASTree download error: \(errno)") 116 | throw error 117 | } 118 | 119 | // Check the result. 120 | XCTAssertEqual(tree.files, [ 121 | LLBDirectoryEntry(name: "a.txt", type: .plainFile, size: 2), 122 | LLBDirectoryEntry(name: "dir", type: .directory, size: 10)]) 123 | 124 | // Export the results. 125 | let tmpdir2 = dir.appending(component: "second") 126 | try fs.createDirectory(tmpdir2) 127 | try LLBCASFileTree.export( 128 | id, 129 | from: db, 130 | to: tmpdir2, 131 | stats: LLBCASFileTree.ExportProgressStatsInt64(), 132 | ctx 133 | ).wait() 134 | 135 | // Check the file was exported. 136 | XCTAssertEqual(try fs.readFileContents(tmpdir2.appending(component: "a.txt")), "hi") 137 | } 138 | } 139 | 140 | XCTAssertNoThrow(try group.syncShutdownGracefully()) 141 | } 142 | 143 | func testImportMissingDirectory() throws { 144 | let group = LLBMakeDefaultDispatchGroup() 145 | let ctx = Context() 146 | 147 | try withTemporaryDirectory(prefix: #function, removeTreeOnDeinit: true) { dir in 148 | let somedir = dir.appending(component: "some") 149 | 150 | // Create sample file system content. 151 | let fs = TSCBasic.localFileSystem 152 | try fs.createDirectory(somedir) 153 | 154 | let nonexistDir = somedir.appending(component: "nonexist") 155 | let db = LLBInMemoryCASDatabase(group: group) 156 | XCTAssertThrowsError(try LLBCASFileTree.import(path: nonexistDir, to: db, options: testOptions, ctx).wait()) { error in 157 | XCTAssertEqual(error as? FileSystemError, FileSystemError(.noEntry, nonexistDir)) 158 | } 159 | } 160 | } 161 | 162 | func testUnicodeImport() throws { 163 | let group = LLBMakeDefaultDispatchGroup() 164 | let ctx = Context() 165 | 166 | try withTemporaryDirectory(prefix: #function, removeTreeOnDeinit: true) { dir in 167 | let target = "你好 你好" 168 | let ret = symlink(target, dir.appending(component: "コカコーラ").pathString) 169 | XCTAssertEqual(ret, 0) 170 | 171 | let db = LLBInMemoryCASDatabase(group: group) 172 | let stats = LLBCASFileTree.ImportProgressStats() 173 | 174 | let id = try LLBCASFileTree.import( 175 | path: dir, to: db, options: testOptions, stats: stats, ctx 176 | ).wait() 177 | 178 | // Get the object. 179 | let tree: LLBCASFileTree 180 | do { 181 | let casObject = try db.get(id, ctx).wait() 182 | tree = try LLBCASFileTree(id: id, object: casObject!) 183 | } catch { 184 | XCTFail("Unexpected CASTree download error: \(errno)") 185 | throw error 186 | } 187 | 188 | // Check the result. 189 | XCTAssertEqual( 190 | tree.files, 191 | [ 192 | LLBDirectoryEntry(name: "コカコーラ", type: .symlink, size: target.utf8.count), 193 | ]) 194 | } 195 | 196 | XCTAssertNoThrow(try group.syncShutdownGracefully()) 197 | } 198 | 199 | 200 | } 201 | -------------------------------------------------------------------------------- /Tests/TSFCASTests/DataIDTests.swift: -------------------------------------------------------------------------------- 1 | // This source file is part of the Swift.org open source project 2 | // 3 | // Copyright (c) 2020 Apple Inc. and the Swift project authors 4 | // Licensed under Apache License v2.0 with Runtime Library Exception 5 | // 6 | // See http://swift.org/LICENSE.txt for license information 7 | // See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 8 | 9 | import XCTest 10 | 11 | import TSFCAS 12 | 13 | 14 | class LLBDataIDTests: XCTestCase { 15 | 16 | /// Check that DataID is codable, and that it uses a flat representation. 17 | func testCodability() throws { 18 | // We have to wrap in an array, as Foundation doesn't allow top-level scalar items. 19 | let id = LLBDataID(directHash: Array("abc def".utf8)) 20 | let json = try JSONEncoder().encode([id]) 21 | XCTAssertEqual(String(decoding: json, as: Unicode.UTF8.self), "[\"0~YWJjIGRlZg==\"]") 22 | XCTAssertEqual(try JSONDecoder().decode([LLBDataID].self, from: json), [id]) 23 | 24 | // Check that invalid JSON is detected. 25 | XCTAssertThrowsError(try JSONDecoder().decode([LLBDataID].self, from: Data("[\"not hex\"]".utf8))) 26 | } 27 | 28 | /// Check that DataID can be parsed and re-serialized. 29 | func testRoundTrip() { 30 | let ids = ["0~YWJjIGRlZg==", "4~YWJjIGRlZg=="] 31 | for string in ids { 32 | guard let id = LLBDataID(string: string) else { 33 | XCTFail("Can't parse LLBDataID") 34 | continue 35 | } 36 | XCTAssertEqual("\(id)", string) 37 | } 38 | } 39 | 40 | } 41 | 42 | -------------------------------------------------------------------------------- /Tests/TSFCASTests/FileBackedCASDatabaseTests.swift: -------------------------------------------------------------------------------- 1 | // This source file is part of the Swift.org open source project 2 | // 3 | // Copyright (c) 2020 Apple Inc. and the Swift project authors 4 | // Licensed under Apache License v2.0 with Runtime Library Exception 5 | // 6 | // See http://swift.org/LICENSE.txt for license information 7 | // See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 8 | 9 | import XCTest 10 | import Dispatch 11 | 12 | import NIO 13 | import TSCBasic 14 | import TSCUtility 15 | 16 | import TSFCAS 17 | 18 | 19 | class FileBackedCASDatabaseTests: XCTestCase { 20 | 21 | var group: LLBFuturesDispatchGroup! 22 | var threadPool: NIOThreadPool! 23 | var fileIO: NonBlockingFileIO! 24 | 25 | override func setUp() { 26 | super.setUp() 27 | group = LLBMakeDefaultDispatchGroup() 28 | threadPool = NIOThreadPool(numberOfThreads: 6) 29 | threadPool.start() 30 | fileIO = NonBlockingFileIO(threadPool: threadPool) 31 | } 32 | 33 | override func tearDown() { 34 | super.tearDown() 35 | try? group.syncShutdownGracefully() 36 | try? threadPool.syncShutdownGracefully() 37 | group = nil 38 | } 39 | 40 | /// ${TMPDIR} or just "/tmp", expressed as AbsolutePath 41 | private var temporaryPath: AbsolutePath { 42 | get throws { 43 | return try AbsolutePath(validating: ProcessInfo.processInfo.environment["TMPDIR", default: "/tmp"]) 44 | } 45 | } 46 | 47 | 48 | func testBasics() throws { 49 | try withTemporaryDirectory(dir: temporaryPath, prefix: "LLBUtilTests" + #function, removeTreeOnDeinit: true) { tmpDir in 50 | let db = LLBFileBackedCASDatabase(group: group, path: tmpDir) 51 | let ctx = Context() 52 | 53 | let id1 = try db.put(data: LLBByteBuffer.withBytes([1, 2, 3]), ctx).wait() 54 | let obj1 = try db.get(id1, ctx).wait()! 55 | XCTAssertEqual(id1, LLBDataID(string: "0~sXfsG_Jt-ztwENRz5tRHE7KbdluZxuYOy_rnQt5JZUM=")) 56 | XCTAssertEqual(obj1.size, 3) 57 | XCTAssertEqual(obj1.refs, []) 58 | XCTAssertEqual(obj1.data, LLBByteBuffer.withBytes([1, 2, 3])) 59 | XCTAssertEqual(try db.contains(id1, ctx).wait(), true) 60 | 61 | let id2 = try db.put(refs: [id1], data: LLBByteBuffer.withBytes([4, 5, 6]), ctx).wait() 62 | let obj2 = try db.get(id2, ctx).wait()! 63 | XCTAssertEqual(id2, LLBDataID(string: "0~udZrZzFHJr8uovWT5dOWtKz95ZqKi-vBkpiH0mJfjM4=")) 64 | XCTAssertEqual(obj2.size, 3) 65 | XCTAssertEqual(obj2.refs, [id1]) 66 | XCTAssertEqual(obj2.data, LLBByteBuffer.withBytes([4, 5, 6])) 67 | 68 | // Check contains on a missing object. 69 | let missingID = try LLBInMemoryCASDatabase(group: group).identify(data: LLBByteBuffer.withBytes([]), ctx).wait() 70 | XCTAssertEqual(try db.contains(missingID, ctx).wait(), false) 71 | } 72 | } 73 | 74 | func testPutStressTest() throws { 75 | try withTemporaryDirectory(dir: temporaryPath, prefix: "LLBUtilTests" + #function, removeTreeOnDeinit: true) { tmpDir in 76 | let db = LLBFileBackedCASDatabase(group: group, path: tmpDir) 77 | let ctx = Context() 78 | let queue = DispatchQueue(label: "sync") 79 | 80 | // Insert one object. 81 | let id1 = try db.put(data: LLBByteBuffer.withBytes([1, 2, 3]), ctx).wait() 82 | 83 | // Insert a bunch of objects concurrently. 84 | // 85 | // We take care here to do this in a way that no references to the 86 | // object data is held (other than in the database). 87 | let allocator = LLBByteBufferAllocator() 88 | func makeData(i: Int, objectSize: Int = 16) -> LLBByteBuffer { 89 | var buffer = allocator.buffer(capacity: objectSize) 90 | for j in 0 ..< objectSize { 91 | buffer.writeInteger(UInt8((i + j) & 0xFF)) 92 | } 93 | return buffer 94 | } 95 | let numObjects = 100 96 | var objects = [LLBDataID?](repeating: nil, count: numObjects) 97 | DispatchQueue.concurrentPerform(iterations: numObjects) { i in 98 | let id = { try! db.put(refs: [id1], data: makeData(i: i), ctx).wait() }() 99 | queue.sync { 100 | objects[i] = id 101 | } 102 | } 103 | 104 | for i in 0 ..< numObjects { 105 | guard let result = try db.get(objects[i]!, ctx).wait() else { 106 | XCTFail("missing expected object") 107 | return 108 | } 109 | XCTAssertEqual(result.refs, [id1]) 110 | XCTAssertEqual(result.data, makeData(i: i)) 111 | } 112 | } 113 | } 114 | 115 | } 116 | -------------------------------------------------------------------------------- /Tests/TSFCASTests/InMemoryCASDatabaseTests.swift: -------------------------------------------------------------------------------- 1 | // This source file is part of the Swift.org open source project 2 | // 3 | // Copyright (c) 2020 Apple Inc. and the Swift project authors 4 | // Licensed under Apache License v2.0 with Runtime Library Exception 5 | // 6 | // See http://swift.org/LICENSE.txt for license information 7 | // See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 8 | 9 | import XCTest 10 | 11 | import Dispatch 12 | 13 | import TSCUtility 14 | 15 | import TSFCAS 16 | 17 | 18 | class InMemoryCASDatabaseTests: XCTestCase { 19 | let group = LLBMakeDefaultDispatchGroup() 20 | 21 | func testBasics() throws { 22 | let db = LLBInMemoryCASDatabase(group: group) 23 | let ctx = Context() 24 | 25 | let id1 = try db.put(data: LLBByteBuffer.withBytes([1, 2, 3]), ctx).wait() 26 | let obj1 = try db.get(id1, ctx).wait()! 27 | XCTAssertEqual(id1, LLBDataID(string: "0~sXfsG_Jt-ztwENRz5tRHE7KbdluZxuYOy_rnQt5JZUM=")) 28 | XCTAssertEqual(obj1.size, 3) 29 | XCTAssertEqual(obj1.refs, []) 30 | XCTAssertEqual(obj1.data, LLBByteBuffer.withBytes([1, 2, 3])) 31 | 32 | let id2 = try db.put(refs: [id1], data: LLBByteBuffer.withBytes([4, 5, 6]), ctx).wait() 33 | let obj2 = try db.get(id2, ctx).wait()! 34 | XCTAssertEqual(id2, LLBDataID(string: "0~udZrZzFHJr8uovWT5dOWtKz95ZqKi-vBkpiH0mJfjM4=")) 35 | XCTAssertEqual(obj2.size, 3) 36 | XCTAssertEqual(obj2.refs, [id1]) 37 | XCTAssertEqual(Array(buffer: obj2.data), [4, 5, 6]) 38 | XCTAssertEqual(try db.contains(id1, ctx).wait(), true) 39 | 40 | // Check contains on a missing object. 41 | let missingID = try LLBInMemoryCASDatabase(group: group).put(data: LLBByteBuffer.withBytes([]), ctx).wait() 42 | XCTAssertEqual(try db.contains(missingID, ctx).wait(), false) 43 | } 44 | 45 | func testPutStressTest() throws { 46 | let db = LLBInMemoryCASDatabase(group: group) 47 | let ctx = Context() 48 | let queue = DispatchQueue(label: "sync") 49 | 50 | // Insert one object. 51 | let id1 = try db.put(data: LLBByteBuffer.withBytes([1, 2, 3]), ctx).wait() 52 | 53 | // Insert a bunch of objects concurrently. 54 | // 55 | // We take care here to do this in a way that no references to the 56 | // object data is held (other than in the database). 57 | let allocator = LLBByteBufferAllocator() 58 | func makeData(i: Int, objectSize: Int = 16) -> LLBByteBuffer { 59 | var buffer = allocator.buffer(capacity: objectSize) 60 | for j in 0 ..< objectSize { 61 | buffer.writeInteger(UInt8((i + j) & 0xFF)) 62 | } 63 | return buffer 64 | } 65 | let numObjects = 100 66 | var objects = [LLBDataID?](repeating: nil, count: numObjects) 67 | DispatchQueue.concurrentPerform(iterations: numObjects) { i in 68 | let id = { try! db.put(refs: [id1], data: makeData(i: i), ctx).wait() }() 69 | queue.sync { 70 | objects[i] = id 71 | } 72 | } 73 | 74 | for i in 0 ..< numObjects { 75 | guard let result = try db.get(objects[i]!, ctx).wait() else { 76 | XCTFail("missing expected object") 77 | return 78 | } 79 | XCTAssertEqual(result.refs, [id1]) 80 | XCTAssertEqual(result.data, makeData(i: i)) 81 | } 82 | } 83 | } 84 | -------------------------------------------------------------------------------- /Tests/TSFCASUtilitiesTests/BufferedStreamWriterTests.swift: -------------------------------------------------------------------------------- 1 | // This source file is part of the Swift.org open source project 2 | // 3 | // Copyright (c) 2020 Apple Inc. and the Swift project authors 4 | // Licensed under Apache License v2.0 with Runtime Library Exception 5 | // 6 | // See http://swift.org/LICENSE.txt for license information 7 | // See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 8 | 9 | import XCTest 10 | 11 | import TSFCAS 12 | import TSFCASUtilities 13 | 14 | import TSCBasic 15 | import TSCUtility 16 | 17 | class BufferedStreamWriterTests: XCTestCase { 18 | let group = LLBMakeDefaultDispatchGroup() 19 | 20 | func testBasics() throws { 21 | let db = LLBInMemoryCASDatabase(group: group) 22 | let ctx = Context() 23 | let allocator = LLBByteBufferAllocator() 24 | 25 | let writer = LLBBufferedStreamWriter(db, bufferSize: 32) 26 | 27 | var buffer = allocator.buffer(capacity: 128) 28 | buffer.writeRepeatingByte(65, count: 16) 29 | 30 | writer.write(data: buffer, channel: 0) 31 | 32 | // Nil because it hasn't buffered out yet. 33 | XCTAssertNil(writer.latestID) 34 | 35 | buffer.clear() 36 | buffer.writeRepeatingByte(65, count: 16) 37 | writer.write(data: buffer, channel: 0) 38 | 39 | // Now there should be an ID because we've reached the buffer size 40 | let dataID = try XCTUnwrap(writer.latestID).wait() 41 | 42 | let reader = LLBCASStreamReader(db) 43 | 44 | var readBuffer = allocator.buffer(capacity: 32) 45 | 46 | var timesCalled = 0 47 | try reader.read(id: dataID, ctx) { (channel, data) -> Bool in 48 | print(data.count) 49 | readBuffer.writeBytes(data) 50 | timesCalled += 1 51 | return true 52 | }.wait() 53 | 54 | XCTAssertEqual(timesCalled, 1) 55 | XCTAssertEqual(Data(readBuffer.readableBytesView), Data(LLBByteBufferView(repeating: 65, count: 32))) 56 | } 57 | 58 | func testDifferentChannels() throws { 59 | let db = LLBInMemoryCASDatabase(group: group) 60 | let ctx = Context() 61 | let allocator = LLBByteBufferAllocator() 62 | 63 | let writer = LLBBufferedStreamWriter(db, bufferSize: 32) 64 | 65 | var buffer = allocator.buffer(capacity: 128) 66 | buffer.writeRepeatingByte(65, count: 16) 67 | 68 | writer.write(data: buffer, channel: 0) 69 | 70 | // Nil because it hasn't buffered out yet. 71 | XCTAssertNil(writer.latestID) 72 | 73 | buffer.clear() 74 | buffer.writeRepeatingByte(65, count: 16) 75 | writer.write(data: buffer, channel: 1) 76 | 77 | // Flush to send the remaining data. 78 | writer.flush() 79 | 80 | // Now there should be an ID because we've reached the buffer size 81 | let dataID = try XCTUnwrap(writer.latestID).wait() 82 | 83 | let reader = LLBCASStreamReader(db) 84 | 85 | var readBuffer = allocator.buffer(capacity: 32) 86 | 87 | var timesCalled = 0 88 | var channelsRead = Set() 89 | try reader.read(id: dataID, ctx) { (channel, data) -> Bool in 90 | readBuffer.writeBytes(data) 91 | channelsRead.insert(channel) 92 | timesCalled += 1 93 | return true 94 | }.wait() 95 | 96 | XCTAssertEqual(timesCalled, 2) 97 | XCTAssertEqual(Data(readBuffer.readableBytesView), Data(LLBByteBufferView(repeating: 65, count: 32))) 98 | XCTAssertEqual(channelsRead, [0, 1]) 99 | } 100 | } 101 | -------------------------------------------------------------------------------- /Tests/TSFCASUtilitiesTests/LinkedListStreamTests.swift: -------------------------------------------------------------------------------- 1 | // This source file is part of the Swift.org open source project 2 | // 3 | // Copyright (c) 2020 Apple Inc. and the Swift project authors 4 | // Licensed under Apache License v2.0 with Runtime Library Exception 5 | // 6 | // See http://swift.org/LICENSE.txt for license information 7 | // See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 8 | 9 | import XCTest 10 | 11 | import TSFCAS 12 | import TSFCASUtilities 13 | import TSFCASFileTree 14 | 15 | import TSCBasic 16 | import TSCUtility 17 | 18 | class LinkedListStreamTests: XCTestCase { 19 | let group = LLBMakeDefaultDispatchGroup() 20 | 21 | func testSingleLine() throws { 22 | let db = LLBInMemoryCASDatabase(group: group) 23 | let ctx = Context() 24 | 25 | var writer = LLBLinkedListStreamWriter(db) 26 | 27 | writer.append(data: LLBByteBuffer(string: "hello, world!"), ctx) 28 | 29 | let reader = LLBCASStreamReader(db) 30 | 31 | let latestID = try writer.latestID!.wait() 32 | 33 | var contentRead = false 34 | try reader.read(id: latestID, ctx) { (channel, data) -> Bool in 35 | let stringData = String(decoding: Data(data), as: UTF8.self) 36 | XCTAssertEqual(stringData, "hello, world!") 37 | contentRead = true 38 | return true 39 | }.wait() 40 | 41 | XCTAssertTrue(contentRead) 42 | } 43 | 44 | func testStreamSingleChannel() throws { 45 | let db = LLBInMemoryCASDatabase(group: group) 46 | let ctx = Context() 47 | 48 | let writeStream = Array(0...20).map { "Stream line \($0)" } 49 | 50 | var writer = LLBLinkedListStreamWriter(db) 51 | 52 | for block in writeStream { 53 | writer.append(data: LLBByteBuffer(string: block), ctx) 54 | } 55 | 56 | let reader = LLBCASStreamReader(db) 57 | 58 | let latestID = try writer.latestID!.wait() 59 | 60 | var readStream = [String]() 61 | 62 | try reader.read(id: latestID, ctx) { (channel, data) -> Bool in 63 | let block = String(decoding: Data(data), as: UTF8.self) 64 | readStream.append(block) 65 | return true 66 | }.wait() 67 | 68 | XCTAssertEqual(readStream, writeStream) 69 | } 70 | 71 | func testStreamMultiChannel() throws { 72 | let db = LLBInMemoryCASDatabase(group: group) 73 | let ctx = Context() 74 | 75 | let writeStream: [(UInt8, String)] = Array(0...20).map { ($0 % 4, "Stream line \($0)") } 76 | 77 | var writer = LLBLinkedListStreamWriter(db) 78 | 79 | for (channel, block) in writeStream { 80 | writer.append(data: LLBByteBuffer(string: block), channel: channel, ctx) 81 | } 82 | 83 | let reader = LLBCASStreamReader(db) 84 | 85 | let latestID = try writer.latestID!.wait() 86 | 87 | var readStream = [String]() 88 | 89 | try reader.read(id: latestID, channels: [0, 1], ctx) { (channel, data) -> Bool in 90 | let block = String(decoding: Data(data), as: UTF8.self) 91 | readStream.append(block) 92 | return true 93 | }.wait() 94 | 95 | let filteredWriteStream = writeStream.filter { $0.0 <= 1 }.map { $0.1 } 96 | 97 | XCTAssertEqual(readStream, filteredWriteStream) 98 | } 99 | 100 | func testStreamAggregateSize() throws { 101 | let db = LLBInMemoryCASDatabase(group: group) 102 | let ctx = Context() 103 | 104 | let writeStream: [(UInt8, String)] = Array(0...1).map { ($0 % 2, "Stream line \($0)") } 105 | 106 | var writer = LLBLinkedListStreamWriter(db) 107 | 108 | for (channel, block) in writeStream { 109 | writer.append(data: LLBByteBuffer(string: block), channel: channel, ctx) 110 | } 111 | 112 | let reader = LLBCASStreamReader(db) 113 | 114 | let latestID = try writer.latestID!.wait() 115 | 116 | let node = try LLBCASFSClient(db).load(latestID, ctx).wait() 117 | 118 | var readLength: Int = 0 119 | try reader.read(id: latestID, ctx) { (channel, data) -> Bool in 120 | readLength += data.count 121 | return true 122 | }.wait() 123 | 124 | XCTAssertEqual(node.size(), readLength) 125 | } 126 | 127 | func testStreamReadLimit() throws { 128 | let db = LLBInMemoryCASDatabase(group: group) 129 | let ctx = Context() 130 | 131 | let writeStream = Array(0...20).map { "Stream line \($0)" } 132 | 133 | var writer = LLBLinkedListStreamWriter(db) 134 | 135 | for block in writeStream { 136 | writer.append(data: LLBByteBuffer(string: block), ctx) 137 | } 138 | 139 | let reader = LLBCASStreamReader(db) 140 | 141 | let latestID = try writer.latestID!.wait() 142 | 143 | var readStream = [String]() 144 | 145 | var stopped = false 146 | try reader.read(id: latestID, ctx) { (channel, data) -> Bool in 147 | // Read only 5 elements 148 | if readStream.count == 5 { 149 | stopped = true 150 | return false 151 | } 152 | guard !stopped else { 153 | XCTFail("Requested to stop but kept receiving data") 154 | return false 155 | } 156 | 157 | let block = String(decoding: Data(data), as: UTF8.self) 158 | readStream.append(block) 159 | return true 160 | }.wait() 161 | 162 | XCTAssertEqual(readStream, Array(writeStream.prefix(5))) 163 | } 164 | 165 | 166 | func testStreamFromPreviousState() throws { 167 | let db = LLBInMemoryCASDatabase(group: group) 168 | let ctx = Context() 169 | 170 | let writeStream = Array(0...20).map { "Stream line \($0)" } 171 | 172 | var writer = LLBLinkedListStreamWriter(db) 173 | 174 | for block in writeStream { 175 | writer.append(data: LLBByteBuffer(string: block), ctx) 176 | } 177 | 178 | let startMarker = try writer.latestID!.wait() 179 | 180 | let writeStream2 = Array(21...40).map { "Stream line \($0)" } 181 | 182 | for block in writeStream2 { 183 | writer.append(data: LLBByteBuffer(string: block), ctx) 184 | } 185 | 186 | let reader = LLBCASStreamReader(db) 187 | 188 | let latestID = try writer.latestID!.wait() 189 | 190 | var readStream = [String]() 191 | 192 | try reader.read(id: latestID, lastReadID: startMarker, ctx) { (channel, data) -> Bool in 193 | let block = String(decoding: Data(data), as: UTF8.self) 194 | readStream.append(block) 195 | return true 196 | }.wait() 197 | 198 | XCTAssertEqual(readStream, writeStream2) 199 | } 200 | } 201 | -------------------------------------------------------------------------------- /Tests/TSFFuturesTests/BatchingFutureOperationQueue.swift: -------------------------------------------------------------------------------- 1 | // 2 | // Copyright © 2019-2021 Apple, Inc. All rights reserved. 3 | // 4 | 5 | import Atomics 6 | import XCTest 7 | 8 | import NIO 9 | import NIOConcurrencyHelpers 10 | 11 | import TSFFutures 12 | 13 | 14 | class BatchingFutureOperationQueueTests: XCTestCase { 15 | 16 | // Test dynamic capacity increase. 17 | func testDynamic() throws { 18 | let group = MultiThreadedEventLoopGroup(numberOfThreads: 1) 19 | defer { try! group.syncShutdownGracefully() } 20 | 21 | let manager = LLBOrderManager(on: group.next(), timeout: .seconds(5)) 22 | 23 | var q = LLBBatchingFutureOperationQueue(name: "foo", group: group, maxConcurrentOperationCount: 1) 24 | 25 | let opsInFlight = ManagedAtomic(0) 26 | 27 | let future1: LLBFuture = q.execute { () -> LLBFuture in 28 | opsInFlight.wrappingIncrement(ordering: .relaxed) 29 | return manager.order(1).flatMap { 30 | manager.order(6) { 31 | opsInFlight.wrappingDecrement(ordering: .relaxed) 32 | } 33 | } 34 | } 35 | 36 | let future2: LLBFuture = q.execute { () -> LLBFuture in 37 | opsInFlight.wrappingIncrement(ordering: .relaxed) 38 | return manager.order(3).flatMap { 39 | manager.order(6) { 40 | opsInFlight.wrappingDecrement(ordering: .relaxed) 41 | } 42 | } 43 | } 44 | 45 | // Wait until future1 adss to opsInFlight. 46 | try manager.order(2).wait() 47 | XCTAssertEqual(opsInFlight.load(ordering: .relaxed), 1) 48 | 49 | // The test breaks without this line. 50 | q.maxOpCount += 1 51 | 52 | try manager.order(4).wait() 53 | XCTAssertEqual(opsInFlight.load(ordering: .relaxed), 2) 54 | try manager.order(5).wait() 55 | 56 | try manager.order(7).wait() 57 | XCTAssertEqual(opsInFlight.load(ordering: .relaxed), 0) 58 | 59 | try future2.wait() 60 | try future1.wait() 61 | 62 | } 63 | 64 | // Test setMaxOpCount on immutable queue. 65 | func testSetMaxConcurrency() throws { 66 | let group = MultiThreadedEventLoopGroup(numberOfThreads: 1) 67 | defer { try! group.syncShutdownGracefully() } 68 | 69 | let q = LLBBatchingFutureOperationQueue(name: "foo", group: group, maxConcurrentOperationCount: 1) 70 | q.setMaxOpCount(q.maxOpCount + 1) 71 | XCTAssertEqual(q.maxOpCount, 2) 72 | } 73 | 74 | } 75 | -------------------------------------------------------------------------------- /Tests/TSFFuturesTests/CancellableFutureTests.swift: -------------------------------------------------------------------------------- 1 | // 2 | // Copyright © 2019-2020 Apple, Inc. All rights reserved. 3 | // 4 | 5 | import Atomics 6 | import XCTest 7 | 8 | import NIO 9 | import NIOConcurrencyHelpers 10 | 11 | import TSFFutures 12 | 13 | class CancellableFutureTests: XCTestCase { 14 | 15 | var group: EventLoopGroup! 16 | 17 | override func setUp() { 18 | super.setUp() 19 | 20 | group = MultiThreadedEventLoopGroup(numberOfThreads: 1) 21 | } 22 | 23 | override func tearDown() { 24 | super.tearDown() 25 | 26 | try! group.syncShutdownGracefully() 27 | group = nil 28 | } 29 | 30 | /// This is a mock for some function that is able to cancel 31 | /// future's underlying operation. 32 | struct Handler: LLBCancelProtocol { 33 | private let called = ManagedAtomic(0) 34 | 35 | var wasCalled: Bool { 36 | return timesCalled > 0 37 | } 38 | 39 | var timesCalled: Int { 40 | return called.load(ordering: .relaxed) 41 | } 42 | 43 | func cancel(reason: String?) { 44 | called.wrappingIncrement(ordering: .relaxed) 45 | } 46 | } 47 | 48 | public enum GenericError: Swift.Error { 49 | case error 50 | } 51 | 52 | func testBasicSuccess() throws { 53 | let promise = group.next().makePromise(of: Void.self) 54 | let handler = Handler() 55 | let cf = LLBCancellableFuture(promise.futureResult, canceller: .init(handler)) 56 | promise.succeed(()) 57 | XCTAssertNoThrow(try promise.futureResult.wait()) 58 | XCTAssertFalse(handler.wasCalled) 59 | cf.canceller.cancel(reason: #function) 60 | // Canceller won't be invoked if the future was already fired. 61 | XCTAssertFalse(handler.wasCalled) 62 | } 63 | 64 | func testBasicFailure() throws { 65 | let promise = group.next().makePromise(of: Void.self) 66 | let handler = Handler() 67 | let cf = LLBCancellableFuture(promise.futureResult, canceller: .init(handler)) 68 | promise.fail(GenericError.error) 69 | XCTAssertThrowsError(try promise.futureResult.wait()) { error in 70 | XCTAssert(error is GenericError) 71 | } 72 | XCTAssertFalse(handler.wasCalled) 73 | cf.canceller.cancel(reason: #function) 74 | // Canceller won't be invoked if the future was already fired. 75 | XCTAssertFalse(handler.wasCalled) 76 | } 77 | 78 | func testBasicCancellation() throws { 79 | let promise = group.next().makePromise(of: Void.self) 80 | let handler = Handler() 81 | let cf = LLBCancellableFuture(promise.futureResult, canceller: .init(handler)) 82 | cf.cancel(reason: #function) 83 | promise.succeed(()) 84 | XCTAssertNoThrow(try promise.futureResult.wait()) 85 | XCTAssertTrue(handler.wasCalled) 86 | } 87 | 88 | func testLateCancellation() throws { 89 | let promise = group.next().makePromise(of: Void.self) 90 | let handler = Handler() 91 | let cf = LLBCancellableFuture(promise.futureResult, canceller: .init(handler)) 92 | promise.succeed(()) 93 | XCTAssertNoThrow(try promise.futureResult.wait()) 94 | cf.cancel(reason: #function) 95 | XCTAssertFalse(handler.wasCalled) 96 | } 97 | 98 | func testDoubleCancellation() throws { 99 | let promise = group.next().makePromise(of: Void.self) 100 | let handler = Handler() 101 | let cf = LLBCancellableFuture(promise.futureResult, canceller: .init(handler)) 102 | cf.cancel(reason: #function) 103 | promise.succeed(()) 104 | XCTAssertNoThrow(try promise.futureResult.wait()) 105 | XCTAssertTrue(handler.wasCalled) 106 | cf.canceller.cancel(reason: #function) 107 | XCTAssertEqual(handler.timesCalled, 1) 108 | } 109 | 110 | func testLateInitialization() throws { 111 | let promise = group.next().makePromise(of: Void.self) 112 | let handler = Handler() 113 | let cf = LLBCancellableFuture(promise.futureResult) 114 | cf.cancel(reason: #function) 115 | // Setting the handler after cancelling. 116 | cf.canceller.set(handler: handler) 117 | promise.succeed(()) 118 | XCTAssertNoThrow(try promise.futureResult.wait()) 119 | XCTAssertTrue(handler.wasCalled) 120 | cf.canceller.cancel(reason: #function) 121 | XCTAssertEqual(handler.timesCalled, 1) 122 | } 123 | 124 | func testLateInitializationAndCancellation() throws { 125 | let promise = group.next().makePromise(of: Void.self) 126 | let handler = Handler() 127 | let cf = LLBCancellableFuture(promise.futureResult) 128 | promise.succeed(()) 129 | XCTAssertNoThrow(try promise.futureResult.wait()) 130 | // Setting the handler after cancelling. 131 | cf.canceller.set(handler: handler) 132 | cf.cancel(reason: #function) 133 | XCTAssertFalse(handler.wasCalled) 134 | } 135 | 136 | } 137 | -------------------------------------------------------------------------------- /Tests/TSFFuturesTests/CancellablePromiseTests.swift: -------------------------------------------------------------------------------- 1 | // 2 | // Copyright © 2019-2020 Apple, Inc. All rights reserved. 3 | // 4 | 5 | import XCTest 6 | 7 | import NIO 8 | import NIOConcurrencyHelpers 9 | 10 | import TSFFutures 11 | 12 | class CancellablePromiseTests: XCTestCase { 13 | 14 | var group: EventLoopGroup! 15 | 16 | override func setUp() { 17 | super.setUp() 18 | 19 | group = MultiThreadedEventLoopGroup(numberOfThreads: 1) 20 | } 21 | 22 | override func tearDown() { 23 | super.tearDown() 24 | 25 | try! group.syncShutdownGracefully() 26 | group = nil 27 | } 28 | 29 | public enum GenericError: Swift.Error { 30 | case error 31 | case error1 32 | case error2 33 | } 34 | 35 | func testBasicSuccess() throws { 36 | let promise = LLBCancellablePromise(on: group.next()) 37 | XCTAssertTrue(promise.succeed(())) 38 | XCTAssertNoThrow(try promise.futureResult.wait()) 39 | } 40 | 41 | func testBasicFailure() throws { 42 | let promise = LLBCancellablePromise(on: group.next()) 43 | XCTAssertTrue(promise.fail(GenericError.error)) 44 | XCTAssertThrowsError(try promise.futureResult.wait()) 45 | } 46 | 47 | func testCancel() throws { 48 | let promise = LLBCancellablePromise(on: group.next()) 49 | XCTAssertTrue(promise.cancel(GenericError.error)) 50 | XCTAssertThrowsError(try promise.futureResult.wait()) { error in 51 | XCTAssert(error is GenericError) 52 | } 53 | XCTAssertFalse(promise.succeed(())) 54 | } 55 | 56 | func testDoubleCancel() throws { 57 | let promise = LLBCancellablePromise(on: group.next()) 58 | XCTAssertTrue(promise.cancel(GenericError.error)) 59 | XCTAssertFalse(promise.cancel(GenericError.error1)) 60 | XCTAssertThrowsError(try promise.futureResult.wait()) { error in 61 | guard case .error? = error as? GenericError else { 62 | XCTFail("Unexpected throw \(error)") 63 | return 64 | } 65 | } 66 | XCTAssertFalse(promise.fail(GenericError.error2)) 67 | } 68 | 69 | func testLeakIsOK() throws { 70 | let _ = LLBCancellablePromise(on: group.next()) 71 | } 72 | 73 | } 74 | -------------------------------------------------------------------------------- /Tests/TSFFuturesTests/CancellerTests.swift: -------------------------------------------------------------------------------- 1 | // 2 | // Copyright © 2019-2020 Apple, Inc. All rights reserved. 3 | // 4 | 5 | import Atomics 6 | import XCTest 7 | 8 | import NIO 9 | import NIOConcurrencyHelpers 10 | 11 | import TSFFutures 12 | 13 | class CancellerTests: XCTestCase { 14 | 15 | var group: EventLoopGroup! 16 | 17 | override func setUp() { 18 | super.setUp() 19 | 20 | group = MultiThreadedEventLoopGroup(numberOfThreads: 1) 21 | } 22 | 23 | override func tearDown() { 24 | super.tearDown() 25 | 26 | try! group.syncShutdownGracefully() 27 | group = nil 28 | } 29 | 30 | /// This is a mock for some function that is able to cancel 31 | /// future's underlying operation. 32 | struct Handler: LLBCancelProtocol { 33 | private let called = ManagedAtomic(0) 34 | 35 | var wasCalled: Bool { 36 | return timesCalled > 0 37 | } 38 | 39 | var timesCalled: Int { 40 | return called.load(ordering: .relaxed) 41 | } 42 | 43 | func cancel(reason: String?) { 44 | called.wrappingIncrement(ordering: .relaxed) 45 | } 46 | } 47 | 48 | func testCancel() throws { 49 | let handler = Handler() 50 | let canceller = LLBCanceller(handler) 51 | XCTAssertFalse(handler.wasCalled) 52 | canceller.cancel(reason: #function) 53 | XCTAssertTrue(handler.wasCalled) 54 | } 55 | 56 | func testDoubleCancellation() throws { 57 | let handler = Handler() 58 | let canceller = LLBCanceller(handler) 59 | canceller.cancel(reason: #function) 60 | XCTAssertTrue(handler.wasCalled) 61 | canceller.cancel(reason: #function) 62 | XCTAssertEqual(handler.timesCalled, 1) 63 | } 64 | 65 | func testLateInitialization() throws { 66 | let handler = Handler() 67 | let canceller = LLBCanceller() 68 | canceller.cancel(reason: #function) 69 | // Setting the handler after cancelling. 70 | canceller.set(handler: handler) 71 | XCTAssertTrue(handler.wasCalled) 72 | canceller.cancel(reason: #function) 73 | XCTAssertEqual(handler.timesCalled, 1) 74 | } 75 | 76 | func testAbandonFirst() throws { 77 | let handler = Handler() 78 | let canceller = LLBCanceller() 79 | canceller.abandon() 80 | canceller.cancel(reason: #function) 81 | canceller.set(handler: handler) 82 | XCTAssertFalse(handler.wasCalled) 83 | } 84 | 85 | func testAbandonLast() throws { 86 | let handler = Handler() 87 | let canceller = LLBCanceller() 88 | canceller.cancel(reason: #function) 89 | canceller.abandon() 90 | canceller.set(handler: handler) 91 | XCTAssertFalse(handler.wasCalled) 92 | } 93 | 94 | } 95 | -------------------------------------------------------------------------------- /Tests/TSFFuturesTests/EventualResultsCacheTests.swift: -------------------------------------------------------------------------------- 1 | // 2 | // Copyright © 2019 Apple, Inc. All rights reserved. 3 | // 4 | 5 | import XCTest 6 | 7 | import NIO 8 | 9 | import TSFFutures 10 | 11 | 12 | class EventualResultsCacheTests: XCTestCase { 13 | 14 | var group: LLBFuturesDispatchGroup! 15 | 16 | override func setUp() { 17 | super.setUp() 18 | 19 | group = LLBMakeDefaultDispatchGroup() 20 | } 21 | 22 | override func tearDown() { 23 | super.tearDown() 24 | 25 | try! group.syncShutdownGracefully() 26 | group = nil 27 | } 28 | 29 | /// Test that we don't re-resolve the cached value 30 | func testSerialCoalescing() throws { 31 | let cache = LLBEventualResultsCache(group: group) 32 | var hits = 0 33 | 34 | let v1Future = cache.value(for: 1) { key in 35 | hits += 1 36 | return group.next().makeSucceededFuture("\(hits)") 37 | } 38 | 39 | let v1 = try v1Future.wait() 40 | 41 | let v2Future = cache.value(for: 1) { key in 42 | XCTFail("Unexpected resolver invocation") 43 | hits += 1 44 | return group.next().makeSucceededFuture("\(hits)") 45 | } 46 | 47 | let v2 = try v2Future.wait() 48 | 49 | XCTAssertEqual(v1, v2) 50 | XCTAssertEqual(hits, 1) 51 | } 52 | 53 | /// Test that we don't re-resolve even if resolution takes time. 54 | func testParallelCoalescing() throws { 55 | let cache = LLBEventualResultsCache(group: group) 56 | var hits = 0 57 | 58 | func resolver(_ key: Int) -> LLBFuture { 59 | hits += 1 60 | let promise = group.next().makePromise(of: String.self) 61 | _ = group.next().scheduleTask(in: TimeAmount.milliseconds(100)) { 62 | promise.succeed("\(hits)") 63 | } 64 | return promise.futureResult 65 | } 66 | 67 | let v1Future = cache.value(for: 1, with: resolver) 68 | let v2Future = cache.value(for: 1, with: resolver) 69 | 70 | XCTAssertEqual(try v1Future.wait(), try v2Future.wait()) 71 | XCTAssertEqual(hits, 1) 72 | } 73 | 74 | /// Test that we don't re-resolve an in-flight value when requesting multiple 75 | func testMultipleValueResolution() throws { 76 | let cache = LLBEventualResultsCache(group: group) 77 | 78 | // Immediate resolution. 79 | _ = try cache.value(for: 0) { key in 80 | return group.next().makeSucceededFuture(0) 81 | }.wait() 82 | 83 | // Delayed resolution. 84 | _ = cache.value(for: 1) { key in 85 | let promise = group.next().makePromise(of: Int.self) 86 | _ = group.next().scheduleTask(in: TimeAmount.milliseconds(100)) { 87 | promise.succeed(key) 88 | } 89 | return promise.futureResult 90 | } 91 | 92 | let futures = cache.values(for: [0, 1, 2, 3]) { keys in 93 | XCTAssertFalse(keys.contains(0), "Unexpected resolver invocation") 94 | XCTAssertFalse(keys.contains(1), "Unexpected resolver invocation") 95 | XCTAssertTrue(keys.contains(2), "Unexpected resolver invocation") 96 | XCTAssertTrue(keys.contains(3), "Unexpected resolver invocation") 97 | return group.next().makeSucceededFuture(keys) 98 | } 99 | 100 | let results = try LLBFuture.whenAllSucceed(futures, on: group.next()).wait() 101 | 102 | XCTAssertEqual([0, 1, 2, 3], results) 103 | } 104 | 105 | } 106 | -------------------------------------------------------------------------------- /Tests/TSFFuturesTests/FutureDeduplicatorTests.swift: -------------------------------------------------------------------------------- 1 | // 2 | // Copyright © 2019 Apple, Inc. All rights reserved. 3 | // 4 | 5 | import XCTest 6 | 7 | import NIO 8 | 9 | import TSFFutures 10 | 11 | 12 | class FutureDeduplicatorTests: XCTestCase { 13 | 14 | var group: LLBFuturesDispatchGroup! 15 | 16 | override func setUp() { 17 | super.setUp() 18 | 19 | group = LLBMakeDefaultDispatchGroup() 20 | } 21 | 22 | override func tearDown() { 23 | super.tearDown() 24 | 25 | try! group.syncShutdownGracefully() 26 | group = nil 27 | } 28 | 29 | /// Test that we don't re-resolve the cached value 30 | func testSerialCoalescing() throws { 31 | let cache = LLBFutureDeduplicator(group: group) 32 | var hits = 0 33 | 34 | let v1Future = cache.value(for: 1) { key in 35 | hits += 1 36 | return group.next().makeSucceededFuture("\(hits)") 37 | } 38 | 39 | let v1 = try v1Future.wait() 40 | 41 | let v2Future = cache.value(for: 1) { key in 42 | hits += 1 43 | return group.next().makeSucceededFuture("\(hits)") 44 | } 45 | 46 | let v2 = try v2Future.wait() 47 | 48 | XCTAssertEqual(hits, 2) 49 | XCTAssertNotEqual(v1, v2) 50 | } 51 | 52 | /// Test that we don't re-resolve even if resolution takes time. 53 | func testParallelCoalescing() throws { 54 | let cache = LLBFutureDeduplicator(group: group) 55 | var hits = 0 56 | 57 | func resolver(_ key: Int) -> LLBFuture { 58 | hits += 1 59 | let promise = group.next().makePromise(of: String.self) 60 | _ = group.next().scheduleTask(in: TimeAmount.milliseconds(100)) { 61 | promise.succeed("\(hits)") 62 | } 63 | return promise.futureResult 64 | } 65 | 66 | let v1Future = cache.value(for: 1, with: resolver) 67 | let v2Future = cache.value(for: 1, with: resolver) 68 | 69 | XCTAssertEqual(try v1Future.wait(), try v2Future.wait()) 70 | XCTAssertEqual(hits, 1) 71 | } 72 | 73 | /// Test that we don't re-resolve an in-flight value when requesting multiple 74 | func testMultipleValueResolution() throws { 75 | let cache = LLBFutureDeduplicator(group: group) 76 | 77 | // Immediate resolution. 78 | _ = try cache.value(for: 0) { key in 79 | return group.next().makeSucceededFuture(0) 80 | }.wait() 81 | 82 | // Delayed resolution. 83 | _ = cache.value(for: 1) { key in 84 | let promise = group.next().makePromise(of: Int.self) 85 | _ = group.next().scheduleTask(in: TimeAmount.milliseconds(100)) { 86 | promise.succeed(key) 87 | } 88 | return promise.futureResult 89 | } 90 | 91 | let futures = cache.values(for: [0, 1, 2, 3]) { keys in 92 | // This has already been resolved once, so we are expected 93 | // to resolve it anew in the `FutureDeduplicator` abstraction. 94 | // See `EventualResultsCache` for a different behavior. 95 | XCTAssertTrue(keys.contains(0), "Unexpected resolver invocation") 96 | XCTAssertFalse(keys.contains(1), "Unexpected resolver invocation") 97 | XCTAssertTrue(keys.contains(2), "Unexpected resolver invocation") 98 | XCTAssertTrue(keys.contains(3), "Unexpected resolver invocation") 99 | return group.next().makeSucceededFuture(keys) 100 | } 101 | 102 | let results = try LLBFuture.whenAllSucceed(futures, on: group.next()).wait() 103 | 104 | XCTAssertEqual([0, 1, 2, 3], results) 105 | } 106 | 107 | } 108 | -------------------------------------------------------------------------------- /Tests/TSFFuturesTests/FutureOperationQueueTests.swift: -------------------------------------------------------------------------------- 1 | // 2 | // Copyright © 2019-2021 Apple, Inc. All rights reserved. 3 | // 4 | 5 | import Atomics 6 | import XCTest 7 | 8 | import NIO 9 | import NIOConcurrencyHelpers 10 | 11 | import TSFFutures 12 | 13 | 14 | class FutureOperationQueueTests: XCTestCase { 15 | func testBasics() throws { 16 | let group = LLBMakeDefaultDispatchGroup() 17 | defer { try! group.syncShutdownGracefully() } 18 | 19 | let loop = group.next() 20 | let p1 = loop.makePromise(of: Bool.self) 21 | let p2 = loop.makePromise(of: Bool.self) 22 | let p3 = loop.makePromise(of: Bool.self) 23 | var p1Started = false 24 | var p2Started = false 25 | var p3Started = false 26 | 27 | let manager = LLBOrderManager(on: group.next()) 28 | 29 | let q = LLBFutureOperationQueue(maxConcurrentOperations: 2) 30 | 31 | // Start the first two operations, they should run immediately. 32 | _ = q.enqueue(on: loop) { () -> LLBFuture in 33 | p1Started = true 34 | return manager.order(2).flatMap { 35 | p1.futureResult 36 | } 37 | } 38 | _ = q.enqueue(on: loop) { () -> LLBFuture in 39 | p2Started = true 40 | return manager.order(1).flatMap { 41 | p2.futureResult 42 | } 43 | } 44 | 45 | // Start the third, it should queue. 46 | _ = q.enqueue(on: loop) { () -> LLBFuture in 47 | p3Started = true 48 | return manager.order(4).flatMap { 49 | p3.futureResult 50 | } 51 | } 52 | 53 | try manager.order(3).wait() 54 | XCTAssertEqual(p1Started, true) 55 | XCTAssertEqual(p2Started, true) 56 | XCTAssertEqual(p3Started, false) 57 | 58 | // Complete the first. 59 | p1.succeed(true) 60 | try manager.order(5).wait() 61 | 62 | // Now p3 should have started. 63 | XCTAssertEqual(p3Started, true) 64 | p2.succeed(true) 65 | p3.succeed(true) 66 | 67 | _ = try! p1.futureResult.wait() 68 | _ = try! p2.futureResult.wait() 69 | _ = try! p3.futureResult.wait() 70 | } 71 | 72 | // Stress test. 73 | func testStress() throws { 74 | let group = MultiThreadedEventLoopGroup(numberOfThreads: 2) 75 | defer { try! group.syncShutdownGracefully() } 76 | 77 | let q = LLBFutureOperationQueue(maxConcurrentOperations: 2) 78 | 79 | let atomic = ManagedAtomic(0) 80 | var futures: [LLBFuture] = [] 81 | let lock = NIOConcurrencyHelpers.NIOLock() 82 | DispatchQueue.concurrentPerform(iterations: 1_000) { i in 83 | let result = q.enqueue(on: group.next()) { () -> LLBFuture in 84 | // Check that we aren't executing more operations than we would want. 85 | let p = group.next().makePromise(of: Bool.self) 86 | let prior = atomic.loadThenWrappingIncrement(ordering: .relaxed) 87 | XCTAssert(prior >= 0 && prior < 2, "saw \(prior + 1) concurrent tasks at start") 88 | p.futureResult.whenComplete { _ in 89 | let prior = atomic.loadThenWrappingDecrement(ordering: .relaxed) 90 | XCTAssert(prior > 0 && prior <= 2, "saw \(prior) concurrent tasks at end") 91 | } 92 | 93 | // Complete the future at some point 94 | group.next().execute { 95 | p.succeed(true) 96 | } 97 | 98 | return p.futureResult 99 | } 100 | 101 | lock.withLockVoid { 102 | futures.append(result) 103 | } 104 | } 105 | 106 | lock.withLockVoid { 107 | for future in futures { 108 | _ = try! future.wait() 109 | } 110 | } 111 | } 112 | 113 | // Test dynamic capacity increase. 114 | func testDynamic() throws { 115 | let group = MultiThreadedEventLoopGroup(numberOfThreads: 1) 116 | defer { try! group.syncShutdownGracefully() } 117 | 118 | let manager = LLBOrderManager(on: group.next(), timeout: .seconds(5)) 119 | 120 | let q = LLBFutureOperationQueue(maxConcurrentOperations: 1) 121 | 122 | let opsInFlight = ManagedAtomic(0) 123 | 124 | let future1: LLBFuture = q.enqueue(on: group.next()) { 125 | opsInFlight.wrappingIncrement(ordering: .relaxed) 126 | return manager.order(1).flatMap { 127 | manager.order(6) { 128 | opsInFlight.wrappingDecrement(ordering: .relaxed) 129 | } 130 | } 131 | } 132 | 133 | let future2: LLBFuture = q.enqueue(on: group.next()) { 134 | opsInFlight.wrappingIncrement(ordering: .relaxed) 135 | return manager.order(3).flatMap { 136 | manager.order(6) { 137 | opsInFlight.wrappingDecrement(ordering: .relaxed) 138 | } 139 | } 140 | } 141 | 142 | // Wait until future1 adss to opsInFlight. 143 | try manager.order(2).wait() 144 | XCTAssertEqual(opsInFlight.load(ordering: .relaxed), 1) 145 | 146 | // The test breaks without this line. 147 | q.maxConcurrentOperations += 1 148 | 149 | try manager.order(4).wait() 150 | XCTAssertEqual(opsInFlight.load(ordering: .relaxed), 2) 151 | try manager.order(5).wait() 152 | 153 | try manager.order(7).wait() 154 | XCTAssertEqual(opsInFlight.load(ordering: .relaxed), 0) 155 | 156 | try future2.wait() 157 | try future1.wait() 158 | } 159 | 160 | } 161 | -------------------------------------------------------------------------------- /Tests/TSFFuturesTests/OrderManagerTests.swift: -------------------------------------------------------------------------------- 1 | // 2 | // Copyright © 2020 Apple, Inc. All rights reserved. 3 | // 4 | 5 | import XCTest 6 | 7 | import NIO 8 | 9 | import TSFFutures 10 | 11 | class OrderManagerTests: XCTestCase { 12 | 13 | func testOrderManagerWithLoop() throws { 14 | let group = MultiThreadedEventLoopGroup(numberOfThreads: 1) 15 | defer { 16 | try! group.syncShutdownGracefully() 17 | } 18 | do { 19 | let manager = LLBOrderManager(on: group.next()) 20 | try manager.reset().wait() 21 | } 22 | } 23 | 24 | } 25 | 26 | -------------------------------------------------------------------------------- /Utilities/build_proto_toolchain.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -eu 2 | # 3 | # This source file is part of the Swift.org open source project 4 | # 5 | # Copyright (c) 2020 Apple Inc. and the Swift project authors 6 | # Licensed under Apache License v2.0 with Runtime Library Exception 7 | # 8 | # See http://swift.org/LICENSE.txt for license information 9 | # See http://swift.org/CONTRIBUTORS.txt for the list of Swift project authors 10 | 11 | PROTOC_ZIP=protoc-21.10-osx-universal_binary.zip 12 | PROTOC_URL="https://github.com/protocolbuffers/protobuf/releases/download/v21.10/$PROTOC_ZIP" 13 | 14 | UTILITIES_DIR="$(dirname "$0")" 15 | TOOLS_DIR="$UTILITIES_DIR/tools" 16 | 17 | mkdir -p "$TOOLS_DIR" 18 | 19 | if [[ ! -f "$UTILITIES_DIR/tools/$PROTOC_ZIP" ]]; then 20 | curl -L "$PROTOC_URL" --output "$TOOLS_DIR/$PROTOC_ZIP" 21 | unzip -o "$TOOLS_DIR/$PROTOC_ZIP" -d "$TOOLS_DIR" 22 | fi 23 | 24 | # Use swift build instead of cloning the repo to make sure that the generated code matches the SwiftProtobuf library 25 | # being used as a dependency in the build. This might be a bit slower, but it's correct. 26 | swift build -c release --product protoc-gen-swift --package-path "$UTILITIES_DIR/.." 27 | 28 | cp "$UTILITIES_DIR"/../.build/release/protoc-gen-swift "$TOOLS_DIR/bin" 29 | --------------------------------------------------------------------------------