├── .github └── workflows │ └── swift.yml ├── .gitignore ├── .swiftlint.yml ├── Examples ├── Mixing │ ├── Package.swift │ └── Sources │ │ └── Mixing │ │ └── main.swift ├── RtmpServer │ ├── Package.swift │ └── Sources │ │ └── RtmpServer │ │ └── main.swift └── Transcoding │ ├── Package.swift │ └── Sources │ └── Transcoding │ └── main.swift ├── LICENSE ├── Package.swift ├── Proto ├── CodedMediaSample.proto ├── Composition.proto ├── Rpc.public.proto └── TimePoint.proto ├── README.md ├── Sources ├── CCUDA │ ├── module.modulemap │ └── shim.h ├── CFreeType │ ├── module.modulemap │ └── shim.h ├── CSwiftVideo │ ├── include │ │ └── CSwiftVideo.h │ └── shim.cpp ├── SwiftVideo │ ├── animator.pic.swift │ ├── animator.soun.swift │ ├── buffer.swift │ ├── bus.swift │ ├── cam.apple.swift │ ├── clock.swift │ ├── composer.swift │ ├── compute.cl.swift │ ├── compute.cuda.swift │ ├── compute.metal.swift │ ├── compute.swift │ ├── dec.audio.apple.swift │ ├── dec.video.apple.swift │ ├── enc.video.apple.swift │ ├── event.swift │ ├── filter.pict.swift │ ├── kernels.cl.swift │ ├── kernels.cuda.swift │ ├── kernels.metal │ ├── live.swift │ ├── mix.audio.swift │ ├── mix.video.swift │ ├── net.flavor.swift │ ├── net.tcp.swift │ ├── playback.audio.apple.swift │ ├── proto │ │ ├── CodedMediaSample.pb.swift │ │ ├── Composition.pb.swift │ │ ├── Rpc.public.pb.swift │ │ └── TimePoint.pb.swift │ ├── repeater.swift │ ├── rpc │ │ └── public.rpc.swift │ ├── rtmp │ │ ├── amf.swift │ │ ├── deserialize.swift │ │ ├── rtmp.swift │ │ ├── serialize.swift │ │ └── states.swift │ ├── sample.audio.swift │ ├── sample.coded.swift │ ├── sample.pict.apple.swift │ ├── sample.pict.linux.swift │ ├── sample.pict.swift │ ├── segmenter.audio.swift │ ├── src.audio.apple.swift │ ├── stats.audio.swift │ ├── stats.swift │ ├── ternary.swift │ └── weak.swift ├── SwiftVideo_FFmpeg │ ├── dec.audio.ffmpeg.swift │ ├── dec.video.ffmpeg.swift │ ├── enc.audio.ffmpeg.swift │ ├── enc.video.ffmpeg.swift │ ├── extension.ffmpeg.swift │ ├── file.ffmpeg.swift │ ├── src.audio.ffmpeg.swift │ └── transcode.swift └── SwiftVideo_Freetype │ └── text.swift ├── TestEnvironment.dockerfile ├── Tests ├── LinuxMain.swift ├── swiftVideoInternalTests │ ├── XCTestManifests.swift │ └── computeTests.swift └── swiftVideoTests │ ├── XCTestManifests.swift │ ├── audioMixTests.swift │ ├── audioSegmenterTests.swift │ ├── busTests.swift │ ├── rtmpTests.swift │ ├── sampleRateConversionTests.swift │ ├── statsTest.swift │ └── timePointTests.swift └── flavor.md /.github/workflows/swift.yml: -------------------------------------------------------------------------------- 1 | name: Swift 2 | 3 | on: [push] 4 | 5 | jobs: 6 | build: 7 | runs-on: ubuntu-latest 8 | container: 9 | image: unpause/swiftvideo:latest 10 | 11 | steps: 12 | - uses: actions/checkout@v1 13 | - name: Run public tests 14 | run: swift test -c release --filter swiftVideoTests 15 | - name: Run internal tests 16 | run: swift test --filter swiftVideoInternalTests 17 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | .build 3 | Package.resolved 4 | ._* 5 | *.o 6 | *.a 7 | *~HEAD* 8 | *~master* 9 | *.profraw 10 | DerivedData/ 11 | Packages/ 12 | *.xcodeproj 13 | version.swift 14 | .fuse_* -------------------------------------------------------------------------------- /.swiftlint.yml: -------------------------------------------------------------------------------- 1 | included: 2 | - Sources 3 | 4 | excluded: 5 | - Sources/CFreeType 6 | - Sources/CSwiftVideo 7 | - Sources/SwiftVideo/proto 8 | 9 | disabled_rules: 10 | - todo 11 | -------------------------------------------------------------------------------- /Examples/Mixing/Package.swift: -------------------------------------------------------------------------------- 1 | // swift-tools-version:5.1 2 | /* 3 | SwiftVideo, Copyright 2019 Unpause SAS 4 | 5 | Licensed under the Apache License, Version 2.0 (the "License"); 6 | you may not use this file except in compliance with the License. 7 | You may obtain a copy of the License at 8 | 9 | http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | Unless required by applicable law or agreed to in writing, software 12 | distributed under the License is distributed on an "AS IS" BASIS, 13 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | See the License for the specific language governing permissions and 15 | limitations under the License. 16 | */ 17 | 18 | import PackageDescription 19 | 20 | let package = Package( 21 | name: "Mixing", 22 | platforms: [ 23 | .macOS(.v10_14), 24 | .iOS("11.0") 25 | ], 26 | products: [.executable(name: "mixing", targets: ["Mixing"])], 27 | dependencies: [.package(path:"../../")], 28 | targets: [.target(name: "Mixing", dependencies: ["SwiftVideo"])] 29 | ) 30 | -------------------------------------------------------------------------------- /Examples/Mixing/Sources/Mixing/main.swift: -------------------------------------------------------------------------------- 1 | /* 2 | SwiftVideo, Copyright 2019 Unpause SAS 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | // This example is going to take advantage of the Composer class that will simplify positioning 18 | // audio and video samples in the mixers. You can use the AudioMixer and VideoMixer directly if 19 | // you wish to manually set the transformation matrices and other properties of the samples. 20 | // We will take a couple of file sources, compose them, and stream them out to an RTMP endpoint. 21 | 22 | // This example requires GPU access. 23 | 24 | import SwiftVideo 25 | import Foundation 26 | import NIO 27 | import NIOExtras 28 | import BrightFutures 29 | 30 | let clock = WallClock() 31 | let compute = try makeComputeContext(forType: .GPU) 32 | let group = MultiThreadedEventLoopGroup(numberOfThreads: 4) 33 | let fullyShutdownPromise: EventLoopPromise = group.next().makePromise() 34 | 35 | let leftFile = "file:///Users/james/dev/test-data/movies/spring.mp4" 36 | let rightFile = "file:///Users/james/dev/test-data/movies/bbc_audio_sync.mp4" 37 | let rtmpDestination = "rtmp://localhost/app/playPath" 38 | 39 | typealias ExamplePublisher = (Tx, Terminal) 40 | typealias ExampleSource = (Terminal, Terminal, Terminal) 41 | 42 | var rtmpPublisher: ExamplePublisher? 43 | 44 | // In order for the composer to bind elements to assets, it must be given access to a pool of samples that the 45 | // elements can pull from and the result will be pushed to. This means that the sources you wish to use must feed 46 | // into these buses. This is not a requirement of the underlying mixers or animators, it is a requirement of this 47 | // particular implementation of a holistic composer that handles binding and rebinding assets to animators to mixers. 48 | let pictureBus = Bus(clock) 49 | let audioBus = Bus(clock) 50 | let codedBus = Bus(clock) 51 | 52 | let width = 1280 53 | let height = 720 54 | 55 | // The SwiftVideo composer as implemented requires a composition manifest to define its initial behavior. 56 | // We will start with a simple composition that takes two inputs and stitches them together side-by-side. 57 | let manifest = RpcMakeComposition.with { config in 58 | config.audio = RpcMixerAudioConfig.with { 59 | $0.channels = 2 60 | $0.sampleRate = 48000 61 | } 62 | config.video = RpcMixerVideoConfig.with { 63 | $0.frameDuration = TimePoint(1000, 30000) 64 | $0.width = Int32(width) 65 | $0.height = Int32(height) 66 | } 67 | config.composition = Composition.with { 68 | $0.initialScene = "scene1" 69 | $0.scenes = ["scene1": Scene.with { 70 | $0.elements = [ 71 | "element1": Element.with { 72 | $0.initialState = "state1" 73 | $0.states = ["state1": ElementState.with { 74 | $0.picAspect = .aspectFill 75 | $0.audioGain = 1.0 76 | $0.picOrigin = .originTopLeft 77 | $0.picPos = Vec3.with { $0.x = 0; $0.y = 0; $0.z = 0 } 78 | $0.transparency = 0.0 79 | $0.size = Vec2.with { $0.x = Float(config.video.width/2); $0.y = Float(config.video.height) } 80 | }] 81 | }, 82 | "element2": Element.with { 83 | $0.initialState = "state1" 84 | $0.states = ["state1": ElementState.with { 85 | $0.picAspect = .aspectFill 86 | $0.audioGain = 1.0 87 | $0.picOrigin = .originTopLeft 88 | $0.picPos = Vec3.with { $0.x = Float(config.video.width/2); $0.y = 0; $0.z = 0 } 89 | $0.transparency = 0.0 90 | $0.size = Vec2.with { $0.x = Float(config.video.width/2); $0.y = Float(config.video.height) } 91 | }] 92 | }] 93 | }] 94 | } 95 | } 96 | 97 | // Finally we create the composer. 98 | let composer = Composer(clock, 99 | assetId: "composer", 100 | workspaceId: "sandbox", 101 | compute: compute, 102 | composition: manifest, 103 | audioBus: audioBus, 104 | pictureBus: pictureBus) 105 | 106 | func makeFileSource(_ url: String, _ ident: String) throws -> ExampleSource { 107 | let src = try FileSource(clock, url: url, assetId: ident, workspaceId: "sandbox") 108 | // Here we are composing transformations: taking the encoded media from the file source, decoding it, and 109 | // preparing the samples in a useful format for the mixers. In the case of audio samples, we are doing a sample 110 | // rate conversion to the format we specify in the manifest. In the case of video samples, we are introducing 111 | // a GPU barrier to copy the texture onto the GPU. 112 | // 113 | // Note that the Composer class actually adds a GPU barrier on the upload side anyway, so it is not mandatory 114 | // to add an upload barrier when using it, but the upload barrier is idempotent so you won't upload the texture 115 | // twice if there are two used on the same sample. 116 | // 117 | // The result of one of the compositions below is a Tx which contains all of the 118 | // transformation functions specified. Tx is equivalent to Terminal 119 | return (src >>> codedBus, 120 | codedBus <<| mediaTypeFilter(.audio) >>> FFmpegAudioDecoder() >>> 121 | AudioSampleRateConversion(48000, 2, .s16i) >>> audioBus, 122 | codedBus <<| mediaTypeFilter(.video) >>> FFmpegVideoDecoder() >>> 123 | GPUBarrierUpload(compute) >>> pictureBus) 124 | } 125 | 126 | let onEnded: LiveOnEnded = { print("\($0) ended") ; rtmpPublisher = nil } 127 | 128 | // RTMP connection established, here we create the encoders and compose the outputs. 129 | let onConnection: LiveOnConnection = { pub, sub in 130 | if let pub = pub, let txn = pub as? Terminal { 131 | // Here we are pulling samples off of our audio and video buses that have been generated by the composer 132 | // There is a GPU barrier to download the textuure from the GPU introduced here. This must be used 133 | // even if using the Composer class because the composer makes no assumptions about where you want 134 | // the texture to live after it's finished with it. 135 | // 136 | // We must also filter the assets coming from the bus so that we only get the samples we want rather than 137 | // all of them. You'll notice as well that when we compose the rtmp publisher at the end of the 138 | // composition here we use a standard composition operator for video (>>>) and for audio we use a 139 | // mapping composition operator (|>>). The mapping operator will take a list of samples and map them to the 140 | // publisher, returning a list of the result type. 141 | rtmpPublisher = (audioBus <<| assetFilter("composer") >>> FFmpegAudioEncoder(.aac, bitrate: 96000) |>> txn, 142 | pictureBus <<| assetFilter("composer") >>> GPUBarrierDownload(compute) >>> FFmpegVideoEncoder(.avc, 143 | bitrate: 3_000_000, frameDuration: manifest.video.frameDuration) >>> txn) 144 | } 145 | return Future { $0(.success(true)) } 146 | } 147 | 148 | let leftFs = try makeFileSource(leftFile, "left") 149 | let rightFs = try makeFileSource(rightFile, "right") 150 | 151 | // We need to bind the file assets to the on-screen elements 152 | composer.bind("left", elementId: "element1") 153 | composer.bind("right", elementId: "element2") 154 | 155 | // Create an RTMP output 156 | let rtmp = Rtmp(clock, onEnded: onEnded, onConnection: onConnection) 157 | 158 | if let url = URL(string: rtmpDestination) { 159 | _ = rtmp.connect(url: url, publishToPeer: true, group: group, workspaceId: "sandbox") 160 | } 161 | 162 | try fullyShutdownPromise.futureResult.wait() 163 | -------------------------------------------------------------------------------- /Examples/RtmpServer/Package.swift: -------------------------------------------------------------------------------- 1 | // swift-tools-version:5.1 2 | /* 3 | SwiftVideo, Copyright 2019 Unpause SAS 4 | 5 | Licensed under the Apache License, Version 2.0 (the "License"); 6 | you may not use this file except in compliance with the License. 7 | You may obtain a copy of the License at 8 | 9 | http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | Unless required by applicable law or agreed to in writing, software 12 | distributed under the License is distributed on an "AS IS" BASIS, 13 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | See the License for the specific language governing permissions and 15 | limitations under the License. 16 | */ 17 | 18 | import PackageDescription 19 | 20 | let package = Package( 21 | name: "RtmpServer", 22 | platforms: [ 23 | .macOS(.v10_14), 24 | .iOS("11.0") 25 | ], 26 | products: [.executable(name: "rtmpServer", targets: ["RtmpServer"])], 27 | dependencies: [.package(path:"../../")], 28 | targets: [.target(name: "RtmpServer", dependencies: ["SwiftVideo"])] 29 | ) 30 | -------------------------------------------------------------------------------- /Examples/RtmpServer/Sources/RtmpServer/main.swift: -------------------------------------------------------------------------------- 1 | import Foundation 2 | import SwiftVideo 3 | import NIO 4 | import NIOExtras 5 | import BrightFutures 6 | 7 | let clock = WallClock() 8 | let group = MultiThreadedEventLoopGroup(numberOfThreads: 4) 9 | let quiesce = ServerQuiescingHelper(group: group) 10 | 11 | var subs: [String: Terminal] = [:] 12 | // onEnded is called when a publish or subscribe session ends. AssetID is the string value that's passed to the closure. 13 | let onEnded: LiveOnEnded = { print("\($0) ended") ; subs.removeValue(forKey: $0) } 14 | 15 | // 16 | // With the Rtmp system, onConnection is called after an RTMP publish or subscribe handshake occurs. The workspaceToken 17 | // contains the "playpath" (or stream key) portion of the URI and the "workspaceId" contains the "app" portion of the 18 | // URI. You must return a future that emits either true or false, depending on if permission is granted to either 19 | // publish or subscribe. Note that the nomenclature "publisher" and "subscriber" are from the perspective of the media 20 | // server, therefore when the peer ispublishing, the server is subscribing. 21 | // 22 | // Asset ID is a generated UUIDv4 that can identify the asset in the system. 23 | // 24 | let onConnection: LiveOnConnection = { pub, sub in 25 | if let pub = pub, let streamKey = pub.workspaceToken() { 26 | print("publisher asking for permission: \(pub.workspaceId())/\(streamKey). Dropping.") 27 | return Future { $0(.success(false)) } 28 | } 29 | if let sub = sub, let src = sub as? Source, let streamKey = sub.workspaceToken() { 30 | print("subscriber asking for permission: \(sub.workspaceId())/\(streamKey)") 31 | 32 | // In order to receive samples, we need to compose the subscriber with a receiver and keep a strong reference. 33 | // to it. Releasing the reference will close the connection an end the stream. 34 | // You can create more sophisticated graphs with decoders, buses, and other components. 35 | subs[sub.assetId()] = src >>> Tx { sample in 36 | print("got sample \(sample.pts().toString()) type = \(sample.mediaType())") 37 | return .nothing(nil) 38 | } 39 | } 40 | return Future { $0(.success(true)) } 41 | } 42 | 43 | let rtmp = Rtmp(clock, onEnded: onEnded, onConnection: onConnection) 44 | 45 | _ = rtmp.serve(host: "0.0.0.0", port: 1935, quiesce: quiesce, group: group) 46 | 47 | print("Listening on 1935") 48 | 49 | _ = try rtmp.wait() 50 | -------------------------------------------------------------------------------- /Examples/Transcoding/Package.swift: -------------------------------------------------------------------------------- 1 | // swift-tools-version:5.1 2 | /* 3 | SwiftVideo, Copyright 2019 Unpause SAS 4 | 5 | Licensed under the Apache License, Version 2.0 (the "License"); 6 | you may not use this file except in compliance with the License. 7 | You may obtain a copy of the License at 8 | 9 | http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | Unless required by applicable law or agreed to in writing, software 12 | distributed under the License is distributed on an "AS IS" BASIS, 13 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | See the License for the specific language governing permissions and 15 | limitations under the License. 16 | */ 17 | 18 | import PackageDescription 19 | 20 | let package = Package( 21 | name: "Transcoding", 22 | platforms: [ 23 | .macOS(.v10_14), 24 | .iOS("11.0") 25 | ], 26 | products: [.executable(name: "transcoding", targets: ["Transcoding"])], 27 | dependencies: [.package(path:"../../")], 28 | targets: [.target(name: "Transcoding", dependencies: ["SwiftVideo"])] 29 | ) 30 | -------------------------------------------------------------------------------- /Examples/Transcoding/Sources/Transcoding/main.swift: -------------------------------------------------------------------------------- 1 | /* 2 | SwiftVideo, Copyright 2019 Unpause SAS 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | // This example is going to use the transcoder functions to transcode a file and output it to an RTMP endpoint. 18 | // The transcoder functions return compositions containing the appropriate operations to transcode a CodedMediaSample 19 | // into one (or more) different CodedMediaSamples of the same media type (audio, video) 20 | 21 | import SwiftVideo 22 | import Foundation 23 | import NIO 24 | import NIOExtras 25 | import BrightFutures 26 | 27 | let clock = WallClock() 28 | let group = MultiThreadedEventLoopGroup(numberOfThreads: 4) 29 | let fullyShutdownPromise: EventLoopPromise = group.next().makePromise() 30 | let codedBus = Bus(clock) 31 | 32 | let inputFile = "file:///Users/james/dev/test-data/movies/spring.mp4" 33 | let rtmpDestination = "rtmp://localhost/app/playPath" 34 | 35 | var fileSource: Terminal? 36 | var videoTranscoder: Terminal? 37 | var audioTranscoder: Tx? 38 | 39 | let onEnded: LiveOnEnded = { 40 | print("\($0) ended") 41 | fileSource = nil 42 | videoTranscoder = nil 43 | audioTranscoder = nil 44 | } 45 | 46 | // RTMP connection established, here we create the encoders and compose the outputs. 47 | let onConnection: LiveOnConnection = { pub, sub in 48 | if let pub = pub, 49 | let publisher = pub as? Terminal { 50 | do { 51 | let src = try FileSource(clock, url: inputFile, assetId: "file", workspaceId: "sandbox") 52 | 53 | // Here we are composing transformations: taking the encoded media from the file source, filtering based 54 | // on the media type (audio or video), and transcoding the samples. You'll notice that when we compose 55 | // the rtmp publisher at the end of the composition here we use a standard composition operator for 56 | // video (>>>) and for audio we use a mapping composition operator (|>>). The mapping operator 57 | // will take a list of samples and map them to the publisher, returning a list of the result type. 58 | videoTranscoder = try codedBus <<| mediaTypeFilter(.video) >>> makeVideoTranscoder(.avc, 59 | bitrate: 3_000_000, keyframeInterval: TimePoint(2000, 1000), newAssetId: "new") >>> publisher 60 | audioTranscoder = try codedBus <<| mediaTypeFilter(.audio) >>> 61 | makeAudioTranscoder(.aac, bitrate: 96_000, sampleRate: 48000, newAssetId: "new") |>> publisher 62 | fileSource = src >>> codedBus 63 | } catch { 64 | print("Exception loading file \(error)") 65 | } 66 | } 67 | return Future { $0(.success(true)) } 68 | } 69 | 70 | // Create an RTMP output 71 | let rtmp = Rtmp(clock, onEnded: onEnded, onConnection: onConnection) 72 | 73 | if let url = URL(string: rtmpDestination) { 74 | _ = rtmp.connect(url: url, publishToPeer: true, group: group, workspaceId: "sandbox") 75 | } 76 | 77 | try fullyShutdownPromise.futureResult.wait() 78 | -------------------------------------------------------------------------------- /Package.swift: -------------------------------------------------------------------------------- 1 | // swift-tools-version:5.2 2 | /* 3 | SwiftVideo, Copyright 2019 Unpause SAS 4 | 5 | Licensed under the Apache License, Version 2.0 (the "License"); 6 | you may not use this file except in compliance with the License. 7 | You may obtain a copy of the License at 8 | 9 | http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | Unless required by applicable law or agreed to in writing, software 12 | distributed under the License is distributed on an "AS IS" BASIS, 13 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | See the License for the specific language governing permissions and 15 | limitations under the License. 16 | */ 17 | 18 | import PackageDescription 19 | import Foundation 20 | 21 | let cudaVer = ProcessInfo.processInfo.environment["CUDA_VER"] 22 | 23 | let cudaTarget: [Target] = cudaVer.map { ver in 24 | [.systemLibrary( 25 | name: "CCUDA", 26 | path: "Sources/CCUDA", 27 | pkgConfig: "cuda-\(ver)")] 28 | } ?? [] 29 | 30 | let dependencies: [Target.Dependency] = cudaVer != nil ? ["CCUDA"] : [] 31 | 32 | let swiftSettings: [SwiftSetting] = (cudaVer != nil ? [.define("GPGPU_CUDA", .when(platforms: [.linux]))] : 33 | [.define("GPGPU_OCL", .when(platforms: [.macOS, .linux]))]) + 34 | [.define("GPGPU_METAL", .when(platforms: [.iOS, .tvOS]))] 35 | 36 | let cSettings: [CSetting] = (cudaVer == nil ? [.define("CL_USE_DEPRECATED_OPENCL_1_2_APIS"), 37 | .define("GPGPU_OCL", .when(platforms: [.macOS, .linux]))] : []) + 38 | [.define("linux", .when(platforms: [.linux]))] 39 | 40 | let linkerSettings: [LinkerSetting] = (cudaVer == nil ? [.linkedLibrary("OpenCL", .when(platforms: [.linux]))] : 41 | [.linkedLibrary("nvrtc", .when(platforms: [.linux]))]) + 42 | [.linkedLibrary("bsd", .when(platforms: [.linux]))] 43 | 44 | let package = Package( 45 | name: "SwiftVideo", 46 | platforms: [ 47 | .macOS(.v10_14), 48 | .iOS("13.1") 49 | ], 50 | products: [ 51 | .library( 52 | name: "SwiftVideo", 53 | targets:["SwiftVideo", "SwiftVideo_FFmpeg", "SwiftVideo_Freetype"]), 54 | .library( 55 | name: "SwiftVideo_Bare", 56 | targets: ["SwiftVideo"]) 57 | ], 58 | dependencies: [ 59 | .package(url: "https://github.com/apple/swift-nio.git", from: "2.9.0"), 60 | .package(url: "https://github.com/apple/swift-nio-extras.git", from: "1.3.1"), 61 | .package(url: "https://github.com/nicklockwood/VectorMath.git", from: "0.4.0"), 62 | .package(url: "https://github.com/Thomvis/BrightFutures.git", from: "8.0.1"), 63 | .package(name: "SwiftProtobuf", url: "https://github.com/apple/swift-protobuf.git", from: "1.7.0"), 64 | .package(url: "https://github.com/apple/swift-nio-ssl.git", from: "2.4.3"), 65 | .package(url: "https://github.com/apple/swift-log.git", from: "1.1.1"), 66 | .package(url: "https://github.com/sunlubo/SwiftFFmpeg", .revision("22b886fd5242c1923f0993c9541768a9a16e33f2")) 67 | ], 68 | targets: [ 69 | .systemLibrary( 70 | name: "CFreeType", 71 | path: "Sources/CFreeType", 72 | pkgConfig: "freetype2", 73 | providers: [.brew(["freetype2"]), .apt(["libfreetype6-dev"])] 74 | ), 75 | .target(name: "CSwiftVideo", 76 | dependencies: [], 77 | cSettings: [ 78 | .define("linux", .when(platforms: [.linux]))], 79 | cxxSettings: [ 80 | .define("linux", .when(platforms: [.linux]))] 81 | ), 82 | .target( 83 | name: "SwiftVideo", 84 | dependencies: [.product(name: "NIO", package: "swift-nio"), 85 | "CSwiftVideo", 86 | .product(name: "NIOSSL", package: "swift-nio-ssl"), 87 | .product(name: "NIOExtras", package: "swift-nio-extras"), 88 | .product(name: "NIOFoundationCompat", package: "swift-nio"), 89 | "VectorMath", 90 | "BrightFutures", 91 | "SwiftProtobuf", 92 | .product(name: "NIOWebSocket", package: "swift-nio"), 93 | .product(name: "NIOHTTP1", package: "swift-nio"), 94 | .product(name: "Logging", package: "swift-log")] + dependencies, 95 | cSettings: cSettings, 96 | swiftSettings: swiftSettings, 97 | linkerSettings: linkerSettings 98 | ), 99 | .target( 100 | name: "SwiftVideo_FFmpeg", 101 | dependencies: ["SwiftVideo", "SwiftFFmpeg"], 102 | cSettings: [ 103 | .define("linux", .when(platforms: [.linux])), 104 | .define("USE_FFMPEG") 105 | ], 106 | swiftSettings: [ 107 | .define("USE_FFMPEG") 108 | ] 109 | ), 110 | .target( 111 | name: "SwiftVideo_Freetype", 112 | dependencies: ["SwiftVideo", "CFreeType"], 113 | cSettings: [ 114 | .define("linux", .when(platforms: [.linux])), 115 | .define("USE_FREETYPE") 116 | ], 117 | swiftSettings: [ 118 | .define("USE_FREETYPE") 119 | ] 120 | ), 121 | .testTarget( 122 | name: "swiftVideoTests", 123 | dependencies: ["SwiftVideo", "CSwiftVideo", "SwiftVideo_FFmpeg"]), 124 | .testTarget( 125 | name: "swiftVideoInternalTests", 126 | dependencies: ["SwiftVideo", "CSwiftVideo"], 127 | swiftSettings: [ 128 | .define("DISABLE_INTERNAL", .when(configuration: .release)) 129 | ]) 130 | ], 131 | swiftLanguageVersions: [.v5], 132 | cxxLanguageStandard: .cxx1z 133 | ) /* package */ 134 | -------------------------------------------------------------------------------- /Proto/CodedMediaSample.proto: -------------------------------------------------------------------------------- 1 | /* 2 | SwiftVideo, Copyright 2019 Unpause SAS 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | syntax = "proto3"; 18 | import "TimePoint.proto"; 19 | option java_package="com.unpause.proto"; 20 | 21 | enum MediaType { 22 | video = 0; 23 | audio = 1; 24 | image = 2; 25 | data = 3; 26 | subtitle = 4; 27 | } 28 | 29 | enum MediaFormat { 30 | avc = 0; 31 | hevc = 1; 32 | aac = 2; 33 | opus = 3; 34 | av1 = 4; 35 | vp8 = 5; 36 | vp9 = 6; 37 | uncompressed = 7; 38 | // image 39 | png = 8; 40 | apng = 9; 41 | jpg = 10; 42 | gif = 11; 43 | // data 44 | klv = 12; 45 | // subtitle 46 | srt = 13; 47 | webvtt = 14; 48 | utf8Text = 15; 49 | } 50 | 51 | enum MediaSourceType { 52 | rtmp = 0; 53 | webrtc = 1; 54 | httpPut = 2; 55 | protobuf = 3; 56 | httpGet = 4; 57 | transcode = 5; 58 | composition = 6; 59 | web = 7; 60 | output = 8; 61 | flavor = 9; 62 | file = 10; 63 | text = 11; 64 | } 65 | 66 | message CodedMediaSampleWire { 67 | TimePoint pts = 1; 68 | TimePoint dts = 2; 69 | TimePoint eventTime = 3; 70 | string idAsset = 4; 71 | string idWorkspace = 5; 72 | string tokenWorkspace = 6; 73 | bytes buffer = 7; 74 | map side = 8; 75 | MediaType mediatype = 9; 76 | MediaFormat mediaformat = 10; 77 | //map eventInfo = 11; 78 | string encoder = 12; 79 | repeated MediaConstituent constituents = 14; 80 | } 81 | 82 | 83 | message MediaConstituent { 84 | string idAsset = 1; 85 | TimePoint pts = 3; 86 | TimePoint dts = 4; 87 | TimePoint duration = 5; 88 | TimePoint normalizedPts = 6; 89 | repeated MediaConstituent constituents = 7; 90 | 91 | } -------------------------------------------------------------------------------- /Proto/Composition.proto: -------------------------------------------------------------------------------- 1 | /* 2 | SwiftVideo, Copyright 2019 Unpause SAS 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | syntax = "proto3"; 18 | option java_package="com.unpause.proto"; 19 | 20 | enum AspectMode { 21 | aspectNone = 0; 22 | aspectFit = 1; 23 | aspectFill = 2; 24 | } 25 | 26 | enum PictureOrigin { 27 | originCenter = 0; 28 | originTopLeft = 1; 29 | } 30 | 31 | enum PictureAnchor { 32 | anchorTopLeft = 0; 33 | anchorTopRight = 1; 34 | anchorBottomLeft = 2; 35 | anchorBottomRight =3; 36 | } 37 | 38 | message Vec2 { 39 | float x = 1; 40 | float y = 2; 41 | } 42 | 43 | message Vec3 { 44 | float x = 1; 45 | float y = 2; 46 | float z = 3; 47 | } 48 | 49 | message Vec4 { 50 | float x = 1; 51 | float y = 2; 52 | float z = 3; 53 | float w = 4; 54 | } 55 | 56 | message ElementState { 57 | Vec3 picPos = 1; 58 | Vec2 size = 2; 59 | Vec2 textureOffset = 3; 60 | float rotation = 4; 61 | float transparency = 5; 62 | float audioGain = 6; 63 | Vec2 audioPos = 7; 64 | AspectMode picAspect = 8; 65 | PictureOrigin picOrigin = 9; 66 | Vec4 fillColor = 10; /* r g b a */ 67 | Vec4 borderSize = 11; /* l t r b */ 68 | bool hidden = 12; 69 | bool muted = 13; 70 | repeated PictureAnchor parentAnchor = 14; 71 | } 72 | 73 | message Element { 74 | map states = 1; 75 | string initialState = 2; 76 | string parent = 3; 77 | } 78 | 79 | message Scene { 80 | map elements = 1; 81 | } 82 | 83 | message Composition { 84 | map scenes = 1; 85 | string initialScene = 3; 86 | } -------------------------------------------------------------------------------- /Proto/Rpc.public.proto: -------------------------------------------------------------------------------- 1 | /* 2 | SwiftVideo, Copyright 2019 Unpause SAS 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | syntax = "proto3"; 18 | import "TimePoint.proto"; 19 | import "CodedMediaSample.proto"; 20 | import "Composition.proto"; 21 | 22 | option java_package="com.unpause.proto"; 23 | 24 | message RpcAssetPermissionRequest { 25 | MediaSourceType sourceType = 1; 26 | MediaType mediaType = 2; 27 | repeated MediaFormat formats = 3; 28 | map metadata = 4; 29 | PermissionRequestType requestType = 5; 30 | 31 | enum PermissionRequestType { 32 | write = 0; 33 | read = 1; 34 | }; 35 | } 36 | 37 | message RpcAssetPermissionResponse { 38 | bool granted = 1; 39 | } 40 | 41 | 42 | message RpcComposerCommand { 43 | message StateSet { 44 | string elementId = 1; 45 | TimePoint duration = 2; 46 | string stateId = 3; 47 | } 48 | message Bind { 49 | string assetId = 1; 50 | string elementId = 2; 51 | } 52 | message Load { 53 | string uri = 1; 54 | string assetId = 2; 55 | string workspaceToken = 3; 56 | bool loop = 4; 57 | bool autoplay = 5; 58 | bool closeOnEnd = 6; // default is to keep asset alive after end so you can play() again. Does nothing with loop = true 59 | } 60 | message Text { 61 | string value = 1; 62 | int32 fontSize = 2; 63 | string fontUrl = 3; 64 | string assetId = 4; 65 | Vec4 color = 5; 66 | } 67 | message Command { 68 | repeated Command after = 1; 69 | int32 ident = 2; 70 | oneof command { 71 | string scene = 3; 72 | StateSet elementState = 4; 73 | Bind bind = 5; 74 | Load loadFile = 6; 75 | string playFile = 7; 76 | string stopFile = 8; 77 | Text setText = 9; 78 | } 79 | } 80 | repeated Command commands = 1; 81 | } 82 | 83 | enum RpcFeatureType { 84 | transcoder = 0; 85 | compositor = 1; 86 | subtractor = 2; 87 | rtmpOutput = 3; 88 | browser = 4; 89 | proprietaryIO = 5; // Used for third-party proprietary integrations 90 | } 91 | 92 | 93 | message RpcEncodeConfig { 94 | MediaFormat format = 1; 95 | int32 bitrate = 2; 96 | TimePoint keyframeInterval = 3; 97 | int32 sampleRate = 4; 98 | int32 channelCount = 5; 99 | int32 bitDepth = 6; 100 | map options = 7; // eg. level, profile 101 | } 102 | 103 | message RpcMixerVideoConfig { 104 | int32 width = 1; 105 | int32 height = 2; 106 | TimePoint frameDuration = 3; 107 | } 108 | 109 | message RpcMixerAudioConfig { 110 | int32 sampleRate = 1; 111 | int32 channels = 2; 112 | } 113 | 114 | message RpcMakeComposition { 115 | string newAssetId = 1; 116 | string chosenInstance = 2; 117 | repeated RpcEncodeConfig outputConfigs = 3; 118 | RpcMixerVideoConfig video = 4; 119 | RpcMixerAudioConfig audio = 5; 120 | repeated RpcFeatureType features = 6; 121 | Composition composition = 7; 122 | int64 epoch = 8; 123 | } -------------------------------------------------------------------------------- /Proto/TimePoint.proto: -------------------------------------------------------------------------------- 1 | /* 2 | SwiftVideo, Copyright 2019 Unpause SAS 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | syntax = "proto3"; 18 | option java_package="com.unpause.proto"; 19 | 20 | message TimePoint { 21 | int64 value = 1; 22 | int64 scale = 2; 23 | } 24 | 25 | message EventError { 26 | string source = 1; 27 | int32 code = 2; 28 | string desc = 3; 29 | TimePoint time = 4; 30 | string assetId = 5; 31 | } 32 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ## SwiftVideo 2 | 3 | [![GitHub Actions](https://img.shields.io/endpoint.svg?url=https%3A%2F%2Factions-badge.atrox.dev%2Funpause-live%2FSwiftVideo%2Fbadge&label=build&logo=none)](https://actions-badge.atrox.dev/unpause-live/SwiftVideo/goto) 4 | ![Swift version](https://img.shields.io/badge/swift-5-orange.svg) 5 | ![License](https://img.shields.io/github/license/unpause-live/SwiftVideo) 6 | 7 | 8 | Video streaming and processing framework for Linux, macOS, and iOS/iPadOS/tvOS. Swift 5.1+ because I'm just opening this up and I really don't feel like dealing with older versions of Swift. 9 | 10 | ### Getting Started 11 | 12 | For now, check the Examples directory for some hints about how to use this framework. I promise I will be creating 13 | documentation to clarify how it all fits together and how you can do useful, interesting things with this framework. 14 | 15 | 16 | #### Using with SwiftPM 17 | 18 | 1. Add `.package(url: "https://github.com/unpause-live/SwiftVideo.git", from: "0.2.0")` to your package dependencies 19 | 2. In your target dependencies, add either `SwiftVideo` or `SwiftVideo_Bare` depending on whether or not you wish to 20 | build with FFmpeg and Freetype support. 21 | 22 | #### Using with Xcode + iOS 23 | 24 | You can use this project in Xcode for iOS as a Swift Package as of 0.2.0. 25 | 26 | 1. Go to File -> Swift Packages -> Add Package Dependency, or in your Project settings go to the Swift Packages tab and press + 27 | 2. Set the package repository URL to https://github.com/unpause-live/SwiftVideo 28 | 3. Select the branch or version you want to reference. Only 0.2.0 and above will be usable on iOS unless you also compile 29 | FFmpeg and Freetype for it. 30 | 4. Choose the `SwiftVideo_Bare` product when prompted. This will build SwiftVideo without FFmpeg and Freetype. If you have built 31 | those libraries for iOS and wish to use them with SwiftVideo, choose the `SwiftVideo` product instead. 32 | 5. If you are using the VideoMixer you will need to include `Sources/SwiftVideo/kernels.metal` in your project directly so that they are included. This will be changed [when Swift 5.3 is released](https://github.com/apple/swift-evolution/blob/master/proposals/0278-package-manager-localized-resources.md). 33 | 34 | ### Current Features 35 | 36 | - RTMP Client and Server 37 | - "Flavor" Client and Server (toy protocol, see flavor.md) 38 | - OpenCL Support 39 | - Metal Support 40 | - Audio Decode/Encode (via FFmpeg and CoreAudio) 41 | - Video Decode/Encode (via FFmpeg and CoreVideo) 42 | - Camera capture (macOS/iOS) 43 | - Text rendering (via FreeType) 44 | - Video Mixer 45 | - Audio Mixer 46 | - Audio Resampler (via FFmpeg+SOX) 47 | 48 | 49 | FFmpeg support is thanks to https://github.com/sunlubo/SwiftFFmpeg 50 | -------------------------------------------------------------------------------- /Sources/CCUDA/module.modulemap: -------------------------------------------------------------------------------- 1 | module CCUDA [system] { 2 | header "shim.h" 3 | link "cuda" 4 | link "nvrtc" 5 | export * 6 | } 7 | -------------------------------------------------------------------------------- /Sources/CCUDA/shim.h: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | #include 4 | #include -------------------------------------------------------------------------------- /Sources/CFreeType/module.modulemap: -------------------------------------------------------------------------------- 1 | module CFreeType [system] { 2 | header "shim.h" 3 | link "freetype" 4 | export * 5 | } -------------------------------------------------------------------------------- /Sources/CFreeType/shim.h: -------------------------------------------------------------------------------- 1 | /* 2 | SwiftVideo, Copyright 2019 Unpause SAS 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | #pragma once 18 | #include 19 | #include FT_FREETYPE_H 20 | #include 21 | #include 22 | #include -------------------------------------------------------------------------------- /Sources/CSwiftVideo/include/CSwiftVideo.h: -------------------------------------------------------------------------------- 1 | /* 2 | SwiftVideo, Copyright 2019 Unpause SAS 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | #pragma once 18 | 19 | #if !defined(__APPLE__) && defined(GPGPU_OCL) 20 | #include 21 | #endif 22 | 23 | #include 24 | #include 25 | 26 | #ifdef __cplusplus 27 | extern "C" { 28 | #endif 29 | 30 | int aac_parse_asc(const void* data, int64_t size, int* channels, int* sample_rate, int* samples_per_frame); 31 | int h264_sps_frame_size(const void* data, int64_t size, int* width, int* height); 32 | 33 | #if defined(linux) 34 | void generateRandomBytes(void* buf, size_t size); 35 | #endif 36 | 37 | uint64_t test_golomb_dec(); 38 | 39 | #ifdef __cplusplus 40 | } 41 | #endif 42 | -------------------------------------------------------------------------------- /Sources/SwiftVideo/animator.soun.swift: -------------------------------------------------------------------------------- 1 | /* 2 | SwiftVideo, Copyright 2019 Unpause SAS 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | import Foundation 18 | import VectorMath 19 | import BrightFutures 20 | 21 | public class SoundAnimator: Tx, Animator { 22 | public init(_ clock: Clock, parent: SoundAnimator? = nil) { 23 | self.clock = clock 24 | self.currentState = nil 25 | self.nextState = nil 26 | self.currentStartTime = nil 27 | self.transitionDuration = nil 28 | self.parent = parent 29 | super.init() 30 | super.set { [weak self] in 31 | guard let strongSelf = self else { 32 | return .gone 33 | } 34 | return strongSelf.impl($0) 35 | } 36 | } 37 | 38 | public func setState(_ state: ElementState, _ duration: TimePoint) -> Future { 39 | return Future { completion in 40 | if self.currentState == nil || duration.value <= 0 { 41 | self.currentState = state 42 | completion(.success(true)) 43 | } else { 44 | let now = self.clock.current() 45 | self.currentStartTime = now 46 | self.clock.schedule(now + duration) { [weak self] _ in 47 | self?.currentState = self?.nextState 48 | self?.nextState = nil 49 | self?.currentStartTime = nil 50 | self?.transitionDuration = nil 51 | completion(.success(true)) 52 | } 53 | self.nextState = state 54 | self.transitionDuration = duration 55 | } 56 | } 57 | } 58 | 59 | func computedState() throws -> ComputedAudioState { 60 | guard let currentState = self.currentState else { 61 | throw AnimatorError.noCurrentState 62 | } 63 | guard let transitionDuration = self.transitionDuration, 64 | let currentStartTime = self.currentStartTime, 65 | let nextState = self.nextState else { 66 | return computeAudioState(currentState, next: nil, pct: nil) 67 | } 68 | let now = self.clock.current() 69 | let pct = seconds(now - currentStartTime) / seconds(transitionDuration) 70 | return computeAudioState(currentState, next: nextState, pct: pct) 71 | } 72 | 73 | func setParent( _ parent: SoundAnimator? ) { 74 | self.parent = parent 75 | } 76 | 77 | private func impl(_ sample: AudioSample) -> EventBox { 78 | guard currentState?.muted == false else { 79 | return .nothing(sample.info()) 80 | } 81 | 82 | do { 83 | let computedState = try self.computedState() 84 | let parentState = try self.parent?.computedState() 85 | let transform = computedState.matrix * (parentState?.matrix ?? Matrix3.identity) * sample.transform 86 | return .just(AudioSample(sample, transform: transform)) 87 | } catch { 88 | return .just(sample) 89 | } 90 | } 91 | var currentState: ElementState? 92 | var nextState: ElementState? 93 | var transitionDuration: TimePoint? 94 | var currentStartTime: TimePoint? 95 | let clock: Clock 96 | weak var parent: SoundAnimator? 97 | } 98 | 99 | struct ComputedAudioState { 100 | let matrix: Matrix3 101 | let gain: Float 102 | } 103 | 104 | func computeAudioState( _ current: ElementState, next: ElementState?, pct: Float?) -> ComputedAudioState { 105 | let state = next.map { next in 106 | guard let pct = pct else { 107 | return current 108 | } 109 | return ElementState.with { 110 | $0.audioGain = interpolate(current.audioGain, next.audioGain, pct) 111 | $0.audioPos = interpolate(current.audioPos, next.audioPos, pct) 112 | } 113 | } ?? current 114 | 115 | return ComputedAudioState(matrix: Matrix3(translation: Vector2(state.audioPos)) * 116 | Matrix3(scale: Vector2(state.audioGain, state.audioGain)), 117 | gain: state.audioGain) 118 | } 119 | -------------------------------------------------------------------------------- /Sources/SwiftVideo/buffer.swift: -------------------------------------------------------------------------------- 1 | /* 2 | SwiftVideo, Copyright 2019 Unpause SAS 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | import NIO 18 | import struct Foundation.Data 19 | 20 | // swiftlint:disable identifier_name 21 | // swiftlint:disable type_name 22 | public enum buffer { 23 | // Concatenate ByteBuffers into one buffer. 24 | // Can be used as a reducer. 25 | // This will potentially cause unintended side-effects if you use an existing buffer as lhs 26 | // because it will be resized and appended to. 27 | // So be aware of ownership and expectations in that case. 28 | public static func concat(_ lhs: ByteBuffer?, _ rhs: ByteBuffer? = nil) -> ByteBuffer? { 29 | var res: ByteBuffer? 30 | if lhs == nil { 31 | if let rhs = rhs { 32 | let allocator = ByteBufferAllocator() 33 | res = allocator.buffer(capacity: rhs.readableBytes) 34 | } 35 | } else if let lhs = lhs { 36 | res = lhs.getSlice(at: lhs.readerIndex, length: lhs.readableBytes) 37 | if let rhs = rhs, var res = res { 38 | let needed = max(rhs.readableBytes - res.writableBytes, 0) 39 | res.reserveCapacity(res.capacity + needed) // mutates 40 | } 41 | } 42 | // Unfortunately no real way to do this in an immutable fashion. 43 | // The ByteBuffer needs the writer index moved forward to enable 44 | // bytes to be readable. 45 | if var res = res, let rhs = rhs { 46 | res.setBuffer(rhs, at: res.readableBytes) 47 | res.moveWriterIndex(forwardBy: rhs.readableBytes) // mutates 48 | return res 49 | } 50 | 51 | return res 52 | } 53 | 54 | public static func readBytes(_ buf: ByteBuffer, length: Int) -> (ByteBuffer?, [UInt8]?) { 55 | // Slice is mutated here and returned as a new ByteBuffer? 56 | var mutBuf = buf.getSlice(at: buf.readerIndex, length: buf.readableBytes) 57 | let bytes = mutBuf?.readBytes(length: length) 58 | return (rebase(mutBuf), bytes) 59 | } 60 | 61 | public static func getSlice(_ buf: ByteBuffer?, _ at: Int, _ length: Int) -> ByteBuffer? { 62 | if let buf = buf { 63 | let total = max(at - buf.readerIndex, 0) &+ length 64 | if total > 0 && total <= buf.readableBytes && length > 0 { 65 | return buf.getSlice(at: at + buf.readerIndex, length: length) 66 | } 67 | } 68 | return nil 69 | } 70 | 71 | public static func getSlice(_ buf: ByteBuffer?, _ length: Int) -> ByteBuffer? { 72 | if let buf = buf { 73 | return getSlice(buf, buf.readerIndex, length) 74 | } 75 | return nil 76 | } 77 | 78 | public static func advancingReader(_ buf: ByteBuffer?, by: Int) -> ByteBuffer? { 79 | if var buf = buf { 80 | if buf.readableBytes >= by { 81 | buf.moveReaderIndex(forwardBy: by) // mutates 82 | return buf//rebase(buf) 83 | } 84 | } 85 | return nil 86 | } 87 | 88 | public static func rebase(_ buf: ByteBuffer?) -> ByteBuffer? { 89 | if var buf = buf { 90 | buf.discardReadBytes() 91 | return buf 92 | } 93 | return buf 94 | } 95 | 96 | public static func fromUnsafeBytes(_ bytes: UnsafePointer?, _ size: Int) -> ByteBuffer { 97 | let allocator = ByteBufferAllocator() 98 | var buf = allocator.buffer(capacity: size) 99 | buf.writeWithUnsafeMutableBytes(minimumWritableBytes: size) { (ptr) -> Int in 100 | guard let dst = ptr.baseAddress, let src = bytes else { 101 | return 0 102 | } 103 | memcpy(dst, src, size) 104 | return size 105 | } 106 | return buf 107 | } 108 | 109 | public static func fromBytes(_ bytes: [UInt8]) -> ByteBuffer { 110 | let allocator = ByteBufferAllocator() 111 | var buf = allocator.buffer(capacity: bytes.count) 112 | buf.writeBytes(bytes) 113 | return buf 114 | } 115 | 116 | public static func fromData(_ data: Data) -> ByteBuffer { 117 | return data.withUnsafeBytes { 118 | buffer.fromUnsafeBytes( 119 | $0.baseAddress.map { $0.bindMemory(to: Int8.self, capacity: data.count) }, 120 | data.count) 121 | } 122 | } 123 | 124 | public static func toData(_ buf: ByteBuffer?) -> Data? { 125 | return buf.flatMap { $0.getData(at: $0.readerIndex, length: $0.readableBytes) } 126 | } 127 | 128 | public static func toDataCopy( _ buf: ByteBuffer?) -> Data? { 129 | if let buf = buf { 130 | return buf.withUnsafeReadableBytes { buf in 131 | buf.baseAddress { Data(bytes: $0, count: buf.count) } <|> nil 132 | } 133 | } 134 | return nil 135 | } 136 | 137 | public static func toByteArray(_ value: T) -> [UInt8] where T: FixedWidthInteger { 138 | var value = value 139 | return withUnsafePointer(to: &value) { 140 | $0.withMemoryRebound(to: UInt8.self, capacity: MemoryLayout.size) { 141 | Array(UnsafeBufferPointer(start: $0, count: MemoryLayout.size)) 142 | } 143 | } 144 | } 145 | } 146 | // swiftlint:enable identifier_name 147 | // swiftlint:enable type_name 148 | -------------------------------------------------------------------------------- /Sources/SwiftVideo/cam.apple.swift: -------------------------------------------------------------------------------- 1 | /* 2 | SwiftVideo, Copyright 2019 Unpause SAS 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | #if os(iOS) || os(macOS) || os(tvOS) 18 | import Foundation 19 | import CoreMedia 20 | import AVFoundation 21 | 22 | public class AppleCamera: Source { 23 | private let session: AVCaptureSession 24 | private let frameDuration: TimePoint 25 | private let clock: Clock 26 | private let queue: DispatchQueue 27 | private var captureDevice: AVCaptureDevice? 28 | private var sampleBufferDel: SampleBufferDel? 29 | private let idAsset: String 30 | 31 | private class SampleBufferDel: NSObject, AVCaptureVideoDataOutputSampleBufferDelegate { 32 | private let callback: (CMSampleBuffer) -> Void 33 | init(callback: @escaping (CMSampleBuffer) -> Void) { 34 | self.callback = callback 35 | } 36 | func captureOutput(_ output: AVCaptureOutput, 37 | didOutput sampleBuffer: CMSampleBuffer, 38 | from connection: AVCaptureConnection) { 39 | self.callback(sampleBuffer) 40 | } 41 | } 42 | 43 | public init(pos: AVCaptureDevice.Position, clock: Clock = WallClock(), frameDuration: TimePoint) { 44 | self.session = AVCaptureSession() 45 | self.frameDuration = frameDuration 46 | self.clock = clock 47 | let assetId = UUID().uuidString 48 | self.queue = DispatchQueue.init(label: "cam.apple.\(assetId)") 49 | self.idAsset = assetId 50 | super.init() 51 | 52 | self.sampleBufferDel = SampleBufferDel { [weak self] in 53 | if let strongSelf = self { 54 | strongSelf.push($0) 55 | } 56 | } 57 | 58 | let permission: (Bool) -> Void = { 59 | [weak self] (granted) in 60 | 61 | guard let strongSelf = self else { 62 | return 63 | } 64 | 65 | if granted && strongSelf.setDevice(pos: pos) { 66 | let output = AVCaptureVideoDataOutput() 67 | strongSelf.session.beginConfiguration() 68 | output.alwaysDiscardsLateVideoFrames = true 69 | 70 | output.videoSettings = 71 | [kCVPixelBufferPixelFormatTypeKey as String: 72 | kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange, 73 | kCVPixelBufferIOSurfacePropertiesKey as String: 74 | [kCVPixelBufferMetalCompatibilityKey as String: true], 75 | kCVPixelBufferOpenGLCompatibilityKey as String: true] 76 | 77 | output.setSampleBufferDelegate(strongSelf.sampleBufferDel, queue: strongSelf.queue) 78 | strongSelf.session.addOutput(output) 79 | strongSelf.session.commitConfiguration() 80 | strongSelf.session.startRunning() 81 | } 82 | } 83 | // swiftlint:disable deployment_target 84 | if #available(iOS 7.0, macOS 10.14, *) { 85 | let access = AVCaptureDevice.authorizationStatus(for: .video) 86 | 87 | if access == .authorized { 88 | permission(true) 89 | } else if access == .notDetermined { 90 | AVCaptureDevice.requestAccess(for: .video, completionHandler: permission) 91 | } 92 | } else { 93 | permission(true) 94 | } 95 | } 96 | 97 | @discardableResult 98 | public func setDevice(pos: AVCaptureDevice.Position) -> Bool { 99 | if let device = { () -> AVCaptureDevice? in 100 | #if os(iOS) 101 | return AVCaptureDevice.default(.builtInWideAngleCamera, for: .video, position: pos) 102 | #else 103 | return AVCaptureDevice.default(for: .video) 104 | #endif 105 | }() { 106 | self.session.beginConfiguration() 107 | self.captureDevice = device 108 | 109 | let currentInput = self.session.inputs.first 110 | 111 | if let cur = currentInput { 112 | self.session.removeInput(cur) 113 | } 114 | do { 115 | try device.lockForConfiguration() 116 | // find the closest frame duration to the requested duration 117 | let frameDuration = device.activeFormat.videoSupportedFrameRateRanges.reduce(CMTime.positiveInfinity) { 118 | Float64(seconds(self.frameDuration)).distance( 119 | to: CMTimeGetSeconds($1.maxFrameDuration)) < CMTimeGetSeconds($0) ? $1.maxFrameDuration : $0 120 | } 121 | 122 | device.activeVideoMinFrameDuration = frameDuration 123 | device.activeVideoMaxFrameDuration = frameDuration 124 | try self.session.addInput(AVCaptureDeviceInput(device: device)) 125 | } catch { 126 | if let cur = currentInput { 127 | self.session.addInput(cur) 128 | } 129 | return false 130 | } 131 | 132 | device.unlockForConfiguration() 133 | self.session.commitConfiguration() 134 | return true 135 | } 136 | return false 137 | } 138 | 139 | public func assetId() -> String { 140 | return idAsset 141 | } 142 | 143 | private func push(_ buf: CMSampleBuffer) { 144 | let pts = CMSampleBufferGetPresentationTimeStamp(buf) 145 | let sample = PictureSample(buf, 146 | assetId: self.idAsset, 147 | workspaceId: "cam.apple", 148 | time: self.clock.current(), 149 | pts: TimePoint(pts.value, Int64(pts.timescale))) 150 | _ = self.emit(sample) 151 | } 152 | } 153 | #endif 154 | -------------------------------------------------------------------------------- /Sources/SwiftVideo/enc.video.apple.swift: -------------------------------------------------------------------------------- 1 | /* 2 | SwiftVideo, Copyright 2019 Unpause SAS 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | #if os(iOS) || os(macOS) || os(tvOS) 18 | import VideoToolbox 19 | import Dispatch 20 | 21 | public class AppleVideoEncoder: Tx { 22 | 23 | enum AppleVideoEncoderError: Error { 24 | case noSession 25 | case invalidSystemVersion 26 | } 27 | 28 | public init(format: MediaFormat, frame: CGSize, bitrate: Int = 500000) { 29 | assert(format == .avc || format == .hevc, "AppleVideoEncoder only supports AVC and HEVC") 30 | self.queue = DispatchQueue.init(label: "cezium.encode.video") 31 | 32 | VTCompressionSessionCreate(allocator: kCFAllocatorDefault, 33 | width: Int32(frame.width), 34 | height: Int32(frame.height), 35 | codecType: format == .avc ? kCMVideoCodecType_H264 : kCMVideoCodecType_HEVC, 36 | encoderSpecification: [kVTCompressionPropertyKey_ExpectedFrameRate: 30] as CFDictionary, 37 | imageBufferAttributes: pixelBufferOptions(frame), 38 | compressedDataAllocator: nil, 39 | outputCallback: nil, 40 | refcon: nil, 41 | compressionSessionOut: &self.session) 42 | self.format = format 43 | super.init() 44 | super.set { 45 | [weak self] in 46 | if let strongSelf = self { 47 | do { 48 | try strongSelf.setBitrate(bitrate) 49 | try strongSelf.enc($0) 50 | } catch AppleVideoEncoderError.invalidSystemVersion { 51 | return .error(EventError("venc", -1, "Invalid System Version", $0.time())) 52 | } catch AppleVideoEncoderError.noSession { 53 | return .error(EventError("venc", -2, "No Media Session", $0.time())) 54 | } catch { 55 | return .error(EventError("venc", -999, "Unexpected Error \(error)", $0.time())) 56 | } 57 | let res = strongSelf.samples 58 | strongSelf.samples.removeAll() 59 | return res.count > 0 ? .just(res) : .nothing($0.info()) 60 | } 61 | return .error(EventError("venc", -1000, "Encoder has gone away")) 62 | } 63 | } 64 | 65 | func setBitrate(_ bitrate: Int) throws { 66 | guard let session = self.session else { 67 | throw AppleVideoEncoderError.noSession 68 | } 69 | let val = bitrate as CFTypeRef 70 | VTSessionSetProperty(session, key: kVTCompressionPropertyKey_AverageBitRate, value: val) 71 | } 72 | 73 | func enc(_ sample: PictureSample) throws { 74 | guard let image = sample.imageBuffer() else { 75 | return 76 | } 77 | guard let session = self.session else { 78 | throw AppleVideoEncoderError.noSession 79 | } 80 | let sampleTime = sample.time() 81 | let assetId = sample.assetId() 82 | let workspaceId = sample.workspaceId() 83 | let workspaceToken = sample.workspaceToken() 84 | let outputHandler: VTCompressionOutputHandler = { [weak self] (status, flags, sample) in 85 | if let strongSelf = self, let smp = sample { 86 | if let buf = CMSampleBufferGetDataBuffer(smp) { 87 | var size: Int = 0 88 | var data: UnsafeMutablePointer? 89 | 90 | CMBlockBufferGetDataPointer(buf, 91 | atOffset: 0, 92 | lengthAtOffsetOut: nil, 93 | totalLengthOut: &size, 94 | dataPointerOut: &data) 95 | 96 | if size > 0 { 97 | let bytebuf = Data(buffer: UnsafeBufferPointer(start: data, count: size)) 98 | let pts = CMSampleBufferGetPresentationTimeStamp(smp) 99 | let dts = CMSampleBufferGetDecodeTimeStamp(smp) 100 | let fmt = CMSampleBufferGetFormatDescription(smp) 101 | var extradata: Data? 102 | if let fmt = fmt, 103 | let extensions = CMFormatDescriptionGetExtensions(fmt) as? [String: AnyObject], 104 | let sampleDesc = extensions["SampleDescriptionExtensionAtoms"] { 105 | switch strongSelf.format { 106 | case .avc: 107 | if let opt = sampleDesc["avcC"], let desc = opt { extradata = desc as? Data } 108 | case .hevc: 109 | if let opt = sampleDesc["hvcC"], let desc = opt { extradata = desc as? Data } 110 | default: () 111 | } 112 | } 113 | let samp = CodedMediaSample(assetId, 114 | workspaceId, 115 | sampleTime, 116 | TimePoint(pts.value, Int64(pts.timescale)), 117 | dts == .invalid ? nil : TimePoint(dts.value, Int64(dts.timescale)), 118 | .video, 119 | strongSelf.format, 120 | bytebuf, 121 | extradata.map { ["config": $0] }, 122 | "cezium.apple", 123 | workspaceToken: workspaceToken) 124 | strongSelf.samples.append(samp) 125 | } 126 | } 127 | } 128 | } 129 | if #available(iOS 9.0, macOS 10.11, tvOS 10.2, *) { 130 | VTCompressionSessionEncodeFrame(session, 131 | imageBuffer: image.pixelBuffer, 132 | presentationTimeStamp: CMTime(value: sample.pts().value, 133 | timescale: Int32(sample.pts().scale)), 134 | duration: CMTime.indefinite, 135 | frameProperties: nil, 136 | infoFlagsOut: nil, 137 | outputHandler: outputHandler) 138 | } else { 139 | throw AppleVideoEncoderError.invalidSystemVersion 140 | } 141 | } 142 | private var samples = [CodedMediaSample]() 143 | private let queue: DispatchQueue 144 | private let format: MediaFormat 145 | private var session: VTCompressionSession? 146 | } 147 | 148 | func pixelBufferOptions(_ frame: CGSize) -> CFDictionary { 149 | #if os(iOS) 150 | return [ 151 | kCVPixelBufferPixelFormatTypeKey as String: kCVPixelFormatType_32BGRA, 152 | kCVPixelBufferWidthKey as String: Int32(frame.width), 153 | kCVPixelBufferHeightKey as String: Int32(frame.height), 154 | kCVPixelBufferOpenGLESCompatibilityKey as String: true, 155 | kCVPixelBufferIOSurfacePropertiesKey as String: [:] 156 | ] as CFDictionary 157 | #else 158 | return [ 159 | kCVPixelBufferPixelFormatTypeKey as String: kCVPixelFormatType_32BGRA, 160 | kCVPixelBufferWidthKey as String: Int32(frame.width), 161 | kCVPixelBufferHeightKey as String: Int32(frame.height), 162 | kCVPixelBufferIOSurfacePropertiesKey as String: [:] 163 | ] as CFDictionary 164 | #endif 165 | } 166 | 167 | #endif 168 | -------------------------------------------------------------------------------- /Sources/SwiftVideo/event.swift: -------------------------------------------------------------------------------- 1 | /* 2 | SwiftVideo, Copyright 2019 Unpause SAS 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | extension EventError { 18 | public init(_ src: String, _ code: Int, _ desc: String? = nil, _ time: TimePoint? = nil, assetId: String? = nil) { 19 | self.source = src 20 | self.code = Int32(code) 21 | if let assetId = assetId { 22 | self.assetID = assetId 23 | } 24 | if let desc = desc { 25 | self.desc = desc 26 | } 27 | if let time = time { 28 | self.time = time 29 | } 30 | } 31 | } 32 | 33 | public typealias EventInfo = StatsReport 34 | 35 | public protocol Event { 36 | func type() -> String 37 | func time() -> TimePoint 38 | func assetId() -> String 39 | func workspaceId() -> String 40 | func workspaceToken() -> String? 41 | func info() -> EventInfo? 42 | } 43 | 44 | extension Array: Event where Element: Event { 45 | public func type() -> String { return "list" } 46 | public func time() -> TimePoint { return self.last { $0.time() } <|> TimePoint(0) } 47 | public func assetId() -> String { return self.last { $0.assetId() } <|> "none" } 48 | public func workspaceId() -> String { return self.last { $0.workspaceId() } <|> "none" } 49 | public func workspaceToken() -> String? { return self.last?.workspaceToken() } 50 | 51 | public func info() -> EventInfo? { return self.reduce(nil) { 52 | guard let acc = $0 else { 53 | return $1.info() 54 | } 55 | guard let val = $1.info() else { 56 | return acc 57 | } 58 | return acc.merging(val) 59 | } 60 | } 61 | } 62 | 63 | public enum EventBox { 64 | case just(T) 65 | case error(EventError) 66 | case nothing(EventInfo?) 67 | case gone // next item in chain is gone, stop calling 68 | } 69 | 70 | extension EventBox { 71 | public func flatMap(_ fun: @escaping (T) -> EventBox) -> EventBox { 72 | switch self { 73 | case .just(let payload): 74 | return fun(payload) 75 | case .error(let error): 76 | return .error(error) 77 | case .nothing(let info): 78 | return .nothing(info) 79 | case .gone: 80 | return .gone 81 | } 82 | } 83 | 84 | public func map(_ fun: @escaping (T) -> U) -> EventBox { 85 | switch self { 86 | case .just(let payload): 87 | return .just(fun(payload)) 88 | case .error(let error): 89 | return .error(error) 90 | case .nothing(let info): 91 | return .nothing(info) 92 | case .gone: 93 | return .gone 94 | } 95 | } 96 | 97 | public func apply(_ fun: EventBox<(T) -> U>) -> EventBox { 98 | switch fun { 99 | case .just(let fun): 100 | return map(fun) 101 | case .error(let error): 102 | return .error(error) 103 | case .nothing(let info): 104 | return .nothing(info) 105 | case .gone: 106 | return .gone 107 | } 108 | } 109 | 110 | public func value() -> T? { 111 | guard case let .just(val) = self else { 112 | return nil 113 | } 114 | return val 115 | } 116 | 117 | public func error() -> EventError? { 118 | guard case let .error(val) = self else { 119 | return nil 120 | } 121 | return val 122 | } 123 | } 124 | 125 | precedencegroup ApplyGroup { associativity: left higherThan: AssignmentPrecedence } 126 | infix operator >>- : ApplyGroup 127 | infix operator <*> : ApplyGroup 128 | 129 | public func <*> (_ lhs: EventBox<(T) -> U>, _ rhs: EventBox) -> EventBox { 130 | return rhs.apply(lhs) 131 | } 132 | 133 | public func >>- (_ lhs: EventBox, _ rhs: @escaping (T) -> EventBox) -> EventBox { 134 | return lhs.flatMap(rhs) 135 | } 136 | 137 | public final class ResultEvent: Event { 138 | public func type() -> String { return "result" } 139 | public func time() -> TimePoint { return timePoint } 140 | public func assetId() -> String { return idAsset } 141 | public func workspaceId() -> String { return idWorkspace } 142 | public func workspaceToken() -> String? { return tokenWorkspace } 143 | public func info() -> EventInfo? { return eventInfo } 144 | 145 | public init(time: TimePoint?, assetId: String?, workspaceId: String?, workspaceToken: String?) { 146 | self.timePoint = time ?? TimePoint(0, 1000) 147 | self.idAsset = assetId ?? "" 148 | self.idWorkspace = workspaceId ?? "" 149 | self.tokenWorkspace = workspaceToken 150 | self.eventInfo = nil 151 | } 152 | let timePoint: TimePoint 153 | let idAsset: String 154 | let idWorkspace: String 155 | let tokenWorkspace: String? 156 | let eventInfo: EventInfo? 157 | } 158 | -------------------------------------------------------------------------------- /Sources/SwiftVideo/filter.pict.swift: -------------------------------------------------------------------------------- 1 | /* 2 | SwiftVideo, Copyright 2019 Unpause SAS 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | import Foundation 18 | import VectorMath 19 | 20 | /* 21 | public class PictureFilter : Tx { 22 | public init(_ clock: Clock, 23 | computeContext: ComputeContext? = nil) { 24 | self.clock = clock 25 | do { 26 | if let context = computeContext { 27 | self.context = createComputeContext(sharing: context) 28 | } else { 29 | self.context = try makeComputeContext(forType: .GPU) 30 | } 31 | } catch { 32 | self.context = nil 33 | } 34 | super.init() 35 | super.set { [weak self] sample in 36 | guard let strongSelf = self else { 37 | return .gone 38 | } 39 | 40 | return .just(sample) 41 | } 42 | } 43 | 44 | let clock: Clock 45 | var context: ComputeContext? 46 | } 47 | */ 48 | -------------------------------------------------------------------------------- /Sources/SwiftVideo/live.swift: -------------------------------------------------------------------------------- 1 | /* 2 | SwiftVideo, Copyright 2019 Unpause SAS 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | import BrightFutures 18 | import VectorMath 19 | 20 | public protocol LiveAsset { 21 | func assetId() -> String 22 | func workspaceId() -> String 23 | func workspaceToken() -> String? 24 | func uuid() -> String 25 | func liveType() -> MediaSourceType 26 | func dialedOut() -> Bool 27 | } 28 | 29 | public protocol LivePublisher: LiveAsset { 30 | func acceptedFormats() -> [MediaFormat] 31 | func uri() -> String? 32 | } 33 | 34 | public protocol LiveSubscriber: LiveAsset { 35 | func suppliedFormats() -> [MediaFormat] 36 | } 37 | 38 | public typealias LiveOnConnection = (LivePublisher?, LiveSubscriber?) -> Future 39 | public typealias LiveOnEnded = (String) -> Void 40 | -------------------------------------------------------------------------------- /Sources/SwiftVideo/mix.video.swift: -------------------------------------------------------------------------------- 1 | /* 2 | SwiftVideo, Copyright 2019 Unpause SAS 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | import Dispatch 18 | import Foundation 19 | import VectorMath 20 | 21 | public class VideoMixer: Source { 22 | public init(_ clock: Clock, 23 | workspaceId: String, 24 | frameDuration: TimePoint, 25 | outputSize: Vector2, 26 | outputFormat: PixelFormat = .nv12, 27 | computeContext: ComputeContext? = nil, 28 | assetId: String? = nil, 29 | statsReport: StatsReport? = nil, 30 | epoch: Int64? = nil) { 31 | 32 | if let computeContext = computeContext { 33 | self.clContext = createComputeContext(sharing: computeContext) 34 | } else { 35 | do { 36 | self.clContext = try makeComputeContext(forType: .GPU) 37 | } catch { 38 | print("Error making compute context!") 39 | } 40 | } 41 | 42 | self.clock = clock 43 | self.result = .nothing(nil) 44 | self.samples = Array(repeating: [String: PictureSample](), count: 2) 45 | self.frameDuration = frameDuration 46 | let now = clock.current() 47 | self.epoch = rescale(epoch.map { clock.fromUnixTime($0) } ?? now, frameDuration.scale) 48 | self.backing = [PictureSample]() 49 | self.backingSize = outputSize 50 | self.backingFormat = outputFormat 51 | self.idWorkspace = workspaceId 52 | let idAsset = assetId ?? UUID().uuidString 53 | self.idAsset = idAsset 54 | self.statsReport = statsReport ?? StatsReport(assetId: idAsset, clock: clock) 55 | self.queue = DispatchQueue(label: "mix.video.\(idAsset)") 56 | super.init() 57 | super.set { [weak self] pic -> EventBox in 58 | guard let strongSelf = self else { 59 | return .gone 60 | } 61 | guard strongSelf.clContext != nil else { 62 | return .error(EventError("mix.video", -1, "No Compute Context", 63 | pic.time(), 64 | assetId: strongSelf.idAsset)) 65 | } 66 | if pic.assetId() != strongSelf.assetId() { 67 | strongSelf.queue.async { [weak self] in 68 | self?.samples[0][pic.revision()] = pic 69 | } 70 | return .nothing(pic.info()) 71 | } else { 72 | return .just(pic) 73 | } 74 | 75 | } 76 | clock.schedule(now + frameDuration) { [weak self] in self?.mix(at: $0) } 77 | } 78 | 79 | public func assetId() -> String { idAsset } 80 | 81 | public func workspaceId() -> String { idWorkspace } 82 | 83 | public func computeContext() -> ComputeContext? { self.clContext } 84 | 85 | deinit { 86 | if let context = self.clContext { 87 | do { 88 | try destroyComputeContext(context) 89 | } catch {} 90 | self.clContext = nil 91 | } 92 | } 93 | 94 | // swiftlint:disable:next identifier_name 95 | private func mix(at: ClockTickEvent) { 96 | let next = at.time() + frameDuration 97 | let pts = at.time() - epoch 98 | clock.schedule(next) { [weak self] in self?.mix(at: $0) } 99 | queue.async { [weak self] in 100 | guard let strongSelf = self, 101 | let ctx = strongSelf.clContext else { 102 | return 103 | } 104 | defer { 105 | strongSelf.samples[1] = strongSelf.samples[0] 106 | strongSelf.samples[0].removeAll(keepingCapacity: true) 107 | } 108 | let result: EventBox 109 | do { 110 | strongSelf.statsReport.endTimer("mix.video.delta") 111 | strongSelf.statsReport.startTimer("mix.video.delta") 112 | strongSelf.statsReport.startTimer("mix.video.compose") 113 | let backing = try strongSelf.getBacking() 114 | let images = strongSelf.samples[0].merging(strongSelf.samples[1]) { lhs, _ in lhs } 115 | .values.sorted { $0.zIndex() < $1.zIndex() } 116 | strongSelf.clContext = try usingContext(ctx) { 117 | let clearKernel = try strongSelf.findKernel(nil, backing) 118 | let ctx = try runComputeKernel($0, images: [PictureSample](), target: backing, kernel: clearKernel) 119 | return try images.reduce(ctx) { 120 | try applyComputeImage($0, 121 | image: $1, 122 | target: backing, 123 | kernel: strongSelf.findKernel($1, backing)) 124 | } 125 | } 126 | strongSelf.statsReport.endTimer("mix.video.compose") 127 | let sample = PictureSample(backing, 128 | pts: pts, 129 | time: at.time(), 130 | eventInfo: strongSelf.statsReport) 131 | _ = strongSelf.emit(sample) 132 | result = .nothing(strongSelf.statsReport) 133 | } catch let error { 134 | print("mix caught error \(error)") 135 | result = .error(EventError("mix.video", -2, "Compute error \(error)", 136 | at.time(), assetId: strongSelf.idAsset)) 137 | } 138 | strongSelf.result = result 139 | } 140 | } 141 | 142 | private func findKernel(_ image: PictureSample?, _ target: PictureSample) throws -> ComputeKernel { 143 | let inp = image { String(describing: $0.pixelFormat()).lowercased() } <|> "clear" 144 | let outp = String(describing: target.pixelFormat()).lowercased() 145 | return try defaultComputeKernelFromString("img_\(inp)_\(outp)") 146 | } 147 | 148 | private func getBacking() throws -> PictureSample { 149 | guard let ctx = clContext else { 150 | throw ComputeError.badContextState(description: "No context") 151 | } 152 | if backing.count < numberBackingImages { 153 | let image = try createPictureSample(self.backingSize, 154 | backingFormat, 155 | assetId: self.assetId(), 156 | workspaceId: self.workspaceId()) 157 | let gpuImage = try uploadComputePicture(ctx, pict: image) 158 | backing.append(gpuImage) 159 | return gpuImage 160 | } else { 161 | let image = backing[currentBacking] 162 | currentBacking = (currentBacking + 1) % backing.count 163 | return image 164 | } 165 | } 166 | 167 | private let numberBackingImages = 10 168 | private let statsReport: StatsReport 169 | 170 | private var backing: [PictureSample] 171 | private var currentBacking = 0 172 | private let backingFormat: PixelFormat 173 | private let backingSize: Vector2 174 | 175 | public let frameDuration: TimePoint 176 | private let clock: Clock 177 | private let epoch: TimePoint 178 | internal let queue: DispatchQueue 179 | private var clContext: ComputeContext? 180 | private var result: EventBox 181 | private var samples: [[String: PictureSample]] 182 | private let idAsset: String 183 | private let idWorkspace: String 184 | } 185 | -------------------------------------------------------------------------------- /Sources/SwiftVideo/playback.audio.apple.swift: -------------------------------------------------------------------------------- 1 | #if os(macOS) || os(iOS) || os(tvOS) 2 | import Foundation 3 | import AudioToolbox 4 | import CoreAudio 5 | 6 | public class AppleAudioPlayback: Terminal { 7 | public init(_ gain: Float = 1.0) { 8 | self.unit = nil 9 | self.samples = [] 10 | self.pts = TimePoint(0, 1000) 11 | super.init() 12 | super.set { [weak self] sample in 13 | guard let strongSelf = self else { 14 | return .gone 15 | } 16 | guard sample.format() == .f32p else { 17 | return .error(EventError("playback.audio.apple", -1, "Currently only .f32p is supported")) 18 | } 19 | if strongSelf.unit == nil { 20 | strongSelf.setupAudioUnit(sample, gain: gain) 21 | } 22 | if (sample.pts()) < (strongSelf.pts-TimePoint(2.0)) { 23 | strongSelf.pts = rescale(sample.pts(), Int64(sample.sampleRate())) 24 | //strongSelf.samples.removeAll(keepingCapacity: true) 25 | strongSelf.ptsOffset = nil 26 | } 27 | strongSelf.samples.append(sample) 28 | strongSelf.samples = strongSelf.samples.filter { ($0.pts() + $0.duration()) > strongSelf.pts } 29 | return .nothing(sample.info()) 30 | } 31 | } 32 | deinit { 33 | if let unit = self.unit { 34 | AudioOutputUnitStop(unit) 35 | } 36 | } 37 | 38 | public func setGain(_ gain: Float) { 39 | if let unit = self.unit { 40 | AudioUnitSetParameter(unit, kHALOutputParam_Volume, kAudioUnitScope_Global, 0, gain, 0) 41 | } 42 | } 43 | 44 | private func setupAudioUnit(_ sample: AudioSample, gain: Float) { 45 | var asbd = AudioStreamBasicDescription(mSampleRate: Float64(sample.sampleRate()), 46 | mFormatID: kAudioFormatLinearPCM, 47 | mFormatFlags: kAudioFormatFlagIsPacked | 48 | kAudioFormatFlagIsFloat | 49 | kAudioFormatFlagIsNonInterleaved, 50 | mBytesPerPacket: 4, 51 | mFramesPerPacket: 1, 52 | mBytesPerFrame: 4, 53 | mChannelsPerFrame: UInt32(sample.numberChannels()), 54 | mBitsPerChannel: 32, 55 | mReserved: 0) 56 | #if os(macOS) 57 | let subtype = kAudioUnitSubType_HALOutput 58 | #else 59 | let subtype = kAudioUnitSubType_RemoteIO 60 | #endif 61 | var desc = AudioComponentDescription(componentType: kAudioUnitType_Output, 62 | componentSubType: subtype, 63 | componentManufacturer: kAudioUnitManufacturer_Apple, 64 | componentFlags: 0, 65 | componentFlagsMask: 0) 66 | var unit: AudioUnit? 67 | 68 | if let component = AudioComponentFindNext(nil, &desc), 69 | AudioComponentInstanceNew(component, &unit) == noErr { 70 | if let unit = unit { 71 | 72 | self.pts = rescale(sample.pts(), Int64(sample.sampleRate())) 73 | self.unit = unit 74 | var callback = AURenderCallbackStruct(inputProc: ioProc, inputProcRefCon: bridge(self)) 75 | var flag: UInt32 = 1 76 | AudioUnitSetProperty(unit, kAudioUnitProperty_SetRenderCallback, 77 | kAudioUnitScope_Global, 0, &callback, UInt32(MemoryLayout.size)) 78 | AudioUnitSetProperty(unit, kAudioOutputUnitProperty_EnableIO, 79 | kAudioUnitScope_Output, 0, &flag, 4) 80 | AudioUnitSetProperty(unit, kAudioUnitProperty_StreamFormat, 81 | kAudioUnitScope_Input, 0, &asbd, UInt32(MemoryLayout.size)) 82 | AudioUnitInitialize(unit) 83 | AudioOutputUnitStart(unit) 84 | AudioUnitSetParameter(unit, kHALOutputParam_Volume, kAudioUnitScope_Global, 0, gain, 0) 85 | } 86 | } 87 | } 88 | 89 | private var unit: AudioUnit? 90 | fileprivate var samples: [AudioSample] 91 | fileprivate var pts: TimePoint 92 | fileprivate var ptsOffset: TimePoint? 93 | } 94 | 95 | //swiftlint:disable:next function_parameter_count 96 | private func ioProc(inRefCon: UnsafeMutableRawPointer, 97 | ioActionFlags: UnsafeMutablePointer, 98 | audioTimestamp: UnsafePointer, 99 | inBusNumber: UInt32, 100 | inNumberFrames: UInt32, 101 | ioData: UnsafeMutablePointer?) -> OSStatus { 102 | guard let buffers = UnsafeMutableAudioBufferListPointer(ioData) else { 103 | return -1 104 | } 105 | let this: AppleAudioPlayback = bridge(from: inRefCon) 106 | if this.ptsOffset == nil { 107 | this.ptsOffset = TimePoint(Int64(audioTimestamp.pointee.mSampleTime), this.pts.scale) 108 | } 109 | //guard let ptsOffset = this.ptsOffset else { 110 | // return -1 111 | //} 112 | let windowStart = this.pts //- ptsOffset 113 | let windowEnd = windowStart + TimePoint(Int64(inNumberFrames), windowStart.scale) 114 | buffers.forEach { 115 | guard let ptr = $0.mData else { 116 | return 117 | } 118 | memset(ptr, 0, Int($0.mDataByteSize)) 119 | } 120 | let samples = Array(this.samples) 121 | samples.forEach { sample in 122 | let sampleStart = rescale(sample.pts(), this.pts.scale) 123 | let sampleEnd = sampleStart + TimePoint(Int64(sample.numberSamples()), this.pts.scale) 124 | if windowEnd >= sampleStart && windowStart < sampleEnd { 125 | let readOffset = min(Int(max(windowStart.value - sampleStart.value, 0)), sample.sampleCount) * 4 126 | let writeOffset = min(Int(max(sampleStart.value - windowStart.value, 0)), Int(inNumberFrames)) * 4 127 | let writeCount = min(sample.sampleCount * 4 - readOffset, Int(inNumberFrames) * 4 - writeOffset) 128 | zip(buffers, sample.data()).forEach { 129 | guard let ptr = $0.0.mData else { 130 | return 131 | } 132 | _ = $0.1.withUnsafeBytes { 133 | guard let readPtr = $0.baseAddress else { 134 | return 135 | } 136 | memcpy(ptr+writeOffset, readPtr+readOffset, writeCount) 137 | } 138 | } 139 | } 140 | } 141 | this.pts = windowEnd 142 | return noErr 143 | } 144 | #endif 145 | -------------------------------------------------------------------------------- /Sources/SwiftVideo/proto/TimePoint.pb.swift: -------------------------------------------------------------------------------- 1 | // DO NOT EDIT. 2 | // 3 | // Generated by the Swift generator plugin for the protocol buffer compiler. 4 | // Source: TimePoint.proto 5 | // 6 | // For information on using the generated types, please see the documenation: 7 | // https://github.com/apple/swift-protobuf/ 8 | 9 | // 10 | //SwiftVideo, Copyright 2019 Unpause SAS 11 | // 12 | //Licensed under the Apache License, Version 2.0 (the "License"); 13 | //you may not use this file except in compliance with the License. 14 | //You may obtain a copy of the License at 15 | // 16 | //http://www.apache.org/licenses/LICENSE-2.0 17 | // 18 | //Unless required by applicable law or agreed to in writing, software 19 | //distributed under the License is distributed on an "AS IS" BASIS, 20 | //WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 21 | //See the License for the specific language governing permissions and 22 | //limitations under the License. 23 | 24 | import Foundation 25 | import SwiftProtobuf 26 | 27 | // If the compiler emits an error on this type, it is because this file 28 | // was generated by a version of the `protoc` Swift plug-in that is 29 | // incompatible with the version of SwiftProtobuf to which you are linking. 30 | // Please ensure that your are building against the same version of the API 31 | // that was used to generate this file. 32 | fileprivate struct _GeneratedWithProtocGenSwiftVersion: SwiftProtobuf.ProtobufAPIVersionCheck { 33 | struct _2: SwiftProtobuf.ProtobufAPIVersion_2 {} 34 | typealias Version = _2 35 | } 36 | 37 | public struct TimePoint { 38 | // SwiftProtobuf.Message conformance is added in an extension below. See the 39 | // `Message` and `Message+*Additions` files in the SwiftProtobuf library for 40 | // methods supported on all messages. 41 | 42 | public var value: Int64 = 0 43 | 44 | public var scale: Int64 = 0 45 | 46 | public var unknownFields = SwiftProtobuf.UnknownStorage() 47 | 48 | public init() {} 49 | } 50 | 51 | public struct EventError { 52 | // SwiftProtobuf.Message conformance is added in an extension below. See the 53 | // `Message` and `Message+*Additions` files in the SwiftProtobuf library for 54 | // methods supported on all messages. 55 | 56 | public var source: String { 57 | get {return _storage._source} 58 | set {_uniqueStorage()._source = newValue} 59 | } 60 | 61 | public var code: Int32 { 62 | get {return _storage._code} 63 | set {_uniqueStorage()._code = newValue} 64 | } 65 | 66 | public var desc: String { 67 | get {return _storage._desc} 68 | set {_uniqueStorage()._desc = newValue} 69 | } 70 | 71 | public var time: TimePoint { 72 | get {return _storage._time ?? TimePoint()} 73 | set {_uniqueStorage()._time = newValue} 74 | } 75 | /// Returns true if `time` has been explicitly set. 76 | public var hasTime: Bool {return _storage._time != nil} 77 | /// Clears the value of `time`. Subsequent reads from it will return its default value. 78 | public mutating func clearTime() {_uniqueStorage()._time = nil} 79 | 80 | public var assetID: String { 81 | get {return _storage._assetID} 82 | set {_uniqueStorage()._assetID = newValue} 83 | } 84 | 85 | public var unknownFields = SwiftProtobuf.UnknownStorage() 86 | 87 | public init() {} 88 | 89 | fileprivate var _storage = _StorageClass.defaultInstance 90 | } 91 | 92 | // MARK: - Code below here is support for the SwiftProtobuf runtime. 93 | 94 | extension TimePoint: SwiftProtobuf.Message, SwiftProtobuf._MessageImplementationBase, SwiftProtobuf._ProtoNameProviding { 95 | public static let protoMessageName: String = "TimePoint" 96 | public static let _protobuf_nameMap: SwiftProtobuf._NameMap = [ 97 | 1: .same(proto: "value"), 98 | 2: .same(proto: "scale"), 99 | ] 100 | 101 | public mutating func decodeMessage(decoder: inout D) throws { 102 | while let fieldNumber = try decoder.nextFieldNumber() { 103 | switch fieldNumber { 104 | case 1: try decoder.decodeSingularInt64Field(value: &self.value) 105 | case 2: try decoder.decodeSingularInt64Field(value: &self.scale) 106 | default: break 107 | } 108 | } 109 | } 110 | 111 | public func traverse(visitor: inout V) throws { 112 | if self.value != 0 { 113 | try visitor.visitSingularInt64Field(value: self.value, fieldNumber: 1) 114 | } 115 | if self.scale != 0 { 116 | try visitor.visitSingularInt64Field(value: self.scale, fieldNumber: 2) 117 | } 118 | try unknownFields.traverse(visitor: &visitor) 119 | } 120 | 121 | public static func ==(lhs: TimePoint, rhs: TimePoint) -> Bool { 122 | if lhs.value != rhs.value {return false} 123 | if lhs.scale != rhs.scale {return false} 124 | if lhs.unknownFields != rhs.unknownFields {return false} 125 | return true 126 | } 127 | } 128 | 129 | extension EventError: SwiftProtobuf.Message, SwiftProtobuf._MessageImplementationBase, SwiftProtobuf._ProtoNameProviding { 130 | public static let protoMessageName: String = "EventError" 131 | public static let _protobuf_nameMap: SwiftProtobuf._NameMap = [ 132 | 1: .same(proto: "source"), 133 | 2: .same(proto: "code"), 134 | 3: .same(proto: "desc"), 135 | 4: .same(proto: "time"), 136 | 5: .same(proto: "assetId"), 137 | ] 138 | 139 | fileprivate class _StorageClass { 140 | var _source: String = String() 141 | var _code: Int32 = 0 142 | var _desc: String = String() 143 | var _time: TimePoint? = nil 144 | var _assetID: String = String() 145 | 146 | static let defaultInstance = _StorageClass() 147 | 148 | private init() {} 149 | 150 | init(copying source: _StorageClass) { 151 | _source = source._source 152 | _code = source._code 153 | _desc = source._desc 154 | _time = source._time 155 | _assetID = source._assetID 156 | } 157 | } 158 | 159 | fileprivate mutating func _uniqueStorage() -> _StorageClass { 160 | if !isKnownUniquelyReferenced(&_storage) { 161 | _storage = _StorageClass(copying: _storage) 162 | } 163 | return _storage 164 | } 165 | 166 | public mutating func decodeMessage(decoder: inout D) throws { 167 | _ = _uniqueStorage() 168 | try withExtendedLifetime(_storage) { (_storage: _StorageClass) in 169 | while let fieldNumber = try decoder.nextFieldNumber() { 170 | switch fieldNumber { 171 | case 1: try decoder.decodeSingularStringField(value: &_storage._source) 172 | case 2: try decoder.decodeSingularInt32Field(value: &_storage._code) 173 | case 3: try decoder.decodeSingularStringField(value: &_storage._desc) 174 | case 4: try decoder.decodeSingularMessageField(value: &_storage._time) 175 | case 5: try decoder.decodeSingularStringField(value: &_storage._assetID) 176 | default: break 177 | } 178 | } 179 | } 180 | } 181 | 182 | public func traverse(visitor: inout V) throws { 183 | try withExtendedLifetime(_storage) { (_storage: _StorageClass) in 184 | if !_storage._source.isEmpty { 185 | try visitor.visitSingularStringField(value: _storage._source, fieldNumber: 1) 186 | } 187 | if _storage._code != 0 { 188 | try visitor.visitSingularInt32Field(value: _storage._code, fieldNumber: 2) 189 | } 190 | if !_storage._desc.isEmpty { 191 | try visitor.visitSingularStringField(value: _storage._desc, fieldNumber: 3) 192 | } 193 | if let v = _storage._time { 194 | try visitor.visitSingularMessageField(value: v, fieldNumber: 4) 195 | } 196 | if !_storage._assetID.isEmpty { 197 | try visitor.visitSingularStringField(value: _storage._assetID, fieldNumber: 5) 198 | } 199 | } 200 | try unknownFields.traverse(visitor: &visitor) 201 | } 202 | 203 | public static func ==(lhs: EventError, rhs: EventError) -> Bool { 204 | if lhs._storage !== rhs._storage { 205 | let storagesAreEqual: Bool = withExtendedLifetime((lhs._storage, rhs._storage)) { (_args: (_StorageClass, _StorageClass)) in 206 | let _storage = _args.0 207 | let rhs_storage = _args.1 208 | if _storage._source != rhs_storage._source {return false} 209 | if _storage._code != rhs_storage._code {return false} 210 | if _storage._desc != rhs_storage._desc {return false} 211 | if _storage._time != rhs_storage._time {return false} 212 | if _storage._assetID != rhs_storage._assetID {return false} 213 | return true 214 | } 215 | if !storagesAreEqual {return false} 216 | } 217 | if lhs.unknownFields != rhs.unknownFields {return false} 218 | return true 219 | } 220 | } 221 | -------------------------------------------------------------------------------- /Sources/SwiftVideo/repeater.swift: -------------------------------------------------------------------------------- 1 | /* 2 | SwiftVideo, Copyright 2019 Unpause SAS 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | import Dispatch 18 | import Foundation 19 | 20 | // Repeats a sample at a specified interval if a new one is not provided. 21 | public class Repeater: AsyncTx { 22 | public init(_ clock: Clock, interval: TimePoint) { 23 | self.clock = clock 24 | self.lastEmit = clock.current() 25 | self.queue = DispatchQueue(label: "repeater.\(UUID().uuidString)") 26 | super.init() 27 | let interval = rescale(interval, clock.current().scale) 28 | super.set { [weak self] sample in 29 | guard let strongSelf = self else { 30 | return .gone 31 | } 32 | let now = strongSelf.clock.current() 33 | strongSelf.queue.sync { 34 | strongSelf.sample = sample 35 | strongSelf.lastEmit = now 36 | } 37 | strongSelf.run(interval) 38 | return .just(sample) 39 | } 40 | } 41 | 42 | private func run(_ interval: TimePoint) { 43 | let now = self.clock.current() 44 | self.clock.schedule(now + interval) { [weak self] evt in 45 | self?.queue.async { 46 | guard let strongSelf = self, let sample = strongSelf.sample else { 47 | return 48 | } 49 | if (strongSelf.lastEmit + interval) <= evt.timePoint { 50 | _ = strongSelf.emit(sample) 51 | strongSelf.lastEmit = evt.timePoint 52 | strongSelf.run(interval) 53 | } 54 | } 55 | } 56 | } 57 | 58 | private let queue: DispatchQueue 59 | private let clock: Clock 60 | private var sample: T? 61 | private var lastEmit: TimePoint 62 | } 63 | -------------------------------------------------------------------------------- /Sources/SwiftVideo/rpc/public.rpc.swift: -------------------------------------------------------------------------------- 1 | /* 2 | SwiftVideo, Copyright 2019 Unpause SAS 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | public enum RpcError: Error { 18 | case timedOut 19 | case gone 20 | case invalidConfiguration 21 | case caught(Error) 22 | case remote(String) 23 | case unknown 24 | } 25 | 26 | extension RpcAssetPermissionResponse { 27 | public init(_ granted: Bool) { 28 | self.granted = granted 29 | } 30 | } 31 | 32 | extension RpcAssetPermissionRequest { 33 | public init(_ sourceType: MediaSourceType, 34 | mediaType: MediaType, 35 | formats: [MediaFormat], 36 | requestType: PermissionRequestType) { 37 | self.sourceType = sourceType 38 | self.mediaType = mediaType 39 | self.formats = formats 40 | self.requestType = requestType 41 | } 42 | } 43 | -------------------------------------------------------------------------------- /Sources/SwiftVideo/sample.audio.swift: -------------------------------------------------------------------------------- 1 | /* 2 | SwiftVideo, Copyright 2019 Unpause SAS 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | import VectorMath 18 | import Foundation 19 | 20 | public enum AudioError: Error { 21 | case invalidFormat 22 | } 23 | 24 | public enum AudioFormat { 25 | case s16i // Signed, 16-bit, interlaced 26 | case s16p // Signed, 16-bit, planar 27 | case f32i // Float, interlaced 28 | case f32p // Float, planar 29 | case f64i // Double, interlaced 30 | case f64p // Double, planar 31 | // Used for internal accumulators: 32 | case s64i // Signed, 64-bit, interlaced 33 | case s64p // Signed, 64-bit, planar 34 | case invalid 35 | } 36 | 37 | public enum AudioChannelLayout { 38 | case mono 39 | case stereo 40 | } 41 | 42 | public class AudioSample: Event { 43 | public init(_ buffers: [Data], 44 | frequency: Int, 45 | channels: Int, 46 | format: AudioFormat, 47 | sampleCount: Int, 48 | time: TimePoint, 49 | pts: TimePoint, 50 | assetId: String, 51 | workspaceId: String, 52 | workspaceToken: String? = nil, 53 | computeBuffers: [ComputeBuffer] = [], 54 | bufferType: BufferType = .cpu, 55 | constituents: [MediaConstituent]? = nil, 56 | eventInfo: EventInfo? = nil) { 57 | self.buffers = buffers 58 | self.frequency = frequency 59 | self.channels = channels 60 | self.sampleCount = sampleCount 61 | self.timePoint = time 62 | self.audioFormat = format 63 | self.presentationTimestamp = pts 64 | self.idAsset = assetId 65 | self.idWorkspace = workspaceId 66 | self.tokenWorkspace = workspaceToken 67 | self.eventInfo = eventInfo 68 | self.transform = Matrix3.identity 69 | self.computeBuffers = computeBuffers 70 | self.buffertype = bufferType 71 | self.mediaConstituents = constituents 72 | } 73 | 74 | public init(_ other: AudioSample, 75 | bufferType: BufferType? = nil, 76 | assetId: String? = nil, 77 | buffers: [Data]? = nil, 78 | frequency: Int? = nil, 79 | channels: Int? = nil, 80 | format: AudioFormat? = nil, 81 | sampleCount: Int? = nil, 82 | pts: TimePoint? = nil, 83 | transform: Matrix3? = nil, 84 | computeBuffers: [ComputeBuffer]? = nil, 85 | constituents: [MediaConstituent]? = nil, 86 | eventInfo: EventInfo? = nil) { 87 | self.buffers = buffers ?? other.buffers 88 | self.computeBuffers = computeBuffers ?? other.computeBuffers 89 | self.buffertype = bufferType ?? other.bufferType() 90 | self.frequency = frequency ?? other.frequency 91 | self.channels = channels ?? other.channels 92 | self.sampleCount = sampleCount ?? other.sampleCount 93 | self.timePoint = other.timePoint 94 | self.audioFormat = format ?? other.audioFormat 95 | self.presentationTimestamp = pts ?? other.presentationTimestamp 96 | self.idAsset = assetId ?? other.idAsset 97 | self.idWorkspace = other.idWorkspace 98 | self.tokenWorkspace = other.tokenWorkspace 99 | self.eventInfo = eventInfo ?? other.eventInfo 100 | self.transform = transform ?? other.transform 101 | self.mediaConstituents = constituents ?? other.mediaConstituents 102 | } 103 | 104 | public func type() -> String { 105 | "soun" 106 | } 107 | public func time() -> TimePoint { 108 | timePoint 109 | } 110 | public func assetId() -> String { 111 | idAsset 112 | } 113 | public func workspaceId() -> String { 114 | idWorkspace 115 | } 116 | public func workspaceToken() -> String? { 117 | tokenWorkspace 118 | } 119 | public func info() -> EventInfo? { 120 | eventInfo 121 | } 122 | public func data() -> [Data] { 123 | buffers 124 | } 125 | public func computeData() -> [ComputeBuffer] { 126 | computeBuffers 127 | } 128 | public func pts() -> TimePoint { 129 | presentationTimestamp 130 | } 131 | public func duration() -> TimePoint { 132 | rescale(TimePoint(Int64(sampleCount), Int64(frequency)), presentationTimestamp.scale) 133 | } 134 | public func sampleRate() -> Int { 135 | frequency 136 | } 137 | public func numberSamples() -> Int { 138 | sampleCount 139 | } 140 | public func numberChannels() -> Int { 141 | channels 142 | } 143 | public func format() -> AudioFormat { 144 | audioFormat 145 | } 146 | public func bufferType() -> BufferType { 147 | buffertype 148 | } 149 | public func constituents() -> [MediaConstituent]? { 150 | self.mediaConstituents 151 | } 152 | 153 | let idAsset: String 154 | let idWorkspace: String 155 | let tokenWorkspace: String? 156 | let mediaConstituents: [MediaConstituent]? 157 | 158 | let buffers: [Data] 159 | let computeBuffers: [ComputeBuffer] 160 | let buffertype: BufferType 161 | let frequency: Int 162 | let channels: Int 163 | let sampleCount: Int 164 | let audioFormat: AudioFormat 165 | let presentationTimestamp: TimePoint 166 | let timePoint: TimePoint 167 | let transform: Matrix3 /// Position is represented by a single 2D circular plane. 168 | /// Maybe in the future we will add elevation, but not now. 169 | /// Gain is represented by the length of a line that starts as (0, 0) -> (0, 1). 170 | let eventInfo: EventInfo? 171 | 172 | } 173 | 174 | func numberOfChannels(_ channelLayout: AudioChannelLayout) -> Int { 175 | switch channelLayout { 176 | case .mono: 177 | return 1 178 | case .stereo: 179 | return 2 180 | } 181 | } 182 | 183 | public func numberOfBuffers(_ format: AudioFormat, _ channelCount: Int) -> Int { 184 | return isPlanar(format) ? channelCount : 1 185 | } 186 | 187 | public func numberOfBuffers(_ format: AudioFormat, _ channelLayout: AudioChannelLayout) -> Int { 188 | return numberOfBuffers(format, numberOfChannels(channelLayout)) 189 | } 190 | 191 | func bytesPerSample(_ format: AudioFormat, _ channelCount: Int) -> Int { 192 | let sampleBytes = { () -> Int in 193 | switch format { 194 | case .s16p, .s16i: 195 | return 2 196 | case .f32p, .f32i: 197 | return 4 198 | case .f64p, .f64i, .s64p, .s64i: 199 | return 8 200 | case .invalid: 201 | return 0 202 | } 203 | }() 204 | return isPlanar(format) ? sampleBytes : sampleBytes * channelCount 205 | } 206 | 207 | func isPlanar(_ format: AudioFormat) -> Bool { 208 | switch format { 209 | case .s16p, .f32p, .f64p, .s64p: 210 | return true 211 | default: 212 | return false 213 | } 214 | } 215 | -------------------------------------------------------------------------------- /Sources/SwiftVideo/sample.coded.swift: -------------------------------------------------------------------------------- 1 | /* 2 | SwiftVideo, Copyright 2019 Unpause SAS 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | import NIO 18 | import NIOFoundationCompat 19 | import Foundation 20 | import VectorMath 21 | import CSwiftVideo 22 | 23 | public enum EncodeError: Error { 24 | case invalidMediaFormat 25 | case invalidPixelFormat 26 | case invalidImageBuffer 27 | case invalidContext 28 | case encoderNotFound 29 | } 30 | 31 | public struct AVCSettings { 32 | public let useBFrames: Bool 33 | public let useHWAccel: Bool 34 | public let profile: String 35 | public let preset: String 36 | public init(useBFrames: Bool? = nil, 37 | useHWAccel: Bool? = nil, 38 | profile: String? = nil, 39 | preset: String? = nil) { 40 | self.useBFrames = useBFrames ?? false 41 | self.useHWAccel = useHWAccel ?? false 42 | self.profile = profile ?? "main" 43 | self.preset = preset ?? "veryfast" 44 | } 45 | } 46 | 47 | public enum EncoderSpecificSettings { 48 | case avc(AVCSettings) 49 | case unused 50 | } 51 | 52 | public struct BasicVideoDescription { 53 | public let size: Vector2 54 | } 55 | 56 | public struct BasicAudioDescription { 57 | public let sampleRate: Float 58 | public let channelCount: Int 59 | public let samplesPerPacket: Int 60 | } 61 | 62 | public enum BasicMediaDescription { 63 | case video(BasicVideoDescription) 64 | case audio(BasicAudioDescription) 65 | } 66 | 67 | public func formatsFilter (_ formats: [MediaFormat]) -> Tx { 68 | return Tx { sample in 69 | if formats.contains(where: { sample.mediaFormat() == $0 }) { 70 | return .just(sample) 71 | } else { 72 | return .nothing(sample.info()) 73 | } 74 | } 75 | } 76 | 77 | public func mediaTypeFilter(_ mediaType: MediaType) -> Tx { 78 | return Tx { sample in 79 | if sample.mediaType() == mediaType { 80 | return .just(sample) 81 | } else { 82 | return .nothing(sample.info()) 83 | } 84 | } 85 | } 86 | 87 | public struct CodedMediaSample { 88 | let eventInfo: EventInfo? 89 | var wire: CodedMediaSampleWire 90 | } 91 | 92 | // 93 | // 94 | // See Proto/CodedMediaSample.proto 95 | extension CodedMediaSample: Event { 96 | public func type() -> String { 97 | return "sample.mediacoded" 98 | } 99 | 100 | public func time() -> TimePoint { 101 | return wire.eventTime 102 | } 103 | 104 | public func assetId() -> String { 105 | return wire.idAsset 106 | } 107 | 108 | public func workspaceId() -> String { 109 | return wire.idWorkspace 110 | } 111 | 112 | public func workspaceToken() -> String? { 113 | return wire.tokenWorkspace 114 | } 115 | 116 | public func mediaType() -> MediaType { 117 | return wire.mediatype 118 | } 119 | 120 | public func mediaFormat() -> MediaFormat { 121 | return wire.mediaformat 122 | } 123 | 124 | public func info() -> EventInfo? { 125 | return eventInfo 126 | } 127 | 128 | public func pts() -> TimePoint { 129 | return wire.pts 130 | } 131 | 132 | public func dts() -> TimePoint { 133 | return wire.dts 134 | } 135 | 136 | public func serializedData() throws -> Data { 137 | return try wire.serializedData() 138 | } 139 | 140 | public func data() -> Data { 141 | return wire.buffer 142 | } 143 | 144 | public func sideData() -> [String: Data] { 145 | return wire.side 146 | } 147 | 148 | public func constituents() -> [MediaConstituent]? { 149 | return wire.constituents 150 | } 151 | 152 | public init(_ assetId: String, 153 | _ workspaceId: String, 154 | _ time: TimePoint, 155 | _ pts: TimePoint, 156 | _ dts: TimePoint?, 157 | _ type: MediaType, 158 | _ format: MediaFormat, 159 | _ data: Data, 160 | _ sideData: [String: Data]?, 161 | _ encoder: String?, 162 | workspaceToken: String? = nil, 163 | eventInfo: EventInfo? = nil, 164 | constituents: [MediaConstituent] = [MediaConstituent]()) { 165 | self.wire = CodedMediaSampleWire() 166 | self.wire.mediatype = type 167 | self.wire.mediaformat = format 168 | self.wire.buffer = data 169 | self.wire.eventTime = time 170 | self.wire.pts = pts 171 | self.wire.dts = dts ?? pts 172 | self.wire.side = sideData ?? [String: Data]() 173 | self.wire.idAsset = assetId 174 | self.wire.idWorkspace = workspaceId 175 | self.wire.tokenWorkspace = workspaceToken ?? "" 176 | self.wire.encoder = encoder ?? "" 177 | self.eventInfo = eventInfo 178 | self.wire.constituents = constituents 179 | } 180 | 181 | public init(_ other: CodedMediaSample, 182 | assetId: String? = nil, 183 | constituents: [MediaConstituent]? = nil, 184 | eventInfo: EventInfo? = nil) { 185 | self.wire = other.wire 186 | self.wire.idAsset = assetId ?? other.assetId() 187 | self.eventInfo = eventInfo ?? other.eventInfo 188 | self.wire.constituents = constituents ?? other.wire.constituents 189 | } 190 | 191 | public init(serializedData: Data, eventInfo: EventInfo? = nil) throws { 192 | self.wire = try CodedMediaSampleWire(serializedData: serializedData) 193 | self.eventInfo = eventInfo 194 | } 195 | } 196 | 197 | enum MediaDescriptionError: Error { 198 | case unsupported 199 | case invalidMetadata 200 | } 201 | 202 | func basicMediaDescription(_ sample: CodedMediaSample) throws -> BasicMediaDescription { 203 | switch sample.mediaFormat() { 204 | case .avc: 205 | let sps = try spsFromAVCDCR(sample) 206 | let (width, height): (Int32, Int32) = sps.withUnsafeBytes { 207 | var width: Int32 = 0 208 | var height: Int32 = 0 209 | h264_sps_frame_size($0.baseAddress, Int64($0.count), &width, &height) 210 | return (width, height) 211 | } 212 | return .video(BasicVideoDescription(size: Vector2(Float(width), Float(height)))) 213 | case .aac: 214 | guard let asc = sample.sideData()["config"] else { 215 | throw MediaDescriptionError.invalidMetadata 216 | } 217 | let (channels, sampleRate, samplesPerPacket): (Int32, Int32, Int32) = asc.withUnsafeBytes { 218 | var channels: Int32 = 0 219 | var sampleRate: Int32 = 0 220 | var samplesPerPacket: Int32 = 0 221 | aac_parse_asc($0.baseAddress, Int64($0.count), &channels, &sampleRate, &samplesPerPacket) 222 | return (channels, sampleRate, samplesPerPacket) 223 | } 224 | return .audio(BasicAudioDescription(sampleRate: Float(sampleRate), 225 | channelCount: Int(channels), 226 | samplesPerPacket: Int(samplesPerPacket))) 227 | default: 228 | throw MediaDescriptionError.unsupported 229 | } 230 | } 231 | 232 | public func isKeyframe(_ sample: CodedMediaSample) -> Bool { 233 | guard sample.mediaType() == .video else { 234 | return true 235 | } 236 | switch sample.mediaFormat() { 237 | case .avc: 238 | return isKeyframeAVC(sample) 239 | case .hevc: 240 | return false 241 | default: 242 | return false 243 | } 244 | } 245 | 246 | // TODO: Add HEVC, VP8/VP9, AV1 247 | private func isKeyframeAVC(_ sample: CodedMediaSample) -> Bool { 248 | guard sample.data().count >= 5 else { 249 | return false 250 | } 251 | return sample.data()[4] & 0x1f == 5 252 | } 253 | 254 | private func spsFromAVCDCR(_ sample: CodedMediaSample) throws -> Data { 255 | guard let record = sample.sideData()["config"], 256 | record.count > 8 else { 257 | throw MediaDescriptionError.invalidMetadata 258 | } 259 | //let spsCount = Int(record[5]) & 0x1f 260 | let size = (Int(record[6]) << 8) | Int(record[7]) 261 | guard record.count > 8 + size else { 262 | throw MediaDescriptionError.invalidMetadata 263 | } 264 | return record[8..<8+size] 265 | } 266 | -------------------------------------------------------------------------------- /Sources/SwiftVideo/sample.pict.swift: -------------------------------------------------------------------------------- 1 | /* 2 | SwiftVideo, Copyright 2019 Unpause SAS 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | import VectorMath 18 | 19 | // TODO: Higher bit-depth formats 20 | public enum PixelFormat { 21 | case nv12 22 | case nv21 23 | case yuvs 24 | case zvuy 25 | case y420p 26 | case y422p 27 | case y444p 28 | case RGBA 29 | case BGRA 30 | case shape 31 | case text 32 | case invalid 33 | } 34 | 35 | // swiftlint:disable identifier_name 36 | public enum Component { 37 | case r 38 | case g 39 | case b 40 | case a 41 | case y 42 | case cr 43 | case cb 44 | } 45 | // swiftlint:enable identifier_name 46 | 47 | public struct Plane { 48 | public init(size: Vector2, stride: Int, bitDepth: Int, components: [Component]) { 49 | self.size = size 50 | self.stride = stride 51 | self.bitDepth = bitDepth 52 | self.components = components 53 | } 54 | public let size: Vector2 55 | public let stride: Int 56 | public let bitDepth: Int 57 | public let components: [Component] 58 | } 59 | 60 | public enum BufferType { 61 | case shared 62 | case cpu 63 | case gpu 64 | case invalid 65 | } 66 | 67 | public protocol PictureEvent: Event { 68 | func pts() -> TimePoint 69 | func matrix() -> Matrix4 70 | func textureMatrix() -> Matrix4 71 | func zIndex() -> Int 72 | func pixelFormat() -> PixelFormat 73 | func bufferType() -> BufferType 74 | func size() -> Vector2 75 | func lock() 76 | func unlock() 77 | func revision() -> String 78 | func fillColor() -> Vector4 79 | func borderMatrix() -> Matrix4 80 | func opacity() -> Float 81 | } 82 | 83 | public func componentsForPlane(_ pixelFormat: PixelFormat, _ idx: Int) -> [Component] { 84 | switch pixelFormat { 85 | case .y420p, .y422p, .y444p: 86 | return [[.y], [.cb], [.cr]][idx] 87 | case .nv12: 88 | return [[.y], [.cb, .cr]][idx] 89 | case .nv21: 90 | return [[.y], [.cr, .cb]][idx] 91 | case .yuvs: 92 | return [.y, .cb, .y, .cr] 93 | case .zvuy: 94 | return [.cb, .y, .cr, .y] 95 | case .BGRA: 96 | return [.b, .g, .r, .a] 97 | case .RGBA: 98 | return [.r, .g, .b, .a] 99 | default: 100 | return [] 101 | } 102 | } 103 | -------------------------------------------------------------------------------- /Sources/SwiftVideo/segmenter.audio.swift: -------------------------------------------------------------------------------- 1 | import Foundation 2 | 3 | public class AudioPacketSegmenter: Tx { 4 | 5 | public init(_ duration: TimePoint) { 6 | self.incoming = [] 7 | self.pts = nil 8 | self.duration = duration 9 | super.init() 10 | super.set { [weak self] sample in 11 | guard let strongSelf = self else { 12 | return .gone 13 | } 14 | strongSelf.incoming.append(sample) 15 | let pts = strongSelf.pts ?? sample.pts() 16 | let result = audioSampleSplit(strongSelf.duration, pts: pts, inSamples: strongSelf.incoming) 17 | strongSelf.pts = result.0 18 | strongSelf.incoming = result.1 19 | //print("result=\(result.2)") 20 | return .just(result.2) 21 | } 22 | } 23 | 24 | var pts: TimePoint? 25 | var incoming: [AudioSample] 26 | let duration: TimePoint 27 | } 28 | private typealias SplitResult = (TimePoint, [AudioSample], [AudioSample]) // final pts, remaining samples, new samples 29 | private func audioSampleSplit(_ duration: TimePoint, 30 | pts: TimePoint, 31 | inSamples: [AudioSample], 32 | outSamples: [AudioSample] = []) -> SplitResult { 33 | guard inSamples.count > 0 else { 34 | return (pts, [], outSamples) 35 | } 36 | 37 | // we are going to try to extract a single segment from the buffer we have built up. 38 | let totalDuration = inSamples.reduce(TimePoint(0)) { $0 + $1.duration() } - (pts - inSamples[0].pts()) 39 | //print("inSamples=\(inSamples) totalDuration=\(totalDuration.toString()) duration=\(duration.toString())") 40 | guard totalDuration >= duration else { 41 | return (pts, inSamples, outSamples) 42 | } 43 | let sampleCount = rescale(duration, Int64(inSamples[0].sampleRate())).value 44 | let sampleBytes = bytesPerSample(inSamples[0].format(), inSamples[0].numberChannels()) 45 | let bufferLength = Int(sampleCount) * sampleBytes 46 | var buffers = (0.. nextPts } 56 | let toCopy = inSamples.filter { $0.pts() <= nextPts } 57 | 58 | toCopy.forEach { 59 | let inOffset = sample.pts() - $0.pts() 60 | let inStartBytes = max(Int(rescale(inOffset, Int64($0.sampleRate())).value) * sampleBytes, 0) 61 | let outOffset = $0.pts() - sample.pts() 62 | let outStartBytes = max(Int(rescale(outOffset, Int64($0.sampleRate())).value) * sampleBytes, 0) 63 | let bytesToCopy = min(bufferLength - outStartBytes, $0.data()[0].count - inStartBytes) 64 | //let outEndBytes = outStartBytes + bytesToCopy 65 | //let inEndBytes = inStartBytes + bytesToCopy 66 | // print("bufferLength = \(bufferLength) data.count = \($0.data()[0].count)") 67 | // print("bytesToCopy = \(bytesToCopy)") 68 | // print("inOffset=\(inOffset.toString()) outOffset=\(outOffset.toString())") 69 | // print("outStartBytes=\(outStartBytes) inStartBytes=\(inStartBytes)") 70 | // print("inPts=\($0.pts().toString())") 71 | if bytesToCopy > 0 { 72 | $0.data().enumerated().forEach { 73 | let idx = $0.offset 74 | var inBuf = $0.element 75 | inBuf.withUnsafeMutableBytes { src in 76 | guard let src = src.baseAddress else { return } 77 | buffers[idx].withUnsafeMutableBytes { dst in 78 | guard let dst = dst.baseAddress else { return } 79 | memcpy(dst+outStartBytes, src+inStartBytes, bytesToCopy) 80 | } 81 | } 82 | } 83 | } 84 | } 85 | return audioSampleSplit(duration, pts: pts + duration, inSamples: remaining, outSamples: outSamples + [sample]) 86 | } 87 | -------------------------------------------------------------------------------- /Sources/SwiftVideo/src.audio.apple.swift: -------------------------------------------------------------------------------- 1 | /* 2 | SwiftVideo, Copyright 2019 Unpause SAS 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | public class AudioSampleRateConversion: Tx { 18 | public init(_ outFrequency: Int, _ outChannelCount: Int, _ outAudioFormat: AudioFormat) { 19 | super.init { 20 | .just($0) 21 | } 22 | } 23 | } 24 | 25 | -------------------------------------------------------------------------------- /Sources/SwiftVideo/stats.audio.swift: -------------------------------------------------------------------------------- 1 | /* 2 | SwiftVideo, Copyright 2019 Unpause SAS 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | import Foundation 18 | 19 | public func audioStats() -> Tx { 20 | return Tx { sample in 21 | let channels = sample.numberChannels() 22 | if let info = sample.info() { 23 | var peak: [Float] = Array(repeating: 0, count: channels) 24 | var rms: [Float] = Array(repeating: 0, count: channels) 25 | switch sample.format() { 26 | case .s16i, .s16p: 27 | var accum: [Int] = Array(repeating: 0, count: channels) 28 | var i16peak: [Int] = Array(repeating: 0, count: channels) 29 | iterate(sample, as: Int16.self) { (channel, sample) in 30 | let val = abs(Int(sample)) 31 | if val > i16peak[channel] { 32 | i16peak[channel] = val 33 | } 34 | let sqr = Int(sample)*Int(sample) 35 | accum[channel] += sqr 36 | } 37 | for idx in 0.. peak[channel] { 45 | peak[channel] = val 46 | } 47 | let sqr = sample*sample 48 | rms[channel] += sqr 49 | } 50 | for idx in 0..(_ sample: AudioSample, as: T.Type, fn: (Int, T) -> Void) { 68 | let planar = isPlanar(sample.format()) 69 | let samples = sample.numberSamples() 70 | let channels = sample.numberChannels() 71 | let sampleBytes = bytesPerSample(sample.format(), channels) 72 | let bufferBytes = sampleBytes * samples 73 | sample.data().enumerated().forEach { (idx, buffer) in 74 | let count = min(buffer.count, bufferBytes) / MemoryLayout.size 75 | guard count > 0 else { 76 | return 77 | } 78 | buffer.withUnsafeBytes { ptr in 79 | let bound = ptr.bindMemory(to: T.self) 80 | for elem in 0..: Group 22 | func (_ input: I?, 23 | _ handler: (lhs: (I) -> O, rhs: () -> O)) -> O { 24 | guard let input = input else { 25 | return handler.rhs() 26 | } 27 | return handler.lhs(input) 28 | } 29 | 30 | infix operator <|>: Group 31 | func <|> (_ lhs: @escaping (I) -> O, 32 | _ rhs: @autoclosure @escaping () -> O) -> ((I) -> O, () -> O) { 33 | return (lhs, rhs) 34 | } 35 | 36 | public extension Collection { 37 | /// Returns the element at the specified index if it is within bounds, otherwise nil. 38 | subscript (safe index: Index) -> Element? { 39 | return indices.contains(index) ? self[index] : nil 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /Sources/SwiftVideo/weak.swift: -------------------------------------------------------------------------------- 1 | /* 2 | SwiftVideo, Copyright 2019 Unpause SAS 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | import Foundation 18 | 19 | public class Weak { 20 | public weak var value: T? 21 | public var uuid: String? 22 | public init (value: T, uuid: String? = nil) { 23 | self.value = value 24 | self.uuid = uuid 25 | } 26 | } 27 | 28 | public func bridge(_ obj: T) -> UnsafeMutableRawPointer { 29 | return Unmanaged.passUnretained(obj).toOpaque() 30 | } 31 | 32 | public func bridge(from ptr: UnsafeMutableRawPointer) -> T { 33 | return Unmanaged.fromOpaque(ptr).takeUnretainedValue() 34 | } 35 | 36 | enum ConversionError: Error { 37 | case cannotConvertToData 38 | } 39 | 40 | extension String { 41 | // swiftlint:disable:next identifier_name 42 | public func toJSON(_ as: T.Type) throws -> T { 43 | guard let data = self.data(using: .utf8, allowLossyConversion: false) else { 44 | throw ConversionError.cannotConvertToData 45 | } 46 | return try JSONDecoder().decode(T.self, from: data) 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /Sources/SwiftVideo_FFmpeg/dec.audio.ffmpeg.swift: -------------------------------------------------------------------------------- 1 | /* 2 | SwiftVideo, Copyright 2019 Unpause SAS 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | import SwiftVideo 18 | import SwiftFFmpeg 19 | import Foundation 20 | import VectorMath 21 | 22 | private let kTimebase: Int64 = 96000 23 | 24 | public class FFmpegAudioDecoder: Tx { 25 | public override init() { 26 | self.codec = nil 27 | self.codecContext = nil 28 | self.extradata = nil 29 | super.init() 30 | super.set { [weak self] in 31 | guard let strongSelf = self else { 32 | return .gone 33 | } 34 | return strongSelf.handle($0) 35 | } 36 | } 37 | deinit { 38 | if extradata != nil { 39 | AVIO.freep(extradata) 40 | } 41 | print("AudioDecoder deinit") 42 | } 43 | private func handle(_ sample: CodedMediaSample) -> EventBox { 44 | guard sample.mediaType() == .audio else { 45 | return .error(EventError("dec.sound.ffmpeg", 46 | -1, 47 | "Only audio samples are supported", 48 | assetId: sample.assetId())) 49 | } 50 | if self.codecContext == nil { 51 | do { 52 | try setupContext(sample) 53 | } catch let err { 54 | return .error(EventError("dec.sound.ffmpeg", 55 | -2, 56 | "Error creating codec context \(err)", 57 | assetId: sample.assetId())) 58 | } 59 | } 60 | do { 61 | return try decode(sample) 62 | } catch let error { 63 | print("decode error \(error)") 64 | return .error(EventError("dec.sound.ffmpeg", 65 | -3, 66 | "Error decoding bitstream \(error)", 67 | assetId: sample.assetId())) 68 | } 69 | } 70 | 71 | private func decode(_ sample: CodedMediaSample) throws -> EventBox { 72 | guard let codecCtx = self.codecContext else { 73 | return .error(EventError("dec.sound.ffmpeg", -4, "No codec context", assetId: sample.assetId())) 74 | } 75 | 76 | guard sample.data().count > 0 else { 77 | return .nothing(sample.info()) 78 | } 79 | 80 | let packetSize = try sendPacket(sample, ctx: codecCtx) 81 | 82 | guard packetSize > 0 else { 83 | return .nothing(sample.info()) 84 | } 85 | 86 | do { 87 | let frame = AVFrame() 88 | try codecCtx.receiveFrame(frame) 89 | 90 | let channelCt = codecCtx.channelCount 91 | let sampleCt = frame.sampleCount 92 | let sampleRate = codecCtx.sampleRate 93 | let (format, bytesPerSample) = sampleFormatFromFrame(frame, 94 | codecCtx.sampleFormat.bytesPerSample, channelCt) 95 | let data = dataFromFrame(frame, bytesPerSample, sampleCt, channelCt) 96 | let pts = self.pts ?? rescale(TimePoint(frame.pts, kTimebase), Int64(sampleRate)) 97 | let dur = TimePoint(Int64(sampleCt), Int64(sampleRate)) 98 | self.pts = pts + dur 99 | let sample = AudioSample(data, 100 | frequency: sampleRate, 101 | channels: channelCt, 102 | format: format, 103 | sampleCount: sampleCt, 104 | time: sample.time(), 105 | pts: pts, 106 | assetId: sample.assetId(), 107 | workspaceId: sample.workspaceId(), 108 | workspaceToken: sample.workspaceToken()) 109 | return .just(sample) 110 | } catch let error as AVError where error == .tryAgain { 111 | return .nothing(sample.info()) 112 | } 113 | } 114 | 115 | private func setupContext(_ sample: CodedMediaSample) throws { 116 | self.codec = { 117 | switch sample.mediaFormat() { 118 | case .aac: return AVCodec.findDecoderByName("libfdk_aac") 119 | case .opus: return AVCodec.findDecoderByName("libopus") 120 | default: return nil 121 | } }() 122 | if let codec = self.codec { 123 | let ctx = AVCodecContext(codec: codec) 124 | self.codecContext = ctx 125 | } else { 126 | print("No codec!") 127 | } 128 | if let context = self.codecContext { 129 | if let sideData = sample.sideData()["config"], 130 | let mem = AVIO.malloc(size: sideData.count + AVConstant.inputBufferPaddingSize) { 131 | let memBuf = UnsafeMutableRawBufferPointer(start: mem, 132 | count: sideData.count + AVConstant.inputBufferPaddingSize) 133 | _ = memBuf.baseAddress.map { 134 | sideData.copyBytes(to: $0.assumingMemoryBound(to: UInt8.self), count: sideData.count) 135 | context.extradata = $0.assumingMemoryBound(to: UInt8.self) 136 | context.extradataSize = sideData.count 137 | } 138 | } 139 | try context.openCodec() 140 | } 141 | } 142 | var pts: TimePoint? 143 | var codec: AVCodec? 144 | var codecContext: AVCodecContext? 145 | var extradata: UnsafeMutableRawPointer? 146 | } 147 | 148 | private func sampleFormatFromFrame(_ frame: AVFrame, _ bytesPerSample: Int, _ channels: Int) -> (AudioFormat, Int) { 149 | switch frame.sampleFormat { 150 | case .int16: 151 | return (.s16i, bytesPerSample * channels) 152 | case .int16Planar: 153 | return (.s16p, bytesPerSample) 154 | case .float: 155 | return (.f32i, bytesPerSample * channels) 156 | case .floatPlanar: 157 | return (.f32p, bytesPerSample) 158 | case .double: 159 | return (.f64i, bytesPerSample * channels) 160 | case .doublePlanar: 161 | return (.f64p, bytesPerSample) 162 | default: 163 | return (.invalid, 0) 164 | } 165 | } 166 | 167 | private func dataFromFrame(_ frame: AVFrame, _ bytesPerSample: Int, _ sampleCt: Int, _ channels: Int) -> [Data] { 168 | guard bytesPerSample > 0, sampleCt > 0 else { 169 | return [] 170 | } 171 | return (0.. Data? in 172 | guard let data = frame.data[idx] else { 173 | return nil 174 | } 175 | if idx == 0 { 176 | return Data(bytesNoCopy: data, count: bytesPerSample * sampleCt, deallocator: .custom({ _, _ in 177 | frame.unref() 178 | })) 179 | } else { 180 | return Data(bytesNoCopy: data, count: bytesPerSample * sampleCt, deallocator: .none) 181 | } 182 | } 183 | } 184 | 185 | private func sendPacket(_ sample: CodedMediaSample, ctx: AVCodecContext) throws -> Int { 186 | let pts = rescale(sample.pts(), kTimebase) 187 | let dts = rescale(sample.dts(), kTimebase) 188 | let packet = AVPacket() 189 | let size = sample.data().count 190 | var data = sample.data() 191 | 192 | try packet.makeWritable() 193 | 194 | data.withUnsafeMutableBytes { 195 | guard let buffer = packet.buffer, 196 | let baseAddress = $0.baseAddress else { 197 | return 198 | } 199 | buffer.realloc(size: size) 200 | memcpy(buffer.data, baseAddress, size) 201 | } 202 | packet.data = packet.buffer?.data 203 | packet.size = size 204 | packet.pts = pts.value 205 | packet.dts = dts.value 206 | 207 | if packet.size > 0 { 208 | try ctx.sendPacket(packet) 209 | } 210 | return packet.size 211 | } 212 | -------------------------------------------------------------------------------- /Sources/SwiftVideo_FFmpeg/dec.video.ffmpeg.swift: -------------------------------------------------------------------------------- 1 | /* 2 | SwiftVideo, Copyright 2019 Unpause SAS 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | import SwiftVideo 18 | import SwiftFFmpeg 19 | import Foundation 20 | import VectorMath 21 | 22 | private let kTimebase: Int64 = 600600 23 | 24 | public class FFmpegVideoDecoder: Tx { 25 | public override init() { 26 | self.codec = nil 27 | self.codecContext = nil 28 | self.extradata = nil 29 | super.init() 30 | super.set { [weak self] in 31 | guard let strongSelf = self else { 32 | return .gone 33 | } 34 | return strongSelf.handle($0) 35 | } 36 | } 37 | deinit { 38 | if extradata != nil { 39 | AVIO.freep(extradata) 40 | } 41 | print("VideoDecoder deinit") 42 | } 43 | private func handle(_ sample: CodedMediaSample) -> EventBox { 44 | guard sample.mediaType() == .video else { 45 | return .error(EventError("dec.video.ffmpeg", 46 | -1, "Only video samples are supported", assetId: sample.assetId())) 47 | } 48 | if self.codecContext == nil { 49 | do { 50 | try setupContext(sample) 51 | } catch let err { 52 | print("Context setup error \(err)") 53 | return .error(EventError("dec.video.ffmpeg", 54 | -2, "Error creating codec context \(err)", assetId: sample.assetId())) 55 | } 56 | } 57 | do { 58 | return try decode(sample) 59 | } catch let error { 60 | print("decode error \(error)") 61 | return .error(EventError("dec.video.ffmpeg", 62 | -3, "Error decoding bitstream \(error)", assetId: sample.assetId())) 63 | } 64 | } 65 | 66 | private func decode(_ sample: CodedMediaSample) throws -> EventBox { 67 | guard let codecCtx = self.codecContext else { 68 | return .error(EventError("dec.video.ffmpeg", -4, "No codec context", assetId: sample.assetId())) 69 | } 70 | 71 | guard sample.data().count > 0 else { 72 | return .nothing(sample.info()) 73 | } 74 | 75 | let pts = rescale(sample.pts(), kTimebase) 76 | let dts = rescale(sample.dts(), kTimebase) 77 | let packet = AVPacket() 78 | let size = sample.data().count 79 | var data = sample.data() 80 | 81 | try packet.makeWritable() 82 | 83 | data.withUnsafeMutableBytes { 84 | guard let buffer = packet.buffer, 85 | let baseAddress = $0.baseAddress else { 86 | return 87 | } 88 | buffer.realloc(size: size) 89 | memcpy(buffer.data, baseAddress, size) 90 | } 91 | packet.data = packet.buffer?.data 92 | packet.size = size 93 | packet.pts = pts.value 94 | packet.dts = dts.value 95 | 96 | if packet.size > 0 { 97 | try codecCtx.sendPacket(packet) 98 | do { 99 | let frame = AVFrame() 100 | try codecCtx.receiveFrame(frame) 101 | return makePictureSample(frame, sample: sample) 102 | } catch let error as AVError where error == .tryAgain { 103 | return .nothing(sample.info()) 104 | } 105 | } 106 | return .nothing(sample.info()) 107 | } 108 | 109 | private func setupContext(_ sample: CodedMediaSample) throws { 110 | self.codec = { 111 | switch sample.mediaFormat() { 112 | case .avc: return AVCodec.findDecoderById(.H264) 113 | case .hevc: return AVCodec.findDecoderById(.HEVC) 114 | case .vp8: return AVCodec.findDecoderById(.VP8) 115 | case .vp9: return AVCodec.findDecoderById(.VP9) 116 | case .png: return AVCodec.findDecoderById(.PNG) 117 | case .apng: return AVCodec.findDecoderById(.APNG) 118 | default: return nil 119 | } }() 120 | if let codec = self.codec { 121 | let ctx = AVCodecContext(codec: codec) 122 | self.codecContext = ctx 123 | } 124 | if let context = self.codecContext { 125 | if let sideData = sample.sideData()["config"], 126 | let mem = AVIO.malloc(size: sideData.count + AVConstant.inputBufferPaddingSize) { 127 | let memBuf = UnsafeMutableRawBufferPointer(start: mem, 128 | count: sideData.count + AVConstant.inputBufferPaddingSize) 129 | _ = memBuf.baseAddress.map { 130 | sideData.copyBytes(to: $0.assumingMemoryBound(to: UInt8.self), count: sideData.count) 131 | context.extradata = $0.assumingMemoryBound(to: UInt8.self) 132 | context.extradataSize = sideData.count 133 | } 134 | } 135 | try context.openCodec() 136 | } 137 | } 138 | 139 | var codec: AVCodec? 140 | var codecContext: AVCodecContext? 141 | var extradata: UnsafeMutableRawPointer? 142 | } 143 | 144 | private func frameData(_ frame: AVFrame) -> [Data] { 145 | (0..<3).compactMap { idx -> Data? in 146 | guard let data = frame.data[idx], frame.linesize[idx] > 0 else { 147 | return nil 148 | } 149 | if idx == 0 { 150 | return Data(bytesNoCopy: data, 151 | count: Int(frame.linesize[idx]) * Int(frame.height), 152 | deallocator: .custom({ _, _ in 153 | frame.unref() 154 | })) 155 | } else { 156 | let height = frame.pixelFormat == .YUV420P ? frame.height / 2 : frame.height 157 | return Data(bytesNoCopy: data, count: Int(frame.linesize[idx]) * Int(height), deallocator: .none) 158 | } 159 | } 160 | } 161 | 162 | private func planeSize(_ frame: AVFrame, _ idx: Int, _ pixelFormat: PixelFormat) -> Vector2 { 163 | switch pixelFormat { 164 | case .y420p, .nv12, .nv21: 165 | return idx == 0 ? Vector2(Float(frame.width), 166 | Float(frame.height)) : Vector2(Float(frame.width/2), Float(frame.height/2)) 167 | case .yuvs, .zvuy, .y444p, .BGRA, .RGBA: 168 | return Vector2(Float(frame.width), Float(frame.height)) 169 | case .y422p: 170 | return idx == 0 ? Vector2(Float(frame.width), 171 | Float(frame.height)) : Vector2(Float(frame.width/2), Float(frame.height)) 172 | default: return Vector2(0, 0) 173 | } 174 | } 175 | 176 | private func framePlanes(_ frame: AVFrame, pixelFormat: PixelFormat) -> [Plane] { 177 | (0..<3).compactMap { idx -> Plane? in 178 | guard frame.linesize[idx] > 0 else { 179 | return nil 180 | } 181 | let size = planeSize(frame, idx, pixelFormat) 182 | let components: [Component] = componentsForPlane(pixelFormat, idx) 183 | return Plane(size: size, stride: Int(frame.linesize[idx]), bitDepth: 8, components: components) 184 | } 185 | } 186 | 187 | private func makePictureSample(_ frame: AVFrame, sample: CodedMediaSample) -> EventBox { 188 | let pixelFormat = fromFfPixelFormat(frame) 189 | let data = frameData(frame) 190 | let planes = framePlanes(frame, pixelFormat: pixelFormat) 191 | do { 192 | let image = try ImageBuffer(pixelFormat: pixelFormat, 193 | bufferType: .cpu, 194 | size: Vector2(Float(frame.width), Float(frame.height)), 195 | buffers: data, 196 | planes: planes) 197 | let pts = TimePoint(frame.pts, kTimebase) 198 | return .just(PictureSample(image, 199 | assetId: sample.assetId(), 200 | workspaceId: sample.workspaceId(), 201 | workspaceToken: sample.workspaceToken(), 202 | time: sample.time(), 203 | pts: pts)) 204 | } catch let error { 205 | print("caught error \(error)") 206 | return .error(EventError("dec.video.ffmpeg", 207 | -5, "Error creating image \(error)", assetId: sample.assetId())) 208 | } 209 | } 210 | 211 | private func fromFfPixelFormat(_ frame: AVFrame) -> PixelFormat { 212 | [.YUV420P: .y420p, 213 | .YUYV422: .yuvs, 214 | .UYVY422: .zvuy, 215 | .YUV422P: .y422p, 216 | .YUV444P: .y444p, 217 | .NV12: .nv12, 218 | .NV21: .nv21, 219 | .BGRA: .BGRA, 220 | .RGBA: .RGBA][frame.pixelFormat] ?? .invalid 221 | } 222 | -------------------------------------------------------------------------------- /Sources/SwiftVideo_FFmpeg/enc.audio.ffmpeg.swift: -------------------------------------------------------------------------------- 1 | /* 2 | SwiftVideo, Copyright 2019 Unpause SAS 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | import SwiftVideo 18 | import SwiftFFmpeg 19 | import Foundation 20 | 21 | public class FFmpegAudioEncoder: Tx { 22 | private let kTimebase: Int64 = 96000 23 | public init(_ format: MediaFormat, 24 | bitrate: Int) { 25 | self.codecContext = nil 26 | self.format = format 27 | self.bitrate = bitrate 28 | self.frameNumber = 0 29 | self.accumulators = [Data]() 30 | self.pts = nil 31 | super.init() 32 | super.set { [weak self] in 33 | guard let strongSelf = self else { 34 | return .gone 35 | } 36 | return strongSelf.handle($0) 37 | } 38 | } 39 | 40 | deinit { 41 | print("AudioEncoder deinit") 42 | } 43 | 44 | private func handle(_ sample: AudioSample) -> EventBox<[CodedMediaSample]> { 45 | if self.codecContext == nil { 46 | do { 47 | try setupContext(sample) 48 | } catch let error { 49 | print("setupContext error \(error)") 50 | return .error(EventError("enc.audio.ffmpeg", 51 | -1, "Codec setup error \(error)", assetId: sample.assetId())) 52 | } 53 | } 54 | //if self.pts == nil { 55 | // self.pts = sample.pts() 56 | //} 57 | return encode(sample) 58 | } 59 | 60 | private func encode(_ sample: AudioSample) -> EventBox<[CodedMediaSample]> { 61 | guard let codecContext = self.codecContext else { 62 | return .nothing(sample.info()) 63 | } 64 | var samples = [CodedMediaSample]() 65 | do { 66 | //print("audio in \(seconds(sample.pts()))") 67 | var frames = try makeAVFrame(sample) 68 | 69 | while frames.count > 0 { 70 | do { 71 | try codecContext.sendFrame(frames[0]) 72 | _ = frames.removeFirst() 73 | } catch let error as AVError where error == .tryAgain {} 74 | 75 | do { 76 | repeat { 77 | let packet = AVPacket() 78 | defer { 79 | packet.unref() 80 | } 81 | try codecContext.receivePacket(packet) 82 | 83 | guard let data = packet.data, packet.size > 0 else { 84 | throw AVError.tryAgain 85 | } 86 | let frameDuration = TimePoint(Int64(codecContext.frameSize), Int64(codecContext.sampleRate)) 87 | let pts = self.pts ?? rescale(sample.pts(), Int64(codecContext.sampleRate)) 88 | self.frameNumber = self.frameNumber &+ 1 89 | let buffer = Data(bytes: data, count: packet.size) 90 | let extradata = codecContext.extradata 91 | .map { Data(bytes: $0, count: codecContext.extradataSize) } 92 | let dts = pts 93 | self.pts = pts + frameDuration 94 | let sidedata: [String: Data]? = extradata.map { ["config": $0] } 95 | let sample = CodedMediaSample(sample.assetId(), 96 | sample.workspaceId(), 97 | sample.time(), // incorrect, needs to be matched with packet 98 | pts, 99 | dts, 100 | .audio, 101 | self.format, 102 | buffer, 103 | sidedata, 104 | "enc.audio.ffmpeg.\(format)", 105 | workspaceToken: sample.workspaceToken(), 106 | eventInfo: sample.info()) 107 | //print("audio out \(seconds(sample.pts()))") 108 | samples.append(sample) 109 | } while true 110 | } catch let error as AVError where error == .tryAgain {} 111 | } 112 | return .just(samples) 113 | } catch let error { 114 | print("error enc.audio.ffmpeg \(error)") 115 | return .error(EventError("enc.audio.ffmpeg", -2, "Encode error \(error)", assetId: sample.assetId())) 116 | } 117 | } 118 | 119 | private func makeAVFrame(_ sample: AudioSample) throws -> [AVFrame] { 120 | guard let codecCtx = self.codecContext else { 121 | throw EncodeError.invalidContext 122 | } 123 | var frames = [AVFrame]() 124 | sample.data().enumerated().forEach { (offset, buffer) in 125 | if self.accumulators.count == offset { 126 | self.accumulators.append(Data(capacity: buffer.count * 2)) 127 | } 128 | self.accumulators[offset].append(buffer) 129 | } 130 | do { 131 | repeat { 132 | let frame = AVFrame() 133 | frame.sampleCount = codecCtx.frameSize 134 | frame.sampleFormat = codecCtx.sampleFormat 135 | frame.channelLayout = codecCtx.channelLayout 136 | 137 | try frame.allocBuffer() 138 | let isPlanar = sample.format() == .s16p || sample.format() == .f32p 139 | try (0..= requiredBytes { 146 | self.accumulators[offset].copyBytes(to: ptr, count: requiredBytes) 147 | if self.accumulators[offset].count > requiredBytes { 148 | self.accumulators[offset] = self.accumulators[offset].advanced(by: requiredBytes) 149 | } else { 150 | self.accumulators[offset].removeAll(keepingCapacity: true) 151 | } 152 | } else { 153 | throw AVError.tryAgain 154 | } 155 | } 156 | frames.append(frame) 157 | } while true 158 | } catch let error as AVError where error == .tryAgain {} 159 | return frames 160 | } 161 | private func setupContext(_ sample: AudioSample) throws { 162 | let name: String = try { 163 | switch format { 164 | case .aac: return "libfdk_aac" 165 | case .opus: return "libopus" 166 | default: throw EncodeError.invalidMediaFormat 167 | } 168 | }() 169 | guard let codec = AVCodec.findEncoderByName(name) else { 170 | throw EncodeError.encoderNotFound 171 | } 172 | let codecContext = AVCodecContext(codec: codec) 173 | codecContext.flags = [.globalHeader] 174 | codecContext.bitRate = Int64(bitrate) 175 | codecContext.sampleRate = sample.sampleRate() 176 | codecContext.sampleFormat = { 177 | switch sample.format() { 178 | case .s16i: 179 | return .int16 180 | case .s16p: 181 | return .int16Planar 182 | case .f32p: 183 | return .floatPlanar 184 | default: 185 | return .int16 186 | } 187 | }() 188 | codecContext.channelLayout = .CHL_STEREO 189 | codecContext.channelCount = sample.numberChannels() 190 | codecContext.frameSize = sample.numberSamples() 191 | try codecContext.openCodec() 192 | self.codecContext = codecContext 193 | } 194 | 195 | let format: MediaFormat 196 | let bitrate: Int 197 | var accumulators: [Data] 198 | var frameNumber: Int64 199 | var pts: TimePoint? 200 | var codecContext: AVCodecContext? 201 | 202 | } 203 | -------------------------------------------------------------------------------- /Sources/SwiftVideo_FFmpeg/extension.ffmpeg.swift: -------------------------------------------------------------------------------- 1 | /* 2 | SwiftVideo, Copyright 2019 Unpause SAS 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | import SwiftVideo 18 | import CFFmpeg 19 | import SwiftFFmpeg 20 | 21 | // swiftlint:disable identifier_name 22 | extension AVCodecID { 23 | public static let OPUS = AV_CODEC_ID_OPUS 24 | public static let SMPTE_KLV = AV_CODEC_ID_SMPTE_KLV 25 | } 26 | 27 | //extension AVMediaType: Hashable {} 28 | 29 | extension AVCodecID: Hashable {} 30 | 31 | extension AVPixelFormat: Hashable {} 32 | 33 | -------------------------------------------------------------------------------- /Sources/SwiftVideo_FFmpeg/file.ffmpeg.swift: -------------------------------------------------------------------------------- 1 | /* 2 | SwiftVideo, Copyright 2019 Unpause SAS 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | import SwiftVideo 18 | import SwiftFFmpeg 19 | import Foundation 20 | 21 | enum FileError: Error { 22 | case unsupported 23 | } 24 | 25 | private struct StreamInfo { 26 | let format: MediaFormat 27 | let type: MediaType 28 | let timebase: TimePoint 29 | let startTime: TimePoint 30 | let extradata: Data? 31 | } 32 | public class FileSource: Source { 33 | public init(_ clock: Clock, 34 | url: String, 35 | assetId: String, 36 | workspaceId: String, 37 | workspaceToken: String? = nil, 38 | repeats: Bool = false, 39 | onEnd: LiveOnEnded? = nil) throws { 40 | let fmtCtx = AVFormatContext() 41 | fmtCtx.flags = [.fastSeek] 42 | try fmtCtx.openInput(url) 43 | try fmtCtx.findStreamInfo() 44 | let streams: [Int: StreamInfo] = 45 | Dictionary(uniqueKeysWithValues: fmtCtx.streams.enumerated().compactMap { (idx, val) in 46 | let codecParams = val.codecParameters 47 | let timebase = TimePoint(Int64(val.timebase.num), Int64(val.timebase.den)) 48 | let codecMap: [AVCodecID: MediaFormat] = 49 | [.H264: .avc, .HEVC: .hevc, .VP8: .vp8, .VP9: .vp9, 50 | .AAC: .aac, .OPUS: .opus, 51 | .PNG: .png, .APNG: .apng] 52 | let typeMap: [AVMediaType: MediaType] = 53 | [.audio: .audio, .video: .video, .data: .data, .subtitle: .subtitle] 54 | let extradata: Data? = { 55 | guard let ptr = codecParams.extradata, codecParams.extradataSize > 0 else { 56 | return nil 57 | } 58 | return Data(bytes: ptr, count: codecParams.extradataSize) 59 | }() 60 | guard let codec = codecMap[codecParams.codecId], let type = typeMap[val.mediaType] else { 61 | return nil 62 | } 63 | // to be used with hls/dash sources 64 | //val.discard = .all 65 | let startTime = 66 | (val.startTime != AVTimestamp.noPTS) ? TimePoint(Int64(val.startTime), timebase.scale) : timebase 67 | return (idx, StreamInfo(format: codec, 68 | type: type, 69 | timebase: timebase, 70 | startTime: startTime, 71 | extradata: extradata)) 72 | }) 73 | if streams.count == 0 { 74 | throw FileError.unsupported 75 | } 76 | self.ctx = fmtCtx 77 | self.clock = clock 78 | self.assetId = assetId 79 | self.fnEnded = onEnd 80 | self.workspaceId = workspaceId 81 | self.workspaceToken = workspaceToken 82 | self.streams = streams 83 | self.repeats = repeats 84 | self.queue = DispatchQueue(label: "file.\(assetId)") 85 | epoch = clock.current() 86 | super.init() 87 | } 88 | 89 | public func formats() -> [MediaFormat] { 90 | return streams.map { 91 | $0.1.format 92 | } 93 | } 94 | 95 | public func play() { 96 | running = true 97 | epoch = clock.current() 98 | self.refill() 99 | } 100 | 101 | public func reset() { 102 | do { 103 | ctx.flush() 104 | // swiftlint:disable:next shorthand_operator 105 | tsBase = tsBase + lastRead 106 | for idx in 0.. { 22 | public init(_ outFrequency: Int, _ outChannelCount: Int, _ outAudioFormat: AudioFormat) { 23 | self.swrCtx = nil 24 | super.init() 25 | super.set { [weak self] sample in 26 | guard let strongSelf = self else { 27 | return .gone 28 | } 29 | if outFrequency == sample.sampleRate() && 30 | outChannelCount == sample.numberChannels() && 31 | outAudioFormat == sample.format() { 32 | return .just(sample) 33 | } 34 | if strongSelf.swrCtx == nil { 35 | strongSelf.pts = rescale(sample.pts(), Int64(outFrequency)) 36 | strongSelf.makeContext(sample, 37 | frequency: outFrequency, 38 | channelCount: outChannelCount, 39 | format: outAudioFormat) 40 | } 41 | return strongSelf.resample(sample, 42 | frequency: outFrequency, 43 | outChannelCount: outChannelCount, 44 | format: outAudioFormat) 45 | } 46 | } 47 | 48 | private func resample(_ sample: AudioSample, 49 | frequency: Int, 50 | outChannelCount: Int, 51 | format: AudioFormat) -> EventBox { 52 | guard let swrCtx = self.swrCtx, let pts = self.pts else { 53 | return .nothing(sample.info()) 54 | } 55 | 56 | let srcSampleRate = Int64(sample.sampleRate()) 57 | let dstSampleRate = Int64(frequency) 58 | let srcSamples = AVSamples(channelCount: sample.numberChannels(), 59 | sampleCount: sample.numberSamples(), 60 | sampleFormat: avSampleFormat(sample.format())) 61 | let srcSampleCount = Int64(swrCtx.getDelay(srcSampleRate) + sample.numberSamples()) 62 | let dstMaxSampleCount = Int(AVMath.rescale(Int64(sample.numberSamples()), dstSampleRate, srcSampleRate, .up)) 63 | let dstSampleCount = Int(AVMath.rescale(srcSampleCount, dstSampleRate, srcSampleRate, .up)) 64 | let dstSamples: AVSamples = { 65 | if dstSampleCount <= dstMaxSampleCount { 66 | return AVSamples(channelCount: outChannelCount, 67 | sampleCount: dstMaxSampleCount, 68 | sampleFormat: avSampleFormat(format), 69 | align: 0) 70 | } else { 71 | return AVSamples(channelCount: outChannelCount, 72 | sampleCount: dstSampleCount, 73 | sampleFormat: avSampleFormat(format), 74 | align: 1) 75 | } 76 | }() 77 | 78 | sample.data().enumerated().forEach { (idx, buffer) in 79 | let bufferSize = min(buffer.count, srcSamples.size) 80 | buffer.withUnsafeBytes { 81 | guard let data = srcSamples.data[idx], let baseAddress = $0.baseAddress else { 82 | return 83 | } 84 | memcpy(data, baseAddress, bufferSize) 85 | } 86 | } 87 | do { 88 | let count = try srcSamples.reformat(using: swrCtx, to: dstSamples) 89 | guard count > 0 else { 90 | return .nothing(sample.info()) 91 | } 92 | let (size, _) = try AVSamples.getBufferSize(channelCount: outChannelCount, 93 | sampleCount: count, 94 | sampleFormat: avSampleFormat(format), 95 | align: 1) 96 | let bufferCount = numberOfBuffers(format, outChannelCount) 97 | let buffers = ((0.. Data? in 98 | guard let data = dstSamples.data[idx] else { 99 | return nil 100 | } 101 | return Data(bytes: data, count: size) 102 | } 103 | self.pts = pts + TimePoint(Int64(count), Int64(frequency)) 104 | let outSample = AudioSample(sample, 105 | bufferType: .cpu, 106 | buffers: buffers, 107 | frequency: frequency, 108 | channels: outChannelCount, 109 | format: format, 110 | sampleCount: count, 111 | pts: pts) 112 | return .just(outSample) 113 | } catch let error { 114 | print("SRC error \(error) \(sample.format()) \(sample.numberChannels()) \(sample.numberSamples())") 115 | return .error(EventError("src.audio.ffmpeg", -1, "conversion error \(error)", 116 | sample.time(), 117 | assetId: sample.assetId())) 118 | } 119 | } 120 | 121 | private func makeContext(_ sample: AudioSample, frequency: Int, channelCount: Int, format: AudioFormat) { 122 | // source 123 | // TODO: Support surround sound for > 2 channels 124 | let srcChannelLayout = sample.numberChannels() == 2 ? AVChannelLayout.CHL_STEREO : AVChannelLayout.CHL_MONO 125 | let srcSampleRate = sample.sampleRate() 126 | let srcSampleFmt = avSampleFormat(sample.format()) 127 | 128 | // destination 129 | let dstChannelLayout = channelCount > 1 ? AVChannelLayout.CHL_STEREO : AVChannelLayout.CHL_MONO 130 | let dstSampleRate = Int64(frequency) 131 | let dstSampleFmt = avSampleFormat(format) 132 | 133 | do { 134 | let ctx = SwrContext() 135 | try ctx.set(srcChannelLayout.rawValue, forKey: "in_channel_layout") 136 | try ctx.set(srcSampleRate, forKey: "in_sample_rate") 137 | try ctx.set(srcSampleFmt, forKey: "in_sample_fmt") 138 | try ctx.set(dstChannelLayout.rawValue, forKey: "out_channel_layout") 139 | try ctx.set(dstSampleRate, forKey: "out_sample_rate") 140 | try ctx.set(dstSampleFmt, forKey: "out_sample_fmt") 141 | try ctx.set("soxr", forKey: "resampler") 142 | try ctx.set(24, forKey: "precision") // Set to 28 for higher bit-depths than 16-bit 143 | try ctx.set(1.0, forKey: "rematrix_maxval") 144 | try ctx.set("triangular", forKey: "dither_method") 145 | 146 | try ctx.initialize() 147 | self.swrCtx = ctx 148 | } catch let error { 149 | print("SwrContext error \(error)") 150 | } 151 | } 152 | private var swrCtx: SwrContext? 153 | private var pts: TimePoint? 154 | } 155 | 156 | private func avSampleFormat(_ fmt: AudioFormat) -> AVSampleFormat { 157 | switch fmt { 158 | case .s16i: 159 | return .int16 160 | case .s16p: 161 | return .int16Planar 162 | case .f32p: 163 | return .floatPlanar 164 | case .f32i: 165 | return .float 166 | case .f64p: 167 | return .doublePlanar 168 | case .f64i: 169 | return .double 170 | default: 171 | return .int16 172 | } 173 | } 174 | -------------------------------------------------------------------------------- /Sources/SwiftVideo_FFmpeg/transcode.swift: -------------------------------------------------------------------------------- 1 | /* 2 | SwiftVideo, Copyright 2019 Unpause SAS 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | import SwiftVideo 18 | 19 | // swiftlint:disable force_cast 20 | 21 | public protocol Renameable: Event { 22 | static func make(_ other: Renameable, 23 | assetId: String, 24 | constituents: [MediaConstituent], 25 | eventInfo: EventInfo?) -> Renameable 26 | func pts() -> TimePoint 27 | func dts() -> TimePoint 28 | func constituents() -> [MediaConstituent]? 29 | } 30 | 31 | private class AssetRenamer: Tx where T: Renameable { 32 | public init(_ assetId: String) { 33 | self.statsReport = nil 34 | super.init() 35 | super.set { [weak self] sample in 36 | guard let strongSelf = self else { 37 | return .gone 38 | } 39 | if strongSelf.statsReport == nil { 40 | strongSelf.statsReport = (sample.info().map { 41 | StatsReport(assetId: assetId, other: $0) } ?? StatsReport(assetId: assetId)) 42 | } 43 | return .just(T.make(sample, 44 | assetId: assetId, 45 | constituents: [MediaConstituent.with { $0.idAsset = sample.assetId() 46 | $0.pts = sample.pts() 47 | $0.dts = sample.dts() 48 | $0.constituents = sample.constituents() ?? [MediaConstituent]() }], 49 | eventInfo: strongSelf.statsReport) as! T) 50 | } 51 | } 52 | 53 | private var statsReport: StatsReport? 54 | } 55 | 56 | extension CodedMediaSample: Renameable { 57 | public static func make(_ other: Renameable, 58 | assetId: String, 59 | constituents: [MediaConstituent], 60 | eventInfo: EventInfo?) -> Renameable { 61 | return CodedMediaSample(other as! CodedMediaSample, 62 | assetId: assetId, constituents: constituents, eventInfo: eventInfo) 63 | } 64 | } 65 | 66 | extension AudioSample: Renameable { 67 | public static func make(_ other: Renameable, 68 | assetId: String, 69 | constituents: [MediaConstituent], 70 | eventInfo: EventInfo?) -> Renameable { 71 | return AudioSample(other as! AudioSample, assetId: assetId, constituents: constituents, eventInfo: eventInfo) 72 | } 73 | public func dts() -> TimePoint { 74 | return pts() 75 | } 76 | } 77 | 78 | extension PictureSample: Renameable { 79 | public static func make(_ other: Renameable, 80 | assetId: String, 81 | constituents: [MediaConstituent], 82 | eventInfo: EventInfo?) -> Renameable { 83 | return PictureSample(other as! PictureSample, 84 | assetId: assetId, constituents: constituents, eventInfo: eventInfo) 85 | } 86 | public func dts() -> TimePoint { 87 | return pts() 88 | } 89 | } 90 | 91 | public func assetRename(_ assetId: String) -> Tx where T: Renameable { 92 | return AssetRenamer(assetId) 93 | } 94 | 95 | public func makeVideoTranscoder(_ fmt: MediaFormat, 96 | bitrate: Int, 97 | keyframeInterval: TimePoint, 98 | newAssetId: String, 99 | settings: EncoderSpecificSettings? = nil) throws 100 | -> Tx { 101 | guard [.avc, .hevc, .vp8, .vp9, .av1].contains(where: { $0 == fmt }) else { 102 | throw EncodeError.invalidMediaFormat 103 | } 104 | 105 | if bitrate > 0 { 106 | return assetRename(newAssetId) >>> FFmpegVideoDecoder() >>> FFmpegVideoEncoder(fmt, 107 | bitrate: bitrate, 108 | keyframeInterval: keyframeInterval, 109 | settings: settings) 110 | } else { 111 | return assetRename(newAssetId) 112 | } 113 | } 114 | 115 | public func makeAudioTranscoder(_ fmt: MediaFormat, 116 | bitrate: Int, 117 | sampleRate: Int, 118 | newAssetId: String) throws -> Tx { 119 | guard [.aac, .opus].contains(where: { $0 == fmt }) else { 120 | throw EncodeError.invalidMediaFormat 121 | } 122 | if bitrate > 0 { 123 | return assetRename(newAssetId) >>> FFmpegAudioDecoder() >>> 124 | AudioSampleRateConversion(sampleRate, 2, .s16i) >>> FFmpegAudioEncoder(fmt, bitrate: bitrate) 125 | } else { 126 | return Tx { .just([$0]) } |>> assetRename(newAssetId) 127 | } 128 | } 129 | 130 | public class TranscodeContainer: AsyncTx { 131 | public init(_ videoTranscodes: [Tx], 132 | _ audioTranscodes: [Tx], _ bus: Bus) { 133 | self.videoTranscoders = [Tx]() 134 | self.audioTranscoders = [Tx]() 135 | super.init() 136 | self.videoTranscoders = videoTranscodes.map { 137 | return bus <<| ($0 >>> Tx { [weak self] in 138 | guard let strongSelf = self else { 139 | return .gone 140 | } 141 | return (strongSelf.emit($0).value() as? CodedMediaSample).map { .just($0) } ?? .nothing($0.info()) 142 | }) 143 | } 144 | self.audioTranscoders = audioTranscodes.map { 145 | return bus <<| ($0 >>> Tx { [weak self] samples in 146 | guard let strongSelf = self else { 147 | return .gone 148 | } 149 | let results = samples.map { 150 | strongSelf.emit($0) 151 | } 152 | return .just(results.compactMap { $0.value() as? CodedMediaSample }) 153 | }) 154 | } 155 | } 156 | var videoTranscoders: [Tx] 157 | var audioTranscoders: [Tx] 158 | } 159 | -------------------------------------------------------------------------------- /Tests/LinuxMain.swift: -------------------------------------------------------------------------------- 1 | /* 2 | SwiftVideo, Copyright 2019 Unpause SAS 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | import XCTest 18 | 19 | import swiftVideoTests 20 | import swiftVideoInternalTests 21 | 22 | var tests = [XCTestCaseEntry]() 23 | tests += swiftVideoTests.allTests() 24 | tests += swiftVideoInternalTests.allTests() 25 | XCTMain(tests) 26 | -------------------------------------------------------------------------------- /Tests/swiftVideoInternalTests/XCTestManifests.swift: -------------------------------------------------------------------------------- 1 | /* 2 | SwiftVideo, Copyright 2019 Unpause SAS 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | import XCTest 18 | 19 | #if !os(macOS) 20 | public func allTests() -> [XCTestCaseEntry] { 21 | #if !DISABLE_INTERNAL 22 | return [ 23 | testCase(computeTests.allTests) 24 | ] 25 | #else 26 | return [] 27 | #endif 28 | } 29 | #endif 30 | -------------------------------------------------------------------------------- /Tests/swiftVideoInternalTests/computeTests.swift: -------------------------------------------------------------------------------- 1 | #if !DISABLE_INTERNAL 2 | import XCTest 3 | import Foundation 4 | import CSwiftVideo 5 | @testable import SwiftVideo 6 | 7 | // swiftlint:disable:next type_name 8 | final class computeTests: XCTestCase { 9 | func defaultKernelSearch() { 10 | let kernels = [ 11 | "img_nv12_nv12", 12 | "img_bgra_nv12", 13 | "img_rgba_nv12", 14 | "img_bgra_bgra", 15 | "img_y420p_y420p", 16 | "img_y420p_nv12", 17 | "img_clear_nv12", 18 | "img_clear_yuvs", 19 | "img_clear_bgra", 20 | "img_clear_rgba", 21 | "img_rgba_y420p", 22 | "img_bgra_y420p", 23 | "img_clear_y420p" 24 | ] 25 | kernels.forEach { 26 | do { 27 | let result = try defaultComputeKernelFromString($0) 28 | // img_clear_rgba uses img_clear_bgra 29 | if $0 != "img_clear_rgba" { 30 | XCTAssertEqual($0, String(describing: result)) 31 | } else { 32 | XCTAssertEqual("img_clear_bgra", String(describing: result)) 33 | } 34 | } catch { 35 | print("Caught error \(error)") 36 | XCTAssertEqual(0, 1) 37 | } 38 | } 39 | } 40 | static var allTests = [ 41 | ("defaultKernelSearch", defaultKernelSearch) 42 | ] 43 | } 44 | #endif 45 | -------------------------------------------------------------------------------- /Tests/swiftVideoTests/XCTestManifests.swift: -------------------------------------------------------------------------------- 1 | /* 2 | SwiftVideo, Copyright 2019 Unpause SAS 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | import XCTest 18 | 19 | #if !os(macOS) 20 | public func allTests() -> [XCTestCaseEntry] { 21 | return [ 22 | //testCase(audioSegmenterTests.allTests), 23 | testCase(statsTests.allTests), 24 | testCase(sampleRateConversionTests.allTests), 25 | testCase(busTests.allTests), 26 | testCase(timePointTests.allTests), 27 | testCase(audioMixTests.allTests), 28 | testCase(rtmpTests.allTests) 29 | ] 30 | } 31 | #endif 32 | -------------------------------------------------------------------------------- /Tests/swiftVideoTests/audioSegmenterTests.swift: -------------------------------------------------------------------------------- 1 | import XCTest 2 | import SwiftVideo 3 | 4 | private func makeSine(_ idx: Int, 5 | _ count: Int, 6 | _ frequency: Int, 7 | _ sampleRate: Int, 8 | amplitude: Float = 1.0) -> [Int16] { 9 | var result = [Int16]() 10 | let freq = Float(frequency) 11 | let sampleRate = Float(sampleRate) 12 | for idx in idx..<(idx+count) { 13 | let pos = Float(idx) 14 | let val = Int16(sin(pos * Float.twoPi * freq / sampleRate) * Float(Int16.max) * amplitude) 15 | result.append(val) 16 | } 17 | return result 18 | } 19 | 20 | final class audioSegmenterTests: XCTestCase { 21 | let duration = TimePoint(60 * 60 * 1000, 1000) 22 | 23 | private func diff(_ lhs: Data, _ rhs: Data, lhsStart: Int = 0, rhsStart: Int = 0, count: Int = Int.max) -> Float { 24 | let byteCount = min(min(lhs.count - lhsStart, rhs.count - rhsStart), count) 25 | var diffs = 0 26 | for i in 0.. TimePoint) { 35 | let time = fn(at) 36 | clock.schedule(time) { [weak self] evt in 37 | self?.recur(clock, evt.time(), fn) 38 | } 39 | } 40 | 41 | private func runner(_ clock: Clock, 42 | _ frameDuration: TimePoint, 43 | _ audioPacketDuration: TimePoint, 44 | _ receiver: Terminal, 45 | _ generator: @escaping (TimePoint) -> EventBox, 46 | latePacketProb: Float = 0.0) { 47 | let segmenter = AudioPacketSegmenter(frameDuration) 48 | let txn = segmenter |>> receiver 49 | recur(clock, TimePoint(0, 48000)) { time in 50 | let sample = generator(time) 51 | let result = sample >>- txn 52 | print("result=\(result)") 53 | let value = Int.random(in: 0..<1000) 54 | let scheduleLate = value < Int(1000.0 * latePacketProb) 55 | return time + audioPacketDuration + (scheduleLate ? audioPacketDuration/2*3 : TimePoint(0, 48000)) 56 | } 57 | print("first step") 58 | clock.step() 59 | while clock.current() < duration { 60 | sleep(1) 61 | print("clock.current = \(clock.current().toString())") 62 | //clock.step() 63 | } 64 | } 65 | 66 | func segmenterTest() { 67 | let audioPacketDuration = TimePoint(1024, 48000) 68 | let frameDuration = TimePoint(960, 48000) 69 | // need to make a sine pattern of numberBuffers * 1024 samples in duration at (sample rate / frame duration) Hz 70 | let numberBuffers = lcm(audioPacketDuration.value, frameDuration.value) / audioPacketDuration.value 71 | let sineFreq = Int(frameDuration.scale / frameDuration.value) 72 | let baseBuffers: [Data] = (0.. { sample in 94 | //print("received sample") 95 | //dump(sample) 96 | clock.step() 97 | guard isFirst == false && sample.pts().value > 960 else { 98 | isFirst = false 99 | clock.step() 100 | return .nothing(nil) 101 | } 102 | let similarity = self.diff(reference, sample.data()[0]) 103 | guard similarity > 0.9 else { // some small differences may be present due to floating point conversion 104 | print("error at timePoint \(sample.pts().toString()) \(pushIdx)") 105 | //try! reference.write(to: URL(string: "file:///home/james/dev/reference.pcm")!) 106 | //try! sample.data()[0].write(to: URL(string: "file:///home/james/dev/sample.pcm")!) 107 | fatalError("reference != sample.data() [\(similarity)]") 108 | } 109 | let targetPts = clock.current() 110 | guard targetPts == sample.pts() else { 111 | fatalError("targetPts != pts \(targetPts.toString()) != \(sample.pts().toString())") 112 | } 113 | //clock.step() 114 | return .nothing(nil) 115 | } 116 | 117 | let generator: (TimePoint) -> EventBox = { pts in 118 | let buffers = [baseBuffers[pushIdx]] 119 | pushIdx = (pushIdx + 1) % baseBuffers.count 120 | let sample = AudioSample(buffers, 121 | frequency: 48000, 122 | channels: 2, 123 | format: .s16i, 124 | sampleCount: Int(audioPacketDuration.value), 125 | time: clock.current(), 126 | pts: pts, 127 | assetId: "blank", 128 | workspaceId: "test") 129 | return .just(sample) 130 | } 131 | 132 | runner(clock, frameDuration, audioPacketDuration, receiver, generator) 133 | } 134 | static var allTests = [ 135 | ("segmenterTest", segmenterTest) 136 | ] 137 | } 138 | -------------------------------------------------------------------------------- /Tests/swiftVideoTests/busTests.swift: -------------------------------------------------------------------------------- 1 | /* 2 | SwiftVideo, Copyright 2019 Unpause SAS 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | import XCTest 18 | import Foundation 19 | import SwiftVideo 20 | import CSwiftVideo 21 | 22 | // swiftlint:disable:next type_name 23 | final class busTests: XCTestCase { 24 | 25 | struct TestEvent: Event { 26 | func type() -> String { 27 | return "test" 28 | } 29 | func time() -> TimePoint { 30 | return TimePoint(0, 1000) 31 | } 32 | func assetId() -> String { 33 | return "assetId" 34 | } 35 | func workspaceId() -> String { 36 | return "workspaceId" 37 | } 38 | func workspaceToken() -> String? { 39 | return "workspaceToken" 40 | } 41 | func info() -> EventInfo? { 42 | return nil 43 | } 44 | init(_ idx: Int) { 45 | self.idx = idx 46 | } 47 | let idx: Int 48 | } 49 | 50 | struct TestEvent2: Event { 51 | func type() -> String { 52 | return "test2" 53 | } 54 | func time() -> TimePoint { 55 | return TimePoint(0, 1000) 56 | } 57 | func assetId() -> String { 58 | return "assetId2" 59 | } 60 | func workspaceId() -> String { 61 | return "workspaceId2" 62 | } 63 | func workspaceToken() -> String? { 64 | return "workspaceToken2" 65 | } 66 | func info() -> EventInfo? { 67 | return nil 68 | } 69 | } 70 | 71 | func busDispatchTest() { 72 | let bus = Bus() 73 | var count: Int = 0 74 | let txn: Tx = Tx { event in 75 | XCTAssertEqual(event.idx, count) 76 | count += 1 77 | return .just(event) 78 | } 79 | let tx2: Tx = Tx { _ in 80 | return .nothing(nil) 81 | } 82 | _ = bus <<| txn 83 | _ = bus <<| tx2 84 | for idx in 0..<100 { 85 | _ = bus.append(.just(TestEvent(idx))) 86 | } 87 | print("appended, waiting") 88 | sleep(3) 89 | XCTAssertEqual(count, 100) 90 | } 91 | 92 | func busFilterTest() { 93 | let bus = HeterogeneousBus() 94 | var count: Int = 0 95 | 96 | let txn: Tx = Tx { event in 97 | XCTAssertEqual(event.idx, count) 98 | count += 1 99 | return .just(event) 100 | } 101 | let tx2: Tx = Tx { _ in 102 | .nothing(nil) 103 | } 104 | let event2 = TestEvent2() 105 | let pipe: Tx = mix() >>> bus 106 | let pipe2: Tx = mix() >>> bus 107 | let rcv = bus <<| filter() >>> txn 108 | let rcv2 = bus <<| filter() >>> tx2 109 | for idx in 0..<100 { 110 | _ = .just(TestEvent(idx)) >>- pipe 111 | _ = .just(event2) >>- pipe2 112 | } 113 | sleep(3) 114 | print("count = \(count)") 115 | XCTAssertEqual(count, 100) 116 | 117 | } 118 | 119 | func golombTest() { 120 | let result = test_golomb_dec() 121 | print("golombTest result=\(result)") 122 | XCTAssertEqual(result, 254) 123 | } 124 | 125 | static var allTests = [ 126 | ("busDispatchTest", busDispatchTest), 127 | ("busFilterTest", busFilterTest), 128 | ("golombTest", golombTest) 129 | ] 130 | } 131 | -------------------------------------------------------------------------------- /Tests/swiftVideoTests/rtmpTests.swift: -------------------------------------------------------------------------------- 1 | /* 2 | SwiftVideo, Copyright 2019 Unpause SAS 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | import XCTest 18 | import Foundation 19 | import SwiftVideo 20 | import BrightFutures 21 | import NIO 22 | import NIOExtras 23 | 24 | public typealias XCTestCaseClosure = (XCTestCase) throws -> Void 25 | 26 | // swiftlint:disable:next type_name 27 | final class rtmpTests: XCTestCase { 28 | 29 | #if os(Linux) 30 | required override init(name: String, testClosure: @escaping XCTestCaseClosure) { 31 | let stepSize = TimePoint(16, 1000) 32 | let clock = StepClock(stepSize: stepSize) 33 | self.stepSize = stepSize 34 | self.sampleInfo = [(TimePoint, Int)]() 35 | self.rtmp = nil 36 | self.clock = clock 37 | self.group = MultiThreadedEventLoopGroup(numberOfThreads: 4) 38 | self.quiesce = ServerQuiescingHelper(group: group) 39 | self.buffers = [1009, 2087, 1447, 2221, 2503, 3001, 4999, 2857, 9973, 8191, 7331, 40 | 3539, 44701, 47701, 65537, 65701, 99989, 99991, 111323].map { 41 | var data = Data(count: $0) 42 | data[4] = 0x5 43 | return data 44 | } 45 | self.currentTs = TimePoint(0, 1000) 46 | super.init(name: name, testClosure: testClosure) 47 | } 48 | 49 | func setupRtmp() { 50 | let bufferSize = TimePoint(0, 1000) 51 | let stepSize = TimePoint(16, 1000) 52 | let onConnection: LiveOnConnection = { [weak self] pub, sub in 53 | if let pub = pub as? Terminal, let strongSelf = self { 54 | strongSelf.publish = pub 55 | strongSelf.clock.schedule(strongSelf.currentTs) { [weak self] in 56 | self?.push($0.time()) 57 | } 58 | strongSelf.sampleInfo.remove(at: 0) // remove first sample that is not sent 59 | for _ in 0...12 { 60 | strongSelf.clock.step() 61 | } 62 | strongSelf.clock.schedule(strongSelf.currentTs) { [weak self] in 63 | self?.push($0.time()) 64 | } 65 | let iterations = bufferSize.value / stepSize.value 66 | for _ in 0...(iterations+1) { 67 | strongSelf.clock.schedule(strongSelf.currentTs + stepSize) { [weak self] in 68 | self?.push($0.time()) 69 | } 70 | strongSelf.clock.step() 71 | } 72 | } 73 | if let sub = sub as? Source { 74 | self?.subscribe = sub 75 | 76 | self?.stx = sub >>> Tx { [weak self] in 77 | self?.recv($0) 78 | 79 | return .nothing($0.info()) 80 | } 81 | } 82 | return Future { $0(.success(true)) } 83 | } 84 | 85 | self.rtmp = Rtmp(self.clock, bufferSize: bufferSize, onEnded: { _ in () }, onConnection: onConnection) 86 | } 87 | 88 | func rtmpTest(_ port: Int, _ duration: TimePoint, offset: TimePoint = TimePoint(0, 1000)) { 89 | currentTs = TimePoint(0, 1000) 90 | self.clock.reset() 91 | self.setupRtmp() 92 | let start = currentTs 93 | let end = start + duration 94 | self.offset = offset 95 | _ = rtmp?.serve(host: "0.0.0.0", port: port, quiesce: self.quiesce, group: self.group) 96 | rtmp?.connect(url: URL(string: "rtmp://localhost:\(port)/hi/hello")!, publishToPeer: true, 97 | group: self.group, workspaceId: "test", assetId: "test") 98 | let wallClock = WallClock() 99 | 100 | while currentTs < end { 101 | let currentReal = seconds(wallClock.current()) 102 | let currentProgress = seconds(currentTs - start) 103 | let rate = currentProgress / currentReal 104 | let progress = currentProgress / seconds(end) 105 | print("[\(rate)x] progress: \(progress * 100)%") 106 | sleep(1) 107 | } 108 | self.subscribe = nil 109 | self.publish = nil 110 | self.sampleInfo.removeAll(keepingCapacity: true) 111 | self.stx = nil 112 | self.rtmp = nil 113 | sleep(1) 114 | } 115 | 116 | func basicTest() { 117 | let duration = TimePoint(60 * 5 * 1000, 1000) 118 | rtmpTest(5001, duration) 119 | } 120 | 121 | func extendedTimestampTest() { 122 | let offset = TimePoint(16777216, 1000) 123 | let duration = TimePoint(60 * 5 * 1000, 1000) 124 | rtmpTest(5002, duration, offset: offset) 125 | } 126 | 127 | func rolloverTest() { 128 | let offset = TimePoint(4294966296, 1000) 129 | let duration = TimePoint(60 * 5 * 1000, 1000) 130 | rtmpTest(5003, duration, offset: offset) 131 | } 132 | 133 | private func push(_ time: TimePoint) { 134 | guard let pub = publish else { 135 | return 136 | } 137 | self.currentTs = time 138 | let idx = Int.random(in: 0..>- pub 145 | } 146 | 147 | private func recv(_ sample: CodedMediaSample) { 148 | guard self.sampleInfo.count > 0 else { 149 | print("sampleinfo == 0") 150 | return 151 | } 152 | let (pts, idx) = self.sampleInfo[0] 153 | if sample.pts() != pts { 154 | fatalError("got packet pts=\(sample.pts().toString()) expected=\(pts.toString())") 155 | } 156 | if sample.data() != buffers[idx] { 157 | fatalError("buffers dont match") 158 | } 159 | self.sampleInfo.remove(at: 0) 160 | 161 | self.clock.schedule(self.currentTs + self.stepSize) { [weak self] in 162 | self?.push($0.time()) 163 | } 164 | self.clock.step() 165 | } 166 | 167 | static var allTests = [ 168 | ("extendedTimestampTest", extendedTimestampTest), 169 | ("basicTest", basicTest), 170 | ("rolloverTest", rolloverTest) 171 | ] 172 | var offset = TimePoint(0, 1000) 173 | var sampleInfo: [(TimePoint, Int)] 174 | let buffers: [Data] 175 | let stepSize: TimePoint 176 | var currentIndex: Int = 0 177 | var currentTs: TimePoint 178 | var shouldExit: Bool = false 179 | var rtmp: Rtmp? 180 | let clock: StepClock 181 | let group: EventLoopGroup 182 | let quiesce: ServerQuiescingHelper 183 | var publish: Terminal? 184 | var subscribe: Source? 185 | var stx: Tx? 186 | #else 187 | static var allTests: [(String, XCTestCaseClosure)] = [] 188 | #endif 189 | } 190 | -------------------------------------------------------------------------------- /Tests/swiftVideoTests/sampleRateConversionTests.swift: -------------------------------------------------------------------------------- 1 | /* 2 | SwiftVideo, Copyright 2019 Unpause SAS 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | import XCTest 18 | import Foundation 19 | import SwiftVideo 20 | import SwiftVideo_FFmpeg 21 | 22 | // swiftlint:disable shorthand_operator 23 | // swiftlint:disable:next type_name 24 | final class sampleRateConversionTests: XCTestCase { 25 | 26 | func sampleCountTest() { 27 | let audioPacketDuration = TimePoint(1024, 44100) 28 | let src = FFmpegAudioSRC(48000, 2, .s16i) 29 | let blank = Data(count: Int(audioPacketDuration.value) * 4) // 1-ch, 32-bit float 30 | let buffers = [blank] 31 | 32 | let clock = StepClock(stepSize: audioPacketDuration) 33 | var pts = TimePoint(0, 44100) 34 | var newPts = TimePoint(0, 48000) 35 | let txn = src >>> Terminal { sample in 36 | XCTAssertEqual(newPts.scale, sample.pts().scale) 37 | XCTAssertEqual(newPts.value, sample.pts().value) 38 | newPts.value = newPts.value + Int64(sample.numberSamples()) 39 | return .nothing(sample.info()) 40 | } 41 | 42 | for _ in 0..<100000 { 43 | let sample = AudioSample(buffers, 44 | frequency: 44100, 45 | channels: 1, 46 | format: .f32p, 47 | sampleCount: Int(audioPacketDuration.value), 48 | time: clock.current(), 49 | pts: pts, 50 | assetId: "blank", 51 | workspaceId: "test") 52 | 53 | EventBox.just(sample) >>- txn 54 | pts = pts + audioPacketDuration 55 | clock.step() 56 | } 57 | 58 | } 59 | 60 | static var allTests = [ 61 | ("sampleCountTest", sampleCountTest) 62 | ] 63 | } 64 | -------------------------------------------------------------------------------- /Tests/swiftVideoTests/statsTest.swift: -------------------------------------------------------------------------------- 1 | /* 2 | SwiftVideo, Copyright 2019 Unpause SAS 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | import XCTest 18 | import Foundation 19 | import SwiftVideo 20 | import CSwiftVideo 21 | 22 | // swiftlint:disable:next type_name 23 | final class statsTests: XCTestCase { 24 | 25 | func statsTest() { 26 | let clock = StepClock(stepSize: TimePoint(1000, 30000)) 27 | let stats = StatsReport(period: TimePoint(5000 * 30, 1000 * 30), clock: clock) 28 | 29 | while clock.current() <= TimePoint(10000 * 30, 1000 * 30) { 30 | stats.addSample("test", 1) 31 | _ = clock.step() 32 | } 33 | sleep(1) 34 | let report = stats.report() 35 | let json = """ 36 | { \"name\": \"test\", \"period\": 5.00, \"type\": \"int\", \"median\": 1, \"mean\": 1.00000, \"peak\": 1, \"low\": 1, \"total\": 150, 37 | \"averagePerSecond\": 30.00000, \"count\": 150 } 38 | """ 39 | guard let reportJson = report?.results["test.5.00"] else { 40 | print("failure...\(clock.current().toString()) report=\(report) results=\(report?.results)") 41 | XCTAssertTrue(false) 42 | fatalError("reportJson missing") 43 | } 44 | XCTAssertEqual(json, reportJson) 45 | } 46 | 47 | static var allTests = [ 48 | ("statsTest", statsTest) 49 | ] 50 | } 51 | -------------------------------------------------------------------------------- /Tests/swiftVideoTests/timePointTests.swift: -------------------------------------------------------------------------------- 1 | /* 2 | SwiftVideo, Copyright 2019 Unpause SAS 3 | 4 | Licensed under the Apache License, Version 2.0 (the "License"); 5 | you may not use this file except in compliance with the License. 6 | You may obtain a copy of the License at 7 | 8 | http://www.apache.org/licenses/LICENSE-2.0 9 | 10 | Unless required by applicable law or agreed to in writing, software 11 | distributed under the License is distributed on an "AS IS" BASIS, 12 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | See the License for the specific language governing permissions and 14 | limitations under the License. 15 | */ 16 | 17 | // swiftlint:disable identifier_name 18 | import XCTest 19 | import Foundation 20 | import SwiftVideo 21 | 22 | // swiftlint:disable:next type_name 23 | final class timePointTests: XCTestCase { 24 | 25 | func rescaleTest() { 26 | let a = TimePoint(2987595, 30000) 27 | let b = TimePoint(9958650, 100000) 28 | let c = rescale(a, b.scale) 29 | XCTAssertEqual(c.value, b.value) 30 | } 31 | 32 | func greaterThanTest() { 33 | let a = TimePoint(2987595, 30000) 34 | let b = TimePoint(9955317, 100000) 35 | XCTAssertEqual(a > b, true) 36 | XCTAssertEqual(b > a, false) 37 | } 38 | func lessThanTest() { 39 | let a = TimePoint(2987595, 30000) 40 | let b = TimePoint(9955317, 100000) 41 | XCTAssertEqual(b < a, true) 42 | XCTAssertEqual(a < b, false) 43 | } 44 | func gteTest() { 45 | let a = TimePoint(2987595, 30000) 46 | let b = TimePoint(9955317, 100000) 47 | XCTAssertEqual(a >= b, true) 48 | XCTAssertEqual(b >= a, false) 49 | } 50 | func lteTest() { 51 | let a = TimePoint(2987595, 30000) 52 | let b = TimePoint(9955317, 100000) 53 | XCTAssertEqual(b <= a, true) 54 | XCTAssertEqual(a <= b, false) 55 | } 56 | func addTest() { 57 | let a = TimePoint(2987595, 30000) 58 | let b = TimePoint(9955317, 100000) 59 | let c = b + TimePoint(1000, 30000) 60 | XCTAssertEqual(a <= c, true) 61 | XCTAssertEqual(a >= c, true) 62 | } 63 | func subTest() { 64 | let a = TimePoint(2957595, 30000) 65 | let b = TimePoint(9855316, 100000) // (2957595-1000) * 10 / 3 66 | let c = a - TimePoint(1000, 30000) 67 | print("\(b.toString()) == \(c.toString())") 68 | XCTAssertEqual(c >= b, true) 69 | XCTAssertEqual(c <= b, true) 70 | } 71 | 72 | func minTest() { 73 | let a = TimePoint(2957595, 30000) 74 | let b = TimePoint(9855316, 100000) 75 | let c = min(a, b) 76 | XCTAssertTrue(c == b) 77 | } 78 | 79 | func maxTest() { 80 | let a = TimePoint(2957595, 30000) 81 | let b = TimePoint(9855316, 100000) 82 | let c = max(a, b) 83 | XCTAssertTrue(c == a) 84 | } 85 | 86 | static var allTests = [ 87 | ("rescaleTest", rescaleTest), 88 | ("greaterThanTest", greaterThanTest), 89 | ("lessThanTest", lessThanTest), 90 | ("gteTest", gteTest), 91 | ("lteTest", lteTest), 92 | ("addTest", addTest), 93 | ("subTest", subTest) 94 | ] 95 | 96 | } 97 | -------------------------------------------------------------------------------- /flavor.md: -------------------------------------------------------------------------------- 1 | "flavor" protocol 2 | ---- 3 | 4 | Codename **F**ast **L**ightweight **A**wesome **V**ide**O** p**R**otocol (the p is silent like in pzebra) (i'm not doing this the british way) 5 | 6 | - Data format: all messages are framed by isobmff-style atoms, `[4-byte size][FourCC type]`. Atoms can have children, and those children 7 | contribute to the size of the parent. Size includes the size and type fields, so an empty atom would have a size of 8. 8 | - Numbers are all little-endian (because most devices are little endian these days fight me) 9 | - This protocol is meant for reliable transports 10 | - Dynamic data types, used with dictionaries: `in32`, `in64`, `fl32`, `fl64`, `bool` (1-byte) (?), `data`, `utf8` 11 | - Dictionaries and arrays are present: 12 | ``` 13 | [size]['dict'] 14 | [11]['utf8']["key"] // Key 15 | [size][FourCC][bytes...] // Value 16 | [size]['utf8'][utf8 bytes] // Key 17 | [size][FourCC][bytes...] // Value 18 | ``` 19 | - Dictionaries can embed any atom in addition to the data types above (which are really just atoms containing data) 20 | - Dictionaries must have a utf8 key value. If you wish to use numerical indices, use a list instead. 21 | - Lists look like this and support any atom: 22 | ``` 23 | [size]['list'] 24 | [size]['itm1']... 25 | ``` 26 | - RPC calls: `sync` and `asyn`, rpc format is as follows 27 | ``` 28 | [size]['sync'] 29 | [call_id int32 generated] 30 | [FourCC call type] 31 | [..call atom..] (if needed) 32 | ``` 33 | - In the case of a sync call, client must respond immediately with a reply. Async may respond later or never, the requester shouldn't depend on it to continue. 34 | ``` 35 | [size]['rply'] 36 | [call_id int32 from request] 37 | [success int32 code] 38 | [size]['dict'][optional dictionary with response data] 39 | ``` 40 | - Media tracks should be given a track id, see below for transfer format 41 | - Default port should be 3751 (or 0xEA7) and uri will be `flavor://server.com/{token}` 42 | 43 | ### Connection process 44 | 45 | 1. Client Connects 46 | 2. Server sends ping: 47 | ``` 48 | [16]['sync'] 49 | [0] 50 | ['ping'] 51 | ``` 52 | 3. Client responds: 53 | ``` 54 | [16]['rply'] 55 | [0] 56 | [0] // 0 response code indicates success 57 | ``` 58 | 59 | Connection is established. 60 | 61 | ### Pull or Push a media stream 62 | 63 | Clients will then be interested in either pushing or pulling a stream, in this case it could send a sync request like this: 64 | 65 | ``` 66 | [56]['sync'] 67 | [1] // generated call_id 68 | ['push'] // could be 'pull' if wanting to recieve a stream the peer vends 69 | [60]['list'] 70 | [12]['in32'][generated streamId] // requesting peer generates the stream id. 71 | [40]['utf8']['SampleUTF8Identifier?ohboy=hi'] // token to identify the stream request. This is a freeform utf8 string that the service can use. 72 | ``` 73 | 74 | Server responds with permission granted or denied: 75 | ``` 76 | [16]['rply'] 77 | [1] // call_id for push request 78 | [0] // Success == granted 79 | ``` 80 | ``` 81 | [55]['rply'] 82 | [1] // call_id for push request 83 | [1] // nonzero error code 84 | [39]['dict'] 85 | [14]['utf8']["reason"] 86 | [17]['utf8']["No Access"] 87 | ``` 88 | 89 | The pusher will send an async track info atom if permission is granted. The pulling side must not send a track info atom, it is an error. The pushing peer should consider the pulling peer misbehaving if it does. 90 | ``` 91 | [size]['asyn'] 92 | [int32 generated call_id] 93 | ['mdia'] 94 | [size]['list'] 95 | [size]['trak'] 96 | [FourCC codec name] 97 | [int32 stream id] 98 | [int32 track id] 99 | [int64 time base] 100 | [bool uses_dts] 101 | [size]['data'][extradata bytes...] // optional 102 | [size]['trak'] 103 | ... 104 | ``` 105 | If multiple streams are requested, they _must_ be given unique track numbers across all strems. If a track's properties need to be updated, it _must_ be overwritten by reusing the same track id as before and changing properties. This should be treated as a discontinuity by the peer. 106 | 107 | The pulling peer _should_ respond with a list of unsupported tracks only if there are any unsupported tracks. In this case, the pushing peer should remove those tracks using an `rmtk` command (see below) and stop sending any associated media data. 108 | ``` 109 | [80]['rply'] 110 | [int32 mdia call_id] 111 | [1] // or other non-zero error code 112 | [65]['dict'] 113 | [14]['utf8']["reason"] 114 | [17]['utf8']["unsupported"] 115 | [14]['utf8']["tracks"] 116 | [20]['list'] 117 | [12]['in32'][1] 118 | ``` 119 | 120 | If tracks are disappearing or to unsubscribe from a track, 121 | ``` 122 | [36]['asyn'] 123 | [int32 generated call_id] 124 | ['rmtk'] 125 | [20]['list'] 126 | [12]['in32'][track_id to remove] 127 | ``` 128 | 129 | ### Transmit media 130 | 131 | ``` 132 | [size]['mdia'] 133 | [track-id int32] 134 | [pts int64] 135 | [dts int64] // if track format requires a dts 136 | [size]['data'][payload data] 137 | ``` 138 | _(use a ts delta similar to rtmp? ... adds extra state to track, though)_ 139 | 140 | ### Saying farewell 141 | 142 | The peer ending the connection should send the following async command 143 | 144 | ``` 145 | [16]['asyn'] 146 | [generated call_id int32] 147 | ['bye!'] 148 | ``` 149 | 150 | ### Other potential commands 151 | 152 | - Encoder metadata 153 | ``` 154 | [65]['asyn'] 155 | [generated call_id int32] 156 | ['meta'] 157 | [49]['dict'] 158 | [15]['utf8']["encoder"] 159 | [26]['utf8']["some sweet encoder"] 160 | ... 161 | ``` 162 | 163 | - Query media support 164 | ``` 165 | [size]['sync'] 166 | [int32 generated call_id] 167 | ['mdqr'] 168 | [size]['list'] 169 | [size]['tksp'] 170 | [FourCC codec name] 171 | [size]['xtra'][extradata bytes...] 172 | [size]['tksp'] 173 | ... 174 | ``` 175 | - in which case the peer will respond with, 176 | ``` 177 | [80]['rply'] 178 | [int32 mdia call_id] 179 | [1] // or other non-zero error code 180 | [65]['dict'] 181 | [14]['utf8']["reason"] 182 | [17]['utf8']["unsupported"] 183 | [14]['utf8']["tracks"] 184 | [20]['list'] 185 | [12]['in32'][1] 186 | ``` 187 | - or a success reply if all tracks are supported 188 | 189 | - Query peer capabilities 190 | ``` 191 | [16]['sync'] 192 | [int32 generated call_id] 193 | ['caps'] 194 | ``` 195 | - Response: 196 | ``` 197 | [162]['rply'] 198 | [int32 caps call_id] 199 | [0] 200 | [146]['dict'] 201 | [12]['utf8']["motd"] 202 | [29]['utf8']["Welcome to flavortown"] 203 | [15]['utf8']["version"] 204 | [12]['in32'][1] 205 | [14]['utf8']["codecs"] 206 | [56]['list'] 207 | [12]['in32']['AVC1'] 208 | [12]['in32']['MP4A '] 209 | [12]['in32']['OPUS'] 210 | [12]['in32']['AV10'] 211 | ``` 212 | 213 | 214 | ## Codecs 215 | 216 | Codecs are passed as 4CharCodes in little-endian (so in a bytestream capture they will appear to be backwards). The codes are as follows: 217 | 218 | - H.264/AVC: `AVC1` 219 | - HEVC: `HVC1` 220 | - VP8: `VP80` 221 | - VP9: `VP90` 222 | - AV1: `AV10` _AV2 will be `AV20`_ 223 | - AAC: `MP4A` 224 | - Opus: `OPUS` 225 | 226 | Obviously this is not an exhaustive list, I've used https://www.codecguide.com/klcp_ability_comparison.htm as a reference to find standardish 4cc codes for codecs, feel free to make a PR to add more codecs and formats (e.g. subtitle formats or other important data) or raise an issue to disagree. 227 | 228 | --------------------------------------------------------------------------------