├── .circleci └── config.yml ├── .github ├── CODEOWNERS └── workflows │ ├── archive.yml │ ├── ghpages.yml │ └── publish.yml ├── .gitignore ├── .note.xml ├── CONTRIBUTING.md ├── LICENSE.md ├── Makefile ├── README.md ├── draft-kpugin-rush.md └── meeting-materials ├── RTP-over-QUIC-IETF111.pdf ├── RUSH Reliable (Unreliable) Streaming protocol.pdf ├── Tunnelling SRT over QUIC IETF111.pptx ├── agenda.2021.07.03.md └── bofreq-frindell-media-over-quic.md /.circleci/config.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | jobs: 3 | build: 4 | docker: 5 | - image: martinthomson/i-d-template:latest 6 | working_directory: ~/draft 7 | 8 | steps: 9 | - run: 10 | name: "Print Configuration" 11 | command: | 12 | xml2rfc --version 13 | gem list -q kramdown-rfc2629 14 | echo -n 'mmark '; mmark --version 15 | 16 | - restore_cache: 17 | name: "Restoring cache - Git" 18 | keys: 19 | - v2-cache-git-{{ .Branch }}-{{ .Revision }} 20 | - v2-cache-git-{{ .Branch }} 21 | - v2-cache-git- 22 | 23 | - restore_cache: 24 | name: "Restoring cache - References" 25 | keys: 26 | - v1-cache-references-{{ epoch }} 27 | - v1-cache-references- 28 | 29 | # Workaround for https://discuss.circleci.com/t/22437 30 | - run: 31 | name: Tag Checkout 32 | command: | 33 | if [ -n "$CIRCLE_TAG" ] && [ -d .git ]; then 34 | remote=$(echo "$CIRCLE_REPOSITORY_URL" | \ 35 | sed -e 's,/^git.github.com:,https://github.com/,') 36 | git fetch -f "$remote" "refs/tags/$CIRCLE_TAG:refs/tags/$CIRCLE_TAG" || \ 37 | (echo 'Removing .git cache for tag build'; rm -rf .git) 38 | fi 39 | 40 | - checkout 41 | 42 | # Build txt and html versions of drafts 43 | - run: 44 | name: "Build Drafts" 45 | command: "make 'CLONE_ARGS=--reference ~/git-reference'" 46 | 47 | # Update editor's copy on gh-pages 48 | - run: 49 | name: "Update GitHub Pages" 50 | command: | 51 | if [ "${CIRCLE_TAG#draft-}" == "$CIRCLE_TAG" ]; then 52 | make gh-pages 53 | fi 54 | 55 | # For tagged builds, upload to the datatracker. 56 | - deploy: 57 | name: "Upload to Datatracker" 58 | command: | 59 | if [ "${CIRCLE_TAG#draft-}" != "$CIRCLE_TAG" ]; then 60 | make upload 61 | fi 62 | 63 | # Archive GitHub Issues 64 | - run: 65 | name: "Archive GitHub Issues" 66 | command: "make archive || make archive DISABLE_ARCHIVE_FETCH=true && make gh-archive" 67 | 68 | # Create and store artifacts 69 | - run: 70 | name: "Create Artifacts" 71 | command: "make artifacts CI_ARTIFACTS=/tmp/artifacts" 72 | 73 | - store_artifacts: 74 | path: /tmp/artifacts 75 | 76 | - run: 77 | name: "Prepare for Caching" 78 | command: "git reflog expire --expire=now --all && git gc --prune=now" 79 | 80 | - save_cache: 81 | name: "Saving Cache - Git" 82 | key: v2-cache-git-{{ .Branch }}-{{ .Revision }} 83 | paths: 84 | - ~/draft/.git 85 | 86 | - save_cache: 87 | name: "Saving Cache - Drafts" 88 | key: v1-cache-references-{{ epoch }} 89 | paths: 90 | - ~/.cache/xml2rfc 91 | 92 | 93 | workflows: 94 | version: 2 95 | build: 96 | jobs: 97 | - build: 98 | filters: 99 | tags: 100 | only: /.*?/ 101 | -------------------------------------------------------------------------------- /.github/CODEOWNERS: -------------------------------------------------------------------------------- 1 | # Automatically generated CODEOWNERS file. 2 | draft-frindell-rush.md afrind@fb.comikir@fb.com 3 | -------------------------------------------------------------------------------- /.github/workflows/archive.yml: -------------------------------------------------------------------------------- 1 | name: "Archive Issues and Pull Requests" 2 | 3 | on: 4 | schedule: 5 | - cron: '0 0 * * 0,2,4' 6 | repository_dispatch: 7 | types: [archive] 8 | workflow_dispatch: 9 | inputs: 10 | archive_full: 11 | description: 'Recreate the archive from scratch' 12 | default: false 13 | type: boolean 14 | 15 | jobs: 16 | build: 17 | name: "Archive Issues and Pull Requests" 18 | runs-on: ubuntu-latest 19 | steps: 20 | - name: "Checkout" 21 | uses: actions/checkout@v4 22 | 23 | # Note: No caching for this build! 24 | 25 | - name: "Update Archive" 26 | uses: martinthomson/i-d-template@v1 27 | env: 28 | ARCHIVE_FULL: ${{ inputs.archive_full }} 29 | with: 30 | make: archive 31 | token: ${{ github.token }} 32 | 33 | - name: "Update GitHub Pages" 34 | uses: martinthomson/i-d-template@v1 35 | with: 36 | make: gh-archive 37 | token: ${{ github.token }} 38 | 39 | - name: "Save Archive" 40 | uses: actions/upload-artifact@v4 41 | with: 42 | path: archive.json 43 | -------------------------------------------------------------------------------- /.github/workflows/ghpages.yml: -------------------------------------------------------------------------------- 1 | name: "Update Editor's Copy" 2 | 3 | on: 4 | push: 5 | paths-ignore: 6 | - README.md 7 | - CONTRIBUTING.md 8 | - LICENSE.md 9 | - .gitignore 10 | pull_request: 11 | paths-ignore: 12 | - README.md 13 | - CONTRIBUTING.md 14 | - LICENSE.md 15 | - .gitignore 16 | 17 | jobs: 18 | build: 19 | name: "Update Editor's Copy" 20 | runs-on: ubuntu-latest 21 | steps: 22 | - name: "Checkout" 23 | uses: actions/checkout@v4 24 | 25 | - name: "Setup" 26 | id: setup 27 | run: date -u "+date=%FT%T" >>"$GITHUB_OUTPUT" 28 | 29 | - name: "Caching" 30 | uses: actions/cache@v4 31 | with: 32 | path: | 33 | .refcache 34 | .venv 35 | .gems 36 | node_modules 37 | .targets.mk 38 | key: i-d-${{ steps.setup.outputs.date }} 39 | restore-keys: i-d- 40 | 41 | - name: "Build Drafts" 42 | uses: martinthomson/i-d-template@v1 43 | with: 44 | token: ${{ github.token }} 45 | 46 | - name: "Update GitHub Pages" 47 | uses: martinthomson/i-d-template@v1 48 | if: ${{ github.event_name == 'push' }} 49 | with: 50 | make: gh-pages 51 | token: ${{ github.token }} 52 | 53 | - name: "Archive Built Drafts" 54 | uses: actions/upload-artifact@v4 55 | with: 56 | path: | 57 | draft-*.html 58 | draft-*.txt 59 | -------------------------------------------------------------------------------- /.github/workflows/publish.yml: -------------------------------------------------------------------------------- 1 | name: "Publish New Draft Version" 2 | 3 | on: 4 | push: 5 | tags: 6 | - "draft-*" 7 | workflow_dispatch: 8 | inputs: 9 | email: 10 | description: "Submitter email" 11 | default: "" 12 | type: string 13 | 14 | jobs: 15 | build: 16 | name: "Publish New Draft Version" 17 | runs-on: ubuntu-latest 18 | steps: 19 | - name: "Checkout" 20 | uses: actions/checkout@v4 21 | 22 | # See https://github.com/actions/checkout/issues/290 23 | - name: "Get Tag Annotations" 24 | run: git fetch -f origin ${{ github.ref }}:${{ github.ref }} 25 | 26 | - name: "Setup" 27 | id: setup 28 | run: date -u "+date=%FT%T" >>"$GITHUB_OUTPUT" 29 | 30 | - name: "Caching" 31 | uses: actions/cache@v4 32 | with: 33 | path: | 34 | .refcache 35 | .venv 36 | .gems 37 | node_modules 38 | .targets.mk 39 | key: i-d-${{ steps.setup.outputs.date }} 40 | restore-keys: i-d- 41 | 42 | - name: "Build Drafts" 43 | uses: martinthomson/i-d-template@v1 44 | with: 45 | token: ${{ github.token }} 46 | 47 | - name: "Upload to Datatracker" 48 | uses: martinthomson/i-d-template@v1 49 | with: 50 | make: upload 51 | env: 52 | UPLOAD_EMAIL: ${{ inputs.email }} 53 | 54 | - name: "Archive Submitted Drafts" 55 | uses: actions/upload-artifact@v4 56 | with: 57 | path: "versioned/draft-*-[0-9][0-9].*" 58 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *~ 2 | /*-[0-9][0-9].xml 3 | archive.json 4 | *.html 5 | *.pdf 6 | *.redxml 7 | .refcache 8 | report.xml 9 | *.swp 10 | .tags 11 | .targets.mk 12 | *.txt 13 | *.upload 14 | venv/ 15 | lib 16 | draft-frindell-rush.xml 17 | -------------------------------------------------------------------------------- /.note.xml: -------------------------------------------------------------------------------- 1 | 2 | Discussion of this document takes place on the 3 | mailing list (), 4 | which is archived at . 5 | Source for this draft and an issue tracker can be found at 6 | . 7 | 8 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing 2 | 3 | This repository relates to activities in the Internet Engineering Task Force 4 | ([IETF](https://www.ietf.org/)). All material in this repository is considered 5 | Contributions to the IETF Standards Process, as defined in the intellectual 6 | property policies of IETF currently designated as 7 | [BCP 78](https://www.rfc-editor.org/info/bcp78), 8 | [BCP 79](https://www.rfc-editor.org/info/bcp79) and the 9 | [IETF Trust Legal Provisions (TLP) Relating to IETF Documents](http://trustee.ietf.org/trust-legal-provisions.html). 10 | 11 | Any edit, commit, pull request, issue, comment or other change made to this 12 | repository constitutes Contributions to the IETF Standards Process 13 | (https://www.ietf.org/). 14 | 15 | You agree to comply with all applicable IETF policies and procedures, including, 16 | BCP 78, 79, the TLP, and the TLP rules regarding code components (e.g. being 17 | subject to a Simplified BSD License) in Contributions. 18 | 19 | 20 | ## Other Resources 21 | 22 | Discussion of this work occurs on the 23 | [avtcore working group mailing list](https://mailarchive.ietf.org/arch/browse/avt/) 24 | ([subscribe](https://www.ietf.org/mailman/listinfo/avt)). In addition to 25 | contributions in GitHub, you are encouraged to participate in discussions there. 26 | 27 | **Note**: Some working groups adopt a policy whereby substantive discussion of 28 | technical issues needs to occur on the mailing list. 29 | 30 | You might also like to familiarize yourself with other 31 | [working group documents](https://datatracker.ietf.org/wg/avt/documents/). 32 | -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | # License 2 | 3 | See the 4 | [guidelines for contributions](https://github.com/afrind/draft-rush/blob/main/CONTRIBUTING.md). 5 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | LIBDIR := lib 2 | include $(LIBDIR)/main.mk 3 | 4 | $(LIBDIR)/main.mk: 5 | ifneq (,$(shell grep "path *= *$(LIBDIR)" .gitmodules 2>/dev/null)) 6 | git submodule sync 7 | git submodule update $(CLONE_ARGS) --init 8 | else 9 | git clone -q --depth 10 $(CLONE_ARGS) \ 10 | -b main https://github.com/martinthomson/i-d-template $(LIBDIR) 11 | endif 12 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # RUSH - Reliable (unreliable) streaming protocol 2 | 3 | This is the working area for the individual Internet-Draft, "RUSH - Reliable (unreliable) streaming protocol". 4 | 5 | * [Editor's Copy](https://afrind.github.io/draft-rush/#go.draft-kpugin-rush.html) 6 | * [Individual Draft](https://datatracker.ietf.org/doc/html/draft-kpugin-rush) 7 | * [Compare Editor's Copy to Individual Draft](https://afrind.github.io/draft-rush/#go.draft-kpugin-rush.diff) 8 | 9 | ## Building the Draft 10 | 11 | Formatted text and HTML versions of the draft can be built using `make`. 12 | 13 | ```sh 14 | $ make 15 | ``` 16 | 17 | This requires that you have the necessary software installed. See 18 | [the instructions](https://github.com/martinthomson/i-d-template/blob/master/doc/SETUP.md). 19 | 20 | 21 | ## Contributing 22 | 23 | See the 24 | [guidelines for contributions](https://github.com/afrind/draft-rush/blob/main/CONTRIBUTING.md). 25 | -------------------------------------------------------------------------------- /draft-kpugin-rush.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: "RUSH - Reliable (unreliable) streaming protocol" 3 | abbrev: "rush" 4 | docname: draft-kpugin-rush-latest 5 | category: info 6 | 7 | ipr: trust200902 8 | area: General 9 | workgroup: TODO Working Group 10 | keyword: Internet-Draft 11 | 12 | stand_alone: yes 13 | smart_quotes: no 14 | pi: [toc, sortrefs, symrefs] 15 | 16 | author: 17 | - 18 | ins: K. Pugin 19 | name: Kirill Pugin 20 | organization: Meta 21 | email: ikir@meta.com 22 | - 23 | ins: N. Garg 24 | name: Nitin Garg 25 | organization: Meta 26 | email: ngarg@meta.com 27 | - 28 | ins: A. Frindell 29 | name: Alan Frindell 30 | organization: Meta 31 | email: afrind@meta.com 32 | - 33 | ins: J. Cenzano 34 | name: Jordi Cenzano 35 | organization: Meta 36 | email: jcenzano@meta.com 37 | - 38 | ins: J. Weissman 39 | name: Jake Weissman 40 | organization: Meta 41 | email: jakeweissman@meta.com 42 | 43 | normative: 44 | RFC2119: 45 | 46 | informative: 47 | 48 | 49 | 50 | --- abstract 51 | 52 | RUSH is an application-level protocol for ingesting live video. 53 | This document describes the protocol and how it maps onto QUIC. 54 | 55 | --- middle 56 | 57 | # Introduction 58 | 59 | RUSH is a bidirectional application level protocol designed for live video 60 | ingestion that runs on top of QUIC. 61 | 62 | RUSH was built as a replacement for RTMP (Real-Time Messaging Protocol) with the 63 | goal to provide support for new audio and video codecs, extensibility in the 64 | form of new message types, and multi-track support. In addition, RUSH gives 65 | applications option to control data delivery guarantees by utilizing QUIC 66 | streams. 67 | 68 | This document describes the RUSH protocol, wire format, and QUIC mapping. 69 | 70 | # Conventions and Definitions 71 | 72 | The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", "SHOULD", 73 | "SHOULD NOT", "RECOMMENDED", "NOT RECOMMENDED", "MAY", and "OPTIONAL" in this 74 | document are to be interpreted as described in BCP 14 {{RFC2119}} {{!RFC8174}} 75 | when, and only when, they appear in all capitals, as shown here. 76 | 77 | Frame/Message: 78 | 79 | : logical unit of information that client and server can exchange 80 | 81 | PTS: 82 | 83 | : presentation timestamp 84 | 85 | 86 | DTS: 87 | 88 | : decoding timestamp 89 | 90 | AAC: 91 | 92 | : advanced audio codec 93 | 94 | NALU: 95 | 96 | : network abstract layer unit 97 | 98 | VPS: 99 | 100 | : video parameter set (H265 video specific NALU) 101 | 102 | SPS: 103 | 104 | : sequence parameter set (H264/H265 video specific NALU) 105 | 106 | PPS: 107 | 108 | : picture parameter set (H264/H265 video specific NALU) 109 | 110 | ADTS header: 111 | 112 | : *Audio Data Transport Stream Header* 113 | 114 | ASC: 115 | 116 | : Audio specific config 117 | 118 | GOP: 119 | 120 | : Group of pictures, specifies the order in which intra- and inter-frames are 121 | arranged. 122 | 123 | 124 | # Theory of Operations 125 | 126 | ## Connection establishment 127 | 128 | In order to live stream using RUSH, the client establishes a QUIC connection 129 | using the ALPN token "rush". 130 | 131 | After the QUIC connection is established, client creates a new bidirectional 132 | QUIC stream, choses starting frame ID and sends `Connect` frame 133 | {{connect-frame}} over that stream. This stream is called the Connect Stream. 134 | 135 | The client sends `mode of operation` setting in `Connect` frame {{connect-frame}} payload. 136 | 137 | One connection SHOULD only be used to send one media stream, for now 1 video and 1 audio track are supported. In the future we could send multiple tracks per stream. 138 | 139 | ## Sending Video Data 140 | 141 | The client can choose to wait for the `ConnectAck` frame {{connect-ack-frame}} 142 | or it can start optimistically sending data immediately after sending the `Connect` frame. 143 | 144 | A track is a logical organization of the data, for example, video can have one 145 | video track, and two audio tracks (for two languages). The client can send data 146 | for multiple tracks simultaneously. 147 | 148 | The encoded audio or video data of each track is serialized into frames (see 149 | {{audio-frame}} or {{video-frame}}) and transmitted from the client to the 150 | server. Each track has its own monotonically increasing frame ID sequence. The 151 | client MUST start with initial frame ID = 1. 152 | 153 | Depending on mode of operation ({{quic-mapping}}), the client sends audio and 154 | video frames on the Connect stream or on a new QUIC stream for each frame. 155 | 156 | In `Multi Stream Mode` ({{multi-stream-mode}}), the client can stop sending a 157 | frame by resetting the corresponding QUIC stream. In this case, there is no 158 | guarantee that the frame was received by the server. 159 | 160 | ## Receiving data 161 | 162 | Upon receiving `Connect` frame {{connect-frame}}, if the server accepts the stream, the server will reply with `ConnectAck` frame {{connect-ack-frame}} and it will prepare to receive audio/video data. 163 | 164 | It's possible that in `Multi Stream Mode` ({{multi-stream-mode}}), the server 165 | receives audio or video data before it receives the `Connect` frame {{connect-frame}}. The 166 | implementation can choose whether to buffer or drop the data. 167 | The audio/video data cannot be interpreted correctly before the arrival of the `Connect` frame {{connect-frame}}. 168 | 169 | In `Single Stream Mode` ({{single-stream-mode}}), it is guaranteed by the transport that 170 | frames arrive into the application layer in order they were sent. 171 | 172 | In `Multi Stream Mode`, it's possible that frames arrive at the application 173 | layer in a different order than they were sent, therefore the server MUST keep 174 | track of last received frame ID for every track that it receives. A gap in the 175 | frame sequence ID on a given track can indicate out of order delivery and the 176 | server MAY wait until missing frames arrive. The server must consider frame lost 177 | if the corresponding QUIC stream was reset. 178 | 179 | Upon detecting a gap in the frame sequence, the server MAY wait for the missing 180 | frames to arrive for an implementation defined time. If missing frames don't 181 | arrive, the server SHOULD consider them lost and continue processing rest of the 182 | frames. For example if the server receives the following frames for track 1: `1 183 | 2 3 5 6` and frame `#4` hasn't arrived after implementation defined timeout, 184 | thee server SHOULD continue processing frames `5` and `6`. 185 | 186 | It is worth highlighting that in multi stream mode there is a need for a de-jitter function (that introduces latency). Also the subsequent processing pipeline should tolerate lost frames, so "holes" in the audio / video streams. 187 | 188 | When the client is done streaming, it sends the `End of Video` frame 189 | ({{end-of-video-frame}}) to indicate to the server that there won't be any more 190 | data sent. 191 | 192 | ## Reconnect 193 | 194 | If the QUIC connection is closed at any point, client MAY reconnect by simply 195 | repeat the `Connection establishment` process ({{connection-establishment}}) and 196 | resume sending the same video where it left off. In order to support 197 | termination of the new connection by a different server, the client SHOULD 198 | resume sending video frames starting with I-frame, to guarantee that the video 199 | track can be decoded from the 1st frame sent. 200 | 201 | Reconnect can be initiated by the server if it needs to "go away" for 202 | maintenance. In this case, the server sends a `GOAWAY` frame ({{goaway-frame}}) 203 | to advise the client to gracefully close the connection. This allows client to 204 | finish sending some data and establish new connection to continue sending 205 | without interruption. 206 | 207 | # Wire Format 208 | 209 | ## Frame Header 210 | 211 | The client and server exchange information using frames. There are different 212 | types of frames and the payload of each frame depends on its type. 213 | 214 | The bytes in the wire are in **big endian** 215 | 216 | Generic frame format: 217 | 218 | ~~~ 219 | 0 1 2 3 4 5 6 7 220 | +--------------------------------------------------------------+ 221 | | Length (64) | 222 | +--------------------------------------------------------------+ 223 | | ID (64) | 224 | +-------+------------------------------------------------------+ 225 | |Type(8)| Payload ... | 226 | +-------+------------------------------------------------------+ 227 | ~~~ 228 | 229 | Length(64)`: 230 | : Each frame starts with length field, 64 bit size that tells size of the frame 231 | in bytes (including predefined fields, so if LENGTH is 100 bytes, then PAYLOAD 232 | length is 100 - 8 - 8 - 1 = 82 bytes). 233 | 234 | ID(64): 235 | : 64 bit frame sequence number, every new frame MUST have a sequence ID greater 236 | than that of the previous frame within the same track. Track ID would be 237 | specified in each frame. If track ID is not specified it's 0 implicitly. 238 | 239 | Type(8): 240 | : 1 byte representing the type of the frame. 241 | 242 | Predefined frame types: 243 | 244 | | Frame Type | Frame | 245 | |------------|-------| 246 | | 0x0 | connect frame| 247 | | 0x1 | connect ack frame | 248 | | 0x2 | reserved | 249 | | 0x3 | reserved | 250 | | 0x4 | end of video frame | 251 | | 0x5 | error frame | 252 | | 0x6 | reserved | 253 | | 0x7 | reserved | 254 | | 0x8 | reserved | 255 | | 0x9 | reserved | 256 | | 0xA | reserved | 257 | | 0XB | reserved | 258 | | 0xC | reserved | 259 | | 0xD | video frame | 260 | | 0xE | reserved | 261 | | 0XF | reserved | 262 | | 0X10 | reserved | 263 | | 0x11 | reserved | 264 | | 0x12 | reserved | 265 | | 0x13 | reserved | 266 | | 0x14 | audio frame | 267 | | 0x15 | GOAWAY frame | 268 | | 0x16 | Timed metadata | 269 | 270 | ## Frames 271 | 272 | ### Connect frame 273 | 274 | ~~~ 275 | +--------------------------------------------------------------+ 276 | | Length (64) | 277 | +--------------------------------------------------------------+ 278 | | ID (64) | 279 | +-------+-------+---------------+---------------+--------------+ 280 | | 0x0 |Version|Video Timescale|Audio Timescale| | 281 | +-------+-------+---------------+---------------+--------------+ 282 | | Live Session ID(64) | 283 | +--------------------------------------------------------------+ 284 | | Payload ... | 285 | +--------------------------------------------------------------+ 286 | ~~~ 287 | 288 | Version (unsigned 8bits): 289 | : version of the protocol (initial version is 0x0). 290 | 291 | Video Timescale(unsigned 16bits): 292 | : timescale for all video frame timestamps on this connection. For instance 25 293 | 294 | Audio Timescale(unsigned 16bits): 295 | : timescale for all audio samples timestamps on this connection, recommended 296 | value same as audio sample rate, for example 44100 297 | 298 | Live Session ID(unsigned 64bits): 299 | : identifier of broadcast, when reconnect, client MUST use the same live session 300 | ID 301 | 302 | Payload: 303 | : application and version specific data that can be used by the server. OPTIONAL 304 | A possible implementation for this could be to add in the payload a UTF-8 encoded JSON data that specifies some parameters that server needs to authenticate / validate that connection, for instance: 305 | ~~~ 306 | payloadBytes = strToJSonUtf8('{"url": "/rtmp/BID?s_bl=1&s_l=3&s_sc=VALID&s_sw=0&s_vt=usr_dev&a=TOKEN"}') 307 | ~~~ 308 | 309 | This frame is used by the client to initiate broadcasting. The client can start 310 | sending other frames immediately after Connect frame {{connect-frame}} without waiting 311 | acknowledgement from the server. 312 | 313 | If server doesn't support VERSION sent by the client, the server sends an Error 314 | frame {{error-frame}} with code `UNSUPPORTED VERSION`. 315 | 316 | If audio timescale or video timescale are 0, the server sends error frame {{error-frame}} with 317 | error code `INVALID FRAME FORMAT` and closes connection. 318 | 319 | If the client receives a Connect frame from the server, the client sends an 320 | Error frame {{error-frame}} with code `TBD`. 321 | 322 | ### Connect Ack frame 323 | 324 | ~~~ 325 | 0 1 2 3 4 5 6 7 326 | +--------------------------------------------------------------+ 327 | | Length (64) = 17 | 328 | +--------------------------------------------------------------+ 329 | | ID (64) | 330 | +-------+------------------------------------------------------+ 331 | | 0x1 | 332 | +-------+ 333 | ~~~ 334 | 335 | The server sends the "Connect Ack" frame in response to "Connect" {{connect-frame}} frame 336 | indicating that server accepts "version" and the stream is authenticated / validated (optional), so it is ready to receive data. 337 | 338 | If the client doesn't receive "Connect Ack" frame from the server within a 339 | timeout, it will close the connection. The timeout value is chosen by the 340 | implementation. 341 | 342 | There can be only one "Connect Ack" frame sent over lifetime of the QUIC 343 | connection. 344 | 345 | If the server receives a Connect Ack frame from the client, the client sends an 346 | Error frame with code `TBD`. 347 | 348 | ### End of Video frame 349 | 350 | ~~~ 351 | +--------------------------------------------------------------+ 352 | | Length (64) = 17 | 353 | +--------------------------------------------------------------+ 354 | | ID (64) | 355 | +-------+------------------------------------------------------+ 356 | | 0x4 | 357 | +-------+ 358 | ~~~ 359 | 360 | End of Video frame is sent by a client when it's done sending data and is about 361 | to close the connection. The server SHOULD ignore all frames sent after that. 362 | 363 | ### Error frame 364 | 365 | ~~~ 366 | +--------------------------------------------------------------+ 367 | | Length (64) = 29 | 368 | +--------------------------------------------------------------+ 369 | | ID (64) | 370 | +-------+------------------------------------------------------+ 371 | | 0x5 | 372 | +-------+------------------------------------------------------+ 373 | | Sequence ID (64) | 374 | +------------------------------+-------------------------------+ 375 | | Error Code (32) | 376 | +------------------------------+ 377 | ~~~ 378 | 379 | Sequence ID(unsigned 64bits): 380 | : ID of the frame sent by the client that error is generated for, ID=0x0 381 | indicates connection level error. 382 | 383 | Error Code(unsigned 32bits): 384 | : Indicates the error code 385 | 386 | Error frame can be sent by the client or the server to indicate that an error 387 | occurred. 388 | 389 | Some errors are fatal and the connection will be closed after sending the Error 390 | frame. 391 | 392 | See section {{connection-errors}} and {{frame-errors}} for more information about error codes 393 | 394 | ### Video frame 395 | 396 | ~~~ 397 | +--------------------------------------------------------------+ 398 | | Length (64) | 399 | +--------------------------------------------------------------+ 400 | | ID (64) | 401 | +-------+-------+----------------------------------------------+ 402 | | 0xD | Codec | 403 | +-------+-------+----------------------------------------------+ 404 | | PTS (64) | 405 | +--------------------------------------------------------------+ 406 | | DTS (64) | 407 | +-------+------------------------------------------------------+ 408 | |TrackID| | 409 | +-------+-------+----------------------------------------------+ 410 | | I Offset | Video Data ... | 411 | +---------------+----------------------------------------------+ 412 | ~~~ 413 | 414 | Codec (unsigned 8bits): 415 | : specifies codec that was used to encode this frame. 416 | 417 | PTS (signed 64bits): 418 | : presentation timestamp in connection video timescale 419 | 420 | DTS (signed 64bits): 421 | : decoding timestamp in connection video timescale 422 | 423 | Supported type of codecs: 424 | 425 | | Type | Codec | 426 | |------|-------| 427 | |0x1| H264| 428 | |0x2| H265| 429 | |0x3| VP8| 430 | |0x4| VP9| 431 | 432 | Track ID (unsigned 8bits): 433 | : ID of the track that this frame is on 434 | 435 | I Offset (unsigned 16bits): 436 | : Distance from sequence ID of the I-frame that is required before this frame 437 | can be decoded. This can be useful to decide if frame can be dropped. 438 | 439 | Video Data: 440 | : variable length field, that carries actual video frame data that is codec 441 | dependent 442 | 443 | For h264/h265 codec, "Video Data" are 1 or more NALUs in AVCC format (4 bytes size header): 444 | 445 | ~~~ 446 | 0 1 2 3 4 5 6 7 447 | +--------------------------------------------------------------+ 448 | | NALU Length (64) | 449 | +--------------------------------------------------------------+ 450 | | NALU Data ... 451 | +--------------------------------------------------------------+ 452 | ~~~ 453 | 454 | EVERY h264 video key-frame MUST start with SPS/PPS NALUs. 455 | EVERY h265 video key-frame MUST start with VPS/SPS/PPS NALUs. 456 | 457 | Binary concatenation of "video data" from consecutive video frames, without data 458 | loss MUST produce VALID h264/h265 bitstream. 459 | 460 | 461 | ### Audio frame 462 | 463 | ~~~ 464 | +--------------------------------------------------------------+ 465 | | Length (64) | 466 | +--------------------------------------------------------------+ 467 | | ID (64) | 468 | +-------+------------------------------------------------------+ 469 | | 0x14 | Codec | 470 | +-------+-------+----------------------------------------------+ 471 | | Timestamp (64) | 472 | +-------+-------+-------+--------------------------------------+ 473 | |TrackID| Header Len | 474 | +-------+-------+-------+--------------------------------------+ 475 | | Header + Audio Data ... 476 | +--------------------------------------------------------------+ 477 | ~~~ 478 | 479 | Codec (unsigned 8bits): 480 | : specifies codec that was used to encode this frame. 481 | 482 | Supported type of codecs: 483 | 484 | | Type| Codec| 485 | |-----|------| 486 | |0x1| AAC| 487 | |0x2| OPUS| 488 | 489 | Timestamp (signed 64bits): 490 | : timestamp of first audio sample in Audio Data. 491 | 492 | Track ID (unsigned 8bits): 493 | : ID of the track that this frame is on 494 | 495 | Header Len (unsigned 16bits): 496 | : Length in bytes of the audio header contained in the first portion of the payload 497 | 498 | Audio Data (variable length field): 499 | : it carries the audio header and 1 or more audio frames that are codec dependent. 500 | 501 | For AAC codec: 502 | - "Audio Data" are 1 or more AAC samples, prefixed with Audio Specific Config (ASC) header defined in `ISO 14496-3` 503 | - Binary concatenation of all AAC samples in "Audio Data" from consecutive audio frames, without data loss MUST produce VALID AAC bitstream. 504 | 505 | For OPUS codec: 506 | - "Audio Data" are 1 or more OPUS samples, prefixed with OPUS header as defined in {{!RFC7845}} 507 | 508 | 509 | ### GOAWAY frame 510 | 511 | ~~~ 512 | 0 1 2 3 4 5 6 7 513 | +--------------------------------------------------------------+ 514 | | 17 | 515 | +--------------------------------------------------------------+ 516 | | ID (64) | 517 | +-------+------------------------------------------------------+ 518 | | 0x15 | 519 | +-------+ 520 | ~~~ 521 | 522 | The GOAWAY frame is used by the server to initiate graceful shutdown of a connection, for example, for server maintenance. 523 | 524 | Upon receiving GOAWAY frame, the client MUST send frames remaining in current GOP and 525 | stop sending new frames on this connection. The client SHOULD establish a new connection and resume sending frames there, so when resume video frame will start with an IDR frame. 526 | 527 | After sending a GOAWAY frame, the server continues processing arriving frames 528 | for an implementation defined time, after which the server SHOULD close 529 | the connection. 530 | 531 | 532 | ### TimedMetadata frame 533 | 534 | ~~~ 535 | +--------------------------------------------------------------+ 536 | | Length (64) | 537 | +--------------------------------------------------------------+ 538 | | ID (64) | 539 | +-------+------------------------------------------------------+ 540 | | 0x16 |TrackID| 541 | +-------+-------+----------------------------------------------+ 542 | | Topic (64) | 543 | +--------------------------------------------------------------+ 544 | | EventMessage (64) | 545 | +-------+------------------------------------------------------+ 546 | | Timestamp (64) | 547 | +-------+------------------------------------------------------+ 548 | | Duration (64) | 549 | +-------+------------------------------------------------------+ 550 | | Payload ... 551 | +--------------------------------------------------------------+ 552 | ~~~ 553 | 554 | Track ID (unsigned 8bits): 555 | : ID of the track that this frame is on 556 | 557 | Timestamp (signed 64bits): 558 | : PTS of the event 559 | 560 | Topic (unsigned 64bits): 561 | : A unique identifier of the app level feature. May be used to decode payload or do other application specific processing 562 | 563 | EventMessage (unsigned 64bits): 564 | : A unique identifier of the event message used for app level events deduplication 565 | 566 | Duration (unsigned 64bits): 567 | : duration of the event in video PTS timescale. Can be 0. 568 | 569 | Payload: 570 | : variable length field. May be used by the app to send additional event metadata. UTF-8 JSON recommended 571 | 572 | ## QUIC Mapping 573 | 574 | One of the main goals of the RUSH protocol was ability to provide applications a 575 | way to control reliability of delivering audio/video data. This is achieved by 576 | using a special mode {{multi-stream-mode}}. 577 | 578 | ### Single Stream Mode 579 | 580 | In single stream mode, RUSH uses one bidirectional QUIC stream to send data and receive 581 | data. Using one stream guarantees reliable, in-order delivery - applications 582 | can rely on QUIC transport layer to retransmit lost packets. The performance 583 | characteristics of this mode are similar to RTMP over TCP. 584 | 585 | ### Multi Stream Mode 586 | 587 | In single stream mode {{single-stream-mode}}, if packet belonging to video frame is lost, all packets sent 588 | after it will not be delivered to application, even though those packets may 589 | have arrived at the server. This introduces head of line blocking and can 590 | negatively impact latency. 591 | 592 | To address this problem, RUSH defines "Multi Stream Mode", in which one QUIC 593 | stream is used per audio/video frame. 594 | 595 | Connection establishment follows the normal procedure by client sending Connect 596 | frame, after that Video and Audio frames are sent using following rules: 597 | 598 | * Each new frame is sent on new bidirectional QUIC stream 599 | * Frames within same track must have IDs that are monotonically increasing, 600 | such that ID(n) = ID(n-1) + 1 601 | 602 | The receiver reconstructs the track using the frames IDs. 603 | 604 | Response Frames (Connect Ack{{connect-ack-frame}} and Error{{error-frame}}), will be in the response stream of the 605 | stream that sent it. 606 | 607 | The client MAY control delivery reliability by setting a delivery timer for 608 | every audio or video frame and reset the QUIC stream when the timer fires. This 609 | will effectively stop retransmissions if the frame wasn't fully delivered in 610 | time. 611 | 612 | Timeout is implementation defined, however future versions of the draft will 613 | define a way to negotiate it. 614 | 615 | # Error Handling 616 | 617 | An endpoint that detects an error SHOULD signal the existence of that error to 618 | its peer. Errors can affect an entire connection (see {{connection-errors}}), 619 | or a single frame (see {{frame-errors}}). 620 | 621 | The most appropriate error code SHOULD be included in the error frame that 622 | signals the error. 623 | 624 | 625 | ## Connection Errors 626 | 627 | Affects the the whole connection: 628 | 629 | 1 - UNSUPPORTED VERSION - indicates that the server doesn't support version 630 | specified in Connect frame 631 | 4- CONNECTION_REJECTED - Indicates the server can not process that connection for any reason 632 | 633 | ## Frame errors 634 | 635 | There are two error codes defined in core protocol that indicate a problem with 636 | a particular frame: 637 | 638 | 2 - UNSUPPORTED CODEC - indicates that the server doesn't support the given 639 | audio or video codec 640 | 641 | 3 - INVALID FRAME FORMAT - indicates that the receiver was not able to parse 642 | the frame or there was an issue with a field's value. 643 | 644 | # Extensions 645 | 646 | RUSH permits extension of the protocol. 647 | 648 | Extensions are permitted to use new frame types ({{wire-format}}), new error 649 | codes ({{error-frame}}), or new audio and video codecs ({{audio-frame}}, 650 | {{video-frame}}). 651 | 652 | Implementations MUST ignore unknown or unsupported values in all extensible 653 | protocol elements, except `codec id`, which returns an UNSUPPORTED CODEC error. 654 | Implementations MUST discard frames that have unknown or unsupported types. 655 | 656 | # Security Considerations 657 | 658 | RUSH protocol relies on security guarantees provided by the transport. 659 | 660 | Implementation SHOULD be prepared to handle cases when sender deliberately sends 661 | frames with gaps in sequence IDs. 662 | 663 | Implementation SHOULD be prepare to handle cases when server never receives 664 | Connect frame ({{connect-frame}}). 665 | 666 | A frame parser MUST ensure that value of frame length field (see 667 | {{frame-header}}) matches actual length of the frame, including the frame 668 | header. 669 | 670 | Implementation SHOULD be prepare to handle cases when sender sends a frame with 671 | large frame length field value. 672 | 673 | 674 | # IANA Considerations 675 | 676 | TODO: add frame type registry, error code registry, audio/video codecs 677 | registry 678 | 679 | 680 | 681 | --- back 682 | 683 | # Acknowledgments 684 | {:numbered="false"} 685 | 686 | This draft is the work of many people: Vlad Shubin, Nitin Garg, Milen Lazarov, 687 | Benny Luo, Nick Ruff, Konstantin Tsoy, Nick Wu. 688 | -------------------------------------------------------------------------------- /meeting-materials/RTP-over-QUIC-IETF111.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/afrind/draft-rush/26092ac2b5fa7036f06694671e168cb3ee5bda86/meeting-materials/RTP-over-QUIC-IETF111.pdf -------------------------------------------------------------------------------- /meeting-materials/RUSH Reliable (Unreliable) Streaming protocol.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/afrind/draft-rush/26092ac2b5fa7036f06694671e168cb3ee5bda86/meeting-materials/RUSH Reliable (Unreliable) Streaming protocol.pdf -------------------------------------------------------------------------------- /meeting-materials/Tunnelling SRT over QUIC IETF111.pptx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/afrind/draft-rush/26092ac2b5fa7036f06694671e168cb3ee5bda86/meeting-materials/Tunnelling SRT over QUIC IETF111.pptx -------------------------------------------------------------------------------- /meeting-materials/agenda.2021.07.03.md: -------------------------------------------------------------------------------- 1 | # Video Ingest over QUIC side meeting 2 | 3 | ## Switched from Bluejeans to Zoom! 4 | 5 | * [Zoom](https://fb.zoom.us/j/96613780902) passcode 061119 6 | * [Minutes](https://codimd.ietf.org/W_IANtwnRLmYOkvfbQoOJA) 7 | * Meeting chat: use zoom chat 8 | 9 | ## Friday, Jul 30, 2021 11:00-12:00 10 | 11 | ### Administrivia 12 | 13 | 5 min total 14 | 15 | * Attendance sheets 16 | * Agenda bashing 17 | 18 | ### Reliable (unreliable) streaming protocol 19 | 20 | Kirill Pugin - 10 min 21 | 22 | https://www.ietf.org/archive/id/draft-kpugin-rush-00.html 23 | 24 | ### SRT over QUIC Datagram 25 | 26 | Maxim Sharabayko - 10 min 27 | 28 | https://haivision.github.io/srt-rfc/draft-sharabayko-srt-over-quic.html 29 | https://tools.ietf.org/html/draft-sharabayko-srt-00 30 | 31 | ### RTP over QUIC Datagram 32 | 33 | Jörg Ott - 10 min 34 | 35 | https://datatracker.ietf.org/doc/html/draft-engelbart-rtp-over-quic 36 | 37 | ### General Discussion 38 | 39 | 15 minutes 40 | 41 | ### Logistics and next steps 42 | 43 | 10 minutes 44 | -------------------------------------------------------------------------------- /meeting-materials/bofreq-frindell-media-over-quic.md: -------------------------------------------------------------------------------- 1 | # Name: Media Over QUIC 2 | # Description 3 | There are video/media related use-cases that do not appear to be well met by existing protocols. 4 | 5 | ## Media Ingest Use Case 6 | Example: live stream broadcast, with later video-on-demand playback. 7 | 8 | **Requirements:** 9 | 1. Have a mechanism to tune latency/quality tradeoffs. 10 | 2. Have a mechanism to allow bitrate adaptation in the encoders. The adaptation can also be based on latency/quality tradeoffs. 11 | 3. Able to support low latency in the range of 500ms - 1s. 12 | 4. Able to support high quality such as 4K HDR 13 | 5. Able to support new video/audio codecs in the future. 14 | 6. Frame-based, not chunk-based, i.e. frames are sent out immediately as they are available. 15 | 7. Able to support live captions and cuepoints for Ad insertions. 16 | 8. Supports switching networks. (ex. WiFi => cellular) 17 | 9. Support “server migration”: a mechanism that allows clients to continue ingest even if servers on the ingest path need to restart for maintenance/etc. 18 | 19 | **Existing Protocols/Challenges:** 20 | 1. RTMP: The standard is a bit stagnant and doesn’t have an official way to add new codecs support. It’s TCP based which has the head-of-line blocking issue and can’t use newer congestion control algorithms such as BBR. 21 | 2. RTC/WebRTC: Doesn’t have an easy way to tune latency/quality tradeoffs and the existing protocol sacrifices too much quality for latency which is not suitable for live streams with premium content. Requires SDP exchange between peers which is unnecessary for client server live streaming use cases. 22 | 3. HTTP based protocols (HLS, DASH): Chunk-based protocols incur more latency. These protocols are defined for delivery, but have also been used for ingestion in some cases as well. 23 | 24 | ## Large-scale Media Playback Use Case 25 | Example: 10k+ viewers watching a live broadcast with different quality networks 26 | 27 | **Requirements:** 28 | 1. Same as media ingest. 29 | 2. Browser playback support. 30 | 3. Able to automatically choose the best rendition for playback. (ABR) 31 | 4. Able to drop media during severe congestion/starvation. 32 | 5. Does not rely on encoder feedback. 33 | 6. Minimal handshake RTTs and fast playback start. 34 | 7. CDN support is possible. 35 | 8. Support for timed metadata 36 | 9. Support for media seeking 37 | 38 | **Existing Protocols/Challenges:** 39 | 1. HTTP based protocols (HLS, DASH): Head-of-line blocking and chunk-based protocols incur more latency. Low latency client-side ABR algorithms are unable to determine if the stream is network bound or encoder bound, limiting their effectiveness. 40 | 2. RTP/WebRTC: 41 | 1. Focused on real-time latency and doesn’t have a way to tune latency/quality tradeoffs. 42 | 2. Peer-to-peer support complicates everything: long handshake, blocked by corporate firewalls, no roaming support, port per connection. Frame dropping requires encoder feedback (Full Intra Request) or minimal reference frames (lower quality) otherwise it causes artifacting. 43 | 3. No b-frame support for higher quality/latency video. 44 | 4. Cannot easily leverage CDN infrastructure. 45 | 5. Tight coupling between media framing and transport. 46 | 3. Others: No browser support. 47 | 48 | ## BoF Goal 49 | The purpose of this BoF is to: 50 | 51 | 1. Determine if there is an existing protocol that can meet these use cases and requirements. 52 | 2. If not, is there an existing protocol that could be easily extended to meet these use cases 53 | 3. If not, is there agreement we should forge a new protocol to meet these use cases 54 | 4. If so, determine whether an existing working group or a new working group is best suited to work on this protocol. 55 | 56 | # Required Details 57 | * Status: not WG Forming 58 | * Responsible AD: Murray Kucherawy 59 | * BOF proponents: Alan Frindell afrind@fb.com, Ying Yin yingyin@google.com 60 | * BOF chairs: Magnus Westerlund, Alan Frindell 61 | * Number of people expected to attend: 50 62 | * Length of session (1 or 2 hours): 2 hours 63 | * Conflicts (whole Areas and/or WGs) 64 | * Chair Conflicts: quic, webtransport, masque, ohai 65 | * Technology Overlap: quic, mops, avtcore, mmusic 66 | * Key Participant Conflict: webtransport, masque 67 | 68 | # Agenda 69 | 1. Administrative Details (5m) 70 | 1. Note Well 71 | 2. Note taker, Jabber relay 72 | 2. Use cases (20m) 73 | 1. Present Media Ingest and Playback use cases not well covered by existing protocols 74 | 2. Discussion of use cases 75 | 3. BoF Questions and Discussion (30m) 76 | 1. Does an existing protocol cover these use cases? 77 | 2. Can an existing protocol be extended to cover these use cases? 78 | 3. Should we create a new protocol to cover these use cases? 79 | 4. Should an existing working group take on this work or should we form a new working group? 80 | 4. Wrap Up and Next Steps (5m) 81 | 82 | # Links to the mailing list, draft charter if any, relevant Internet-Drafts, etc. 83 | * Mailing List: https://www.ietf.org/mailman/listinfo/moq 84 | * Draft charter: Not WG Forming 85 | * Relevant drafts: 86 | * Use Cases: 87 | * https://fiestajetsam.github.io/2021/08/22/quic-video.html 88 | * Solutions 89 | * RUSH - https://datatracker.ietf.org/doc/draft-kpugin-rush/ 90 | * WARP - https://www.ietf.org/archive/id/draft-lcurley-warp-00.html, https://www.youtube.com/watch?v=hG0nmy3Otg4 91 | --------------------------------------------------------------------------------