├── .circleci └── config.yml ├── .credo.exs ├── .formatter.exs ├── .github ├── ISSUE_TEMPLATE │ └── please--open-new-issues-in-membranefranework-membrane_core.md └── workflows │ ├── on_issue_opened.yaml │ └── on_pr_opened.yaml ├── .gitignore ├── LICENSE ├── README.md ├── config ├── config.exs ├── dev.exs ├── prod.exs └── test.exs ├── lib └── membrane │ ├── rtcp │ ├── app_packet.ex │ ├── bye_packet.ex │ ├── feedback_packet.ex │ ├── feedback_packet │ │ ├── afb.ex │ │ ├── fir.ex │ │ └── pli.ex │ ├── header.ex │ ├── packet.ex │ ├── parser.ex │ ├── receiver.ex │ ├── receiver_report │ │ ├── stats.ex │ │ ├── stats_event.ex │ │ └── stats_request_event.ex │ ├── receiver_report_packet.ex │ ├── report_packet_block.ex │ ├── sdes_packet.ex │ ├── sender_report_packet.ex │ ├── transport_feedback_packet.ex │ └── transport_feedback_packet │ │ ├── nack.ex │ │ └── twcc.ex │ ├── rtcp_event.ex │ ├── rtp │ ├── demuxer.ex │ ├── demuxer │ │ └── jitter_buffer.ex │ ├── depayloader_bin.ex │ ├── header.ex │ ├── header_extension.ex │ ├── header_generator.ex │ ├── inbound_packet_tracker.ex │ ├── jitter_buffer.ex │ ├── jitter_buffer │ │ ├── buffer_store.ex │ │ └── record.ex │ ├── metrics.ex │ ├── muxer.ex │ ├── outbound_rtx_controller.ex │ ├── outbound_tracking_serializer.ex │ ├── packet.ex │ ├── packet_payload_type.ex │ ├── packets_discarded_event.ex │ ├── parser.ex │ ├── payload_format_resolver.ex │ ├── payloader_bin.ex │ ├── retransmission_request_event.ex │ ├── rtx_parser.ex │ ├── sequence_number_tracker.ex │ ├── serializer │ │ └── stats.ex │ ├── session │ │ └── sender_report.ex │ ├── session_bin.ex │ ├── session_bin │ │ └── rtx_info.ex │ ├── silence_discarder.ex │ ├── ssrc_router.ex │ ├── ssrc_router │ │ └── streams_info.ex │ ├── stream_receive_bin.ex │ ├── stream_send_bin.ex │ ├── tcp_decapsulator.ex │ ├── tcp_encapsulator.ex │ ├── twcc_receiver.ex │ ├── twcc_receiver │ │ └── packet_info_store.ex │ ├── twcc_sender.ex │ ├── twcc_sender │ │ ├── congestion_control.ex │ │ └── receiver_rate.ex │ ├── utils.ex │ ├── vad.ex │ ├── vad │ │ ├── audio_level_queue.ex │ │ ├── is_speaking_estimator.ex │ │ └── vad_params.ex │ └── vad_event.ex │ ├── srtcp │ └── decryptor.ex │ └── srtp │ ├── decryptor.ex │ └── encryptor.ex ├── mix.exs ├── mix.lock └── test ├── fixtures ├── rtcp │ ├── malformed.hex │ ├── packets.hex │ ├── single_packet.hex │ ├── twcc_feedbacks.hex │ ├── twcc_malformed_feedbacks.hex │ └── with_unknown_pt.hex └── rtp │ ├── h264 │ └── bun.h264 │ ├── rtp_packet.bin │ ├── rtp_packet_payload.bin │ ├── rtp_packet_with_padding.bin │ └── session │ ├── demo.pcap │ ├── demo_audio_video_rtp3.pcap │ ├── demo_rtp.pcap │ ├── h264_before_sr.pcap │ ├── h264_before_sr2.pcap │ └── srtp.pcap ├── membrane ├── rtcp │ ├── afb_packet_test.exs │ ├── compound_packet_test.exs │ ├── nack_packet_test.exs │ ├── parser_test.exs │ └── twcc_test.exs └── rtp │ ├── demuxer_muxer_integration_test.exs │ ├── demuxer_test.exs │ ├── inbound_packet_tracker_test.exs │ ├── jitter_buffer │ ├── buffer_store_test.exs │ ├── pipeline_test.exs │ └── timestamps_calculation_test.exs │ ├── jitter_buffer_test.exs │ ├── muxer_demuxer_integration_test.exs │ ├── muxer_test.exs │ ├── outbound_rtx_controller_test.exs │ ├── packet_payload_type_test.exs │ ├── packet_test.exs │ ├── parser_test.exs │ ├── pipeline_test.exs │ ├── rtx_parser_test.exs │ ├── sender_report_test.exs │ ├── sequence_number_tracker_test.exs │ ├── session_bin_test.exs │ ├── ssrc_router_test.exs │ ├── stream_receive_bin_test.exs │ ├── stream_send_bin_test.exs │ ├── twcc_receiver │ └── packet_info_store_test.exs │ ├── twcc_receiver_test.exs │ ├── twcc_sender │ └── congestion_control_test.exs │ ├── utils_test.exs │ ├── vad │ ├── audio_level_queue_test.exs │ └── is_speaking_estimator_test.exs │ └── vad_test.exs ├── support ├── buffer_factory.ex ├── rtcp_fixtures.ex ├── rtp_fixtures.ex └── test_source.ex └── test_helper.exs /.circleci/config.yml: -------------------------------------------------------------------------------- 1 | version: 2.1 2 | orbs: 3 | elixir: membraneframework/elixir@1 4 | 5 | workflows: 6 | version: 2 7 | build: 8 | jobs: 9 | - elixir/build_test: 10 | cache-version: 2 11 | filters: &filters 12 | tags: 13 | only: /v.*/ 14 | - elixir/test: 15 | cache-version: 2 16 | filters: 17 | <<: *filters 18 | - elixir/lint: 19 | cache-version: 2 20 | filters: 21 | <<: *filters 22 | - elixir/hex_publish: 23 | cache-version: 2 24 | requires: 25 | - elixir/build_test 26 | - elixir/test 27 | - elixir/lint 28 | context: 29 | - Deployment 30 | filters: 31 | branches: 32 | ignore: /.*/ 33 | tags: 34 | only: /v.*/ 35 | -------------------------------------------------------------------------------- /.formatter.exs: -------------------------------------------------------------------------------- 1 | [ 2 | inputs: [ 3 | "{lib,test,config}/**/*.{ex,exs}", 4 | ".formatter.exs", 5 | "*.exs" 6 | ], 7 | import_deps: [:membrane_core] 8 | ] 9 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/please--open-new-issues-in-membranefranework-membrane_core.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Please, open new issues in membranefranework/membrane_core 3 | about: New issues related to this repo should be opened there 4 | title: "[DO NOT OPEN]" 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | Please, do not open this issue here. Open it in the [membrane_core](https://github.com/membraneframework/membrane_core) repository instead. 11 | 12 | Thanks for helping us grow :) 13 | -------------------------------------------------------------------------------- /.github/workflows/on_issue_opened.yaml: -------------------------------------------------------------------------------- 1 | name: 'Close issue when opened' 2 | on: 3 | issues: 4 | types: 5 | - opened 6 | jobs: 7 | close: 8 | runs-on: ubuntu-latest 9 | steps: 10 | - name: Checkout membrane_core 11 | uses: actions/checkout@v3 12 | with: 13 | repository: membraneframework/membrane_core 14 | - name: Close issue 15 | uses: ./.github/actions/close_issue 16 | with: 17 | GITHUB_TOKEN: ${{ secrets.MEMBRANEFRAMEWORKADMIN_TOKEN }} 18 | ISSUE_URL: ${{ github.event.issue.html_url }} 19 | ISSUE_NUMBER: ${{ github.event.issue.number }} 20 | REPOSITORY: ${{ github.repository }} 21 | -------------------------------------------------------------------------------- /.github/workflows/on_pr_opened.yaml: -------------------------------------------------------------------------------- 1 | name: Add PR to Smackore project board, if the author is from outside Membrane Team 2 | on: 3 | pull_request_target: 4 | types: 5 | - opened 6 | jobs: 7 | maybe_add_to_project_board: 8 | runs-on: ubuntu-latest 9 | steps: 10 | - name: Checkout membrane_core 11 | uses: actions/checkout@v3 12 | with: 13 | repository: membraneframework/membrane_core 14 | - name: Puts PR in "New PRs by community" column in the Smackore project, if the author is from outside Membrane Team 15 | uses: ./.github/actions/add_pr_to_smackore_board 16 | with: 17 | GITHUB_TOKEN: ${{ secrets.MEMBRANEFRAMEWORKADMIN_TOKEN }} 18 | AUTHOR_LOGIN: ${{ github.event.pull_request.user.login }} 19 | PR_URL: ${{ github.event.pull_request.html_url }} 20 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | compile_commands.json 2 | .gdb_history 3 | bundlex.sh 4 | bundlex.bat 5 | 6 | # Dir generated by tmp_dir ExUnit tag 7 | /tmp/ 8 | 9 | # Created by https://www.gitignore.io/api/c,vim,linux,macos,elixir,windows,visualstudiocode 10 | # Edit at https://www.gitignore.io/?templates=c,vim,linux,macos,elixir,windows,visualstudiocode 11 | 12 | ### C ### 13 | # Prerequisites 14 | *.d 15 | 16 | # Object files 17 | *.o 18 | *.ko 19 | *.obj 20 | *.elf 21 | 22 | # Linker output 23 | *.ilk 24 | *.map 25 | *.exp 26 | 27 | # Precompiled Headers 28 | *.gch 29 | *.pch 30 | 31 | # Libraries 32 | *.lib 33 | *.a 34 | *.la 35 | *.lo 36 | 37 | # Shared objects (inc. Windows DLLs) 38 | *.dll 39 | *.so 40 | *.so.* 41 | *.dylib 42 | 43 | # Executables 44 | *.exe 45 | *.out 46 | *.app 47 | *.i*86 48 | *.x86_64 49 | *.hex 50 | 51 | # Debug files 52 | *.dSYM/ 53 | *.su 54 | *.idb 55 | *.pdb 56 | 57 | # Kernel Module Compile Results 58 | *.mod* 59 | *.cmd 60 | .tmp_versions/ 61 | modules.order 62 | Module.symvers 63 | Mkfile.old 64 | dkms.conf 65 | 66 | ### Elixir ### 67 | /_build 68 | /cover 69 | /deps 70 | /doc 71 | /.fetch 72 | erl_crash.dump 73 | *.ez 74 | *.beam 75 | /config/*.secret.exs 76 | .elixir_ls/ 77 | 78 | ### Elixir Patch ### 79 | 80 | ### Linux ### 81 | *~ 82 | 83 | # temporary files which can be created if a process still has a handle open of a deleted file 84 | .fuse_hidden* 85 | 86 | # KDE directory preferences 87 | .directory 88 | 89 | # Linux trash folder which might appear on any partition or disk 90 | .Trash-* 91 | 92 | # .nfs files are created when an open file is removed but is still being accessed 93 | .nfs* 94 | 95 | ### macOS ### 96 | # General 97 | .DS_Store 98 | .AppleDouble 99 | .LSOverride 100 | 101 | # Icon must end with two \r 102 | Icon 103 | 104 | # Thumbnails 105 | ._* 106 | 107 | # Files that might appear in the root of a volume 108 | .DocumentRevisions-V100 109 | .fseventsd 110 | .Spotlight-V100 111 | .TemporaryItems 112 | .Trashes 113 | .VolumeIcon.icns 114 | .com.apple.timemachine.donotpresent 115 | 116 | # Directories potentially created on remote AFP share 117 | .AppleDB 118 | .AppleDesktop 119 | Network Trash Folder 120 | Temporary Items 121 | .apdisk 122 | 123 | ### Vim ### 124 | # Swap 125 | [._]*.s[a-v][a-z] 126 | [._]*.sw[a-p] 127 | [._]s[a-rt-v][a-z] 128 | [._]ss[a-gi-z] 129 | [._]sw[a-p] 130 | 131 | # Session 132 | Session.vim 133 | Sessionx.vim 134 | 135 | # Temporary 136 | .netrwhist 137 | # Auto-generated tag files 138 | tags 139 | # Persistent undo 140 | [._]*.un~ 141 | 142 | # Coc configuration directory 143 | .vim 144 | 145 | ### VisualStudioCode ### 146 | .vscode/* 147 | !.vscode/settings.json 148 | !.vscode/tasks.json 149 | !.vscode/launch.json 150 | !.vscode/extensions.json 151 | 152 | ### VisualStudioCode Patch ### 153 | # Ignore all local history of files 154 | .history 155 | 156 | ### Windows ### 157 | # Windows thumbnail cache files 158 | Thumbs.db 159 | Thumbs.db:encryptable 160 | ehthumbs.db 161 | ehthumbs_vista.db 162 | 163 | # Dump file 164 | *.stackdump 165 | 166 | # Folder config file 167 | [Dd]esktop.ini 168 | 169 | # Recycle Bin used on file shares 170 | $RECYCLE.BIN/ 171 | 172 | # Windows Installer files 173 | *.cab 174 | *.msi 175 | *.msix 176 | *.msm 177 | *.msp 178 | 179 | # Windows shortcuts 180 | *.lnk 181 | 182 | # End of https://www.gitignore.io/api/c,vim,linux,macos,elixir,windows,visualstudiocode 183 | 184 | !test/fixtures/**/*.hex 185 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Membrane RTP plugin 2 | 3 | [![Hex.pm](https://img.shields.io/hexpm/v/membrane_rtp_plugin.svg)](https://hex.pm/packages/membrane_rtp_plugin) 4 | [![API Docs](https://img.shields.io/badge/api-docs-yellow.svg?style=flat)](https://hexdocs.pm/membrane_rtp_plugin/) 5 | [![CircleCI](https://circleci.com/gh/membraneframework/membrane_rtp_plugin.svg?style=svg)](https://circleci.com/gh/membraneframework/membrane_rtp_plugin) 6 | 7 | This package provides bins and elements for sending and receiving RTP/SRTP and RTCP/SRTCP streams. 8 | 9 | It is a part of [Membrane Multimedia Framework](https://membraneframework.org). 10 | 11 | ## Installation 12 | 13 | The package can be installed by adding `membrane_rtp_plugin` to your list of dependencies in `mix.exs`: 14 | 15 | ```elixir 16 | def deps do 17 | [ 18 | {:membrane_rtp_plugin, "~> 0.31.0"}, 19 | {:ex_libsrtp, ">= 0.0.0"} # required only if SRTP/SRTCP support is needed 20 | ] 21 | end 22 | ``` 23 | 24 | If SRTP/SRTCP support is needed, one has to install `libsrtp` to their system. 25 | 26 | ### MacOS 27 | 28 | Run `brew install srtp` 29 | 30 | ### Ubuntu 31 | 32 | Run `apt install libsrtp2-dev` 33 | 34 | ### Other 35 | 36 | For more details and manual installation, see [ExLibSRTP HexDocs](https://hexdocs.pm/ex_libsrtp/readme.html). 37 | 38 | ## Usage 39 | 40 | For usage examples, check the [RTP demo](https://github.com/membraneframework/membrane_demo/tree/master/rtp). 41 | 42 | The docs can be found at [HexDocs](https://hexdocs.pm/membrane_rtp_plugin). 43 | 44 | ## Copyright and License 45 | 46 | Copyright 2020, [Software Mansion](https://swmansion.com/?utm_source=git&utm_medium=readme&utm_campaign=membrane_rtp_plugin) 47 | 48 | [![Software Mansion](https://logo.swmansion.com/logo?color=white&variant=desktop&width=200&tag=membrane-github)](https://swmansion.com/?utm_source=git&utm_medium=readme&utm_campaign=membrane_rtp_plugin) 49 | 50 | Licensed under the [Apache License, Version 2.0](LICENSE) 51 | -------------------------------------------------------------------------------- /config/config.exs: -------------------------------------------------------------------------------- 1 | import Config 2 | 3 | config :membrane_rtp_plugin, 4 | fir_throttle_duration_ms: 500, 5 | vad_estimation_parameters: %{ 6 | immediate: %{ 7 | subunits: 1, 8 | score_threshold: 0, 9 | lambda: 1 10 | }, 11 | medium: %{ 12 | subunits: 10, 13 | score_threshold: 20, 14 | subunit_threshold: 1, 15 | lambda: 24 16 | }, 17 | long: %{ 18 | subunits: 7, 19 | score_threshold: 20, 20 | subunit_threshold: 3, 21 | lambda: 47 22 | } 23 | } 24 | 25 | import_config "#{config_env()}.exs" 26 | -------------------------------------------------------------------------------- /config/dev.exs: -------------------------------------------------------------------------------- 1 | import Config 2 | 3 | # place for dev compile time env variables 4 | -------------------------------------------------------------------------------- /config/prod.exs: -------------------------------------------------------------------------------- 1 | import Config 2 | 3 | # place for production compile time env variables 4 | -------------------------------------------------------------------------------- /config/test.exs: -------------------------------------------------------------------------------- 1 | import Config 2 | 3 | config :membrane_rtp_plugin, 4 | vad_estimation_parameters: %{ 5 | immediate: %{ 6 | subunits: 2, 7 | score_threshold: 0.1, 8 | lambda: 1 9 | }, 10 | medium: %{ 11 | subunits: 2, 12 | score_threshold: 0.1, 13 | subunit_threshold: 2, 14 | lambda: 1 15 | }, 16 | long: %{ 17 | subunits: 2, 18 | score_threshold: 0.1, 19 | subunit_threshold: 1, 20 | lambda: 1 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /lib/membrane/rtcp/app_packet.ex: -------------------------------------------------------------------------------- 1 | defmodule Membrane.RTCP.AppPacket do 2 | @moduledoc """ 3 | Parses RTCP Application-defined (APP) packets 4 | defined in [RFC3550](https://tools.ietf.org/html/rfc3550#section-6.7) 5 | """ 6 | 7 | @behaviour Membrane.RTCP.Packet 8 | 9 | defstruct [:subtype, :ssrc, :name, :data] 10 | 11 | @type t :: %__MODULE__{ 12 | subtype: non_neg_integer(), 13 | ssrc: non_neg_integer(), 14 | name: String.t(), 15 | data: binary() 16 | } 17 | 18 | @impl true 19 | def decode(<>, subtype) do 20 | {:ok, 21 | %__MODULE__{ 22 | subtype: subtype, 23 | ssrc: ssrc, 24 | name: name, 25 | data: data 26 | }} 27 | end 28 | 29 | @impl true 30 | def encode(%__MODULE__{ 31 | subtype: subtype, 32 | ssrc: ssrc, 33 | name: name, 34 | data: data 35 | }) do 36 | {<>, subtype} 37 | end 38 | end 39 | -------------------------------------------------------------------------------- /lib/membrane/rtcp/bye_packet.ex: -------------------------------------------------------------------------------- 1 | defmodule Membrane.RTCP.ByePacket do 2 | @moduledoc """ 3 | Parses and constructs RTCP BYE packets defined in 4 | [RFC3550](https://tools.ietf.org/html/rfc3550#section-6.6) 5 | """ 6 | 7 | @behaviour Membrane.RTCP.Packet 8 | 9 | @type t :: %__MODULE__{ 10 | ssrcs: [non_neg_integer()], 11 | reason: String.t() | nil 12 | } 13 | 14 | defstruct [:ssrcs, :reason] 15 | 16 | @impl true 17 | def decode(packet, count) do 18 | ssrcs_size = count * 4 19 | <> = packet 20 | 21 | ssrcs = 22 | ssrcs 23 | |> Bunch.Binary.chunk_every(4) 24 | |> Enum.map(&:binary.decode_unsigned(&1)) 25 | 26 | result = %__MODULE__{ssrcs: ssrcs, reason: make_reason(reason)} 27 | {:ok, result} 28 | end 29 | 30 | @impl true 31 | def encode(%{ssrcs: ssrcs, reason: reason}) do 32 | count = ssrcs |> length() 33 | ssrcs = ssrcs |> Enum.map_join(&<<&1::32>>) 34 | 35 | reason = 36 | case reason do 37 | nil -> 38 | <<>> 39 | 40 | other -> 41 | length = String.length(other) 42 | <> 43 | end 44 | 45 | {ssrcs <> reason, count} 46 | end 47 | 48 | defp make_reason(<<>>), do: nil 49 | defp make_reason(<<_length::8, reason::binary>>), do: reason 50 | end 51 | -------------------------------------------------------------------------------- /lib/membrane/rtcp/feedback_packet.ex: -------------------------------------------------------------------------------- 1 | defmodule Membrane.RTCP.FeedbackPacket do 2 | @moduledoc """ 3 | Abstraction and generic encoding/decoding functionality for 4 | [RTCP payload-specific feedback packets](https://datatracker.ietf.org/doc/html/rfc5104#section-4.3). 5 | """ 6 | 7 | @behaviour Membrane.RTCP.Packet 8 | 9 | alias Membrane.RTP 10 | 11 | @enforce_keys [:origin_ssrc, :payload] 12 | defstruct @enforce_keys ++ [target_ssrc: 0] 13 | 14 | @type t :: %__MODULE__{ 15 | origin_ssrc: RTP.ssrc(), 16 | target_ssrc: RTP.ssrc(), 17 | payload: struct 18 | } 19 | 20 | @callback decode(binary()) :: {:ok, struct()} | {:error, any()} 21 | 22 | @callback encode(struct()) :: binary() 23 | 24 | @packet_type_payload BiMap.new(%{ 25 | 1 => __MODULE__.PLI, 26 | 4 => __MODULE__.FIR, 27 | 15 => __MODULE__.AFB 28 | }) 29 | 30 | @impl true 31 | def decode(<>, packet_type) do 32 | with {:ok, module} <- BiMap.fetch(@packet_type_payload, packet_type), 33 | {:ok, payload} <- module.decode(payload) do 34 | {:ok, %__MODULE__{origin_ssrc: origin_ssrc, target_ssrc: target_ssrc, payload: payload}} 35 | else 36 | :error -> {:error, {:unknown_feedback_packet_type, packet_type}} 37 | {:error, reason} -> {:error, reason} 38 | end 39 | end 40 | 41 | @impl true 42 | def decode(_binary, _packet_type) do 43 | {:error, :malformed_packet} 44 | end 45 | 46 | @impl true 47 | def encode(%__MODULE__{} = packet) do 48 | %module{} = packet.payload 49 | packet_type = BiMap.fetch_key!(@packet_type_payload, module) 50 | 51 | {<>, 52 | packet_type} 53 | end 54 | end 55 | -------------------------------------------------------------------------------- /lib/membrane/rtcp/feedback_packet/afb.ex: -------------------------------------------------------------------------------- 1 | defmodule Membrane.RTCP.FeedbackPacket.AFB do 2 | @moduledoc """ 3 | [Application Layer Feedback](https://datatracker.ietf.org/doc/html/rfc4585#section-6.4) packets. 4 | 5 | They use PT=PSFB (206) & FMT=15. 6 | Since the message must be handled at the application layer, the struct simply wraps a binary content of message 7 | """ 8 | 9 | @behaviour Membrane.RTCP.FeedbackPacket 10 | 11 | @enforce_keys [:message] 12 | defstruct @enforce_keys 13 | 14 | @impl true 15 | def decode(binary) do 16 | {:ok, %__MODULE__{message: binary}} 17 | end 18 | 19 | @impl true 20 | def encode(%__MODULE__{message: message}) when is_binary(message) do 21 | message 22 | end 23 | end 24 | -------------------------------------------------------------------------------- /lib/membrane/rtcp/feedback_packet/fir.ex: -------------------------------------------------------------------------------- 1 | defmodule Membrane.RTCP.FeedbackPacket.FIR do 2 | @moduledoc """ 3 | Encodes and decodes [Full Intra Request](https://datatracker.ietf.org/doc/html/rfc5104#section-4.3.1) packets. 4 | """ 5 | @behaviour Membrane.RTCP.FeedbackPacket 6 | 7 | @enforce_keys [:target_ssrc, :seq_num] 8 | defstruct @enforce_keys 9 | 10 | @impl true 11 | def encode(%__MODULE__{} = packet) do 12 | <> 13 | end 14 | 15 | @impl true 16 | def encode(packets) when is_list(packets) do 17 | Enum.map_join(packets, &encode/1) 18 | end 19 | 20 | @impl true 21 | def decode(binary) do 22 | do_decode(binary, []) 23 | end 24 | 25 | defp do_decode(<<>>, acc) do 26 | {:ok, Enum.reverse(acc)} 27 | end 28 | 29 | defp do_decode(<>, acc) do 30 | do_decode(rest, [%__MODULE__{target_ssrc: target_ssrc, seq_num: seq_num} | acc]) 31 | end 32 | 33 | defp do_decode(_binary, _acc) do 34 | {:error, :malformed_packet} 35 | end 36 | end 37 | -------------------------------------------------------------------------------- /lib/membrane/rtcp/feedback_packet/pli.ex: -------------------------------------------------------------------------------- 1 | defmodule Membrane.RTCP.FeedbackPacket.PLI do 2 | @moduledoc """ 3 | Encodes and decodes [Picture Loss Indication](https://datatracker.ietf.org/doc/html/rfc4585#section-6.3.1) packets. 4 | """ 5 | 6 | @behaviour Membrane.RTCP.FeedbackPacket 7 | 8 | defstruct [] 9 | 10 | @impl true 11 | def decode(_binary) do 12 | {:ok, %__MODULE__{}} 13 | end 14 | 15 | @impl true 16 | def encode(_packet) do 17 | <<>> 18 | end 19 | end 20 | -------------------------------------------------------------------------------- /lib/membrane/rtcp/header.ex: -------------------------------------------------------------------------------- 1 | defmodule Membrane.RTCP.Header do 2 | @moduledoc """ 3 | Struct describing 32-bit header common to all RTCP packets 4 | """ 5 | 6 | @enforce_keys [:packet_type, :packet_specific] 7 | defstruct @enforce_keys 8 | 9 | @type packet_type_t :: 200..206 10 | 11 | @type packet_specific_t :: non_neg_integer() 12 | 13 | @type t :: %__MODULE__{ 14 | packet_specific: packet_specific_t(), 15 | packet_type: packet_type_t() 16 | } 17 | 18 | @spec parse(binary()) :: 19 | {:ok, %{header: t(), padding?: boolean, body_size: pos_integer}} 20 | | :error 21 | def parse(<<2::2, padding::1, packet_specific::5, pt::8, length::16>>) do 22 | {:ok, 23 | %{ 24 | header: %__MODULE__{ 25 | packet_specific: packet_specific, 26 | packet_type: pt 27 | }, 28 | padding?: padding == 1, 29 | body_size: length * 4 30 | }} 31 | end 32 | 33 | def parse(_binary) do 34 | :error 35 | end 36 | 37 | @spec serialize(t(), body_size: pos_integer(), padding?: boolean()) :: binary() 38 | def serialize(%__MODULE__{} = header, opts) do 39 | padding = if Keyword.get(opts, :padding?), do: 1, else: 0 40 | size = Keyword.fetch!(opts, :body_size) 41 | 42 | unless rem(size, 4) == 0 do 43 | raise "RTCP packet body size must be divisible by 4, got: #{size}" 44 | end 45 | 46 | <<2::2, padding::1, header.packet_specific::5, header.packet_type::8, div(size, 4)::16>> 47 | end 48 | end 49 | -------------------------------------------------------------------------------- /lib/membrane/rtcp/packet.ex: -------------------------------------------------------------------------------- 1 | defmodule Membrane.RTCP.Packet do 2 | @moduledoc """ 3 | Functions common to all RTCP Packets 4 | """ 5 | require Membrane.Logger 6 | 7 | alias Membrane.RTCP.{ 8 | AppPacket, 9 | ByePacket, 10 | FeedbackPacket, 11 | Header, 12 | ReceiverReportPacket, 13 | SdesPacket, 14 | SenderReportPacket, 15 | TransportFeedbackPacket 16 | } 17 | 18 | alias Membrane.RTP 19 | 20 | @type t :: 21 | AppPacket.t() 22 | | ByePacket.t() 23 | | FeedbackPacket.t() 24 | | TransportFeedbackPacket.t() 25 | | ReceiverReportPacket.t() 26 | | SenderReportPacket.t() 27 | | SdesPacket.t() 28 | 29 | @doc """ 30 | Decodes binary with packet body (without header) into packet struct. Used by `parse/1` 31 | """ 32 | @callback decode(binary(), packet_specific :: Header.packet_specific_t()) :: 33 | {:ok, struct()} | {:error, atom()} 34 | 35 | @doc """ 36 | Encodes packet struct into the tuple used by `serialize/1` 37 | """ 38 | @callback encode(struct()) :: {body :: binary(), packet_specific :: Header.packet_specific_t()} 39 | 40 | @packet_type_module BiMap.new(%{ 41 | 200 => SenderReportPacket, 42 | 201 => ReceiverReportPacket, 43 | 202 => SdesPacket, 44 | 203 => ByePacket, 45 | 204 => AppPacket, 46 | 205 => TransportFeedbackPacket, 47 | 206 => FeedbackPacket 48 | }) 49 | 50 | @doc """ 51 | Converts packet structure into binary 52 | """ 53 | @spec serialize(t() | [t()]) :: binary() 54 | def serialize(%packet_module{} = packet) do 55 | {body, packet_specific} = packet_module.encode(packet) 56 | packet_type = BiMap.fetch_key!(@packet_type_module, packet_module) 57 | 58 | header = 59 | %Header{packet_type: packet_type, packet_specific: packet_specific} 60 | |> Header.serialize(body_size: byte_size(body)) 61 | 62 | header <> body 63 | end 64 | 65 | def serialize(packets) when is_list(packets) do 66 | Enum.map_join(packets, &serialize/1) 67 | end 68 | 69 | @spec parse(binary()) :: {:ok, [t()]} | {:error, :malformed_packet} 70 | def parse(packets) do 71 | do_parse(packets, []) 72 | end 73 | 74 | defp do_parse(<<>>, acc), do: {:ok, Enum.reverse(acc)} 75 | 76 | defp do_parse(<>, acc) do 77 | with {:ok, %{header: header, body_size: length, padding?: padding?}} <- 78 | Header.parse(raw_header), 79 | <> <- body_and_rest, 80 | {:ok, {body, _padding}} <- RTP.Utils.strip_padding(body, padding?) do 81 | case parse_packet(body, header) do 82 | {:ok, packet} -> 83 | do_parse(rest, [packet | acc]) 84 | 85 | {:error, :unknown_packet_type} -> 86 | Membrane.Logger.debug(""" 87 | Ignoring rtcp packet with packet type #{header.packet_type}: 88 | #{inspect(raw_header <> body_and_rest, limit: :infinity)} 89 | Reason: :unknown_packet_type 90 | """) 91 | 92 | do_parse(rest, acc) 93 | 94 | error -> 95 | error 96 | end 97 | else 98 | _error -> {:error, :malformed_packet} 99 | end 100 | end 101 | 102 | defp do_parse(_binary, _acc), do: {:error, :malformed_packet} 103 | 104 | defp parse_packet(body, %Header{} = header) do 105 | case BiMap.fetch(@packet_type_module, header.packet_type) do 106 | {:ok, packet_module} -> 107 | packet_module.decode(body, header.packet_specific) 108 | 109 | :error -> 110 | {:error, :unknown_packet_type} 111 | end 112 | end 113 | end 114 | -------------------------------------------------------------------------------- /lib/membrane/rtcp/parser.ex: -------------------------------------------------------------------------------- 1 | defmodule Membrane.RTCP.Parser do 2 | @moduledoc """ 3 | Element responsible for receiving raw RTCP packets, parsing them and emitting proper RTCP events. 4 | """ 5 | 6 | use Membrane.Filter, flow_control_hints?: false 7 | 8 | require Membrane.Logger 9 | alias Membrane.Buffer 10 | alias Membrane.{RemoteStream, RTCP, RTCPEvent} 11 | 12 | def_input_pad :input, 13 | accepted_format: %RemoteStream{type: :packetized, content_format: cf} when cf in [nil, RTCP], 14 | flow_control: :auto 15 | 16 | def_output_pad :output, accepted_format: RTCP, flow_control: :auto 17 | 18 | def_output_pad :receiver_report_output, 19 | flow_control: :push, 20 | accepted_format: %RemoteStream{type: :packetized, content_format: RTCP} 21 | 22 | @impl true 23 | def handle_init(_ctx, _opts) do 24 | {[], %{}} 25 | end 26 | 27 | @impl true 28 | def handle_playing(_ctx, state) do 29 | stream_format = %RemoteStream{type: :packetized, content_format: RTCP} 30 | {[stream_format: {:receiver_report_output, stream_format}], state} 31 | end 32 | 33 | @impl true 34 | def handle_stream_format(:input, _stream_format, _ctx, state) do 35 | {[stream_format: {:output, %RTCP{}}], state} 36 | end 37 | 38 | @impl true 39 | def handle_buffer(:input, %Buffer{payload: payload, metadata: metadata}, _ctx, state) do 40 | payload 41 | |> RTCP.Packet.parse() 42 | |> case do 43 | {:ok, packets} -> 44 | actions = process_packets(packets, metadata) 45 | {actions, state} 46 | 47 | {:error, reason} -> 48 | Membrane.Logger.warning(""" 49 | Couldn't parse rtcp packet: 50 | #{inspect(payload, limit: :infinity)} 51 | Reason: #{inspect(reason)}. Ignoring packet. 52 | """) 53 | 54 | {[], state} 55 | end 56 | end 57 | 58 | @impl true 59 | def handle_event(:output, %RTCPEvent{} = event, _ctx, state) do 60 | buffer = %Buffer{payload: RTCP.Packet.serialize(event.rtcp)} 61 | 62 | {[buffer: {:receiver_report_output, buffer}], state} 63 | end 64 | 65 | @impl true 66 | def handle_event(pad, event, ctx, state), do: super(pad, event, ctx, state) 67 | 68 | defp process_packets(rtcp, metadata) do 69 | Enum.flat_map(rtcp, &process_rtcp(&1, metadata)) 70 | end 71 | 72 | defp process_rtcp(%RTCP.FeedbackPacket{payload: %keyframe_request{}} = packet, metadata) 73 | when keyframe_request in [RTCP.FeedbackPacket.FIR, RTCP.FeedbackPacket.PLI] do 74 | event = to_rtcp_event(packet, packet.target_ssrc, metadata) 75 | [event: {:output, event}] 76 | end 77 | 78 | defp process_rtcp( 79 | %RTCP.TransportFeedbackPacket{ 80 | media_ssrc: ssrc, 81 | payload: %RTCP.TransportFeedbackPacket.NACK{lost_packet_ids: lost_packets} 82 | } = packet, 83 | metadata 84 | ) do 85 | Membrane.Logger.debug( 86 | "SSRC #{ssrc} reported loss of #{length(lost_packets)} packet(s): #{inspect(lost_packets)}" 87 | ) 88 | 89 | event = to_rtcp_event(packet, ssrc, metadata) 90 | [event: {:output, event}] 91 | end 92 | 93 | defp process_rtcp( 94 | %RTCP.TransportFeedbackPacket{payload: %RTCP.TransportFeedbackPacket.TWCC{} = feedback}, 95 | _metadata 96 | ) do 97 | [notify_parent: {:twcc_feedback, feedback}] 98 | end 99 | 100 | defp process_rtcp(%RTCP.SenderReportPacket{ssrc: ssrc} = packet, metadata) do 101 | event = to_rtcp_event(packet, ssrc, metadata) 102 | [event: {:output, event}] 103 | end 104 | 105 | defp process_rtcp(%RTCP.ReceiverReportPacket{reports: reports}, metadata) do 106 | reports 107 | |> Enum.map(fn report -> 108 | event = to_rtcp_event(report, report.ssrc, metadata) 109 | {:event, {:output, event}} 110 | end) 111 | end 112 | 113 | defp process_rtcp( 114 | %RTCP.FeedbackPacket{ 115 | payload: %RTCP.FeedbackPacket.AFB{message: "REMB" <> _remb_data} 116 | }, 117 | _metadata 118 | ) do 119 | # maybe TODO: handle REMB extension 120 | # Even though we do not support REMB and do not advertise such support in SDP, 121 | # browsers ignore that and send REMB packets for video as part of sender report ¯\_(ツ)_/¯ 122 | [] 123 | end 124 | 125 | defp process_rtcp(%RTCP.ByePacket{ssrcs: ssrcs}, _metadata) do 126 | Membrane.Logger.debug("SSRCs #{inspect(ssrcs)} are leaving (received RTCP Bye)") 127 | [] 128 | end 129 | 130 | defp process_rtcp(%RTCP.SdesPacket{}, _metadata) do 131 | # We don't care about SdesPacket, usually included in compound packet with SenderReportPacket or ReceiverReportPacket 132 | [] 133 | end 134 | 135 | defp process_rtcp(unknown_packet, metadata) do 136 | Membrane.Logger.warning(""" 137 | Unhandled RTCP packet 138 | #{inspect(unknown_packet, pretty: true, limit: :infinity)} 139 | #{inspect(metadata, pretty: true)} 140 | """) 141 | 142 | [] 143 | end 144 | 145 | defp to_rtcp_event(rtcp_packet, ssrc, metadata) do 146 | %RTCPEvent{ 147 | rtcp: rtcp_packet, 148 | ssrcs: [ssrc], 149 | arrival_timestamp: Map.get(metadata, :arrival_ts, Membrane.Time.vm_time()) 150 | } 151 | end 152 | end 153 | -------------------------------------------------------------------------------- /lib/membrane/rtcp/receiver_report/stats.ex: -------------------------------------------------------------------------------- 1 | defmodule Membrane.RTCP.ReceiverReport.Stats do 2 | @moduledoc """ 3 | JitterBuffer stats that can be used for Receiver report generation 4 | """ 5 | 6 | @enforce_keys [:fraction_lost, :total_lost, :highest_seq_num, :interarrival_jitter] 7 | 8 | defstruct @enforce_keys 9 | 10 | @type t :: 11 | %__MODULE__{ 12 | fraction_lost: float(), 13 | total_lost: non_neg_integer(), 14 | highest_seq_num: Membrane.RTP.JitterBuffer.packet_index(), 15 | interarrival_jitter: non_neg_integer() 16 | } 17 | | :no_stats 18 | end 19 | -------------------------------------------------------------------------------- /lib/membrane/rtcp/receiver_report/stats_event.ex: -------------------------------------------------------------------------------- 1 | defmodule Membrane.RTCP.ReceiverReport.StatsEvent do 2 | @moduledoc """ 3 | Event carrying statistics for a receiver report. 4 | """ 5 | 6 | @derive Membrane.EventProtocol 7 | @enforce_keys [:stats] 8 | defstruct @enforce_keys 9 | 10 | @type t :: %__MODULE__{ 11 | stats: Membrane.RTCP.ReceiverReport.Stats.t() 12 | } 13 | end 14 | -------------------------------------------------------------------------------- /lib/membrane/rtcp/receiver_report/stats_request_event.ex: -------------------------------------------------------------------------------- 1 | defmodule Membrane.RTCP.ReceiverReport.StatsRequestEvent do 2 | @moduledoc """ 3 | Event to be sent to jitter buffer to request statistics. 4 | """ 5 | @derive Membrane.EventProtocol 6 | defstruct [] 7 | @type t :: %__MODULE__{} 8 | end 9 | -------------------------------------------------------------------------------- /lib/membrane/rtcp/receiver_report_packet.ex: -------------------------------------------------------------------------------- 1 | defmodule Membrane.RTCP.ReceiverReportPacket do 2 | @moduledoc """ 3 | Parses and constructs RTCP Receiver Report defined in 4 | [RFC3550](https://tools.ietf.org/html/rfc3550#section-6.4.2) 5 | """ 6 | 7 | @behaviour Membrane.RTCP.Packet 8 | 9 | alias Membrane.RTCP.ReportPacketBlock 10 | 11 | defstruct [:ssrc, :reports] 12 | 13 | @type t :: %__MODULE__{ 14 | ssrc: non_neg_integer(), 15 | reports: [ReportPacketBlock.t()] 16 | } 17 | 18 | @impl true 19 | def encode(report) do 20 | blocks = report.reports |> Enum.map_join(&ReportPacketBlock.encode/1) 21 | body = <> <> blocks 22 | reports_count = report.reports |> length() 23 | {body, reports_count} 24 | end 25 | 26 | @impl true 27 | def decode(<>, reports_count) do 28 | with {:ok, reports} <- ReportPacketBlock.decode(blocks), 29 | true <- reports_count == length(reports) do 30 | {:ok, %__MODULE__{ssrc: ssrc, reports: reports}} 31 | else 32 | false -> {:error, :invalid_reports_count} 33 | err -> err 34 | end 35 | end 36 | end 37 | -------------------------------------------------------------------------------- /lib/membrane/rtcp/report_packet_block.ex: -------------------------------------------------------------------------------- 1 | defmodule Membrane.RTCP.ReportPacketBlock do 2 | @moduledoc """ 3 | Parses and constructs report blocks, which are parts of RTCP Sender and Receiver Reports 4 | defined in [RFC3550](https://tools.ietf.org/html/rfc3550#section-6.4) 5 | """ 6 | 7 | @enforce_keys [ 8 | :ssrc, 9 | :fraction_lost, 10 | :total_lost, 11 | :highest_seq_num, 12 | :interarrival_jitter, 13 | :last_sr_timestamp, 14 | :delay_since_sr 15 | ] 16 | 17 | defstruct @enforce_keys 18 | 19 | @type t() :: %__MODULE__{ 20 | ssrc: non_neg_integer(), 21 | fraction_lost: float(), 22 | total_lost: non_neg_integer(), 23 | highest_seq_num: non_neg_integer(), 24 | interarrival_jitter: non_neg_integer(), 25 | last_sr_timestamp: non_neg_integer(), 26 | delay_since_sr: non_neg_integer() 27 | } 28 | 29 | @spec encode(t()) :: binary() 30 | def encode(block) do 31 | %{ 32 | ssrc: ssrc, 33 | fraction_lost: fraction_lost, 34 | total_lost: total_lost, 35 | highest_seq_num: max_seq_num, 36 | interarrival_jitter: jitter, 37 | last_sr_timestamp: last_sr_timestamp, 38 | delay_since_sr: delay_since_sr 39 | } = block 40 | 41 | fixed_point_fraction = round(fraction_lost * 256) 42 | 43 | <> 45 | end 46 | 47 | @spec decode(binary()) :: {:ok, [t()]} | {:error, :invalid_report_block} 48 | def decode(blocks), do: decode(blocks, []) 49 | 50 | defp decode(<<>>, acc), do: {:ok, acc} 51 | 52 | defp decode( 53 | <>, 55 | acc 56 | ) do 57 | data = %__MODULE__{ 58 | ssrc: ssrc, 59 | fraction_lost: fraction_lost / 256, 60 | total_lost: total_lost, 61 | highest_seq_num: max_seq_num, 62 | interarrival_jitter: jitter, 63 | last_sr_timestamp: last_sr_timestamp, 64 | delay_since_sr: delay_since_sr 65 | } 66 | 67 | decode(rest, [data | acc]) 68 | end 69 | 70 | defp decode(_blocks, _acc), do: {:error, :invalid_report_block} 71 | end 72 | -------------------------------------------------------------------------------- /lib/membrane/rtcp/sdes_packet.ex: -------------------------------------------------------------------------------- 1 | defmodule Membrane.RTCP.SdesPacket do 2 | @moduledoc """ 3 | Parses Source Description (SDES) RTCP Packets defined in 4 | [RFC3550](https://tools.ietf.org/html/rfc3550#section-6.5) 5 | """ 6 | 7 | @behaviour Membrane.RTCP.Packet 8 | 9 | defmodule Chunk do 10 | @moduledoc false 11 | 12 | @type t :: %__MODULE__{ 13 | cname: String.t() | nil, 14 | name: String.t() | nil, 15 | email: String.t() | nil, 16 | phone: String.t() | nil, 17 | loc: String.t() | nil, 18 | tool: String.t() | nil, 19 | note: String.t() | nil, 20 | priv: {String.t(), String.t()} | nil 21 | } 22 | defstruct [ 23 | :cname, 24 | :name, 25 | :email, 26 | :phone, 27 | :loc, 28 | :tool, 29 | :note, 30 | :priv 31 | ] 32 | end 33 | 34 | defstruct chunks: [] 35 | 36 | @type t() :: %__MODULE__{ 37 | chunks: %{ 38 | required(Membrane.RTP.ssrc()) => Keyword.t() 39 | } 40 | } 41 | 42 | @sdes_type_atom %{ 43 | 1 => :cname, 44 | 2 => :name, 45 | 3 => :email, 46 | 4 => :phone, 47 | 5 => :loc, 48 | 6 => :tool, 49 | 7 => :note, 50 | 8 => :priv 51 | } 52 | 53 | @sdes_type_from_atom %{ 54 | :cname => 1, 55 | :name => 2, 56 | :email => 3, 57 | :phone => 4, 58 | :loc => 5, 59 | :tool => 6, 60 | :note => 7, 61 | :priv => 8 62 | } 63 | 64 | @impl true 65 | def decode(packet, ssrc_count) do 66 | with {:ok, chunks} <- parse_chunks(packet, []), 67 | true <- ssrc_count == length(chunks) do 68 | chunks = 69 | chunks |> Enum.into(%{}, fn {ssrc, items} -> {ssrc, struct(__MODULE__.Chunk, items)} end) 70 | 71 | {:ok, %__MODULE__{chunks: chunks}} 72 | else 73 | {:error, _reason} = err -> err 74 | false -> {:error, :invalid_ssrc_count} 75 | end 76 | end 77 | 78 | defp parse_chunks(<<>>, acc), do: {:ok, acc} 79 | 80 | defp parse_chunks(<>, acc) do 81 | with {:ok, items, rest} <- parse_items(rest) do 82 | parse_chunks(rest, [{ssrc, items} | acc]) 83 | end 84 | end 85 | 86 | defp parse_items(sdes_item, acc \\ []) 87 | 88 | # null item denoting end of the list 89 | defp parse_items(<<0::8, rest::binary>>, acc) do 90 | # skip padding unitil next 32-bit boundary 91 | to_skip = rest |> bit_size |> rem(32) 92 | <<_skipped::size(to_skip), next_chunk::binary>> = rest 93 | {:ok, acc, next_chunk} 94 | end 95 | 96 | # PRIV item 97 | defp parse_items(<<8::8, length::8, content::binary-size(length), rest::binary>>, acc) do 98 | <> = content 99 | item = {@sdes_type_atom[8], {prefix, value}} 100 | parse_items(rest, [item | acc]) 101 | end 102 | 103 | defp parse_items(<>, acc) 104 | when si_type in 0..7 do 105 | item = {@sdes_type_atom[si_type], value} 106 | parse_items(rest, [item | acc]) 107 | end 108 | 109 | defp parse_items(<<_si_type::8, _payload::binary>>, _acc) do 110 | {:error, :unknown_si_type} 111 | end 112 | 113 | @impl true 114 | def encode(%__MODULE__{chunks: chunks}) do 115 | chunk_list = chunks |> Enum.to_list() 116 | 117 | body = 118 | chunk_list 119 | |> Enum.map_join(fn {ssrc, chunk} -> 120 | <> 121 | end) 122 | 123 | {body, length(chunk_list)} 124 | end 125 | 126 | @spec encode_chunk(chunk :: Chunk.t()) :: binary() 127 | defp encode_chunk(chunk) do 128 | body = 129 | chunk 130 | |> Map.from_struct() 131 | |> Enum.map_join(fn 132 | {_key, nil} -> 133 | <<>> 134 | 135 | {:priv, {prefix, value}} -> 136 | prefix_len = byte_size(prefix) 137 | total_len = 1 + prefix_len + byte_size(value) 138 | <<8::8, total_len::8, prefix_len::8, prefix::binary, value::binary>> 139 | 140 | {key, value} -> 141 | <<@sdes_type_from_atom[key]::8, byte_size(value)::8, value::binary>> 142 | end) 143 | 144 | pad_bits = 32 - (body |> bit_size() |> rem(32)) 145 | end_marker = <<0::size(pad_bits)>> 146 | 147 | body <> end_marker 148 | end 149 | end 150 | -------------------------------------------------------------------------------- /lib/membrane/rtcp/sender_report_packet.ex: -------------------------------------------------------------------------------- 1 | defmodule Membrane.RTCP.SenderReportPacket do 2 | @moduledoc """ 3 | Parses and constructs RTCP Sender Report defined in 4 | [RFC3550](https://tools.ietf.org/html/rfc3550#section-6.4.1) 5 | """ 6 | 7 | @behaviour Membrane.RTCP.Packet 8 | 9 | alias Membrane.RTCP.ReportPacketBlock 10 | alias Membrane.RTP 11 | 12 | defstruct [:ssrc, :reports, :sender_info] 13 | 14 | @type sender_info_t :: %{ 15 | wallclock_timestamp: Membrane.Time.t(), 16 | rtp_timestamp: non_neg_integer(), 17 | sender_packet_count: non_neg_integer(), 18 | sender_octet_count: non_neg_integer() 19 | } 20 | 21 | @type t :: %__MODULE__{ 22 | ssrc: RTP.ssrc(), 23 | reports: [ReportPacketBlock.t()], 24 | sender_info: sender_info_t() 25 | } 26 | 27 | @impl true 28 | def encode(report) do 29 | sender_info = encode_sender_info(report.sender_info) 30 | blocks = report.reports |> Enum.map_join(&ReportPacketBlock.encode/1) 31 | 32 | reports_count = report.reports |> length() 33 | 34 | body = <> <> sender_info <> blocks 35 | 36 | {body, reports_count} 37 | end 38 | 39 | defp encode_sender_info(sender_info) do 40 | ntp_timestamp = Membrane.Time.to_ntp_timestamp(sender_info.wallclock_timestamp) 41 | 42 | <> 44 | end 45 | 46 | @impl true 47 | def decode( 48 | <>, 50 | reports_count 51 | ) do 52 | wallclock_ts = Membrane.Time.from_ntp_timestamp(ntp_timestamp) 53 | 54 | sender_info = %{ 55 | wallclock_timestamp: wallclock_ts, 56 | rtp_timestamp: rtp_time, 57 | sender_packet_count: packet_count, 58 | sender_octet_count: octet_count 59 | } 60 | 61 | with {:ok, reports} <- ReportPacketBlock.decode(blocks), 62 | true <- reports_count == length(reports) do 63 | {:ok, %__MODULE__{ssrc: ssrc, reports: reports, sender_info: sender_info}} 64 | else 65 | false -> {:error, :invalid_reports_count} 66 | err -> err 67 | end 68 | end 69 | 70 | def decode(_packet, _reports_count) do 71 | {:error, :sr_too_short} 72 | end 73 | end 74 | -------------------------------------------------------------------------------- /lib/membrane/rtcp/transport_feedback_packet.ex: -------------------------------------------------------------------------------- 1 | defmodule Membrane.RTCP.TransportFeedbackPacket do 2 | @moduledoc """ 3 | Abstraction and generic encoding/decoding functionality for 4 | [RTCP transport layer feedback packets](https://datatracker.ietf.org/doc/html/rfc4585#section-6.1). 5 | """ 6 | 7 | @behaviour Membrane.RTCP.Packet 8 | 9 | alias Membrane.RTP 10 | 11 | @enforce_keys [:sender_ssrc, :media_ssrc, :payload] 12 | defstruct @enforce_keys 13 | 14 | @type t :: %__MODULE__{ 15 | sender_ssrc: RTP.ssrc(), 16 | media_ssrc: RTP.ssrc(), 17 | payload: struct 18 | } 19 | 20 | @callback decode(binary()) :: {:ok, struct()} | {:error, any()} 21 | 22 | @callback encode(struct()) :: binary() 23 | 24 | @packet_type_payload BiMap.new(%{ 25 | 1 => __MODULE__.NACK, 26 | 15 => __MODULE__.TWCC 27 | }) 28 | 29 | @impl true 30 | def decode(<>, packet_type) do 31 | with {:ok, module} <- BiMap.fetch(@packet_type_payload, packet_type), 32 | {:ok, payload} <- module.decode(payload) do 33 | {:ok, %__MODULE__{sender_ssrc: sender_ssrc, media_ssrc: media_ssrc, payload: payload}} 34 | else 35 | :error -> {:error, {:unknown_feedback_packet_type, packet_type}} 36 | {:error, reason} -> {:error, reason} 37 | end 38 | end 39 | 40 | @impl true 41 | def decode(_binary, _packet_type) do 42 | {:error, :malformed_packet} 43 | end 44 | 45 | @impl true 46 | def encode(%__MODULE__{} = packet) do 47 | %module{} = packet.payload 48 | packet_type = BiMap.fetch_key!(@packet_type_payload, module) 49 | 50 | {<>, 51 | packet_type} 52 | end 53 | end 54 | -------------------------------------------------------------------------------- /lib/membrane/rtcp/transport_feedback_packet/nack.ex: -------------------------------------------------------------------------------- 1 | defmodule Membrane.RTCP.TransportFeedbackPacket.NACK do 2 | @moduledoc """ 3 | Generic Negative Acknowledgment packet that informs about lost RTP packet(s) 4 | 5 | Quoting [RFC4585](https://datatracker.ietf.org/doc/html/rfc4585#section-6.2.1): 6 | The Generic NACK is used to indicate the loss of one or more RTP packets. 7 | The lost packet(s) are identified by the means of a packet identifier and a bit mask. 8 | 9 | The Feedback Control Information (FCI) field has the following Syntax (Figure 4): 10 | 11 | ```txt 12 | 0 1 2 3 13 | 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 14 | +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ 15 | | PID | BLP | 16 | +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ 17 | 18 | Figure 4: Syntax for the Generic NACK message 19 | 20 | Packet ID (PID): 16 bits 21 | The PID field is used to specify a lost packet. The PID field 22 | refers to the RTP sequence number of the lost packet. 23 | 24 | 25 | bitmask of following lost packets (BLP): 16 bits 26 | The BLP allows for reporting losses of any of the 16 RTP packets 27 | immediately following the RTP packet indicated by the PID. The 28 | BLP's definition is identical to that given in [6]. Denoting the 29 | BLP's least significant bit as bit 1, and its most significant bit 30 | as bit 16, then bit i of the bit mask is set to 1 if the receiver 31 | has not received RTP packet number (PID+i) (modulo 2^16) and 32 | indicates this packet is lost; bit i is set to 0 otherwise. Note 33 | that the sender MUST NOT assume that a receiver has received a 34 | packet because its bit mask was set to 0. For example, the least 35 | significant bit of the BLP would be set to 1 if the packet 36 | corresponding to the PID and the following packet have been lost. 37 | However, the sender cannot infer that packets PID+2 through PID+16 38 | have been received simply because bits 2 through 15 of the BLP are 39 | 0; all the sender knows is that the receiver has not reported them 40 | as lost at this time. 41 | 42 | ``` 43 | Implementation based on https://datatracker.ietf.org/doc/html/rfc4585#section-6.2.1 44 | and https://datatracker.ietf.org/doc/html/rfc2032#section-5.2.2 45 | """ 46 | 47 | @behaviour Membrane.RTCP.TransportFeedbackPacket 48 | 49 | import Bitwise 50 | 51 | defstruct lost_packet_ids: [] 52 | 53 | @impl true 54 | def decode(nack_fci) do 55 | for <> do 56 | next_lost_packets = 57 | 0..15 58 | |> Enum.map(fn i -> 59 | if (bit_mask >>> i &&& 1) == 1 do 60 | mod_16bit(packet_id + i + 1) 61 | else 62 | nil 63 | end 64 | end) 65 | |> Enum.reject(&is_nil/1) 66 | 67 | [packet_id | next_lost_packets] 68 | end 69 | |> then(&{:ok, %__MODULE__{lost_packet_ids: List.flatten(&1)}}) 70 | end 71 | 72 | @impl true 73 | def encode(%__MODULE__{lost_packet_ids: lost_packet_ids}) do 74 | # TODO: This code does handle rollover, so 65_535 and 0 will be put in separate FCIs 75 | # That's not optimal, but rare and still correctly reports the lost packets 76 | ids_to_encode = Enum.sort(lost_packet_ids) 77 | 78 | # code splitting ids into tuples with reference_id and a list of following ids 79 | # greater by at most 16. They will fit into one FCI. 80 | chunk_fun = fn 81 | # Initial step - initilaize reference_id 82 | id, nil -> 83 | {:cont, {id, []}} 84 | 85 | # Id to group in the same FCI - no chunk emitted 86 | id, {reference_id, rest} when id > reference_id and id - reference_id <= 16 -> 87 | {:cont, {reference_id, [id | rest]}} 88 | 89 | # Id that should start a next FCI - emit a chunk and set a new reference_id 90 | id, {reference_id, rest} when id - reference_id > 16 -> 91 | {:cont, {reference_id, rest}, {id, []}} 92 | end 93 | 94 | # Emit a chunk with what has been gathered 95 | after_fun = fn acc -> {:cont, acc, nil} end 96 | 97 | ids_to_encode 98 | |> Enum.chunk_while(nil, chunk_fun, after_fun) 99 | |> Enum.map_join(&encode_fci/1) 100 | end 101 | 102 | defp encode_fci({reference_id, ids}) when is_integer(reference_id) do 103 | bit_mask = 104 | ids 105 | |> Enum.reduce(0, fn id, acc -> 106 | # ID must be between reference_id + 1 and reference_id + 16 107 | # we set bit 0 for reference_id + 1 and 15 for reference_id + 16 108 | bit_to_set = id - reference_id - 1 109 | bor(acc, 1 <<< bit_to_set) 110 | end) 111 | 112 | <> 113 | end 114 | 115 | defp mod_16bit(number), do: number &&& 0xFFFF 116 | end 117 | -------------------------------------------------------------------------------- /lib/membrane/rtcp_event.ex: -------------------------------------------------------------------------------- 1 | defmodule Membrane.RTCPEvent do 2 | @moduledoc """ 3 | Event carrying a parsed RTCP packet. 4 | """ 5 | @derive Membrane.EventProtocol 6 | 7 | @enforce_keys [:rtcp] 8 | 9 | defstruct @enforce_keys ++ [ssrcs: [], arrival_timestamp: nil] 10 | 11 | @type t :: %__MODULE__{ 12 | rtcp: Membrane.RTCP.Packet.t(), 13 | ssrcs: [Membrane.RTP.ssrc()], 14 | arrival_timestamp: Membrane.Time.t() 15 | } 16 | end 17 | -------------------------------------------------------------------------------- /lib/membrane/rtp/depayloader_bin.ex: -------------------------------------------------------------------------------- 1 | defmodule Membrane.RTP.DepayloaderBin do 2 | @moduledoc """ 3 | Modules responsible for reordering incoming RTP packets using a jitter buffer 4 | to later depayload packet's payload from RTP format. 5 | """ 6 | 7 | use Membrane.Bin 8 | 9 | alias Membrane.RTP 10 | alias Membrane.RTP.JitterBuffer 11 | 12 | def_options depayloader: [ 13 | spec: module(), 14 | description: "Depayloader module that should be used for incoming stream" 15 | ], 16 | clock_rate: [ 17 | spec: RTP.clock_rate() 18 | ] 19 | 20 | def_input_pad :input, 21 | accepted_format: RTP 22 | 23 | def_output_pad :output, 24 | accepted_format: _any 25 | 26 | @impl true 27 | def handle_init(_ctx, opts) do 28 | structure = 29 | bin_input() 30 | |> child(:jitter_buffer, %JitterBuffer{clock_rate: opts.clock_rate}) 31 | |> child(:depayloader, opts.depayloader) 32 | |> bin_output() 33 | 34 | {[spec: structure], %{}} 35 | end 36 | end 37 | -------------------------------------------------------------------------------- /lib/membrane/rtp/header.ex: -------------------------------------------------------------------------------- 1 | defmodule Membrane.RTP.Header do 2 | @moduledoc """ 3 | Describes RTP Header defined in [RFC3550](https://tools.ietf.org/html/rfc3550#page-13) 4 | 5 | ``` 6 | 0 1 2 3 7 | 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 8 | +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ 9 | |V=2|P|X| CC |M| PT | sequence number | 10 | +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ 11 | | timestamp | 12 | +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ 13 | | synchronization source (SSRC) identifier | 14 | +=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+=+ 15 | | contributing source (CSRC) identifiers | 16 | | .... | 17 | +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ 18 | ``` 19 | """ 20 | 21 | alias Membrane.RTP 22 | 23 | @typedoc """ 24 | This field identifies the version of RTP. The version defined by this specification is 2. 25 | """ 26 | @type version :: 2 27 | 28 | @typedoc """ 29 | The interpretation of the marker is defined by a profile 30 | """ 31 | @type marker :: boolean() 32 | 33 | @typedoc """ 34 | Timestamp of a packet in ticks of clock according to `t:RTP.clock_rate_t/0`. 35 | 36 | Its initial value is random, so it should not be interpreted as an absolute time, but rather used to calculate 37 | time difference from other timestamps. 38 | """ 39 | @type timestamp_t() :: non_neg_integer() 40 | 41 | @typedoc """ 42 | A 16-bit integer sequential number of a packet. 43 | 44 | Its initial value should be random. 45 | """ 46 | @type sequence_number_t() :: non_neg_integer() 47 | 48 | @type t :: %__MODULE__{ 49 | version: version(), 50 | ssrc: RTP.ssrc(), 51 | marker: marker(), 52 | payload_type: RTP.payload_type(), 53 | timestamp: timestamp_t(), 54 | sequence_number: sequence_number_t(), 55 | csrcs: [RTP.ssrc()], 56 | extensions: [__MODULE__.Extension.t()] 57 | } 58 | 59 | @enforce_keys [ 60 | :ssrc, 61 | :payload_type, 62 | :timestamp, 63 | :sequence_number 64 | ] 65 | defstruct @enforce_keys ++ 66 | [ 67 | version: 2, 68 | marker: false, 69 | csrcs: [], 70 | extensions: [] 71 | ] 72 | end 73 | -------------------------------------------------------------------------------- /lib/membrane/rtp/header_extension.ex: -------------------------------------------------------------------------------- 1 | defmodule Membrane.RTP.Header.Extension do 2 | @moduledoc """ 3 | Describes RTP Header Extension defined in [RFC8285](https://datatracker.ietf.org/doc/html/rfc8285) 4 | and provides common functions for interacting with extensions placed in buffers. 5 | 6 | ``` 7 | 0 1 2 8 | 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 ... 9 | +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+- 10 | | ID | len | data (len+1 bytes) ... 11 | +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+- 12 | ``` 13 | """ 14 | alias Membrane.Buffer 15 | alias Membrane.RTP.SessionBin 16 | 17 | @enforce_keys [:identifier, :data] 18 | defstruct @enforce_keys 19 | 20 | @type identifier_t :: 1..14 | SessionBin.rtp_extension_name_t() 21 | 22 | @type t :: %__MODULE__{ 23 | identifier: identifier_t(), 24 | data: binary() 25 | } 26 | 27 | @spec find(Buffer.t(), identifier_t()) :: t() | nil 28 | def find(%Buffer{metadata: %{rtp: %{extensions: extensions}}}, identifier) do 29 | Enum.find(extensions, &(&1.identifier == identifier)) 30 | end 31 | 32 | @spec put(Buffer.t(), t()) :: Buffer.t() 33 | def put(buffer, extension) do 34 | Bunch.Struct.update_in(buffer, [:metadata, :rtp, :extensions], &[extension | &1]) 35 | end 36 | 37 | @spec delete(Buffer.t(), identifier_t()) :: Buffer.t() 38 | def delete(buffer, identifier) do 39 | Bunch.Struct.update_in( 40 | buffer, 41 | [:metadata, :rtp, :extensions], 42 | &Enum.reject(&1, fn extension -> extension.identifier == identifier end) 43 | ) 44 | end 45 | 46 | @spec pop(Buffer.t(), identifier_t()) :: {t() | nil, Buffer.t()} 47 | def pop(buffer, identifier) do 48 | case find(buffer, identifier) do 49 | nil -> {nil, buffer} 50 | extension -> {extension, delete(buffer, identifier)} 51 | end 52 | end 53 | end 54 | -------------------------------------------------------------------------------- /lib/membrane/rtp/header_generator.ex: -------------------------------------------------------------------------------- 1 | defmodule Membrane.RTP.HeaderGenerator do 2 | @moduledoc """ 3 | Given following RTP payloads and their minimal metadata, creates their proper header information, 4 | incrementing timestamps and sequence numbers for each packet. Header information then is put 5 | inside buffer's metadata under `:rtp` key. 6 | 7 | Accepts the following metadata under `:rtp` key: `:marker`, `:csrcs`, `:extensions`. 8 | See `Membrane.RTP.Header` for their meaning and specifications. 9 | """ 10 | use Membrane.Filter 11 | 12 | require Bitwise 13 | alias Membrane.RTP 14 | 15 | @max_seq_num Bitwise.bsl(1, 16) - 1 16 | @max_timestamp Bitwise.bsl(1, 32) - 1 17 | 18 | def_input_pad :input, accepted_format: RTP, flow_control: :auto 19 | 20 | def_output_pad :output, accepted_format: RTP, flow_control: :auto 21 | 22 | def_options ssrc: [spec: RTP.ssrc()], 23 | payload_type: [spec: RTP.payload_type()], 24 | clock_rate: [spec: RTP.clock_rate()] 25 | 26 | defmodule State do 27 | @moduledoc false 28 | use Bunch.Access 29 | 30 | defstruct [ 31 | :ssrc, 32 | :payload_type, 33 | :clock_rate, 34 | sequence_number: 0, 35 | init_timestamp: 0 36 | ] 37 | 38 | @type t :: %__MODULE__{ 39 | ssrc: RTP.ssrc(), 40 | payload_type: RTP.payload_type(), 41 | clock_rate: RTP.clock_rate(), 42 | sequence_number: non_neg_integer(), 43 | init_timestamp: non_neg_integer() 44 | } 45 | end 46 | 47 | @impl true 48 | def handle_init(_ctx, options) do 49 | state = 50 | options 51 | |> Map.from_struct() 52 | |> Map.merge(%{ 53 | sequence_number: Enum.random(0..@max_seq_num), 54 | init_timestamp: Enum.random(0..@max_timestamp) 55 | }) 56 | 57 | {[], struct!(State, state)} 58 | end 59 | 60 | @impl true 61 | def handle_stream_format(:input, _stream_format, _ctx, state) do 62 | {[stream_format: {:output, %RTP{}}], state} 63 | end 64 | 65 | @impl true 66 | def handle_buffer(:input, buffer, _ctx, state) do 67 | {rtp_metadata, metadata} = Map.pop(buffer.metadata, :rtp, %{}) 68 | 69 | rtp_offset = 70 | buffer.pts 71 | |> Numbers.mult(state.clock_rate) 72 | |> Membrane.Time.as_seconds(:round) 73 | 74 | rtp_timestamp = rem(state.init_timestamp + rtp_offset, @max_timestamp + 1) 75 | 76 | state = Map.update!(state, :sequence_number, &rem(&1 + 1, @max_seq_num + 1)) 77 | 78 | header = %{ 79 | ssrc: state.ssrc, 80 | marker: Map.get(rtp_metadata, :marker, false), 81 | payload_type: state.payload_type, 82 | timestamp: rtp_timestamp, 83 | sequence_number: state.sequence_number, 84 | csrcs: Map.get(rtp_metadata, :csrcs, []), 85 | extensions: Map.get(rtp_metadata, :extensions, []) 86 | } 87 | 88 | buffer = %Membrane.Buffer{ 89 | buffer 90 | | metadata: Map.put(metadata, :rtp, header), 91 | payload: buffer.payload 92 | } 93 | 94 | {[buffer: {:output, buffer}], state} 95 | end 96 | end 97 | -------------------------------------------------------------------------------- /lib/membrane/rtp/jitter_buffer/record.ex: -------------------------------------------------------------------------------- 1 | defmodule Membrane.RTP.JitterBuffer.Record do 2 | @moduledoc false 3 | 4 | # Describes a structure that is stored in the BufferStore. 5 | 6 | alias Membrane.RTP.JitterBuffer 7 | @enforce_keys [:index, :timestamp, :buffer] 8 | defstruct @enforce_keys 9 | 10 | @type t :: %__MODULE__{ 11 | index: JitterBuffer.packet_index(), 12 | timestamp: Membrane.Time.t(), 13 | buffer: Membrane.Buffer.t() 14 | } 15 | 16 | @spec new(Membrane.Buffer.t(), JitterBuffer.packet_index()) :: t() 17 | def new(buffer, index) do 18 | %__MODULE__{ 19 | index: index, 20 | timestamp: Membrane.Time.monotonic_time(), 21 | buffer: buffer 22 | } 23 | end 24 | 25 | @doc """ 26 | Compares two records. 27 | 28 | Returns true if the first record is older than the second one. 29 | """ 30 | # Designed to use with Heap: https://gitlab.com/jimsy/heap/blob/master/lib/heap.ex#L71 31 | @spec rtp_comparator(t(), t()) :: boolean() 32 | def rtp_comparator(%__MODULE__{index: l_index}, %__MODULE__{index: r_index}), 33 | do: l_index < r_index 34 | end 35 | -------------------------------------------------------------------------------- /lib/membrane/rtp/metrics.ex: -------------------------------------------------------------------------------- 1 | defmodule Membrane.RTP.Metrics do 2 | @moduledoc """ 3 | Defines list of metrics, that can be aggregated based on events from membrane_rtp_plugin. 4 | """ 5 | 6 | alias Telemetry.Metrics 7 | 8 | @doc """ 9 | Returns list of metrics, that can be aggregated based on events from membrane_rtp_plugin. 10 | """ 11 | @spec metrics() :: [Metrics.t()] 12 | def metrics() do 13 | [ 14 | Metrics.counter( 15 | "inbound-rtp.packets", 16 | event_name: [Membrane.RTP, :packet, :arrival] 17 | ), 18 | Metrics.sum( 19 | "inbound-rtp.bytes_received", 20 | event_name: [Membrane.RTP, :packet, :arrival], 21 | measurement: :bytes 22 | ), 23 | Metrics.last_value( 24 | "inbound-rtp.encoding", 25 | event_name: [Membrane.RTP, :inbound_track, :new], 26 | measurement: :encoding 27 | ), 28 | Metrics.last_value( 29 | "inbound-rtp.ssrc", 30 | event_name: [Membrane.RTP, :inbound_track, :new], 31 | measurement: :ssrc 32 | ), 33 | Metrics.counter( 34 | "inbound-rtp.markers_received", 35 | event_name: [Membrane.RTP, :rtp, :marker_received] 36 | ), 37 | Metrics.counter( 38 | "outbound-rtp.markers_sent", 39 | event_name: [Membrane.RTP, :rtp, :marker_sent] 40 | ), 41 | Metrics.counter( 42 | "rtcp.total_packets_received", 43 | event_name: [Membrane.RTP, :rtcp, :arrival] 44 | ), 45 | Metrics.counter( 46 | "rtcp.total_packets_sent", 47 | event_name: [Membrane.RTP, :rtcp, :sent] 48 | ), 49 | Metrics.counter( 50 | "rtcp.nack_sent", 51 | event_name: [Membrane.RTP, :rtcp, :nack, :sent] 52 | ), 53 | Metrics.counter( 54 | "rtcp.fir_sent", 55 | event_name: [Membrane.RTP, :rtcp, :fir, :sent] 56 | ), 57 | Metrics.counter( 58 | "rtcp.sender_reports_sent", 59 | event_name: [Membrane.RTP, :rtcp, :sender_report, :sent] 60 | ), 61 | Metrics.counter( 62 | "rtcp.receiver_reports_sent", 63 | event_name: [Membrane.RTP, :rtcp, :receiver_report, :sent] 64 | ), 65 | Metrics.counter( 66 | "rtcp.nack_received", 67 | event_name: [Membrane.RTP, :rtcp, :nack, :arrival] 68 | ), 69 | Metrics.counter( 70 | "rtcp.fir_received", 71 | event_name: [Membrane.RTP, :rtcp, :fir, :arrival] 72 | ), 73 | Metrics.counter( 74 | "rtcp.pli_received", 75 | event_name: [Membrane.RTP, :rtcp, :pli, :arrival] 76 | ), 77 | Metrics.counter( 78 | "rtcp.sender_reports_received", 79 | event_name: [Membrane.RTP, :rtcp, :sender_report, :arrival] 80 | ), 81 | Metrics.counter( 82 | "rtcp.receiver_reports_received", 83 | event_name: [Membrane.RTP, :rtcp, :receiver_report, :arrival] 84 | ), 85 | Metrics.sum( 86 | "outbound-rtp.rtx_sent", 87 | event_name: [Membrane.RTP, :rtx, :sent], 88 | measurement: :amount 89 | ), 90 | Metrics.counter( 91 | "outbound-rtp.packets", 92 | event_name: [Membrane.RTP, :packet, :sent] 93 | ), 94 | Metrics.sum( 95 | "outbound-rtp.bytes", 96 | event_name: [Membrane.RTP, :packet, :sent], 97 | measurement: :bytes 98 | ) 99 | ] 100 | end 101 | end 102 | -------------------------------------------------------------------------------- /lib/membrane/rtp/outbound_rtx_controller.ex: -------------------------------------------------------------------------------- 1 | defmodule Membrane.RTP.OutboundRtxController do 2 | use Membrane.Filter 3 | 4 | require Membrane.Logger 5 | require Membrane.TelemetryMetrics 6 | 7 | alias Membrane.RTP.RetransmissionRequestEvent 8 | 9 | def_options telemetry_label: [ 10 | spec: Membrane.TelemetryMetrics.label(), 11 | default: [] 12 | ] 13 | 14 | def_input_pad :input, 15 | availability: :always, 16 | flow_control: :auto, 17 | accepted_format: _any 18 | 19 | def_output_pad :output, 20 | availability: :always, 21 | flow_control: :auto, 22 | accepted_format: _any 23 | 24 | @max_store_size 300 25 | @min_rtx_interval 10 26 | 27 | @retransmission_telemetry_event [Membrane.RTP, :rtx, :sent] 28 | 29 | @doc false 30 | @spec max_store_size() :: pos_integer() 31 | def max_store_size(), do: @max_store_size 32 | 33 | @impl true 34 | def handle_init(_ctx, opts) do 35 | Membrane.TelemetryMetrics.register(@retransmission_telemetry_event, opts.telemetry_label) 36 | {[], %{telemetry_label: opts.telemetry_label, store: %{}}} 37 | end 38 | 39 | @impl true 40 | def handle_buffer(:input, buffer, _ctx, state) when byte_size(buffer.payload) > 0 do 41 | idx = seq_num_to_index(buffer.metadata.rtp.sequence_number) 42 | 43 | state = put_in(state, [:store, idx], {nil, buffer}) 44 | 45 | {[forward: buffer], state} 46 | end 47 | 48 | @impl true 49 | def handle_buffer(:input, buffer, _ctx, state), do: {[forward: buffer], state} 50 | 51 | @impl true 52 | def handle_event( 53 | :input, 54 | %RetransmissionRequestEvent{packet_ids: sequence_numbers}, 55 | _ctx, 56 | state 57 | ) do 58 | Membrane.Logger.debug( 59 | "Got RTX request of size #{length(sequence_numbers)}: #{inspect(sequence_numbers)}" 60 | ) 61 | 62 | now = System.monotonic_time(:millisecond) 63 | 64 | {buffers, store} = 65 | Enum.map_reduce(sequence_numbers, state.store, fn seq_num, store -> 66 | maybe_retransmit(seq_num, now, store) 67 | end) 68 | 69 | buffers_to_retransmit = Enum.reject(buffers, &is_nil/1) 70 | retransmissions_count = length(buffers_to_retransmit) 71 | 72 | unless retransmissions_count == 0 do 73 | Membrane.Logger.debug( 74 | "Retransmitting #{retransmissions_count} buffer(s): #{inspect(Enum.map(buffers_to_retransmit, & &1.metadata.rtp.sequence_number))}" 75 | ) 76 | 77 | Membrane.TelemetryMetrics.execute( 78 | @retransmission_telemetry_event, 79 | %{amount: retransmissions_count}, 80 | %{}, 81 | state.telemetry_label 82 | ) 83 | end 84 | 85 | {[buffer: {:output, buffers_to_retransmit}], %{state | store: store}} 86 | end 87 | 88 | @impl true 89 | def handle_event(pad, event, ctx, state), do: super(pad, event, ctx, state) 90 | 91 | defp seq_num_to_index(seq_num), do: rem(seq_num, @max_store_size) 92 | 93 | defp maybe_retransmit(seq_num, now, store) do 94 | idx = rem(seq_num, @max_store_size) 95 | {last_rtx_time, buffer} = Map.get(store, idx, {nil, nil}) 96 | 97 | if buffer != nil and buffer.metadata.rtp.sequence_number == seq_num and 98 | min_rtx_interval_elapsed?(last_rtx_time, now) do 99 | store = Map.put(store, idx, {now, buffer}) 100 | {buffer, store} 101 | else 102 | {nil, store} 103 | end 104 | end 105 | 106 | defp min_rtx_interval_elapsed?(nil, _now), do: true 107 | defp min_rtx_interval_elapsed?(last_rtx_time, now), do: now - last_rtx_time >= @min_rtx_interval 108 | end 109 | -------------------------------------------------------------------------------- /lib/membrane/rtp/packet_payload_type.ex: -------------------------------------------------------------------------------- 1 | defmodule Membrane.RTP.Packet.PayloadType do 2 | @moduledoc """ 3 | This module contains utility to translate numerical payload type into an atom value. 4 | """ 5 | 6 | alias Membrane.RTP 7 | 8 | @doc """ 9 | Gets the name of used encoding from numerical payload type according to [RFC3551](https://tools.ietf.org/html/rfc3551#page-32). 10 | For quick reference check [datasheet](https://www.iana.org/assignments/rtp-parameters/rtp-parameters.xhtml). 11 | """ 12 | @spec get_encoding_name(payload_type :: RTP.payload_type()) :: 13 | RTP.static_encoding_name() | :dynamic 14 | def get_encoding_name(type) 15 | def get_encoding_name(0), do: :PCMU 16 | def get_encoding_name(3), do: :GSM 17 | def get_encoding_name(4), do: :G732 18 | def get_encoding_name(5), do: :DVI4 19 | def get_encoding_name(6), do: :DVI4 20 | def get_encoding_name(7), do: :LPC 21 | def get_encoding_name(8), do: :PCMA 22 | def get_encoding_name(9), do: :G722 23 | def get_encoding_name(10), do: :L16 24 | def get_encoding_name(11), do: :L16 25 | def get_encoding_name(12), do: :QCELP 26 | def get_encoding_name(13), do: :CN 27 | def get_encoding_name(14), do: :MPA 28 | def get_encoding_name(15), do: :G728 29 | def get_encoding_name(16), do: :DVI4 30 | def get_encoding_name(17), do: :DVI4 31 | def get_encoding_name(18), do: :G729 32 | def get_encoding_name(25), do: :CELB 33 | def get_encoding_name(26), do: :JPEG 34 | def get_encoding_name(28), do: :NV 35 | def get_encoding_name(31), do: :H261 36 | def get_encoding_name(32), do: :MPV 37 | def get_encoding_name(33), do: :MP2T 38 | def get_encoding_name(34), do: :H263 39 | 40 | def get_encoding_name(payload_type) when payload_type in 96..127, do: :dynamic 41 | 42 | @doc """ 43 | Gets the clock rate from numerical payload type according to [RFC3551](https://tools.ietf.org/html/rfc3551#page-32). 44 | For quick reference check [datasheet](https://www.iana.org/assignments/rtp-parameters/rtp-parameters.xhtml). 45 | """ 46 | @spec get_clock_rate(payload_type :: RTP.payload_type()) :: 47 | RTP.clock_rate() | :dynamic 48 | def get_clock_rate(type) 49 | def get_clock_rate(0), do: 8000 50 | def get_clock_rate(3), do: 8000 51 | def get_clock_rate(4), do: 8000 52 | def get_clock_rate(5), do: 8000 53 | def get_clock_rate(6), do: 16_000 54 | def get_clock_rate(7), do: 8000 55 | def get_clock_rate(8), do: 8000 56 | def get_clock_rate(9), do: 8000 57 | def get_clock_rate(10), do: 44_100 58 | def get_clock_rate(11), do: 44_100 59 | def get_clock_rate(12), do: 8000 60 | def get_clock_rate(13), do: 8000 61 | def get_clock_rate(14), do: 90_000 62 | def get_clock_rate(15), do: 8000 63 | def get_clock_rate(16), do: 11_025 64 | def get_clock_rate(17), do: 22_050 65 | def get_clock_rate(18), do: 8000 66 | def get_clock_rate(25), do: 90_000 67 | def get_clock_rate(26), do: 90_000 68 | def get_clock_rate(28), do: 90_000 69 | def get_clock_rate(31), do: 90_000 70 | def get_clock_rate(32), do: 90_000 71 | def get_clock_rate(33), do: 90_000 72 | def get_clock_rate(34), do: 90_000 73 | 74 | def get_clock_rate(payload_type) when payload_type in 96..127, do: :dynamic 75 | 76 | @doc """ 77 | Checks if numerical payload type should be assigned to format type dynamically. 78 | """ 79 | @spec dynamic?(payload_type :: RTP.payload_type()) :: boolean() 80 | def dynamic?(payload_type) when payload_type in 96..127, do: true 81 | def dynamic?(_payload_type), do: false 82 | end 83 | -------------------------------------------------------------------------------- /lib/membrane/rtp/packets_discarded_event.ex: -------------------------------------------------------------------------------- 1 | defmodule Membrane.RTP.PacketsDiscardedEvent do 2 | @moduledoc """ 3 | Event carrying information about how many packets has been discarded by some element. 4 | """ 5 | @derive Membrane.EventProtocol 6 | 7 | defstruct discarded: 0 8 | 9 | @type t :: %__MODULE__{ 10 | discarded: non_neg_integer() 11 | } 12 | end 13 | -------------------------------------------------------------------------------- /lib/membrane/rtp/parser.ex: -------------------------------------------------------------------------------- 1 | defmodule Membrane.RTP.Parser do 2 | @moduledoc """ 3 | Identifies RTP/RTCP packets, then tries to parse RTP packet (parsing header and preparing payload) 4 | and forwards RTCP packet to `:rtcp_output` pad unchanged. 5 | 6 | ## Encrypted packets 7 | In case of SRTP/SRTCP the parser tries to parse just the header of the RTP packet as the packet's payload 8 | is encrypted and must be passed as a whole to the decryptor. The whole packet remains unchanged but 9 | the parsed header gets attached to `Membrane.Buffer`'s metadata. 10 | 11 | SRTP is treated the same as RTCP and all packets gets forwarded to `:rtcp_output` pad. 12 | 13 | ## Parsed packets 14 | In both cases, encrypted and unencryptd, parsed header is put into the metadata field in `Membrane.Buffer` under `:rtp` key. 15 | with the following metadata `:timestamp`, `:sequence_number`, `:ssrc`, `:payload_type`, 16 | `:marker`, `:extension`. See `Membrane.RTP.Header` for their meaning and specifications. 17 | """ 18 | 19 | use Membrane.Filter, flow_control_hints?: false 20 | 21 | require Membrane.Logger 22 | alias Membrane.Buffer 23 | alias Membrane.{RemoteStream, RTCP, RTCPEvent, RTP} 24 | 25 | @metadata_fields [ 26 | :timestamp, 27 | :sequence_number, 28 | :ssrc, 29 | :csrcs, 30 | :payload_type, 31 | :marker, 32 | :extensions 33 | ] 34 | 35 | def_options secure?: [ 36 | type: :boolean, 37 | default: false, 38 | description: """ 39 | Specifies whether Parser should expect packets that are encrypted or not. 40 | Requires adding [srtp](https://github.com/membraneframework/elixir_libsrtp) dependency to work. 41 | """ 42 | ] 43 | 44 | def_input_pad :input, 45 | accepted_format: 46 | %RemoteStream{type: :packetized, content_format: cf} when cf in [nil, RTP, RTCP], 47 | flow_control: :auto 48 | 49 | def_output_pad :output, accepted_format: RTP, flow_control: :auto 50 | 51 | def_output_pad :rtcp_output, 52 | flow_control: :push, 53 | accepted_format: %RemoteStream{content_format: RTCP, type: :packetized}, 54 | availability: :on_request 55 | 56 | @impl true 57 | def handle_init(_ctx, opts) do 58 | {[], %{rtcp_output_pad: nil, secure?: opts.secure?}} 59 | end 60 | 61 | @impl true 62 | def handle_stream_format(:input, _stream_format, _ctx, state) do 63 | {[stream_format: {:output, %RTP{}}], state} 64 | end 65 | 66 | @impl true 67 | def handle_pad_added(Pad.ref(:rtcp_output, _ref) = pad, %{playback_state: :playing}, state) do 68 | actions = [stream_format: {pad, %RemoteStream{content_format: RTCP, type: :packetized}}] 69 | {actions, %{state | rtcp_output_pad: pad}} 70 | end 71 | 72 | @impl true 73 | def handle_pad_added(Pad.ref(:rtcp_output, _ref) = pad, _ctx, state) do 74 | {[], %{state | rtcp_output_pad: pad}} 75 | end 76 | 77 | @impl true 78 | def handle_playing(_ctx, %{rtcp_output_pad: pad} = state) when pad != nil do 79 | actions = [stream_format: {pad, %RemoteStream{content_format: RTCP, type: :packetized}}] 80 | {actions, state} 81 | end 82 | 83 | @impl true 84 | def handle_playing(ctx, state) do 85 | super(ctx, state) 86 | end 87 | 88 | @impl true 89 | def handle_buffer(:input, %Buffer{payload: payload, metadata: metadata} = buffer, _ctx, state) do 90 | with :rtp <- RTP.Packet.identify(payload), 91 | {:ok, 92 | %{packet: packet, padding_size: padding_size, total_header_size: total_header_size}} <- 93 | RTP.Packet.parse(payload, state.secure?) do 94 | %RTP.Packet{payload: payload, header: header} = packet 95 | 96 | rtp = 97 | header 98 | |> Map.take(@metadata_fields) 99 | |> Map.merge(%{padding_size: padding_size, total_header_size: total_header_size}) 100 | 101 | metadata = Map.put(metadata, :rtp, rtp) 102 | {[buffer: {:output, %Buffer{payload: payload, metadata: metadata}}], state} 103 | else 104 | :rtcp -> 105 | case state.rtcp_output_pad do 106 | nil -> {[], state} 107 | pad -> {[buffer: {pad, buffer}], state} 108 | end 109 | 110 | {:error, reason} -> 111 | Membrane.Logger.debug(""" 112 | Couldn't parse rtp packet: 113 | #{inspect(payload, limit: :infinity)} 114 | Reason: #{inspect(reason)}. Ignoring packet. 115 | """) 116 | 117 | {[], state} 118 | end 119 | end 120 | 121 | @impl true 122 | def handle_event(:output, %RTCPEvent{} = event, _ctx, state) do 123 | case state.rtcp_output_pad do 124 | nil -> 125 | {[], state} 126 | 127 | pad -> 128 | {[event: {pad, event}], state} 129 | end 130 | end 131 | 132 | @impl true 133 | def handle_event(pad, event, ctx, state), do: super(pad, event, ctx, state) 134 | end 135 | -------------------------------------------------------------------------------- /lib/membrane/rtp/payload_format_resolver.ex: -------------------------------------------------------------------------------- 1 | defmodule Membrane.RTP.PayloadFormatResolver do 2 | @moduledoc """ 3 | Wrapper over `Membrane.RTP.PayloadFormat` that returns payloaders and depayloaders, or an error 4 | if they can't be resolved. 5 | """ 6 | 7 | alias Membrane.RTP 8 | alias Membrane.RTP.PayloadFormat 9 | 10 | @type encoding_mapper_t :: %{RTP.encoding_name() => module()} 11 | 12 | @doc """ 13 | Tries to resolve a depayloader based on given encoding. 14 | """ 15 | @spec depayloader(RTP.encoding_name()) :: {:ok, module()} | :error 16 | def depayloader(encoding) do 17 | case PayloadFormat.get(encoding).depayloader do 18 | nil -> :error 19 | depayloader -> {:ok, depayloader} 20 | end 21 | end 22 | 23 | @doc """ 24 | Tries to resolve a payloader based on given encoding. 25 | """ 26 | @spec payloader(RTP.encoding_name()) :: {:ok, module()} | :error 27 | def payloader(encoding) do 28 | case PayloadFormat.get(encoding).payloader do 29 | nil -> :error 30 | payloader -> {:ok, payloader} 31 | end 32 | end 33 | 34 | @spec keyframe_detector(atom()) :: {:ok, (binary() -> boolean())} | :error 35 | def keyframe_detector(encoding) do 36 | case PayloadFormat.get(encoding).keyframe_detector do 37 | nil -> :error 38 | keyframe_detector -> {:ok, keyframe_detector} 39 | end 40 | end 41 | 42 | @spec frame_detector(atom()) :: {:ok, (binary() -> boolean())} | :error 43 | def frame_detector(encoding) do 44 | case PayloadFormat.get(encoding).frame_detector do 45 | nil -> :error 46 | frame_detector -> {:ok, frame_detector} 47 | end 48 | end 49 | end 50 | -------------------------------------------------------------------------------- /lib/membrane/rtp/payloader_bin.ex: -------------------------------------------------------------------------------- 1 | defmodule Membrane.RTP.PayloaderBin do 2 | @moduledoc """ 3 | Module responsible for payloading a stream to RTP format and preparing RTP headers. 4 | """ 5 | 6 | use Membrane.Bin 7 | 8 | alias Membrane.RTP 9 | 10 | def_input_pad :input, accepted_format: _any 11 | 12 | def_output_pad :output, accepted_format: RTP 13 | 14 | def_options payloader: [ 15 | spec: module(), 16 | description: "Payloader module used for payloading a stream to RTP format" 17 | ], 18 | ssrc: [spec: RTP.ssrc()], 19 | payload_type: [spec: RTP.payload_type()], 20 | clock_rate: [spec: RTP.clock_rate()] 21 | 22 | @impl true 23 | def handle_init(_ctx, opts) do 24 | structure = 25 | bin_input() 26 | |> child(:payloader, opts.payloader) 27 | |> child(:header_generator, %RTP.HeaderGenerator{ 28 | ssrc: opts.ssrc, 29 | payload_type: opts.payload_type, 30 | clock_rate: opts.clock_rate 31 | }) 32 | |> bin_output() 33 | 34 | {[spec: structure], %{}} 35 | end 36 | end 37 | -------------------------------------------------------------------------------- /lib/membrane/rtp/retransmission_request_event.ex: -------------------------------------------------------------------------------- 1 | defmodule Membrane.RTP.RetransmissionRequestEvent do 2 | @moduledoc """ 3 | An event used to request a retransmission of packet(s). 4 | 5 | In the element responsible for sending NACKs it will be turned into 6 | `Membrane.RTCP.TransportFeedbackPacket.NACK` and sent to the RTP sender 7 | """ 8 | @derive Membrane.EventProtocol 9 | @enforce_keys [:packet_ids] 10 | defstruct @enforce_keys 11 | end 12 | -------------------------------------------------------------------------------- /lib/membrane/rtp/rtx_parser.ex: -------------------------------------------------------------------------------- 1 | defmodule Membrane.RTP.RTXParser do 2 | @moduledoc """ 3 | An element responsible for handling retransmission packets (`rtx`) defined in 4 | [RFC 4588](https://datatracker.ietf.org/doc/html/rfc4588#section-4). 5 | 6 | It parses RTX packet and recreates the lost packet by stripping rtx header from buffer's payload 7 | and updating rtp metadata. The changed fields are: 8 | * `sequence_number` - set to value transported in rtx header 9 | * `payload_type` - set via `original_payload_type` option 10 | * if `rid_id` and `repaired_rid_id` are provided, the former replaces the latter in a matching `:extensions` entry 11 | """ 12 | 13 | use Membrane.Filter 14 | 15 | require Membrane.Logger 16 | 17 | alias Membrane.{Buffer, RTP} 18 | 19 | def_input_pad :input, accepted_format: RTP, flow_control: :auto 20 | def_output_pad :output, accepted_format: RTP, flow_control: :auto 21 | 22 | def_options original_payload_type: [ 23 | description: 24 | "Payload type of original RTP stream that is retransmitted via the parsed RTX stream" 25 | ], 26 | repaired_rid_id: [ 27 | spec: RTP.Header.Extension.identifier_t(), 28 | description: 29 | "The numerical ID of an extension carrying repaired-rid that will be rewritten into rid", 30 | default: nil 31 | ], 32 | rid_id: [ 33 | spec: RTP.Header.Extension.identifier_t(), 34 | description: 35 | "The numerical ID of an extension carrying rid, will replace repaired_rid_id", 36 | default: nil 37 | ] 38 | 39 | @impl true 40 | def handle_init(_ctx, opts) do 41 | {[], Map.from_struct(opts)} 42 | end 43 | 44 | @impl true 45 | def handle_stream_format(:input, rtp_format, _ctx, state) do 46 | {[forward: rtp_format], state} 47 | end 48 | 49 | @impl true 50 | def handle_buffer(:input, %Buffer{payload: payload} = buffer, _ctx, state) 51 | when byte_size(payload) >= 2 do 52 | <> = payload 53 | 54 | Membrane.Logger.debug( 55 | "[RTX SSRC: #{buffer.metadata.rtp.ssrc}] got retransmitted packet with seq_num #{original_seq_num}" 56 | ) 57 | 58 | extensions = 59 | buffer.metadata.rtp.extensions 60 | |> maybe_rewrite_rid_ext_id(state) 61 | 62 | recreated_buffer = %Buffer{ 63 | buffer 64 | | payload: original_payload, 65 | metadata: %{ 66 | rtp: %{ 67 | buffer.metadata.rtp 68 | | extensions: extensions, 69 | sequence_number: original_seq_num, 70 | payload_type: state.original_payload_type 71 | } 72 | } 73 | } 74 | 75 | {[buffer: {:output, recreated_buffer}], state} 76 | end 77 | 78 | @impl true 79 | def handle_buffer(:input, %Buffer{payload: payload, metadata: metadata}, _ctx, state) do 80 | # Ignore empty buffers, most likely used for bandwidth estimation 81 | if byte_size(payload) > 0 do 82 | Membrane.Logger.warning( 83 | "Received invalid RTX buffer with sequence_number #{metadata.rtp.sequence_number}" 84 | ) 85 | end 86 | 87 | {[], state} 88 | end 89 | 90 | defp maybe_rewrite_rid_ext_id(extensions, %{repaired_rid_id: rrid, rid_id: rid}) 91 | when rrid != nil and rid != nil do 92 | extensions 93 | |> Enum.map(fn 94 | %{identifier: ^rrid} = ext -> %{ext | identifier: rid} 95 | ext -> ext 96 | end) 97 | end 98 | 99 | defp maybe_rewrite_rid_ext_id(extensions, _state) do 100 | extensions 101 | end 102 | end 103 | -------------------------------------------------------------------------------- /lib/membrane/rtp/sequence_number_tracker.ex: -------------------------------------------------------------------------------- 1 | defmodule Membrane.RTP.SequenceNumberTracker do 2 | @moduledoc """ 3 | A module used to map 16-bit sequence number to a continuous index without rollovers. 4 | It also detectes repeated and lost packets 5 | """ 6 | 7 | require Bitwise 8 | 9 | alias Membrane.RTP.Utils 10 | 11 | @seq_num_bits 16 12 | @rollover_modulus Bitwise.bsl(1, @seq_num_bits) 13 | 14 | @type t() :: %__MODULE__{ 15 | highest_seen_index: nil | non_neg_integer() 16 | } 17 | defstruct highest_seen_index: nil 18 | 19 | @doc """ 20 | Initializes new SequenceNumberTracker 21 | """ 22 | @spec new() :: t() 23 | def new(), do: %__MODULE__{} 24 | 25 | @doc """ 26 | Main function of the Tracker that returns a difference to the newest package and packet's monotonic index 27 | """ 28 | @spec track(t(), Membrane.RTP.Header.sequence_number_t()) :: {integer(), non_neg_integer(), t()} 29 | def track(%__MODULE__{highest_seen_index: nil} = tracker, 0) do 30 | # Start from roc = 1 to allow late sequence numbers 31 | {1, @rollover_modulus, %__MODULE__{tracker | highest_seen_index: @rollover_modulus}} 32 | end 33 | 34 | def track(%__MODULE__{highest_seen_index: nil} = tracker, seq_num) do 35 | {1, seq_num, %__MODULE__{tracker | highest_seen_index: seq_num}} 36 | end 37 | 38 | def track(%__MODULE__{highest_seen_index: reference_index} = tracker, seq_num) do 39 | reference_seq_num = rem(reference_index, @rollover_modulus) 40 | reference_roc = div(reference_index, @rollover_modulus) 41 | 42 | incoming_roc = 43 | case Utils.from_which_rollover(reference_seq_num, seq_num, @rollover_modulus) do 44 | :current -> reference_roc 45 | :previous -> reference_roc - 1 46 | :next -> reference_roc + 1 47 | end 48 | 49 | incoming_index = seq_num + Bitwise.bsl(incoming_roc, @seq_num_bits) 50 | 51 | diff = incoming_index - reference_index 52 | 53 | tracker = 54 | if diff > 0 do 55 | %__MODULE__{tracker | highest_seen_index: incoming_index} 56 | else 57 | tracker 58 | end 59 | 60 | {diff, incoming_index, tracker} 61 | end 62 | end 63 | -------------------------------------------------------------------------------- /lib/membrane/rtp/serializer/stats.ex: -------------------------------------------------------------------------------- 1 | defmodule Membrane.RTP.Serializer.Stats do 2 | @moduledoc """ 3 | Serializer stats that can be used for Sender Report(SR) generation 4 | """ 5 | use Bunch.Access 6 | 7 | @enforce_keys [ 8 | :timestamp, 9 | :rtp_timestamp, 10 | :sender_packet_count, 11 | :sender_octet_count 12 | ] 13 | 14 | defstruct @enforce_keys 15 | 16 | @type t :: 17 | %__MODULE__{ 18 | timestamp: non_neg_integer(), 19 | rtp_timestamp: non_neg_integer(), 20 | sender_packet_count: non_neg_integer(), 21 | sender_octet_count: non_neg_integer() 22 | } 23 | | :no_stats 24 | end 25 | -------------------------------------------------------------------------------- /lib/membrane/rtp/session/sender_report.ex: -------------------------------------------------------------------------------- 1 | defmodule Membrane.RTP.Session.SenderReport do 2 | @moduledoc false 3 | require Bitwise 4 | require Membrane.Logger 5 | alias Membrane.{RTCP, RTP, Time} 6 | 7 | @timestamp_limit Bitwise.bsl(1, 32) 8 | 9 | @typedoc """ 10 | Mapping from `ssrc` to its sender statistics. 11 | """ 12 | @type sender_stats_t :: %{ 13 | non_neg_integer() => %{ 14 | clock_rate: non_neg_integer(), 15 | timestamp: non_neg_integer(), 16 | rtp_timestamp: non_neg_integer(), 17 | sender_packet_count: non_neg_integer(), 18 | sender_octet_count: non_neg_integer() 19 | } 20 | } 21 | 22 | defmodule Data do 23 | @moduledoc false 24 | @type t :: %__MODULE__{ 25 | senders_ssrcs: MapSet.t(RTP.ssrc()), 26 | stats: %{ 27 | RTP.ssrc() => RTP.Serializer.Stats.t() 28 | } 29 | } 30 | 31 | defstruct senders_ssrcs: MapSet.new(), 32 | stats: %{} 33 | end 34 | 35 | @type maybe_report_t :: {:report, RTCP.Packet.t()} | :no_report 36 | 37 | @spec init_report(ssrcs :: MapSet.t(RTP.ssrc()), report_data :: Data.t()) :: 38 | {MapSet.t(RTP.ssrc()), Data.t()} 39 | def init_report(ssrcs, %Data{senders_ssrcs: senders_ssrcs} = report_data) 40 | when senders_ssrcs == %MapSet{} do 41 | senders_stats = 42 | report_data.stats |> Bunch.KVEnum.filter_by_keys(&MapSet.member?(ssrcs, &1)) |> Map.new() 43 | 44 | report_data = %{ 45 | report_data 46 | | senders_ssrcs: ssrcs, 47 | stats: senders_stats 48 | } 49 | 50 | {ssrcs, report_data} 51 | end 52 | 53 | @spec flush_report(Data.t()) :: {maybe_report_t(), Data.t()} 54 | def flush_report(report_data) do 55 | if Enum.empty?(report_data.senders_ssrcs) do 56 | {:no_report, report_data} 57 | else 58 | Membrane.Logger.debug( 59 | "Not received sender stats from ssrcs: #{Enum.join(report_data.senders_ssrcs, ", ")}" 60 | ) 61 | 62 | case generate_report(report_data.stats) do 63 | [] -> 64 | {:no_report, report_data} 65 | 66 | sender_reports -> 67 | {{:report, sender_reports}, %{report_data | senders_ssrcs: MapSet.new(), stats: %{}}} 68 | end 69 | end 70 | end 71 | 72 | @spec handle_stats(RTP.Serializer.Stats.t(), RTP.ssrc(), Data.t()) :: 73 | {maybe_report_t(), Data.t()} 74 | def handle_stats(stats, sender_ssrc, data) do 75 | senders_ssrcs = MapSet.delete(data.senders_ssrcs, sender_ssrc) 76 | 77 | data = %{data | stats: Map.put(data.stats, sender_ssrc, stats), senders_ssrcs: senders_ssrcs} 78 | 79 | if Enum.empty?(senders_ssrcs) do 80 | case generate_report(data.stats) do 81 | [] -> {:no_report, data} 82 | sender_reports -> {{:report, sender_reports}, data} 83 | end 84 | else 85 | {:no_report, data} 86 | end 87 | end 88 | 89 | @spec generate_report(sender_stats_t()) :: [RTCP.SenderReportPacket.t()] 90 | def generate_report(stats) do 91 | stats 92 | |> Enum.filter(fn {_k, v} -> v != :no_stats end) 93 | |> Enum.flat_map(fn {sender_ssrc, sender_stats} -> 94 | generate_sender_report(sender_ssrc, sender_stats) 95 | end) 96 | end 97 | 98 | defp generate_sender_report(sender_ssrc, sender_stats) do 99 | timestamp = Time.vm_time() 100 | 101 | rtp_offset = 102 | (timestamp - sender_stats.timestamp) 103 | |> Numbers.mult(sender_stats.clock_rate) 104 | |> Time.as_seconds(:round) 105 | 106 | rtp_timestamp = rem(sender_stats.rtp_timestamp + rtp_offset, @timestamp_limit) 107 | 108 | sender_info = %{ 109 | wallclock_timestamp: timestamp, 110 | rtp_timestamp: rtp_timestamp, 111 | sender_packet_count: sender_stats.sender_packet_count, 112 | sender_octet_count: sender_stats.sender_octet_count 113 | } 114 | 115 | [%RTCP.SenderReportPacket{ssrc: sender_ssrc, reports: [], sender_info: sender_info}] 116 | end 117 | end 118 | -------------------------------------------------------------------------------- /lib/membrane/rtp/session_bin/rtx_info.ex: -------------------------------------------------------------------------------- 1 | defmodule Membrane.RTP.SessionBin.RTXInfo do 2 | @moduledoc """ 3 | A struct used to inform SessionBin about availability of RTX stream 4 | with data allowing to set up proper part of a pipeline 5 | """ 6 | @enforce_keys [:ssrc, :original_ssrc, :original_payload_type] 7 | defstruct @enforce_keys ++ [:rid_id, :repaired_rid_id] 8 | end 9 | -------------------------------------------------------------------------------- /lib/membrane/rtp/silence_discarder.ex: -------------------------------------------------------------------------------- 1 | defmodule Membrane.RTP.SilenceDiscarder do 2 | @moduledoc """ 3 | Element responsible for dropping silent audio packets. 4 | 5 | For a packet to be discarded it needs to contain a `RTP.Header.Extension` struct with identifier equal to `vad_id` in 6 | its extensions list. The header extension will contain information about audio level (VAD extension is required). 7 | The element will only drop packets whose audio level is above given silence threshold (muted audio is of value 127). 8 | 9 | `#{__MODULE__}` will drop as many silent packets as possible and on reaching dropping limit it will send the current buffer, 10 | reset dropped packets counter and emit `Membrane.RTP.DroppedPacketEvent` with a number of packets that have been dropped until that point. 11 | The event gets sent on both reaching dropping limit and when a non-silent packet arrives. 12 | """ 13 | use Membrane.Filter 14 | 15 | alias Membrane.RTP.{Header, PacketsDiscardedEvent} 16 | 17 | def_input_pad :input, accepted_format: _any, flow_control: :auto 18 | def_output_pad :output, accepted_format: _any, flow_control: :auto 19 | 20 | def_options max_consecutive_drops: [ 21 | spec: non_neg_integer() | :infinity, 22 | default: 1000, 23 | description: """ 24 | A number indicating how many consecutive silent packets can be dropped before 25 | a single packet will be passed and dropped packet event will we emitted. 26 | 27 | Passing a single packets once in a while is necessary for element such as jitter buffer or encryptor as they can update their ROCs 28 | based on sequence numbers and when we drop to many packets we may roll it over. 29 | """ 30 | ], 31 | silence_threshold: [ 32 | spec: 1..127, 33 | default: 127, 34 | description: """ 35 | Audio level threshold that will be compared against incoming packets. Packet will be dropped if its audio level 36 | is above or equal to the given threshold. 37 | """ 38 | ], 39 | vad_id: [ 40 | spec: 1..14, 41 | default: 1, 42 | description: """ 43 | ID of a VAD extension. 44 | """ 45 | ] 46 | 47 | @impl true 48 | def handle_init(_ctx, opts) do 49 | {[], Map.from_struct(opts) |> Map.put(:dropped, 0)} 50 | end 51 | 52 | @impl true 53 | def handle_event(pad, other, ctx, state), do: super(pad, other, ctx, state) 54 | 55 | @impl true 56 | def handle_buffer( 57 | :input, 58 | buffer, 59 | _ctx, 60 | %{dropped: dropped, max_consecutive_drops: max_drops} = state 61 | ) 62 | when dropped == max_drops do 63 | stop_dropping(buffer, state) 64 | end 65 | 66 | @impl true 67 | def handle_buffer(:input, buffer, _ctx, state) do 68 | buffer 69 | |> Header.Extension.find(state.vad_id) 70 | |> handle_vad(buffer, state) 71 | end 72 | 73 | defp handle_vad(nil, buffer, state), do: {[buffer: {:output, buffer}], state} 74 | 75 | defp handle_vad(vad, buffer, state) do 76 | %{dropped: dropped, silence_threshold: silence_threshold} = state 77 | <<_v::1, audio_level::7>> = vad.data 78 | 79 | cond do 80 | audio_level >= silence_threshold -> 81 | {[], %{state | dropped: dropped + 1}} 82 | 83 | dropped > 0 -> 84 | stop_dropping(buffer, state) 85 | 86 | true -> 87 | {[buffer: {:output, buffer}], state} 88 | end 89 | end 90 | 91 | defp stop_dropping(buffer, state) do 92 | {[ 93 | event: {:output, %PacketsDiscardedEvent{discarded: state.dropped}}, 94 | buffer: {:output, buffer} 95 | ], %{state | dropped: 0}} 96 | end 97 | end 98 | -------------------------------------------------------------------------------- /lib/membrane/rtp/ssrc_router/streams_info.ex: -------------------------------------------------------------------------------- 1 | defmodule Membrane.RTP.SSRCRouter.StreamsInfo do 2 | @moduledoc """ 3 | A struct sent as a message to `Membrane.RTP.SSRCRouter` with info about known streams 4 | and streams that will be identified by extensions and should not be reported until they are present 5 | 6 | In particular, simulcast tracks need a RID extension in header in order to be handled. 7 | """ 8 | 9 | alias Membrane.RTP 10 | 11 | @type t() :: %__MODULE__{ 12 | accept_ssrcs: [RTP.ssrc()], 13 | require_extensions: %{RTP.payload_type() => [RTP.Header.Extension.identifier_t()]} 14 | } 15 | @enforce_keys [:require_extensions] 16 | defstruct @enforce_keys ++ [accept_ssrcs: []] 17 | end 18 | -------------------------------------------------------------------------------- /lib/membrane/rtp/stream_receive_bin.ex: -------------------------------------------------------------------------------- 1 | defmodule Membrane.RTP.StreamReceiveBin do 2 | @moduledoc """ 3 | This bin gets a parsed RTP stream on input and outputs raw media stream. 4 | 5 | Its responsibility is to depayload the RTP stream and compensate the 6 | jitter. 7 | """ 8 | 9 | use Membrane.Bin 10 | 11 | alias Membrane.{RTCP, RTP, SRTP} 12 | 13 | def_options srtp_policies: [ 14 | spec: [ExLibSRTP.Policy.t()], 15 | default: [] 16 | ], 17 | secure?: [ 18 | spec: boolean(), 19 | default: false 20 | ], 21 | extensions: [ 22 | spec: [RTP.SessionBin.extension_t()], 23 | default: [] 24 | ], 25 | clock_rate: [ 26 | spec: RTP.clock_rate() 27 | ], 28 | depayloader: [spec: module() | nil], 29 | local_ssrc: [spec: RTP.ssrc()], 30 | remote_ssrc: [spec: RTP.ssrc()], 31 | rtcp_report_interval: [spec: Membrane.Time.t() | nil], 32 | telemetry_label: [ 33 | spec: [{atom(), any()}], 34 | default: [] 35 | ] 36 | 37 | def_input_pad :input, accepted_format: _any 38 | def_output_pad :output, accepted_format: _any 39 | 40 | @impl true 41 | def handle_init(_ctx, opts) do 42 | if opts.secure? and not Code.ensure_loaded?(ExLibSRTP), 43 | do: raise("Optional dependency :ex_libsrtp is required when using secure? option") 44 | 45 | add_decryptor = 46 | &child(&1, :decryptor, struct(SRTP.Decryptor, %{policies: opts.srtp_policies})) 47 | 48 | add_depayloader_bin = 49 | &child(&1, :depayloader, %RTP.DepayloaderBin{ 50 | depayloader: opts.depayloader, 51 | clock_rate: opts.clock_rate 52 | }) 53 | 54 | structure = 55 | bin_input() 56 | |> add_extensions(opts.extensions) 57 | |> child(:rtcp_receiver, %RTCP.Receiver{ 58 | local_ssrc: opts.local_ssrc, 59 | remote_ssrc: opts.remote_ssrc, 60 | report_interval: opts.rtcp_report_interval, 61 | telemetry_label: opts.telemetry_label 62 | }) 63 | |> child(:packet_tracker, %RTP.InboundPacketTracker{ 64 | clock_rate: opts.clock_rate, 65 | repair_sequence_numbers?: true 66 | }) 67 | |> then(if opts.secure?, do: add_decryptor, else: & &1) 68 | |> then(if opts.depayloader, do: add_depayloader_bin, else: & &1) 69 | |> bin_output() 70 | 71 | {[spec: structure], %{}} 72 | end 73 | 74 | defp add_extensions(link_builder, extensions) do 75 | Enum.reduce(extensions, link_builder, fn {extension_name, extension}, builder -> 76 | builder |> child(extension_name, extension) 77 | end) 78 | end 79 | end 80 | -------------------------------------------------------------------------------- /lib/membrane/rtp/stream_send_bin.ex: -------------------------------------------------------------------------------- 1 | defmodule Membrane.RTP.StreamSendBin do 2 | @moduledoc """ 3 | Bin payloading and serializing media stream to RTP. 4 | """ 5 | use Membrane.Bin 6 | alias Membrane.RTP 7 | 8 | def_input_pad :input, accepted_format: _any 9 | 10 | def_input_pad :rtcp_input, 11 | availability: :on_request, 12 | accepted_format: _any 13 | 14 | def_output_pad :output, accepted_format: _any 15 | 16 | def_output_pad :rtcp_output, 17 | availability: :on_request, 18 | accepted_format: _any 19 | 20 | def_options payloader: [default: nil, spec: module], 21 | payload_type: [spec: RTP.payload_type()], 22 | ssrc: [spec: RTP.ssrc()], 23 | clock_rate: [spec: RTP.clock_rate()], 24 | rtcp_report_interval: [spec: Membrane.Time.t() | nil], 25 | rtp_extension_mapping: [ 26 | default: nil, 27 | spec: RTP.SessionBin.rtp_extension_mapping_t() 28 | ], 29 | telemetry_label: [ 30 | spec: Membrane.TelemetryMetrics.label(), 31 | default: [] 32 | ] 33 | 34 | @impl true 35 | def handle_init(_ctx, opts) do 36 | use_payloader = !is_nil(opts.payloader) 37 | 38 | add_payloader_bin = 39 | &child(&1, :payloader, %RTP.PayloaderBin{ 40 | payloader: opts.payloader, 41 | ssrc: opts.ssrc, 42 | clock_rate: opts.clock_rate, 43 | payload_type: opts.payload_type 44 | }) 45 | 46 | structure = 47 | bin_input() 48 | |> then(if use_payloader, do: add_payloader_bin, else: & &1) 49 | |> child(:packet_tracker, %RTP.OutboundTrackingSerializer{ 50 | ssrc: opts.ssrc, 51 | payload_type: opts.payload_type, 52 | clock_rate: opts.clock_rate, 53 | extension_mapping: opts.rtp_extension_mapping || %{}, 54 | telemetry_label: opts.telemetry_label 55 | }) 56 | |> bin_output() 57 | 58 | {[spec: structure], %{ssrc: opts.ssrc, rtcp_report_interval: opts.rtcp_report_interval}} 59 | end 60 | 61 | @impl true 62 | def handle_playing(_context, %{rtcp_report_interval: nil} = state) do 63 | {[], state} 64 | end 65 | 66 | @impl true 67 | def handle_playing(_ctx, state) do 68 | {[start_timer: {:report_timer, state.rtcp_report_interval}], state} 69 | end 70 | 71 | @impl true 72 | def handle_pad_added(Pad.ref(:rtcp_output, _id) = pad, _ctx, state) do 73 | structure = 74 | get_child(:packet_tracker) 75 | |> via_out(:rtcp_output) 76 | |> bin_output(pad) 77 | 78 | {[spec: structure], state} 79 | end 80 | 81 | @impl true 82 | def handle_pad_added(Pad.ref(:rtcp_input, _id) = pad, _ctx, state) do 83 | structure = [ 84 | bin_input(pad) 85 | |> via_in(:rtcp_input) 86 | |> get_child(:packet_tracker) 87 | ] 88 | 89 | {[spec: structure], state} 90 | end 91 | 92 | @impl true 93 | def handle_tick(:report_timer, _ctx, state) do 94 | {[notify_child: {:packet_tracker, :send_stats}], state} 95 | end 96 | end 97 | -------------------------------------------------------------------------------- /lib/membrane/rtp/tcp_decapsulator.ex: -------------------------------------------------------------------------------- 1 | defmodule Membrane.RTP.TCP.Decapsulator do 2 | @moduledoc """ 3 | This element provides functionality of packetizing bytestream from TCP 4 | into RTP and RTCP Packets. The encapsulation is described in RFC 4571. 5 | 6 | Packets in the stream will have the following structure: 7 | [Length :: 2 bytes][packet :: bytes] 8 | """ 9 | use Membrane.Filter 10 | 11 | alias Membrane.{Buffer, RemoteStream, RTP} 12 | 13 | def_input_pad :input, accepted_format: %RemoteStream{type: :bytestream} 14 | 15 | def_output_pad :output, accepted_format: %RemoteStream{type: :packetized, content_format: RTP} 16 | 17 | @impl true 18 | def handle_init(_ctx, _opts) do 19 | {[], %{unprocessed_data: <<>>}} 20 | end 21 | 22 | @impl true 23 | def handle_playing(_ctx, state) do 24 | stream_format = %RemoteStream{type: :packetized, content_format: RTP} 25 | {[stream_format: {:output, stream_format}], state} 26 | end 27 | 28 | @impl true 29 | def handle_stream_format(:input, _stream_format, _ctx, state) do 30 | {[], state} 31 | end 32 | 33 | @impl true 34 | def handle_buffer(:input, %Buffer{payload: payload, metadata: metadata}, _ctx, state) do 35 | packets_binary = state.unprocessed_data <> payload 36 | 37 | {unprocessed_data, complete_packets_binaries} = get_complete_packets(packets_binary) 38 | 39 | packets_buffers = 40 | Enum.map(complete_packets_binaries, &%Buffer{payload: &1, metadata: metadata}) 41 | 42 | {[buffer: {:output, packets_buffers}], %{state | unprocessed_data: unprocessed_data}} 43 | end 44 | 45 | @spec get_complete_packets(binary()) :: 46 | {unprocessed_data :: binary(), complete_packets :: [binary()]} 47 | defp get_complete_packets(packets_binary, complete_packets \\ []) 48 | 49 | defp get_complete_packets( 50 | <>, 51 | complete_packets 52 | ) do 53 | get_complete_packets(rest, [payload | complete_packets]) 54 | end 55 | 56 | defp get_complete_packets(unprocessed_data, complete_packets) do 57 | {unprocessed_data, Enum.reverse(complete_packets)} 58 | end 59 | end 60 | -------------------------------------------------------------------------------- /lib/membrane/rtp/tcp_encapsulator.ex: -------------------------------------------------------------------------------- 1 | defmodule Membrane.RTP.TCP.Encapsulator do 2 | @moduledoc """ 3 | This element provides functionality of serializing RTP and RTCP packets into a bytestream 4 | that can be send over TCP connection. The encapsulation is described in RFC 4571. 5 | 6 | Packets in the stream will have the following structure: 7 | [Length :: 2 bytes][packet :: bytes] 8 | """ 9 | use Membrane.Filter 10 | 11 | alias Membrane.{Buffer, RemoteStream, RTP} 12 | 13 | def_input_pad :input, accepted_format: %RemoteStream{type: :packetized, content_format: RTP} 14 | 15 | def_output_pad :output, accepted_format: %RemoteStream{type: :bytestream} 16 | 17 | @impl true 18 | def handle_init(_ctx, _opts) do 19 | {[], %{}} 20 | end 21 | 22 | @impl true 23 | def handle_playing(_ctx, state) do 24 | stream_format = %RemoteStream{type: :bytestream} 25 | {[stream_format: {:output, stream_format}], state} 26 | end 27 | 28 | @impl true 29 | def handle_stream_format(:input, _stream_format, _ctx, state) do 30 | {[], state} 31 | end 32 | 33 | @impl true 34 | def handle_buffer(:input, %Buffer{payload: payload, metadata: metadata}, _ctx, state) do 35 | buffer = %Buffer{ 36 | payload: <>, 37 | metadata: metadata 38 | } 39 | 40 | {[buffer: {:output, [buffer]}], state} 41 | end 42 | end 43 | -------------------------------------------------------------------------------- /lib/membrane/rtp/twcc_receiver/packet_info_store.ex: -------------------------------------------------------------------------------- 1 | defmodule Membrane.RTP.TWCCReceiver.PacketInfoStore do 2 | @moduledoc false 3 | 4 | # The module stores TWCC sequence number along with their arrival timestamps, handling sequence 5 | # number rollovers if necessary. Stored packet info can used for generating statistics used for 6 | # assembling a TWCC feedback packet. 7 | 8 | require Bitwise 9 | alias Membrane.RTP.Utils 10 | alias Membrane.Time 11 | 12 | defstruct base_seq_num: nil, 13 | max_seq_num: nil, 14 | seq_to_timestamp: %{} 15 | 16 | @type t :: %__MODULE__{ 17 | base_seq_num: non_neg_integer(), 18 | max_seq_num: non_neg_integer(), 19 | seq_to_timestamp: %{non_neg_integer() => Time.t()} 20 | } 21 | 22 | @type stats_t :: %{ 23 | base_seq_num: non_neg_integer(), 24 | packet_status_count: non_neg_integer(), 25 | receive_deltas: [Time.t() | :not_received], 26 | reference_time: Time.t() 27 | } 28 | 29 | @seq_number_limit Bitwise.bsl(1, 16) 30 | 31 | @spec empty?(__MODULE__.t()) :: boolean 32 | def empty?(%__MODULE__{base_seq_num: base_seq_num}), do: base_seq_num == nil 33 | 34 | @spec insert_packet_info(__MODULE__.t(), non_neg_integer()) :: __MODULE__.t() 35 | def insert_packet_info(store, seq_num) do 36 | arrival_ts = Time.vm_time() 37 | {store, seq_num} = maybe_handle_rollover(store, seq_num) 38 | 39 | %{ 40 | store 41 | | base_seq_num: min(store.base_seq_num, seq_num) || seq_num, 42 | max_seq_num: max(store.max_seq_num, seq_num) || seq_num, 43 | seq_to_timestamp: Map.put(store.seq_to_timestamp, seq_num, arrival_ts) 44 | } 45 | end 46 | 47 | @spec get_stats(__MODULE__.t()) :: stats_t() 48 | def get_stats(store) do 49 | {reference_time, receive_deltas} = make_receive_deltas(store) 50 | packet_status_count = store.max_seq_num - store.base_seq_num + 1 51 | 52 | %{ 53 | base_seq_num: store.base_seq_num, 54 | packet_status_count: packet_status_count, 55 | reference_time: reference_time, 56 | receive_deltas: receive_deltas 57 | } 58 | end 59 | 60 | defp maybe_handle_rollover(store, new_seq_num) do 61 | %{ 62 | base_seq_num: base_seq_num, 63 | max_seq_num: max_seq_num, 64 | seq_to_timestamp: seq_to_timestamp 65 | } = store 66 | 67 | case Utils.from_which_rollover(base_seq_num, new_seq_num, @seq_number_limit) do 68 | :current -> 69 | {store, new_seq_num} 70 | 71 | :next -> 72 | {store, new_seq_num + @seq_number_limit} 73 | 74 | :previous -> 75 | shifted_seq_to_timestamp = 76 | Map.new(seq_to_timestamp, fn {seq_num, timestamp} -> 77 | {seq_num + @seq_number_limit, timestamp} 78 | end) 79 | 80 | store = %{ 81 | store 82 | | base_seq_num: new_seq_num, 83 | max_seq_num: max_seq_num + @seq_number_limit, 84 | seq_to_timestamp: shifted_seq_to_timestamp 85 | } 86 | 87 | {store, new_seq_num} 88 | end 89 | end 90 | 91 | defp make_receive_deltas(store) do 92 | %{ 93 | base_seq_num: base_seq_num, 94 | max_seq_num: max_seq_num, 95 | seq_to_timestamp: seq_to_timestamp 96 | } = store 97 | 98 | # reference time has to be in 64ms resolution 99 | # https://datatracker.ietf.org/doc/html/draft-holmer-rmcat-transport-wide-cc-extensions-01#section-3.1 100 | reference_time = 101 | seq_to_timestamp 102 | |> Map.fetch!(base_seq_num) 103 | |> make_divisible_by_64ms() 104 | 105 | receive_deltas = 106 | base_seq_num..max_seq_num 107 | |> Enum.map_reduce(reference_time, fn seq_num, previous_timestamp -> 108 | case Map.get(seq_to_timestamp, seq_num) do 109 | nil -> 110 | {:not_received, previous_timestamp} 111 | 112 | timestamp -> 113 | delta = timestamp - previous_timestamp 114 | {delta, timestamp} 115 | end 116 | end) 117 | |> elem(0) 118 | 119 | {reference_time, receive_deltas} 120 | end 121 | 122 | defp make_divisible_by_64ms(timestamp) do 123 | timestamp - rem(timestamp, Time.milliseconds(64)) 124 | end 125 | end 126 | -------------------------------------------------------------------------------- /lib/membrane/rtp/twcc_sender/receiver_rate.ex: -------------------------------------------------------------------------------- 1 | defmodule Membrane.RTP.TWCCSender.ReceiverRate do 2 | @moduledoc false 3 | # module responsible for calculating bitrate 4 | # received by a receiver in last `window` time 5 | # (referred to as R_hat in the GCC draft, sec. 5.5) 6 | 7 | alias Membrane.Time 8 | 9 | @type t() :: %__MODULE__{ 10 | # estimated bitrate in bps 11 | value: float() | nil, 12 | # time window for measuring the received bitrate, between [0.5, 1]s (reffered to as "T" in the draft) 13 | window: Time.t(), 14 | # accumulator for packets and their timestamps that have been received in last `window` time 15 | packets_received: Qex.t({Time.t(), pos_integer()}) 16 | } 17 | 18 | @enforce_keys [:window] 19 | defstruct @enforce_keys ++ [:value, packets_received: Qex.new()] 20 | 21 | @spec new(Time.t()) :: t() 22 | def new(window), do: %__MODULE__{window: window} 23 | 24 | @spec update(t(), Time.t(), [Time.t() | :not_received], [pos_integer()]) :: t() 25 | def update(%__MODULE__{value: nil} = rr, reference_time, receive_deltas, packet_sizes) do 26 | packets_received = resolve_receive_deltas(receive_deltas, reference_time, packet_sizes) 27 | 28 | packets_received = Qex.join(rr.packets_received, packets_received) 29 | 30 | {first_packet_timestamp, _first_packet_size} = Qex.first!(packets_received) 31 | {last_packet_timestamp, _last_packet_size} = Qex.last!(packets_received) 32 | 33 | if last_packet_timestamp - first_packet_timestamp >= rr.window do 34 | do_update(rr, packets_received) 35 | else 36 | %__MODULE__{rr | packets_received: packets_received} 37 | end 38 | end 39 | 40 | def update(%__MODULE__{} = rr, reference_time, receive_deltas, packet_sizes) do 41 | packets_received = resolve_receive_deltas(receive_deltas, reference_time, packet_sizes) 42 | do_update(rr, packets_received) 43 | end 44 | 45 | defp do_update(rr, packets_received) do 46 | {last_packet_timestamp, _last_packet_size} = Qex.last!(packets_received) 47 | threshold = last_packet_timestamp - rr.window 48 | 49 | packets_received = 50 | rr.packets_received 51 | |> Qex.join(packets_received) 52 | |> Enum.drop_while(fn {timestamp, _size} -> timestamp < threshold end) 53 | |> Qex.new() 54 | 55 | received_sizes_sum = 56 | Enum.reduce(packets_received, 0, fn {_timestamp, size}, acc -> acc + size end) 57 | 58 | value = 1 / (Time.as_milliseconds(rr.window, :round) / 1000) * received_sizes_sum 59 | 60 | %__MODULE__{rr | value: value, packets_received: packets_received} 61 | end 62 | 63 | defp resolve_receive_deltas(receive_deltas, reference_time, packet_sizes) do 64 | receive_deltas 65 | |> Enum.zip(packet_sizes) 66 | |> Enum.reject(fn {delta, _size} -> delta == :not_received end) 67 | |> Enum.map_reduce(reference_time, fn {recv_delta, size}, prev_timestamp -> 68 | receive_timestamp = prev_timestamp + recv_delta 69 | {{receive_timestamp, size}, receive_timestamp} 70 | end) 71 | # take the packets_received 72 | |> elem(0) 73 | |> Qex.new() 74 | end 75 | end 76 | -------------------------------------------------------------------------------- /lib/membrane/rtp/utils.ex: -------------------------------------------------------------------------------- 1 | defmodule Membrane.RTP.Utils do 2 | @moduledoc false 3 | 4 | alias Membrane.RTP.Packet 5 | 6 | @spec strip_padding(binary, padding_present? :: boolean) :: 7 | {:ok, {binary, padding_size :: non_neg_integer()}} | :error 8 | def strip_padding(binary, padding_present?) 9 | def strip_padding(binary, false), do: {:ok, {binary, 0}} 10 | 11 | def strip_padding(binary, true) do 12 | with size when size > 0 <- byte_size(binary), 13 | padding_size = :binary.last(binary), 14 | payload_size = byte_size(binary) - padding_size, 15 | <> <- 16 | binary do 17 | {:ok, {stripped_payload, padding_size}} 18 | else 19 | _error -> :error 20 | end 21 | end 22 | 23 | @spec generate_padding(Packet.padding_size()) :: binary() 24 | def generate_padding(0), do: <<>> 25 | 26 | def generate_padding(padding_size) when padding_size in 1..255 do 27 | zeros_no = padding_size - 1 28 | <<0::size(zeros_no)-unit(8), padding_size>> 29 | end 30 | 31 | @spec from_which_rollover(number() | nil, number(), number()) :: :current | :previous | :next 32 | def from_which_rollover(previous_value, new_value, rollover_length) 33 | 34 | def from_which_rollover(nil, _new, _rollover_length), do: :current 35 | 36 | def from_which_rollover(previous_value, new_value, rollover_length) do 37 | # a) current rollover 38 | distance_if_current = abs(previous_value - new_value) 39 | # b) new_value is from the previous rollover 40 | distance_if_previous = abs(previous_value - (new_value - rollover_length)) 41 | # c) new_value is in the next rollover 42 | distance_if_next = abs(previous_value - (new_value + rollover_length)) 43 | 44 | [ 45 | {:current, distance_if_current}, 46 | {:previous, distance_if_previous}, 47 | {:next, distance_if_next} 48 | ] 49 | |> Enum.min_by(fn {_atom, distance} -> distance end) 50 | |> then(fn {result, _value} -> result end) 51 | end 52 | end 53 | -------------------------------------------------------------------------------- /lib/membrane/rtp/vad.ex: -------------------------------------------------------------------------------- 1 | defmodule Membrane.RTP.VAD do 2 | @moduledoc """ 3 | Vad based on audio level sent in RTP header. 4 | 5 | To make this module work appropriate RTP header extension has to be set in the SDP offer/answer. 6 | 7 | Sends `Membrane.RTP.VadEvent` when a score from `Membrane.RTP.Vad.IsSpeakingEstimator` changes. 8 | 9 | A more detailed explanation of how the VAD algorithm can be found in the `Membrane.RTP.Vad.IsSpeakingEstimator` module. 10 | 11 | Buffers that are processed by this element may or may not have been processed by 12 | a depayloader and passed through a jitter buffer. If they have not, then the only timestamp 13 | available for time comparison is the RTP timestamp. 14 | 15 | When calculating the epoch of the timestamp, we need to account for 32bit integer wrapping. 16 | * `:current` - the difference between timestamps is low: the timestamp has not wrapped around. 17 | * `:next` - the timestamp has wrapped around to 0. To simplify queue processing we reset the state. 18 | * `:prev` - the timestamp has recently wrapped around. We might receive an out-of-order packet 19 | from before the rollover, which we ignore. 20 | """ 21 | use Membrane.Filter 22 | 23 | alias Membrane.RTP.{Header, Utils, VadEvent} 24 | alias Membrane.RTP.Vad.{AudioLevelQueue, IsSpeakingEstimator} 25 | 26 | def_input_pad :input, availability: :always, accepted_format: _any, flow_control: :auto 27 | 28 | def_output_pad :output, availability: :always, accepted_format: _any, flow_control: :auto 29 | 30 | def_options vad_id: [ 31 | spec: 1..14, 32 | description: "ID of VAD header extension." 33 | ], 34 | vad_threshold: [ 35 | spec: -127..0, 36 | default: -32, 37 | description: """ 38 | Audio level in dBov representing vad threshold. 39 | Values above are considered to represent voice activity. 40 | Value -127 represents digital silence. 41 | """ 42 | ] 43 | 44 | @impl true 45 | def handle_init(_ctx, opts) do 46 | state = %{ 47 | vad_id: opts.vad_id, 48 | audio_levels: AudioLevelQueue.new(), 49 | vad: :silence, 50 | current_timestamp: nil, 51 | vad_threshold: opts.vad_threshold + 127 52 | } 53 | 54 | {[], state} 55 | end 56 | 57 | @impl true 58 | def handle_buffer(:input, %Membrane.Buffer{} = buffer, _ctx, state) do 59 | {extension, buffer} = Header.Extension.pop(buffer, state.vad_id) 60 | handle_if_present(buffer, extension, state) 61 | end 62 | 63 | defp handle_if_present(buffer, nil, state), do: {[buffer: {:output, buffer}], state} 64 | 65 | @timestamp_limit Bitwise.bsl(1, 32) 66 | 67 | defp handle_if_present(buffer, extension, state) do 68 | <<_v::1, level::7>> = extension.data 69 | 70 | new_extension = %Header.Extension{ 71 | identifier: :vad, 72 | data: extension.data 73 | } 74 | 75 | buffer = Header.Extension.put(buffer, new_extension) 76 | 77 | rtp_timestamp = buffer.metadata.rtp.timestamp 78 | rollover = Utils.from_which_rollover(state.current_timestamp, rtp_timestamp, @timestamp_limit) 79 | current_timestamp = state.current_timestamp || 0 80 | 81 | cond do 82 | rollover == :current && rtp_timestamp > current_timestamp -> 83 | handle_vad(buffer, rtp_timestamp, level, state) 84 | 85 | rollover == :next -> 86 | {[], state} = handle_init(%{}, %{state | vad_threshold: state.vad_threshold - 127}) 87 | {[buffer: {:output, buffer}], state} 88 | 89 | true -> 90 | {[buffer: {:output, buffer}], state} 91 | end 92 | end 93 | 94 | defp handle_vad(buffer, rtp_timestamp, level_in_dbov, state) do 95 | level_in_db = 127 - level_in_dbov 96 | updated_audio_levels = AudioLevelQueue.add(state.audio_levels, level_in_db) 97 | 98 | vad_estimation = 99 | updated_audio_levels 100 | |> AudioLevelQueue.to_list() 101 | |> IsSpeakingEstimator.estimate_is_speaking(state.vad_threshold) 102 | 103 | actions = [buffer: {:output, buffer}] ++ maybe_send_event(vad_estimation, state) 104 | 105 | state = %{ 106 | state 107 | | current_timestamp: rtp_timestamp, 108 | audio_levels: updated_audio_levels, 109 | vad: vad_estimation 110 | } 111 | 112 | {actions, state} 113 | end 114 | 115 | defp maybe_send_event(audio_levels_vad, state) do 116 | if vad_state_has_changed(state.vad, audio_levels_vad) do 117 | [event: {:output, %VadEvent{vad: audio_levels_vad}}] 118 | else 119 | [] 120 | end 121 | end 122 | 123 | defp vad_state_has_changed(old_value, new_value), do: old_value != new_value 124 | end 125 | -------------------------------------------------------------------------------- /lib/membrane/rtp/vad/audio_level_queue.ex: -------------------------------------------------------------------------------- 1 | defmodule Membrane.RTP.Vad.AudioLevelQueue do 2 | @moduledoc false 3 | 4 | # The queue contains audio levels for VAD implementation. It is used as an input of IsSpeakingEstimator.estimate_is_speaking. 5 | # This structure builds on top of a simple FIFO Erlang queue by having a fixed max number of elements. 6 | 7 | # The newest element in always appended to the front and popped out from its rear, so `to_list/1` returns the most recent element as the head of a list. 8 | # The length of a list can be obtained in O(1) time. 9 | 10 | alias Membrane.RTP.Vad.VadParams 11 | 12 | @target_audio_level_length VadParams.target_levels_length() 13 | 14 | @enforce_keys [:levels, :length] 15 | defstruct [:levels, :length] 16 | 17 | @typedoc """ 18 | A type for storing information about a fixed number of recent audio levels. 19 | 20 | `:levels` - erlang queue which stores at most @target_audio_level_length elements 21 | `:length` - number of elements 22 | """ 23 | 24 | @type t() :: %__MODULE__{ 25 | levels: :queue.queue(non_neg_integer()), 26 | length: non_neg_integer() 27 | } 28 | 29 | @doc """ 30 | Creates new AudioLevelQueue. 31 | """ 32 | @spec new(Enum.t()) :: t() 33 | def new(init_data \\ []) do 34 | levels = 35 | init_data 36 | |> Enum.take(@target_audio_level_length) 37 | |> Enum.to_list() 38 | |> :queue.from_list() 39 | 40 | %__MODULE__{levels: levels, length: :queue.len(levels)} 41 | end 42 | 43 | @doc """ 44 | Given a AudioLevelQueue and level value it returns a queue with the level value on front 45 | 46 | The function also reduces the size of the queue if the maximum size has been reached. 47 | It does so by dropping the oldest level. 48 | """ 49 | @spec add(t(), non_neg_integer) :: t() 50 | def add(%__MODULE__{length: @target_audio_level_length} = old_queue, level) do 51 | levels = :queue.in_r(level, :queue.drop_r(old_queue.levels)) 52 | %__MODULE__{old_queue | levels: levels} 53 | end 54 | 55 | def add(%__MODULE__{levels: old_levels, length: length}, level) do 56 | %__MODULE__{levels: :queue.in_r(level, old_levels), length: length + 1} 57 | end 58 | 59 | @doc """ 60 | Given an AudioLevelQueue it returns a list. 61 | """ 62 | @spec to_list(t()) :: [non_neg_integer()] 63 | def to_list(%__MODULE__{levels: levels}), do: :queue.to_list(levels) 64 | end 65 | -------------------------------------------------------------------------------- /lib/membrane/rtp/vad/vad_params.ex: -------------------------------------------------------------------------------- 1 | defmodule Membrane.RTP.Vad.VadParams do 2 | @moduledoc false 3 | 4 | # The information about selected parameters used in multiple modules concerning VAD. 5 | # Additionally it computes the `target_levels_length` which is the number audio levels needed for proper voice activity estimation. It is directly dependent on the VAD parameters. 6 | 7 | # The parameters values are different for each environment and can be tweaked in the configuration files placed in `config` directory. 8 | 9 | # The meaning of the parameters is described in the `Membrane.RTP.Vad.IsSpeakingEstimator`. 10 | 11 | @default_parameters %{ 12 | immediate: %{ 13 | subunits: 1, 14 | score_threshold: 0, 15 | lambda: 1 16 | }, 17 | medium: %{ 18 | subunits: 10, 19 | score_threshold: 20, 20 | subunit_threshold: 1, 21 | lambda: 24 22 | }, 23 | long: %{ 24 | subunits: 7, 25 | score_threshold: 20, 26 | subunit_threshold: 3, 27 | lambda: 47 28 | } 29 | } 30 | 31 | @params Application.compile_env( 32 | :membrane_rtp_plugin, 33 | :vad_estimation_parameters, 34 | @default_parameters 35 | ) 36 | 37 | @immediate @params.immediate 38 | @medium @params.medium 39 | @long @params.long 40 | 41 | @spec vad_params() :: map() 42 | def vad_params(), do: @params 43 | 44 | @spec immediate() :: map() 45 | def immediate(), do: @immediate 46 | 47 | @spec medium() :: map() 48 | def medium(), do: @medium 49 | 50 | @spec long() :: map() 51 | def long(), do: @long 52 | 53 | @spec target_levels_length() :: pos_integer() 54 | def target_levels_length(), do: @immediate.subunits * @medium.subunits * @long.subunits 55 | end 56 | -------------------------------------------------------------------------------- /lib/membrane/rtp/vad_event.ex: -------------------------------------------------------------------------------- 1 | defmodule Membrane.RTP.VadEvent do 2 | @moduledoc """ 3 | An event informing about Voice Activity Detection status changes 4 | """ 5 | 6 | @derive Membrane.EventProtocol 7 | @enforce_keys [:vad] 8 | defstruct @enforce_keys 9 | 10 | @typedoc """ 11 | Type describing the structure of the Voice Activity Detection event. 12 | 13 | - `:vad` - contains information about VAD status. Indicates either speech or silence. 14 | For details on voice activity detection algorithm, refer to `Membrane.RTP.VAD` 15 | """ 16 | @type t() :: %__MODULE__{ 17 | vad: :speech | :silence 18 | } 19 | end 20 | -------------------------------------------------------------------------------- /lib/membrane/srtcp/decryptor.ex: -------------------------------------------------------------------------------- 1 | if Code.ensure_loaded?(ExLibSRTP) do 2 | defmodule Membrane.SRTCP.Decryptor do 3 | @moduledoc """ 4 | Converts SRTCP packets to plain RTCP. 5 | 6 | Requires adding [libsrtp](https://github.com/membraneframework/elixir_libsrtp) dependency to work. 7 | """ 8 | use Membrane.Filter 9 | 10 | require Membrane.Logger 11 | 12 | alias Membrane.Buffer 13 | alias Membrane.SRTP 14 | 15 | def_input_pad :input, accepted_format: _any, flow_control: :auto 16 | def_output_pad :output, accepted_format: _any, flow_control: :auto 17 | 18 | def_options policies: [ 19 | spec: [ExLibSRTP.Policy.t()], 20 | description: """ 21 | List of SRTP policies to use for decrypting packets. 22 | See `t:ExLibSRTP.Policy.t/0` for details. 23 | """ 24 | ] 25 | 26 | @impl true 27 | def handle_init(_ctx, options) do 28 | {[], Map.from_struct(options) |> Map.merge(%{srtp: nil})} 29 | end 30 | 31 | @impl true 32 | def handle_setup(_ctx, state) do 33 | srtp = ExLibSRTP.new() 34 | 35 | state.policies 36 | |> Bunch.listify() 37 | |> Enum.each(&ExLibSRTP.add_stream(srtp, &1)) 38 | 39 | {[], %{state | srtp: srtp}} 40 | end 41 | 42 | @impl true 43 | def handle_buffer(:input, buffer, _ctx, state) do 44 | case ExLibSRTP.unprotect_rtcp(state.srtp, buffer.payload) do 45 | {:ok, payload} -> 46 | {[buffer: {:output, %Buffer{buffer | payload: payload}}], state} 47 | 48 | {:error, reason} when reason in [:replay_fail, :replay_old] -> 49 | Membrane.Logger.debug(""" 50 | Couldn't unprotect srtcp packet: 51 | #{inspect(buffer.payload, limit: :infinity)} 52 | Reason: #{inspect(reason)}. Ignoring packet. 53 | """) 54 | 55 | {[], state} 56 | 57 | {:error, reason} -> 58 | raise "Couldn't unprotect SRTCP packet due to #{inspect(reason)}" 59 | end 60 | end 61 | 62 | @impl true 63 | def handle_event(_pad, %SRTP.KeyingMaterialEvent{} = event, _ctx, %{policies: []} = state) do 64 | {:ok, crypto_profile} = 65 | ExLibSRTP.Policy.crypto_profile_from_dtls_srtp_protection_profile( 66 | event.protection_profile 67 | ) 68 | 69 | policy = %ExLibSRTP.Policy{ 70 | ssrc: :any_inbound, 71 | key: event.remote_keying_material, 72 | rtp: crypto_profile, 73 | rtcp: crypto_profile 74 | } 75 | 76 | :ok = ExLibSRTP.add_stream(state.srtp, policy) 77 | {[], %{state | policies: [policy]}} 78 | end 79 | 80 | @impl true 81 | def handle_event(_pad, %SRTP.KeyingMaterialEvent{}, _ctx, state) do 82 | Membrane.Logger.warning("Got unexpected SRTP.KeyingMaterialEvent. Ignoring.") 83 | {[], state} 84 | end 85 | 86 | @impl true 87 | def handle_event(pad, event, ctx, state), do: super(pad, event, ctx, state) 88 | end 89 | end 90 | -------------------------------------------------------------------------------- /lib/membrane/srtp/decryptor.ex: -------------------------------------------------------------------------------- 1 | if Code.ensure_loaded?(ExLibSRTP) do 2 | defmodule Membrane.SRTP.Decryptor do 3 | @moduledoc """ 4 | Converts SRTP packets to plain RTP. 5 | 6 | Decryptor expects that buffers passed to `handle_buffer/4` have already parsed headers 7 | in the metadata field as they contain information about header length. The header 8 | length is needed to avoid parsing the header twice in case of any elements preceding 9 | the decryptor needed the information to e.g. drop the packet before reaching the decryptor. 10 | `ExLibSRTP` expects a valid SRTP packet containing the header, after decryption, the 11 | payload binary again includes the header. The header's length simply allows stripping 12 | the header without any additional parsing. 13 | 14 | Requires adding [srtp](https://github.com/membraneframework/elixir_libsrtp) dependency to work. 15 | """ 16 | use Membrane.Filter 17 | 18 | require Membrane.Logger 19 | 20 | alias Membrane.Buffer 21 | alias Membrane.RTP.Utils 22 | alias Membrane.SRTP 23 | 24 | def_input_pad :input, accepted_format: _any, flow_control: :auto 25 | def_output_pad :output, accepted_format: _any, flow_control: :auto 26 | 27 | def_options policies: [ 28 | spec: [ExLibSRTP.Policy.t()], 29 | description: """ 30 | List of SRTP policies to use for decrypting packets. 31 | See `t:ExLibSRTP.Policy.t/0` for details. 32 | """ 33 | ] 34 | 35 | @impl true 36 | def handle_init(_ctx, %__MODULE__{policies: policies}) do 37 | state = %{ 38 | policies: policies, 39 | srtp: nil 40 | } 41 | 42 | {[], state} 43 | end 44 | 45 | @impl true 46 | def handle_setup(_ctx, state) do 47 | srtp = ExLibSRTP.new() 48 | 49 | state.policies 50 | |> Bunch.listify() 51 | |> Enum.each(&ExLibSRTP.add_stream(srtp, &1)) 52 | 53 | {[], %{state | srtp: srtp}} 54 | end 55 | 56 | @impl true 57 | def handle_event(_pad, %SRTP.KeyingMaterialEvent{} = event, _ctx, %{policies: []} = state) do 58 | {:ok, crypto_profile} = 59 | ExLibSRTP.Policy.crypto_profile_from_dtls_srtp_protection_profile( 60 | event.protection_profile 61 | ) 62 | 63 | policy = %ExLibSRTP.Policy{ 64 | ssrc: :any_inbound, 65 | key: event.remote_keying_material, 66 | rtp: crypto_profile, 67 | rtcp: crypto_profile 68 | } 69 | 70 | :ok = ExLibSRTP.add_stream(state.srtp, policy) 71 | {[], Map.put(state, :policies, [policy])} 72 | end 73 | 74 | @impl true 75 | def handle_event(_pad, %SRTP.KeyingMaterialEvent{}, _ctx, state) do 76 | Membrane.Logger.warning("Got unexpected SRTP.KeyingMaterialEvent. Ignoring.") 77 | {[], state} 78 | end 79 | 80 | @impl true 81 | def handle_event(pad, other, ctx, state), do: super(pad, other, ctx, state) 82 | 83 | @impl true 84 | def handle_buffer(:input, buffer, _ctx, state) do 85 | %Buffer{ 86 | payload: payload, 87 | metadata: %{rtp: %{total_header_size: total_header_size}} 88 | } = buffer 89 | 90 | state.srtp 91 | |> ExLibSRTP.unprotect(payload) 92 | |> case do 93 | {:ok, payload} -> 94 | # decrypted payload contains the header that we can simply strip without any parsing as we know its length 95 | remaining_header_size = total_header_size * 8 - 3 96 | 97 | <<_ver::2, has_padding::1, _header::bitstring-size(remaining_header_size), 98 | payload::binary>> = payload 99 | 100 | {:ok, {payload, padding_size}} = Utils.strip_padding(payload, has_padding == 1) 101 | metadata = put_in(buffer.metadata, [:rtp, :padding_size], padding_size) 102 | buffer = %Buffer{buffer | metadata: metadata, payload: payload} 103 | 104 | {[buffer: {:output, buffer}], state} 105 | 106 | {:error, reason} when reason in [:replay_fail, :replay_old] -> 107 | Membrane.Logger.debug(""" 108 | Couldn't unprotect srtp packet: 109 | #{inspect(payload, limit: :infinity)} 110 | Reason: #{inspect(reason)}. Ignoring packet. 111 | """) 112 | 113 | {[], state} 114 | 115 | {:error, reason} -> 116 | raise "Couldn't unprotect SRTP packet due to #{inspect(reason)}" 117 | end 118 | end 119 | end 120 | end 121 | -------------------------------------------------------------------------------- /lib/membrane/srtp/encryptor.ex: -------------------------------------------------------------------------------- 1 | if Code.ensure_loaded?(ExLibSRTP) do 2 | defmodule Membrane.SRTP.Encryptor do 3 | @moduledoc """ 4 | Converts plain RTP packets to SRTP. 5 | 6 | Requires adding [srtp](https://github.com/membraneframework/elixir_libsrtp) dependency to work. 7 | """ 8 | use Membrane.Filter 9 | 10 | require Membrane.Logger 11 | 12 | alias Membrane.{Buffer, RTP, SRTP} 13 | 14 | def_input_pad :input, accepted_format: _any, flow_control: :auto 15 | def_output_pad :output, accepted_format: _any, flow_control: :auto 16 | 17 | defguardp is_protection_error_fatal(type, reason) 18 | when type == :rtcp or 19 | (type == :rtp and reason not in [:replay_fail, :replay_old]) 20 | 21 | def_options policies: [ 22 | spec: [ExLibSRTP.Policy.t()], 23 | default: [], 24 | description: """ 25 | List of SRTP policies to use for encrypting packets. 26 | See `t:ExLibSRTP.Policy.t/0` for details. 27 | """ 28 | ] 29 | 30 | @impl true 31 | def handle_init(_ctx, %__MODULE__{policies: policies}) do 32 | state = %{ 33 | policies: policies, 34 | srtp: nil, 35 | queue: [] 36 | } 37 | 38 | {[], state} 39 | end 40 | 41 | @impl true 42 | def handle_setup(_ctx, state) do 43 | srtp = ExLibSRTP.new() 44 | 45 | state.policies 46 | |> Bunch.listify() 47 | |> Enum.each(&ExLibSRTP.add_stream(srtp, &1)) 48 | 49 | {[], %{state | srtp: srtp}} 50 | end 51 | 52 | @impl true 53 | def handle_start_of_stream(:input, _ctx, state) do 54 | if state.policies == [] do 55 | # TODO: remove when dynamic switching between automatic and manual demands will be supported 56 | {[start_timer: {:policy_timer, Membrane.Time.seconds(5)}], state} 57 | else 58 | {[], state} 59 | end 60 | end 61 | 62 | @impl true 63 | def handle_tick(:policy_timer, ctx, state) do 64 | if state.policies != [] or ctx.pads.input.end_of_stream? do 65 | {[stop_timer: :policy_timer], state} 66 | else 67 | raise "No SRTP policies arrived in 5 seconds" 68 | end 69 | end 70 | 71 | @impl true 72 | def handle_event(_pad, %SRTP.KeyingMaterialEvent{} = event, _ctx, %{policies: []} = state) do 73 | {:ok, crypto_profile} = 74 | ExLibSRTP.Policy.crypto_profile_from_dtls_srtp_protection_profile( 75 | event.protection_profile 76 | ) 77 | 78 | policy = %ExLibSRTP.Policy{ 79 | ssrc: :any_outbound, 80 | key: event.local_keying_material, 81 | rtp: crypto_profile, 82 | rtcp: crypto_profile 83 | } 84 | 85 | :ok = ExLibSRTP.add_stream(state.srtp, policy) 86 | buffers = state.queue |> Enum.reverse() |> Enum.flat_map(&protect_buffer(&1, state.srtp)) 87 | {[buffer: {:output, buffers}], %{Map.put(state, :policies, [policy]) | queue: []}} 88 | end 89 | 90 | @impl true 91 | def handle_event(_pad, %SRTP.KeyingMaterialEvent{}, _ctx, state) do 92 | Membrane.Logger.warning("Got unexpected SRTP.KeyingMaterialEvent. Ignoring.") 93 | {[], state} 94 | end 95 | 96 | @impl true 97 | def handle_event(pad, other, ctx, state), do: super(pad, other, ctx, state) 98 | 99 | @impl true 100 | def handle_buffer(:input, buffer, _ctx, %{policies: []} = state) do 101 | {[], Map.update!(state, :queue, &[buffer | &1])} 102 | end 103 | 104 | @impl true 105 | def handle_buffer(:input, buffer, _ctx, state) do 106 | {[buffer: {:output, protect_buffer(buffer, state.srtp)}], state} 107 | end 108 | 109 | defp protect_buffer(buffer, srtp) do 110 | %Buffer{payload: payload} = buffer 111 | packet_type = RTP.Packet.identify(payload) 112 | 113 | protection_result = 114 | case packet_type do 115 | :rtp -> ExLibSRTP.protect(srtp, payload) 116 | :rtcp -> ExLibSRTP.protect_rtcp(srtp, payload) 117 | end 118 | 119 | case protection_result do 120 | {:ok, payload} -> 121 | [%Buffer{buffer | payload: payload}] 122 | 123 | {:error, reason} when is_protection_error_fatal(packet_type, reason) -> 124 | raise "Failed to protect #{inspect(packet_type)} due to unhandled error #{reason}" 125 | 126 | {:error, reason} -> 127 | Membrane.Logger.warning("Ignoring #{inspect(packet_type)} packet due to `#{reason}`") 128 | [] 129 | end 130 | end 131 | end 132 | end 133 | -------------------------------------------------------------------------------- /mix.exs: -------------------------------------------------------------------------------- 1 | defmodule Membrane.RTP.Plugin.MixProject do 2 | use Mix.Project 3 | 4 | @version "0.31.0" 5 | @github_url "https://github.com/membraneframework/membrane_rtp_plugin" 6 | 7 | def project do 8 | [ 9 | app: :membrane_rtp_plugin, 10 | version: @version, 11 | elixir: "~> 1.12", 12 | elixirc_paths: elixirc_paths(Mix.env()), 13 | deps: deps(), 14 | dialyzer: dialyzer(), 15 | 16 | # hex 17 | description: "Membrane Multimedia Framework plugin for RTP", 18 | package: package(), 19 | 20 | # docs 21 | name: "Membrane RTP plugin", 22 | source_url: @github_url, 23 | homepage_url: "https://membraneframework.org", 24 | docs: docs() 25 | ] 26 | end 27 | 28 | def application do 29 | [ 30 | extra_applications: [] 31 | ] 32 | end 33 | 34 | defp elixirc_paths(:test), do: ["lib", "test/support"] 35 | defp elixirc_paths(_env), do: ["lib"] 36 | 37 | defp deps do 38 | [ 39 | {:membrane_core, "~> 1.0"}, 40 | {:membrane_rtp_format, "~> 0.11.0"}, 41 | {:membrane_funnel_plugin, "~> 0.9.0"}, 42 | {:membrane_telemetry_metrics, "~> 0.1.0"}, 43 | {:ex_libsrtp, "~> 0.6.0 or ~> 0.7.0", optional: true}, 44 | {:ex_rtp, "~> 0.4.0"}, 45 | {:ex_rtcp, "~> 0.4.0"}, 46 | {:qex, "~> 0.5.1"}, 47 | {:bunch, "~> 1.5"}, 48 | {:heap, "~> 2.0.2"}, 49 | {:bimap, "~> 1.2"}, 50 | 51 | # Test 52 | {:membrane_rtp_h264_plugin, "~> 0.20.1", only: :test}, 53 | {:membrane_rtp_aac_plugin, "~> 0.9.3", only: :test}, 54 | {:membrane_rtp_mpegaudio_plugin, "~> 0.14.1", only: :test}, 55 | {:membrane_h264_ffmpeg_plugin, "~> 0.31.0", only: :test}, 56 | {:membrane_h26x_plugin, "~> 0.10.2", only: :test}, 57 | {:membrane_aac_plugin, "~> 0.19.0", only: :test}, 58 | {:membrane_mp4_plugin, "~> 0.35.0", only: :test}, 59 | {:membrane_pcap_plugin, 60 | github: "membraneframework/membrane_pcap_plugin", tag: "v0.9.0", only: :test}, 61 | {:membrane_hackney_plugin, "~> 0.11.0", only: :test}, 62 | {:membrane_realtimer_plugin, "~> 0.10.1", only: :test}, 63 | 64 | # Dev 65 | {:ex_doc, "~> 0.28", only: :dev, runtime: false}, 66 | {:dialyxir, "~> 1.1", only: :dev, runtime: false}, 67 | {:credo, "~> 1.5", only: :dev, runtime: false} 68 | ] 69 | end 70 | 71 | defp dialyzer() do 72 | opts = [ 73 | plt_add_apps: [:ex_libsrtp], 74 | flags: [:error_handling] 75 | ] 76 | 77 | if System.get_env("CI") == "true" do 78 | # Store PLTs in cacheable directory for CI 79 | [plt_local_path: "priv/plts", plt_core_path: "priv/plts"] ++ opts 80 | else 81 | opts 82 | end 83 | end 84 | 85 | defp package do 86 | [ 87 | maintainers: ["Membrane Team"], 88 | licenses: ["Apache-2.0"], 89 | links: %{ 90 | "GitHub" => @github_url, 91 | "Membrane Framework Homepage" => "https://membraneframework.org" 92 | } 93 | ] 94 | end 95 | 96 | defp docs do 97 | [ 98 | main: "readme", 99 | extras: ["README.md", LICENSE: [title: License]], 100 | formatters: ["html"], 101 | source_ref: "v#{@version}", 102 | nest_modules_by_prefix: [ 103 | Membrane.RTP, 104 | Membrane.RTCP, 105 | Membrane.SRTP, 106 | Membrane.SRTCP 107 | ], 108 | groups_for_modules: [ 109 | "RTP session": [~r/^Membrane\.RTP\.Session/], 110 | RTP: [~r/^Membrane\.RTP/], 111 | RTCP: [~r/^Membrane\.RTCP/], 112 | SRTP: [~r/^Membrane\.SRTP/] 113 | ] 114 | ] 115 | end 116 | end 117 | -------------------------------------------------------------------------------- /test/fixtures/rtcp/malformed.hex: -------------------------------------------------------------------------------- 1 | 80C80006F063E69AE4D120205B7E0AC725706FDA00000F1900452EDA81CA0006F063E69A011076304A496348495066446833647559430000FFFF0003F063E69A2A0000010000012C8FCE0005F063E69A0000000052454D4201009700A4B38285 -------------------------------------------------------------------------------- /test/fixtures/rtcp/packets.hex: -------------------------------------------------------------------------------- 1 | 80c8000662dbefd0e24d497f597a56de127379780000004200002bcf81ca000c62dbefd0011c757365723234363533333039313040686f73742d343437656231366606094753747265616d6572000000 2 | 80c8000662dbefd0e24d4984ca148ba8127722910000012c0000d5c981ca000c62dbefd0011c757365723234363533333039313040686f73742d343437656231366606094753747265616d6572000000 3 | 80c8000662dbefd0e24d498a957bfa4c127b08c50000022600018df781ca000c62dbefd0011c757365723234363533333039313040686f73742d343437656231366606094753747265616d6572000000 4 | 80c8000662dbefd0e24d498e59cb89d6127d91aa000002c80002060c81ca000c62dbefd0011c757365723234363533333039313040686f73742d343437656231366606094753747265616d6572000000 5 | 80c8000662dbefd0e24d4991f9ee99a61280023800000364000279cc81ca000c62dbefd0011c757365723234363533333039313040686f73742d343437656231366606094753747265616d6572000000 6 | 80c8000662dbefd0e24d4994ac3265ad1281d2b5000003d80002d01a81ca000c62dbefd0011c757365723234363533333039313040686f73742d343437656231366606094753747265616d6572000000 7 | -------------------------------------------------------------------------------- /test/fixtures/rtcp/single_packet.hex: -------------------------------------------------------------------------------- 1 | 80C8000651F22349E1FA920539DB22D0211F26C00000009E00006CC381CB000151F22349 2 | -------------------------------------------------------------------------------- /test/fixtures/rtcp/twcc_feedbacks.hex: -------------------------------------------------------------------------------- 1 | 772f00051ed96e0bd6504058feb80000 2 | 849300061ed9c273c0148408 3 | 0bb4000f1ed8fd0c2008e5554400000c08180004fffc341c041c0010 4 | 518c00151ed89d0ed655d556d551a80cfff0041000100c0000000008fff81c0028300c04 -------------------------------------------------------------------------------- /test/fixtures/rtcp/twcc_malformed_feedbacks.hex: -------------------------------------------------------------------------------- 1 | 772f00051ed96e0bd750405800000000 2 | 772f000a1ed96e0bd6504058feb80000 3 | 772f00051ed96e0bd6504058feb8000000000080 4 | 0bb4000f1ed8fd0c6008e5554400000c08180004fffc341c041c0010 -------------------------------------------------------------------------------- /test/fixtures/rtcp/with_unknown_pt.hex: -------------------------------------------------------------------------------- 1 | 80C80006F063E69AE4D120205B7E0AC725706FDA00000F1900452EDA81CA0006F063E69A011076304A49634849506644683364755943000080FF0003F063E69A2A0000010000012C8FCE0005F063E69A0000000052454D4201009700A4B38285 -------------------------------------------------------------------------------- /test/fixtures/rtp/h264/bun.h264: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/membraneframework/membrane_rtp_plugin/30c9f60b0fe92c6194444f7975b0ac43d0d0d1ff/test/fixtures/rtp/h264/bun.h264 -------------------------------------------------------------------------------- /test/fixtures/rtp/rtp_packet.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/membraneframework/membrane_rtp_plugin/30c9f60b0fe92c6194444f7975b0ac43d0d0d1ff/test/fixtures/rtp/rtp_packet.bin -------------------------------------------------------------------------------- /test/fixtures/rtp/rtp_packet_payload.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/membraneframework/membrane_rtp_plugin/30c9f60b0fe92c6194444f7975b0ac43d0d0d1ff/test/fixtures/rtp/rtp_packet_payload.bin -------------------------------------------------------------------------------- /test/fixtures/rtp/rtp_packet_with_padding.bin: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/membraneframework/membrane_rtp_plugin/30c9f60b0fe92c6194444f7975b0ac43d0d0d1ff/test/fixtures/rtp/rtp_packet_with_padding.bin -------------------------------------------------------------------------------- /test/fixtures/rtp/session/demo.pcap: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/membraneframework/membrane_rtp_plugin/30c9f60b0fe92c6194444f7975b0ac43d0d0d1ff/test/fixtures/rtp/session/demo.pcap -------------------------------------------------------------------------------- /test/fixtures/rtp/session/demo_audio_video_rtp3.pcap: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/membraneframework/membrane_rtp_plugin/30c9f60b0fe92c6194444f7975b0ac43d0d0d1ff/test/fixtures/rtp/session/demo_audio_video_rtp3.pcap -------------------------------------------------------------------------------- /test/fixtures/rtp/session/demo_rtp.pcap: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/membraneframework/membrane_rtp_plugin/30c9f60b0fe92c6194444f7975b0ac43d0d0d1ff/test/fixtures/rtp/session/demo_rtp.pcap -------------------------------------------------------------------------------- /test/fixtures/rtp/session/h264_before_sr.pcap: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/membraneframework/membrane_rtp_plugin/30c9f60b0fe92c6194444f7975b0ac43d0d0d1ff/test/fixtures/rtp/session/h264_before_sr.pcap -------------------------------------------------------------------------------- /test/fixtures/rtp/session/h264_before_sr2.pcap: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/membraneframework/membrane_rtp_plugin/30c9f60b0fe92c6194444f7975b0ac43d0d0d1ff/test/fixtures/rtp/session/h264_before_sr2.pcap -------------------------------------------------------------------------------- /test/fixtures/rtp/session/srtp.pcap: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/membraneframework/membrane_rtp_plugin/30c9f60b0fe92c6194444f7975b0ac43d0d0d1ff/test/fixtures/rtp/session/srtp.pcap -------------------------------------------------------------------------------- /test/membrane/rtcp/afb_packet_test.exs: -------------------------------------------------------------------------------- 1 | defmodule Membrane.RTCP.FeedbackPacket.AFBTest do 2 | use ExUnit.Case, async: true 3 | 4 | alias Membrane.RTCP.{FeedbackPacket, Fixtures, Packet, SdesPacket, SenderReportPacket} 5 | 6 | test "AFB Packet can be parsed" do 7 | assert {:ok, [%SenderReportPacket{}, %SdesPacket{}, %FeedbackPacket{payload: fb_payload}]} = 8 | Packet.parse(Fixtures.compound_sr_sdes_remb()) 9 | 10 | assert %FeedbackPacket.AFB{message: binary} = fb_payload 11 | assert "REMB" <> _rest = binary 12 | end 13 | end 14 | -------------------------------------------------------------------------------- /test/membrane/rtcp/compound_packet_test.exs: -------------------------------------------------------------------------------- 1 | defmodule Membrane.RTCP.PacketTest do 2 | use ExUnit.Case, async: true 3 | 4 | alias Membrane.RTCP 5 | alias Membrane.RTCP.Fixtures 6 | 7 | @sample_ssrc 1_374_823_241 8 | 9 | test "compound packets parsing" do 10 | packet = Fixtures.sample_packet_binary() 11 | assert {:ok, packets} = RTCP.Packet.parse(packet) 12 | 13 | assert %RTCP.SenderReportPacket{reports: [], sender_info: %{}, ssrc: @sample_ssrc} = 14 | hd(packets) 15 | 16 | assert %RTCP.ByePacket{ssrcs: [@sample_ssrc]} = Enum.at(packets, 1) 17 | end 18 | 19 | test "reconstructed packets are (almost) equal to original packets" do 20 | packet = Fixtures.sample_packet_binary() 21 | assert {:ok, packets} = RTCP.Packet.parse(packet) 22 | regenerated_packet = RTCP.Packet.serialize(packets) 23 | 24 | <> = packet 25 | 26 | assert <<^head::binary-size(12), ntp_lsw::32, ^tail::binary>> = regenerated_packet 27 | 28 | # The least significant word of NTP timestamp (fractional part) might differ due to rounding errors 29 | assert_in_delta ntp_lsw, ref_ntp_lsw, 10 30 | end 31 | 32 | describe "when a malformed packet is contained" do 33 | test "returns an error" do 34 | packet = Fixtures.malformed_packet_binary() 35 | assert {:error, :malformed_packet} = RTCP.Packet.parse(packet) 36 | end 37 | end 38 | 39 | describe "with unknown packet types" do 40 | test "returns only the parsable packet types" do 41 | packet = Fixtures.with_unknown_packet_type() 42 | assert {:ok, packets} = RTCP.Packet.parse(packet) 43 | 44 | assert [ 45 | %RTCP.SenderReportPacket{}, 46 | %RTCP.SdesPacket{}, 47 | %RTCP.FeedbackPacket{} 48 | ] = packets 49 | end 50 | end 51 | end 52 | -------------------------------------------------------------------------------- /test/membrane/rtcp/nack_packet_test.exs: -------------------------------------------------------------------------------- 1 | defmodule Membrane.RTCP.TransportFeedbackPacket.NACKTest do 2 | use ExUnit.Case, async: true 3 | 4 | alias Membrane.RTCP.TransportFeedbackPacket.NACK 5 | 6 | @examples %{ 7 | # BLP == 0, only sequence number passed via PID lost 8 | "one ID lost" => {<<2137::16, 0::16>>, [2137]}, 9 | # All bits of BLP set to 1, lost packets from PID to PID+16 10 | "17 consecutive IDs lost" => {<<2137::16, 0xFFFF::16>>, 2137..(2137 + 16)//1}, 11 | # Two least significant bits of BLP set to 1, lost packets from PID to PID+2 12 | "3 consecutive IDs lost" => {<<2137::16, 0::14, 0b11::2>>, 2137..(2137 + 2)//1}, 13 | # PID, PID + 2 and PID + 4 missing 14 | "3 IDs with step 2 lost" => {<<2137::16, 0::12, 1::1, 0::1, 1::1, 0::1>>, [2137, 2139, 2141]}, 15 | # Two FCIs combined to report 18 packets lost 16 | "18 consecutive IDs lost" => 17 | {<<2137::16, 0xFFFF::16, 2137 + 17::16, 0::16>>, 2137..(2137 + 17)//1} 18 | } 19 | 20 | describe "NACK.decode/1 decodes" do 21 | for name <- Map.keys(@examples) do 22 | test name do 23 | {binary, expected_ids} = Map.fetch!(@examples, unquote(name)) 24 | assert {:ok, %NACK{lost_packet_ids: ids}} = NACK.decode(binary) 25 | assert ids == Enum.to_list(expected_ids) 26 | end 27 | end 28 | 29 | test "wrapping up ids" do 30 | assert {:ok, %NACK{lost_packet_ids: ids}} = NACK.decode(<<65_535::16, 0xFFFF::16>>) 31 | assert ids == [65_535 | Enum.to_list(0..15//1)] 32 | end 33 | end 34 | 35 | describe "NACK.encode/1 encodes" do 36 | for name <- Map.keys(@examples) do 37 | test name do 38 | {expected_binary, ids} = Map.fetch!(@examples, unquote(name)) 39 | assert NACK.encode(%NACK{lost_packet_ids: ids}) == expected_binary 40 | end 41 | end 42 | 43 | test "wrapping up ids" do 44 | lost_ids = [65_535 | Enum.to_list(0..15//1)] 45 | # Current implementation splits ids into two FCIs on sequence number wrap 46 | # expected_binary = <<0::16, 0b0111::4, 0xFFF::12>> <> <<0xFFFF::16, 0::16>> 47 | 48 | assert {:ok, %NACK{lost_packet_ids: reported_lost}} = 49 | NACK.encode(%NACK{lost_packet_ids: lost_ids}) |> NACK.decode() 50 | 51 | assert MapSet.equal?(MapSet.new(lost_ids), MapSet.new(reported_lost)) 52 | end 53 | end 54 | end 55 | -------------------------------------------------------------------------------- /test/membrane/rtcp/parser_test.exs: -------------------------------------------------------------------------------- 1 | defmodule Membrane.RTCP.ParserTest do 2 | use ExUnit.Case, async: true 3 | 4 | alias Membrane.RTCP.{FeedbackPacket, Fixtures, Parser, SenderReportPacket} 5 | alias Membrane.{Buffer, RTCPEvent} 6 | 7 | test "Handles SR with REMB" do 8 | buffer = %Buffer{payload: Fixtures.compound_sr_sdes_remb(), metadata: %{arrival_ts: 2137}} 9 | state = %{} 10 | assert {events, ^state} = Parser.handle_buffer(:input, buffer, %{}, state) 11 | assert [event: {:output, %RTCPEvent{} = rtcp_event}] = events 12 | assert rtcp_event.arrival_timestamp == 2137 13 | assert %SenderReportPacket{} = rtcp_event.rtcp 14 | end 15 | 16 | test "Handles PLI" do 17 | buffer = %Buffer{payload: Fixtures.pli_packet(), metadata: %{arrival_ts: 2137}} 18 | state = %{} 19 | assert {events, ^state} = Parser.handle_buffer(:input, buffer, %{}, state) 20 | assert [event: {:output, %RTCPEvent{} = rtcp_event}] = events 21 | assert rtcp_event.arrival_timestamp == 2137 22 | assert %{rtcp: %FeedbackPacket{} = feedback} = rtcp_event 23 | 24 | fixture_contents = Fixtures.pli_contents() 25 | assert feedback.target_ssrc == fixture_contents.target_ssrc 26 | assert feedback.origin_ssrc == fixture_contents.origin_ssrc 27 | assert feedback.payload == %FeedbackPacket.PLI{} 28 | end 29 | end 30 | -------------------------------------------------------------------------------- /test/membrane/rtcp/twcc_test.exs: -------------------------------------------------------------------------------- 1 | defmodule Membrane.RTCP.TWCCTest do 2 | use ExUnit.Case, async: true 3 | 4 | alias Membrane.RTCP.Fixtures 5 | alias Membrane.RTCP.TransportFeedbackPacket.TWCC 6 | 7 | describe "TWCC module" do 8 | test "encodes and decodes valid feedbacks" do 9 | encoded_feedbacks = Fixtures.twcc_feedbacks() 10 | expected_feedbacks = Fixtures.twcc_feedbacks_contents() 11 | 12 | encoded_feedbacks 13 | |> Enum.zip(expected_feedbacks) 14 | |> Enum.each(fn {encoded, expected} -> 15 | assert {:ok, expected} == TWCC.decode(encoded) 16 | assert encoded == TWCC.encode(expected) 17 | end) 18 | end 19 | 20 | test "does not decode malformed feedbacks" do 21 | encoded_feedbacks = Fixtures.twcc_malformed_feedbacks() 22 | 23 | Enum.each(encoded_feedbacks, fn encoded -> 24 | assert {:error, :malformed_packet} == TWCC.decode(encoded) 25 | end) 26 | end 27 | end 28 | end 29 | -------------------------------------------------------------------------------- /test/membrane/rtp/demuxer_muxer_integration_test.exs: -------------------------------------------------------------------------------- 1 | defmodule Membrane.RTP.DemuxerMuxerTest do 2 | @moduledoc false 3 | use ExUnit.Case 4 | import Membrane.Testing.Assertions 5 | alias Membrane.RTP 6 | alias Membrane.Testing 7 | 8 | @rtp_input %{ 9 | pcap_path: "test/fixtures/rtp/session/demo_rtp.pcap", 10 | packets: 862 11 | } 12 | 13 | @max_sequence_number Bitwise.bsl(1, 16) - 1 14 | @max_timestamp Bitwise.bsl(1, 32) - 1 15 | 16 | defmodule ReferencePipeline do 17 | use Membrane.Pipeline 18 | 19 | @impl true 20 | def handle_init(_ctx, opts) do 21 | spec = 22 | child(:source, %Membrane.Pcap.Source{path: opts.input_path}) 23 | |> child(:sink, Membrane.Testing.Sink) 24 | 25 | {[spec: spec], %{}} 26 | end 27 | end 28 | 29 | defmodule DynamicSubjectPipeline do 30 | use Membrane.Pipeline 31 | 32 | @impl true 33 | def handle_init(_ctx, opts) do 34 | spec = [ 35 | child(:source, %Membrane.Pcap.Source{path: opts.input_path}) 36 | |> child(:rtp_demuxer, Membrane.RTP.Demuxer), 37 | child(:rtp_muxer, Membrane.RTP.Muxer) 38 | |> child(:sink, Membrane.Testing.Sink) 39 | ] 40 | 41 | {[spec: spec], %{}} 42 | end 43 | 44 | @impl true 45 | def handle_child_notification({:new_rtp_stream, %{ssrc: ssrc}}, :rtp_demuxer, _ctx, state) do 46 | spec = 47 | get_child(:rtp_demuxer) 48 | |> via_out(:output, options: [stream_id: {:ssrc, ssrc}, jitter_buffer_latency: 0]) 49 | |> get_child(:rtp_muxer) 50 | 51 | {[spec: spec], state} 52 | end 53 | end 54 | 55 | defmodule UpfrontSubjectPipeline do 56 | use Membrane.Pipeline 57 | 58 | @impl true 59 | def handle_init(_ctx, opts) do 60 | spec = [ 61 | child(:source, %Membrane.Pcap.Source{path: opts.input_path}) 62 | |> child(:rtp_demuxer, Membrane.RTP.Demuxer) 63 | |> via_out(:output, options: [stream_id: {:encoding_name, :H264}]) 64 | |> child(:rtp_muxer, Membrane.RTP.Muxer) 65 | |> child(:sink, Membrane.Testing.Sink), 66 | get_child(:rtp_demuxer) 67 | |> via_out(:output, options: [stream_id: {:encoding_name, :AAC}]) 68 | |> get_child(:rtp_muxer) 69 | ] 70 | 71 | {[spec: spec], %{}} 72 | end 73 | end 74 | 75 | describe "Demuxed and muxed stream is the same as unchanged one" do 76 | test "when using a dynamically linking pipeline" do 77 | perform_test(DynamicSubjectPipeline) 78 | end 79 | 80 | test "when using a pipeline linked upfront" do 81 | perform_test(UpfrontSubjectPipeline) 82 | end 83 | end 84 | 85 | defp perform_test(subject_pipeline_module) do 86 | reference_pipeline = 87 | Testing.Pipeline.start_supervised!( 88 | module: ReferencePipeline, 89 | custom_args: %{input_path: @rtp_input.pcap_path} 90 | ) 91 | 92 | subject_pipeline = 93 | Testing.Pipeline.start_supervised!( 94 | module: subject_pipeline_module, 95 | custom_args: %{input_path: @rtp_input.pcap_path} 96 | ) 97 | 98 | assert_start_of_stream(reference_pipeline, :sink) 99 | assert_start_of_stream(subject_pipeline, :sink) 100 | 101 | reference_normalized_packets = get_normalized_packets(reference_pipeline, @rtp_input.packets) 102 | subject_normalized_packets = get_normalized_packets(subject_pipeline, @rtp_input.packets) 103 | 104 | assert reference_normalized_packets == subject_normalized_packets 105 | 106 | assert_end_of_stream(reference_pipeline, :sink) 107 | assert_end_of_stream(subject_pipeline, :sink) 108 | Testing.Pipeline.terminate(reference_pipeline) 109 | Testing.Pipeline.terminate(subject_pipeline) 110 | end 111 | 112 | @spec get_normalized_packets(pid(), non_neg_integer()) :: 113 | %{RTP.encoding_name() => [ExRTP.Packet.t()]} 114 | defp get_normalized_packets(pipeline, buffers_amount) do 115 | Enum.map(1..buffers_amount, fn _i -> 116 | assert_sink_buffer(pipeline, :sink, %Membrane.Buffer{ 117 | payload: payload 118 | }) 119 | 120 | {:ok, packet} = ExRTP.Packet.decode(payload) 121 | 122 | packet 123 | end) 124 | |> Enum.group_by(& &1.payload_type) 125 | |> Map.new(fn {payload_type, packets} -> 126 | %{encoding_name: encoding_name} = RTP.PayloadFormat.get_payload_type_mapping(payload_type) 127 | 128 | %{ssrc: ssrc, sequence_number: first_sequence_number, timestamp: first_timestamp} = 129 | List.first(packets) 130 | 131 | normalized_packets = 132 | packets 133 | |> Enum.map(fn packet -> 134 | %{ 135 | packet 136 | | ssrc: packet.ssrc - ssrc, 137 | # modulo to account for wrapping 138 | sequence_number: 139 | Integer.mod( 140 | packet.sequence_number - first_sequence_number, 141 | @max_sequence_number + 1 142 | ), 143 | # round to ignore insignificant differences in timestamps 144 | timestamp: 145 | round(Integer.mod(packet.timestamp - first_timestamp, @max_timestamp + 1) / 10) 146 | } 147 | end) 148 | 149 | {encoding_name, normalized_packets} 150 | end) 151 | end 152 | end 153 | -------------------------------------------------------------------------------- /test/membrane/rtp/inbound_packet_tracker_test.exs: -------------------------------------------------------------------------------- 1 | defmodule Membrane.RTP.InboundPacketTrackerTest do 2 | use ExUnit.Case, async: true 3 | use Bunch 4 | 5 | require Bitwise 6 | alias Membrane.RTP.BufferFactory 7 | alias Membrane.RTP.InboundPacketTracker 8 | 9 | @max_seq_number Bitwise.bsl(1, 16) - 1 10 | @base_seq_number BufferFactory.base_seq_number() 11 | 12 | describe "InboundPacketTracker should" do 13 | setup do 14 | buffer = BufferFactory.sample_buffer(@base_seq_number) 15 | 16 | state = %InboundPacketTracker.State{ 17 | clock_rate: BufferFactory.clock_rate(), 18 | repair_sequence_numbers?: true 19 | } 20 | 21 | [state: state, buffer: buffer] 22 | end 23 | 24 | test "update stats accordingly when receiving new buffers", %{state: state, buffer: buffer} do 25 | ts = ~U[2020-06-19 19:06:00Z] |> DateTime.to_unix() |> Membrane.Time.seconds() 26 | 27 | timestamped_buf = put_in(buffer.metadata[:arrival_ts], ts) 28 | 29 | assert {_actions, state} = 30 | InboundPacketTracker.handle_buffer(:input, timestamped_buf, nil, state) 31 | 32 | assert state.jitter == 0.0 33 | 34 | assert state.transit == 35 | Membrane.Time.as_seconds(ts, :round) * state.clock_rate - 36 | timestamped_buf.metadata.rtp.timestamp 37 | 38 | buffer = BufferFactory.sample_buffer(@base_seq_number + 1) 39 | 40 | arrival_ts_increment = 41 | div(BufferFactory.timestamp_increment(), state.clock_rate) |> Membrane.Time.seconds() 42 | 43 | packet_delay = 1 |> Membrane.Time.seconds() 44 | 45 | timestamped_buf = 46 | put_in(buffer.metadata[:arrival_ts], ts + arrival_ts_increment + packet_delay) 47 | 48 | assert {_actions, state} = 49 | InboundPacketTracker.handle_buffer(:input, timestamped_buf, nil, state) 50 | 51 | # 16 is defined by RFC 52 | assert state.jitter == 53 | Membrane.Time.as_seconds(packet_delay, :round) * state.clock_rate / 16 54 | 55 | assert state.transit == 56 | Membrane.Time.as_seconds(ts + arrival_ts_increment + packet_delay, :round) * 57 | state.clock_rate - 58 | timestamped_buf.metadata.rtp.timestamp 59 | end 60 | 61 | test "update packet's sequence number if there have been discarded packets", %{state: state} do 62 | state = %InboundPacketTracker.State{state | discarded: 10} 63 | 64 | # in sequence number range 65 | buffer = BufferFactory.sample_buffer(100) 66 | 67 | assert {[buffer: {:output, buffer}], state} = 68 | InboundPacketTracker.handle_buffer(:input, buffer, nil, state) 69 | 70 | assert buffer.metadata.rtp.sequence_number == 90 71 | 72 | # border case where sequence number over rolled 73 | buffer = BufferFactory.sample_buffer(5) 74 | 75 | assert {[buffer: {:output, buffer}], state} = 76 | InboundPacketTracker.handle_buffer(:input, buffer, nil, state) 77 | 78 | assert buffer.metadata.rtp.sequence_number == @max_seq_number - 5 + 1 79 | 80 | state = %InboundPacketTracker.State{state | repair_sequence_numbers?: false} 81 | buffer = BufferFactory.sample_buffer(100) 82 | 83 | assert {[buffer: {:output, buffer}], _state} = 84 | InboundPacketTracker.handle_buffer(:input, buffer, nil, state) 85 | 86 | assert buffer.metadata.rtp.sequence_number == 100 87 | end 88 | end 89 | end 90 | -------------------------------------------------------------------------------- /test/membrane/rtp/jitter_buffer/pipeline_test.exs: -------------------------------------------------------------------------------- 1 | defmodule Membrane.RTP.JitterBuffer.PipelineTest do 2 | use ExUnit.Case 3 | 4 | import Membrane.Testing.Assertions 5 | 6 | alias Membrane.RTP.BufferFactory 7 | alias Membrane.RTP.JitterBuffer, as: RTPJitterBuffer 8 | alias Membrane.Testing 9 | 10 | @seq_number_limit 65_536 11 | 12 | defmodule PushTestingSrc do 13 | use Membrane.Source 14 | alias Membrane.RTP.BufferFactory 15 | 16 | @seq_number_limit 65_536 17 | 18 | def_output_pad :output, accepted_format: _any, flow_control: :push 19 | 20 | def_options buffer_num: [spec: non_neg_integer()], 21 | buffer_delay_ms: [spec: non_neg_integer()], 22 | max_latency: [spec: non_neg_integer()] 23 | 24 | @impl true 25 | def handle_playing( 26 | _ctx, 27 | %{ 28 | buffer_delay_ms: delay_ms, 29 | buffer_num: buffer_num, 30 | max_latency: max_latency 31 | } = state 32 | ) do 33 | now = System.monotonic_time(:millisecond) 34 | 35 | 1..buffer_num 36 | |> Enum.each(fn n -> 37 | time = 38 | cond do 39 | # Delay less than max latency 40 | rem(n, 15) == 0 -> n * delay_ms + div(max_latency, 2) 41 | # Delay more than max latency 42 | rem(n, 19) == 0 -> n * delay_ms + max_latency * 2 43 | true -> n * delay_ms 44 | end 45 | 46 | if rem(n, 50) < 30 or rem(n, 50) > 32 do 47 | seq_number = rem(n, @seq_number_limit) 48 | Process.send_after(self(), {:push_buffer, seq_number}, now + time, abs: true) 49 | end 50 | end) 51 | 52 | {[stream_format: {:output, %Membrane.RTP{}}], state} 53 | end 54 | 55 | @impl true 56 | def handle_info({:push_buffer, n}, _ctx, state) do 57 | actions = [action_from_number(n)] 58 | {actions, state} 59 | end 60 | 61 | defp action_from_number(element), 62 | do: {:buffer, {:output, BufferFactory.sample_buffer(element)}} 63 | end 64 | 65 | test "Jitter Buffer works in a Pipeline with small latency" do 66 | test_pipeline(300, 10, 200 |> Membrane.Time.milliseconds()) 67 | end 68 | 69 | test "Jitter Buffer works in a Pipeline with large latency" do 70 | test_pipeline(100, 30, 1000 |> Membrane.Time.milliseconds()) 71 | end 72 | 73 | @tag :long_running 74 | @tag timeout: 70_000 * 10 + 10_000 75 | test "Jitter Buffer works in a long-running Pipeline with small latency" do 76 | test_pipeline(70_000, 10, 100 |> Membrane.Time.milliseconds()) 77 | end 78 | 79 | defp test_pipeline(buffers, buffer_delay_ms, latency) do 80 | import Membrane.ChildrenSpec 81 | 82 | latency_ms = latency |> Membrane.Time.as_milliseconds(:round) 83 | 84 | structure = 85 | child(:source, %PushTestingSrc{ 86 | buffer_num: buffers, 87 | buffer_delay_ms: buffer_delay_ms, 88 | max_latency: latency_ms 89 | }) 90 | |> via_in(:input, target_queue_size: 50) 91 | |> child(:buffer, %RTPJitterBuffer{latency: latency, clock_rate: 8000}) 92 | |> child(:sink, Testing.Sink) 93 | 94 | pipeline = Testing.Pipeline.start_link_supervised!(spec: structure) 95 | 96 | assert_pipeline_setup(pipeline) 97 | 98 | timeout = latency_ms + buffer_delay_ms + 200 99 | assert_start_of_stream(pipeline, :buffer, :input, 5000) 100 | assert_start_of_stream(pipeline, :sink, :input, timeout) 101 | 102 | Enum.each(1..buffers, fn n -> 103 | cond do 104 | rem(n, 50) >= 30 and rem(n, 50) <= 32 -> 105 | assert_sink_event(pipeline, :sink, %Membrane.Event.Discontinuity{}, timeout) 106 | 107 | rem(n, 19) == 0 and rem(n, 15) != 0 -> 108 | assert_sink_event(pipeline, :sink, %Membrane.Event.Discontinuity{}, timeout) 109 | 110 | true -> 111 | seq_num = rem(n, @seq_number_limit) 112 | 113 | assert_sink_buffer( 114 | pipeline, 115 | :sink, 116 | %Membrane.Buffer{ 117 | metadata: %{rtp: %{sequence_number: ^seq_num}}, 118 | payload: _ 119 | }, 120 | timeout 121 | ) 122 | end 123 | end) 124 | 125 | Membrane.Pipeline.terminate(pipeline) 126 | end 127 | end 128 | -------------------------------------------------------------------------------- /test/membrane/rtp/jitter_buffer/timestamps_calculation_test.exs: -------------------------------------------------------------------------------- 1 | defmodule Membrane.RTP.JitterBuffer.TimestampsCalculationTest do 2 | use ExUnit.Case, async: true 3 | use Bunch 4 | 5 | alias Membrane.RTP.BufferFactory 6 | alias Membrane.RTP.JitterBuffer 7 | 8 | @max_rtp_timestamp 0xFFFFFFFF 9 | 10 | defp action_buffers(actions) do 11 | actions 12 | |> Enum.filter(&match?({:buffer, _}, &1)) 13 | |> Enum.map(fn {:buffer, {:output, buffer}} -> buffer end) 14 | end 15 | 16 | defp buffers_timestamps(buffers) do 17 | Enum.map(buffers, & &1.pts) 18 | end 19 | 20 | defp process_and_receive_buffer_timestamps(buffers, state) do 21 | state = 22 | Enum.reduce(buffers, state, fn buffer, state -> 23 | {[], state} = JitterBuffer.handle_buffer(:input, buffer, %{}, state) 24 | state 25 | end) 26 | 27 | {actions, _state} = JitterBuffer.handle_info(:send_buffers, %{}, state) 28 | 29 | actions 30 | |> action_buffers() 31 | |> buffers_timestamps() 32 | end 33 | 34 | defp update_buffer_timestamp(buffer, timestamp) do 35 | %{buffer | metadata: put_in(buffer.metadata, [:rtp, :timestamp], timestamp)} 36 | end 37 | 38 | defp overflowed_timestamp() do 39 | BufferFactory.timestamp_increment() 40 | end 41 | 42 | # the higher multiplier the closer timestamp will be to overflow 43 | defp almost_overflowed_timestamp(multiplier) when multiplier in 1..10 do 44 | @max_rtp_timestamp - (10 - multiplier - 1) * div(BufferFactory.timestamp_increment(), 10) 45 | end 46 | 47 | describe "JitterBuffer correctly calculates timestamps for buffers that" do 48 | setup do 49 | {[], state} = 50 | JitterBuffer.handle_init(nil, %JitterBuffer{clock_rate: BufferFactory.clock_rate()}) 51 | 52 | seq_num = BufferFactory.base_seq_number() 53 | buffer1 = BufferFactory.sample_buffer(seq_num + 1) 54 | buffer2 = BufferFactory.sample_buffer(seq_num + 2) 55 | 56 | [state: state, buffer1: buffer1, buffer2: buffer2] 57 | end 58 | 59 | test "come in first", ctx do 60 | %{buffer1: buffer1, buffer2: buffer2} = ctx 61 | 62 | [timestamp1, timestamp2] = 63 | [buffer1, buffer2] 64 | |> process_and_receive_buffer_timestamps(ctx.state) 65 | 66 | assert Ratio.equal?(timestamp1, 0) 67 | assert Ratio.gt?(timestamp2, 0) 68 | end 69 | 70 | test "have monotonic timestamps within proper range", ctx do 71 | %{buffer1: buffer1, buffer2: buffer2} = ctx 72 | 73 | assert buffer1.metadata.rtp.timestamp < buffer2.metadata.rtp.timestamp 74 | 75 | [timestamp1, timestamp2] = 76 | [buffer1, buffer2] 77 | |> process_and_receive_buffer_timestamps(ctx.state) 78 | 79 | assert timestamp1 < timestamp2 80 | end 81 | 82 | test "have non-monotonic timestamps within proper range", ctx do 83 | %{buffer1: buffer1, buffer2: buffer2} = ctx 84 | 85 | # switch buffers rtp timestamps so that buffer1 has smaller timestamp than buffer 2 86 | {buffer1, buffer2} = { 87 | update_buffer_timestamp(buffer1, buffer2.metadata.rtp.timestamp), 88 | update_buffer_timestamp(buffer2, buffer1.metadata.rtp.timestamp) 89 | } 90 | 91 | assert buffer1.metadata.rtp.timestamp > buffer2.metadata.rtp.timestamp 92 | 93 | [timestamp1, timestamp2] = 94 | [buffer1, buffer2] 95 | |> process_and_receive_buffer_timestamps(ctx.state) 96 | 97 | assert timestamp1 > timestamp2 98 | end 99 | 100 | test "have monotonic timestamps that are about to overflow", ctx do 101 | %{buffer1: buffer1, buffer2: buffer2} = ctx 102 | 103 | # both buffers are really close to overflow 104 | buffer1 = update_buffer_timestamp(buffer1, almost_overflowed_timestamp(1)) 105 | buffer2 = update_buffer_timestamp(buffer2, almost_overflowed_timestamp(2)) 106 | 107 | assert buffer1.metadata.rtp.timestamp < buffer2.metadata.rtp.timestamp 108 | 109 | [timestamp1, timestamp2] = 110 | [buffer1, buffer2] 111 | |> process_and_receive_buffer_timestamps(ctx.state) 112 | 113 | assert timestamp1 < timestamp2 114 | 115 | # buffer2 has already overflowed 116 | buffer1 = update_buffer_timestamp(buffer1, almost_overflowed_timestamp(1)) 117 | buffer2 = update_buffer_timestamp(buffer2, overflowed_timestamp()) 118 | 119 | assert buffer1.metadata.rtp.timestamp > buffer2.metadata.rtp.timestamp 120 | 121 | [timestamp1, timestamp2] = 122 | [buffer1, buffer2] 123 | |> process_and_receive_buffer_timestamps(ctx.state) 124 | 125 | assert timestamp1 < timestamp2 126 | end 127 | 128 | test "have non-monotonic timestamps that are about to overflow", ctx do 129 | %{buffer1: buffer1, buffer2: buffer2} = ctx 130 | 131 | # buffer1 overflowed while buffer2 has not, buffer2's calculated timestamp should be lower than buffer1's 132 | buffer1 = update_buffer_timestamp(buffer1, overflowed_timestamp()) 133 | buffer2 = update_buffer_timestamp(buffer2, almost_overflowed_timestamp(1)) 134 | 135 | assert buffer1.metadata.rtp.timestamp < buffer2.metadata.rtp.timestamp 136 | 137 | [timestamp1, timestamp2] = 138 | [buffer1, buffer2] 139 | |> process_and_receive_buffer_timestamps(ctx.state) 140 | 141 | # buffer2's timestamp must be smaller than buffer1's 142 | assert timestamp2 < timestamp1 143 | end 144 | end 145 | end 146 | -------------------------------------------------------------------------------- /test/membrane/rtp/jitter_buffer_test.exs: -------------------------------------------------------------------------------- 1 | defmodule Membrane.RTP.JitterBufferTest do 2 | use ExUnit.Case, async: true 3 | 4 | alias Membrane.RTP.BufferFactory 5 | alias Membrane.RTP.JitterBuffer 6 | alias Membrane.RTP.JitterBuffer.{BufferStore, Record, State} 7 | 8 | @base_seq_number BufferFactory.base_seq_number() 9 | 10 | setup_all do 11 | buffer = BufferFactory.sample_buffer(@base_seq_number) 12 | 13 | state = %State{ 14 | clock_rate: BufferFactory.clock_rate(), 15 | store: %BufferStore{}, 16 | latency: 10 |> Membrane.Time.milliseconds() 17 | } 18 | 19 | [state: state, buffer: buffer] 20 | end 21 | 22 | describe "When JitterBuffer is in waiting state" do 23 | setup %{state: state} do 24 | [state: %{state | waiting?: true}] 25 | end 26 | 27 | test "start of stream starts timer that changes state", %{state: state} do 28 | assert {[], state} = JitterBuffer.handle_start_of_stream(:input, %{}, state) 29 | assert_receive message, (state.latency |> Membrane.Time.as_milliseconds(:round)) + 5 30 | assert {[], final_state} = JitterBuffer.handle_info(message, %{}, state) 31 | assert final_state.waiting? == false 32 | end 33 | 34 | test "any new buffer is kept", %{state: state, buffer: buffer} do 35 | assert BufferStore.dump(state.store) == [] 36 | assert {[], state} = JitterBuffer.handle_buffer(:input, buffer, nil, state) 37 | 38 | assert %State{store: store} = state 39 | assert {%Record{buffer: ^buffer}, new_store} = BufferStore.flush_one(store) 40 | assert BufferStore.dump(new_store) == [] 41 | end 42 | end 43 | 44 | describe "When new buffer arrives when not waiting and already pushed some buffer" do 45 | setup %{state: state} do 46 | flush_index = @base_seq_number - 1 47 | store = %{state.store | flush_index: flush_index, highest_incoming_index: flush_index} 48 | [state: %{state | waiting?: false, store: store}] 49 | end 50 | 51 | test "outputs it immediately if it is in order", %{state: state, buffer: buffer} do 52 | assert {[buffer: {:output, ^buffer}], state} = 53 | JitterBuffer.handle_buffer(:input, buffer, nil, state) 54 | 55 | assert %JitterBuffer.State{store: store} = state 56 | assert BufferStore.dump(store) == [] 57 | end 58 | 59 | test "refuses to add that packet when it comes too late", %{state: state} do 60 | late_buffer = BufferFactory.sample_buffer(@base_seq_number - 2) 61 | assert {[], new_state} = JitterBuffer.handle_buffer(:input, late_buffer, nil, state) 62 | assert new_state == state 63 | end 64 | 65 | test "adds it and when it fills the gap, returns all buffers in order", %{state: state} do 66 | first_buffer = BufferFactory.sample_buffer(@base_seq_number) 67 | second_buffer = BufferFactory.sample_buffer(@base_seq_number + 1) 68 | third_buffer = BufferFactory.sample_buffer(@base_seq_number + 2) 69 | 70 | flush_index = @base_seq_number - 1 71 | 72 | store = %BufferStore{ 73 | state.store 74 | | flush_index: flush_index, 75 | highest_incoming_index: flush_index 76 | } 77 | 78 | store = 79 | with {:ok, store} <- BufferStore.insert_buffer(store, second_buffer), 80 | {:ok, store} <- BufferStore.insert_buffer(store, third_buffer) do 81 | store 82 | end 83 | 84 | state = %State{state | store: store} 85 | 86 | assert {commands, %State{store: result_store}} = 87 | JitterBuffer.handle_buffer(:input, first_buffer, nil, state) 88 | 89 | buffers = commands |> Keyword.get_values(:buffer) |> Enum.map(fn {:output, buf} -> buf end) 90 | 91 | assert [^first_buffer, ^second_buffer, ^third_buffer] = buffers 92 | assert BufferStore.dump(result_store) == [] 93 | end 94 | end 95 | 96 | describe "When latency pasess without filling the gap, JitterBuffer" do 97 | test "outputs discontinuity and late buffer", %{state: state, buffer: buffer} do 98 | flush_index = @base_seq_number - 2 99 | 100 | store = %BufferStore{ 101 | state.store 102 | | flush_index: flush_index, 103 | highest_incoming_index: flush_index 104 | } 105 | 106 | state = %{state | store: store, waiting?: false} 107 | 108 | assert {commands, state} = JitterBuffer.handle_buffer(:input, buffer, nil, state) 109 | assert commands |> Keyword.get(:buffer) == nil 110 | assert is_reference(state.max_latency_timer) 111 | assert_receive message, (state.latency |> Membrane.Time.as_milliseconds(:round)) + 20 112 | 113 | assert {actions, _state} = JitterBuffer.handle_info(message, %{}, state) 114 | 115 | assert [event: event, buffer: buffer_action] = actions 116 | assert event == {:output, %Membrane.Event.Discontinuity{}} 117 | assert buffer_action == {:output, buffer} 118 | end 119 | end 120 | 121 | describe "When event arrives" do 122 | test "dumps store if event was end of stream", %{state: state, buffer: buffer} do 123 | flush_index = @base_seq_number - 2 124 | 125 | store = %BufferStore{ 126 | state.store 127 | | flush_index: flush_index, 128 | highest_incoming_index: flush_index 129 | } 130 | 131 | {:ok, store} = BufferStore.insert_buffer(store, buffer) 132 | state = %{state | store: store} 133 | assert {actions, _state} = JitterBuffer.handle_end_of_stream(:input, nil, state) 134 | 135 | assert [event: event, buffer: buffer_action, end_of_stream: :output] = actions 136 | assert event == {:output, %Membrane.Event.Discontinuity{}} 137 | assert buffer_action == {:output, buffer} 138 | end 139 | end 140 | end 141 | -------------------------------------------------------------------------------- /test/membrane/rtp/muxer_demuxer_integration_test.exs: -------------------------------------------------------------------------------- 1 | defmodule Membrane.RTP.MuxerDemuxerTest do 2 | @moduledoc false 3 | use ExUnit.Case 4 | import Membrane.Testing.Assertions 5 | alias Membrane.Testing 6 | 7 | @input_path "test/fixtures/rtp/h264/bun.h264" 8 | 9 | defmodule MuxerDemuxerPipeline do 10 | use Membrane.Pipeline 11 | 12 | @impl true 13 | def handle_init(_ctx, opts) do 14 | spec = [ 15 | child(:source, %Membrane.File.Source{location: opts.input_path}) 16 | |> child(:h264_parser, %Membrane.H264.Parser{ 17 | output_alignment: :nalu, 18 | generate_best_effort_timestamps: %{framerate: {120, 1}} 19 | }) 20 | |> child(:realtimer, Membrane.Realtimer) 21 | |> child(:rtp_h264_payloader, Membrane.RTP.H264.Payloader) 22 | |> child(:rtp_muxer, %Membrane.RTP.Muxer{srtp: opts.srtp}) 23 | |> child(:rtp_demuxer, %Membrane.RTP.Demuxer{srtp: opts.srtp}) 24 | |> via_out(:output, 25 | options: [stream_id: {:encoding_name, :H264}] 26 | ) 27 | |> child(:rtp_h264_depayloader, Membrane.RTP.H264.Depayloader) 28 | |> child(:sink, %Membrane.File.Sink{location: opts.output_path}) 29 | ] 30 | 31 | {[spec: spec], %{}} 32 | end 33 | end 34 | 35 | describe "Muxed and demuxed stream is the same as unchanged one" do 36 | @tag :tmp_dir 37 | test "when not using SRTP encryption", %{tmp_dir: tmp_dir} do 38 | perform_test(false, tmp_dir) 39 | end 40 | 41 | @tag :tmp_dir 42 | test "when using SRTP encryption", %{tmp_dir: tmp_dir} do 43 | policy = %ExLibSRTP.Policy{ssrc: :any_inbound, key: String.duplicate("a", 30)} 44 | perform_test([policy], tmp_dir) 45 | end 46 | end 47 | 48 | defp perform_test(srtp, tmp_dir) do 49 | output_path = Path.join(tmp_dir, "output.h264") 50 | 51 | pipeline = 52 | Testing.Pipeline.start_supervised!( 53 | module: MuxerDemuxerPipeline, 54 | custom_args: %{input_path: @input_path, output_path: output_path, srtp: srtp} 55 | ) 56 | 57 | assert_start_of_stream(pipeline, :sink) 58 | assert_end_of_stream(pipeline, :sink, :input, 10_000) 59 | 60 | assert File.read!(@input_path) == File.read!(output_path) 61 | 62 | Testing.Pipeline.terminate(pipeline) 63 | end 64 | end 65 | -------------------------------------------------------------------------------- /test/membrane/rtp/muxer_test.exs: -------------------------------------------------------------------------------- 1 | defmodule Membrane.RTP.MuxerTest do 2 | @moduledoc false 3 | use ExUnit.Case 4 | import Membrane.Testing.Assertions 5 | alias Membrane.Testing 6 | 7 | @rtp_output %{ 8 | video: %{payload_type: 96, packets: 1054}, 9 | audio: %{payload_type: 127, packets: 431} 10 | } 11 | 12 | defmodule Pipeline do 13 | use Membrane.Pipeline 14 | 15 | @impl true 16 | def handle_init(_ctx, opts) do 17 | spec = [ 18 | child(:hackney_source, %Membrane.Hackney.Source{ 19 | location: 20 | "https://raw.githubusercontent.com/membraneframework/static/gh-pages/samples/big-buck-bunny/bun10s.mp4", 21 | hackney_opts: [follow_redirect: true] 22 | }) 23 | |> child(:mp4_demuxer, Membrane.MP4.Demuxer.ISOM) 24 | |> via_out(:output, options: [kind: :video]) 25 | |> child(:h264_parser, %Membrane.H264.Parser{ 26 | output_stream_structure: :annexb, 27 | output_alignment: :nalu 28 | }) 29 | |> child(:h264_payloader, Membrane.RTP.H264.Payloader) 30 | |> child(:rtp_muxer, %Membrane.RTP.Muxer{srtp: opts.srtp}) 31 | |> child(:sink, Membrane.Testing.Sink), 32 | get_child(:mp4_demuxer) 33 | |> via_out(:output, options: [kind: :audio]) 34 | |> child(:aac_parser, %Membrane.AAC.Parser{out_encapsulation: :none}) 35 | |> child(:aac_payloader, %Membrane.RTP.AAC.Payloader{mode: :hbr, frames_per_packet: 1}) 36 | |> get_child(:rtp_muxer) 37 | ] 38 | 39 | {[spec: spec], %{}} 40 | end 41 | end 42 | 43 | describe "Muxer muxes correct amount of packets" do 44 | test "when encrypting the stream with SRTP" do 45 | policy = %ExLibSRTP.Policy{ssrc: :any_inbound, key: String.duplicate("b", 30)} 46 | perform_test([policy]) 47 | end 48 | 49 | test "when not encrypting the stream with SRTP" do 50 | perform_test(false) 51 | end 52 | end 53 | 54 | defp perform_test(srtp) do 55 | pipeline = 56 | Testing.Pipeline.start_supervised!(module: Pipeline, custom_args: %{srtp: srtp}) 57 | 58 | %{audio: %{payload_type: audio_payload_type}, video: %{payload_type: video_payload_type}} = 59 | @rtp_output 60 | 61 | assert_start_of_stream(pipeline, :sink) 62 | 63 | 1..@rtp_output.video.packets 64 | |> Enum.each(fn _i -> 65 | assert_sink_buffer(pipeline, :sink, %Membrane.Buffer{ 66 | metadata: %{rtp: %ExRTP.Packet{payload_type: ^video_payload_type}} 67 | }) 68 | end) 69 | 70 | 1..@rtp_output.audio.packets 71 | |> Enum.each(fn _i -> 72 | assert_sink_buffer(pipeline, :sink, %Membrane.Buffer{ 73 | metadata: %{rtp: %ExRTP.Packet{payload_type: ^audio_payload_type}} 74 | }) 75 | end) 76 | 77 | assert_end_of_stream(pipeline, :sink) 78 | Testing.Pipeline.terminate(pipeline) 79 | end 80 | end 81 | -------------------------------------------------------------------------------- /test/membrane/rtp/outbound_rtx_controller_test.exs: -------------------------------------------------------------------------------- 1 | defmodule Membrane.RTP.OutboundRtxControllerTest do 2 | use ExUnit.Case, async: true 3 | 4 | alias Membrane.RTP.{BufferFactory, RetransmissionRequestEvent} 5 | alias Membrane.RTP.OutboundRtxController, as: RTX 6 | 7 | @base_seq_number BufferFactory.base_seq_number() 8 | @max_store_size RTX.max_store_size() 9 | 10 | test "store size is limited" do 11 | assert {[], state} = RTX.handle_init(%{}, %RTX{}) 12 | 13 | buffs = Enum.map(0..400, &BufferFactory.sample_buffer(@base_seq_number + &1)) 14 | 15 | state = 16 | Enum.reduce(buffs, state, fn buffer, state -> 17 | assert {[forward: ^buffer], state} = RTX.handle_buffer(:input, buffer, %{}, state) 18 | state 19 | end) 20 | 21 | assert map_size(state.store) <= @max_store_size 22 | end 23 | 24 | test "retransmit stored buffer" do 25 | state = init() 26 | 27 | rtx_sn = [@base_seq_number + 98, @base_seq_number + 99] 28 | rtx_event = %RetransmissionRequestEvent{packet_ids: rtx_sn} 29 | rtx_buffers = Enum.map(rtx_sn, &BufferFactory.sample_buffer(&1)) 30 | 31 | assert {[buffer: {:output, ^rtx_buffers}], _state} = 32 | RTX.handle_event(:input, rtx_event, %{}, state) 33 | end 34 | 35 | test "ignore packets not present" do 36 | state = init() 37 | 38 | rtx_event = %RetransmissionRequestEvent{ 39 | packet_ids: [@base_seq_number + 101, @base_seq_number + 102] 40 | } 41 | 42 | assert {actions, _state} = RTX.handle_event(:input, rtx_event, %{}, state) 43 | 44 | bufs = for {:buffer, {:output, bufs}} <- actions, buf <- bufs, do: buf 45 | assert Enum.empty?(bufs) 46 | end 47 | 48 | test "ignore instant doubled RTX request" do 49 | state = init() 50 | 51 | rtx_sn = [@base_seq_number + 98, @base_seq_number + 99] 52 | rtx_event = %RetransmissionRequestEvent{packet_ids: rtx_sn} 53 | rtx_buffers = Enum.map(rtx_sn, &BufferFactory.sample_buffer(&1)) 54 | 55 | assert {[buffer: {:output, ^rtx_buffers}], state} = 56 | RTX.handle_event(:input, rtx_event, %{}, state) 57 | 58 | assert {actions, _state} = RTX.handle_event(:input, rtx_event, %{}, state) 59 | 60 | bufs = for {:buffer, {:output, bufs}} <- actions, buf <- bufs, do: buf 61 | assert Enum.empty?(bufs) 62 | end 63 | 64 | defp init(received_bufs \\ 100) do 65 | {[], state} = RTX.handle_init(%{}, %RTX{}) 66 | 67 | buffs = Enum.map(0..received_bufs, &BufferFactory.sample_buffer(@base_seq_number + &1)) 68 | 69 | Enum.reduce(buffs, state, fn buffer, state -> 70 | {[forward: ^buffer], state} = RTX.handle_buffer(:input, buffer, %{}, state) 71 | state 72 | end) 73 | end 74 | end 75 | -------------------------------------------------------------------------------- /test/membrane/rtp/packet_payload_type_test.exs: -------------------------------------------------------------------------------- 1 | defmodule Membrane.RTP.Packet.PayloadTypeTest do 2 | use ExUnit.Case 3 | 4 | alias Membrane.RTP.Packet.PayloadType 5 | 6 | describe "Payload type params decoder" do 7 | test "raises an error when trying to decode non existent payload type" do 8 | assert_raise FunctionClauseError, fn -> 9 | PayloadType.get_encoding_name(128) 10 | end 11 | 12 | assert_raise FunctionClauseError, fn -> 13 | PayloadType.get_clock_rate(128) 14 | end 15 | end 16 | 17 | # Payload identifiers 96–127 are for dynamic payload types 18 | test "returns `:dynamic` when in dynamic range" do 19 | Enum.each(96..127, fn elem -> 20 | assert PayloadType.get_encoding_name(elem) == :dynamic 21 | assert PayloadType.get_clock_rate(elem) == :dynamic 22 | end) 23 | end 24 | 25 | test "returns an atom and clock rate when in static type range" do 26 | static_types = [0] ++ Enum.to_list(3..18) ++ [25, 26, 28] ++ Enum.to_list(31..34) 27 | 28 | Enum.each(static_types, fn elem -> 29 | assert is_atom(PayloadType.get_encoding_name(elem)) 30 | assert is_integer(PayloadType.get_clock_rate(elem)) 31 | end) 32 | end 33 | end 34 | end 35 | -------------------------------------------------------------------------------- /test/membrane/rtp/packet_test.exs: -------------------------------------------------------------------------------- 1 | defmodule Membrane.RTP.PacketTest do 2 | use ExUnit.Case 3 | 4 | alias Membrane.RTP.{Fixtures, Header, Packet} 5 | 6 | @encrypted? false 7 | 8 | test "parses and serializes valid packets" do 9 | packet = Fixtures.sample_packet() 10 | 11 | assert {:ok, %{packet: ^packet}} = Packet.parse(Fixtures.sample_packet_binary(), @encrypted?) 12 | 13 | assert Packet.serialize(Fixtures.sample_packet()) == Fixtures.sample_packet_binary() 14 | end 15 | 16 | test "returns error when version is not supported" do 17 | assert Packet.parse(<<1::2, 1233::1022>>, @encrypted?) == {:error, :wrong_version} 18 | end 19 | 20 | test "returns error when packet is too short" do 21 | assert Packet.parse(<<128, 127, 0, 0, 1>>, @encrypted?) == {:error, :malformed_packet} 22 | end 23 | 24 | test "parses and serializes csrcs correctly" do 25 | <> = Fixtures.sample_packet_binary() 26 | packet_binary = <> 27 | 28 | packet = %Packet{ 29 | Fixtures.sample_packet() 30 | | header: %Header{Fixtures.sample_header() | csrcs: [12, 21]} 31 | } 32 | 33 | assert {:ok, %{packet: ^packet}} = Packet.parse(packet_binary, @encrypted?) 34 | assert Packet.serialize(packet) == packet_binary 35 | end 36 | 37 | test "generates padding" do 38 | ref_packet = Fixtures.sample_packet_binary_with_padding() 39 | test_packet = Fixtures.sample_packet() 40 | assert ref_packet == Packet.serialize(test_packet, padding_size: 20) 41 | end 42 | 43 | test "ignores padding" do 44 | ref_packet = Fixtures.sample_packet() 45 | test_packet = Fixtures.sample_packet_binary_with_padding() 46 | 47 | assert {:ok, %{packet: ^ref_packet, padding_size: 20}} = 48 | Packet.parse(test_packet, @encrypted?) 49 | end 50 | 51 | test "reads and serializes extension header" do 52 | extension_header = <<0xBE, 0xDE, 1::16, 1::4, 0::4, 0xBE, 0::16>> 53 | 54 | expected_parsed_extensions = [ 55 | %Membrane.RTP.Header.Extension{data: <<0xBE>>, identifier: 1} 56 | ] 57 | 58 | # Extension is stored on 4th bit of header 59 | <> = 60 | Fixtures.sample_packet_binary() 61 | 62 | # Glueing data back together with extension header in place 63 | packet_binary = <> 64 | 65 | packet = %Packet{ 66 | Fixtures.sample_packet() 67 | | header: %Header{Fixtures.sample_header() | extensions: expected_parsed_extensions} 68 | } 69 | 70 | assert {:ok, %{packet: ^packet}} = Packet.parse(packet_binary, @encrypted?) 71 | assert Packet.serialize(packet) == packet_binary 72 | end 73 | 74 | test "Serialize and then parse return same packet" do 75 | packet = %Packet{ 76 | Fixtures.sample_packet() 77 | | header: %Header{ 78 | Fixtures.sample_header() 79 | | extensions: [ 80 | %Membrane.RTP.Header.Extension{data: <<0xBE, 0>>, identifier: 1}, 81 | %Membrane.RTP.Header.Extension{data: <<0xDE, 0>>, identifier: 2} 82 | ] 83 | } 84 | } 85 | 86 | serialized = Packet.serialize(packet) 87 | 88 | {:ok, %{packet: parsed}} = Packet.parse(serialized, @encrypted?) 89 | 90 | assert ^packet = parsed 91 | assert Packet.serialize(parsed) == serialized 92 | end 93 | end 94 | -------------------------------------------------------------------------------- /test/membrane/rtp/parser_test.exs: -------------------------------------------------------------------------------- 1 | defmodule Membrane.RTP.ParserTest do 2 | use ExUnit.Case 3 | 4 | import Membrane.ChildrenSpec 5 | import Membrane.Testing.Assertions 6 | 7 | alias Membrane.Buffer 8 | alias Membrane.RTP.{Fixtures, Parser} 9 | alias Membrane.Testing.{Pipeline, Sink, Source} 10 | 11 | @buffer_receive_timeout 1000 12 | 13 | describe "Parser" do 14 | test "parses a packet" do 15 | state = %{secure?: false} 16 | packet = Fixtures.sample_packet_binary() 17 | 18 | assert Parser.handle_buffer(:input, %Buffer{payload: packet}, nil, state) == 19 | {[ 20 | buffer: 21 | {:output, 22 | %Membrane.Buffer{ 23 | metadata: %{ 24 | rtp: %{ 25 | sequence_number: 3983, 26 | timestamp: 1_653_702_647, 27 | payload_type: 14, 28 | ssrc: 3_919_876_492, 29 | csrcs: [], 30 | extensions: [], 31 | marker: false, 32 | padding_size: 0, 33 | total_header_size: 12 34 | } 35 | }, 36 | payload: Fixtures.sample_packet_payload() 37 | }} 38 | ], state} 39 | 40 | packet = Fixtures.sample_packet_binary_with_padding() 41 | 42 | assert Parser.handle_buffer(:input, %Buffer{payload: packet}, nil, state) == 43 | {[ 44 | buffer: 45 | {:output, 46 | %Membrane.Buffer{ 47 | metadata: %{ 48 | rtp: %{ 49 | sequence_number: 3983, 50 | timestamp: 1_653_702_647, 51 | payload_type: 14, 52 | ssrc: 3_919_876_492, 53 | csrcs: [], 54 | extensions: [], 55 | marker: false, 56 | padding_size: 20, 57 | total_header_size: 12 58 | } 59 | }, 60 | payload: Fixtures.sample_packet_payload() 61 | }} 62 | ], state} 63 | end 64 | 65 | test "buffers when parsing an RTCP packet" do 66 | state = %{secure?: false, rtcp_output_pad: :rtcp_output} 67 | buffer = Fixtures.sample_rtcp_buffer() 68 | 69 | assert Parser.handle_buffer(:input, buffer, nil, state) == 70 | {[buffer: {:rtcp_output, buffer}], state} 71 | end 72 | 73 | test "works in pipeline" do 74 | test_data_base = 1..100 75 | test_data = Fixtures.fake_packet_list(test_data_base) 76 | 77 | pipeline = 78 | Pipeline.start_link_supervised!( 79 | spec: 80 | child(:source, %Source{ 81 | output: test_data, 82 | stream_format: %Membrane.RemoteStream{ 83 | type: :packetized, 84 | content_format: Membrane.RTP 85 | } 86 | }) 87 | |> child(:parser, Parser) 88 | |> child(:sink, Sink) 89 | ) 90 | 91 | Enum.each(test_data_base, fn _test_data -> 92 | assert_sink_buffer(pipeline, :sink, %Buffer{}, @buffer_receive_timeout) 93 | end) 94 | 95 | Pipeline.terminate(pipeline) 96 | end 97 | end 98 | end 99 | -------------------------------------------------------------------------------- /test/membrane/rtp/pipeline_test.exs: -------------------------------------------------------------------------------- 1 | defmodule Membrane.RTP.PipelineTest do 2 | use ExUnit.Case 3 | 4 | import Membrane.ChildrenSpec 5 | import Membrane.Testing.Assertions 6 | 7 | alias Membrane.Buffer 8 | alias Membrane.RemoteStream 9 | alias Membrane.RemoteStream 10 | alias Membrane.RTP 11 | alias Membrane.RTP.{Fixtures, Parser} 12 | alias Membrane.Testing.{Pipeline, Sink, Source} 13 | 14 | @buffer_receive_timeout 1000 15 | 16 | test "Pipeline decodes set of RTP packets" do 17 | test_data_base = 1..100 18 | test_data = Fixtures.fake_packet_list(test_data_base) 19 | 20 | pipeline = 21 | Pipeline.start_link_supervised!( 22 | spec: 23 | child(:source, %Source{ 24 | output: test_data, 25 | stream_format: %RemoteStream{type: :packetized, content_format: RTP} 26 | }) 27 | |> child(:parser, Parser) 28 | |> child(:sink, Sink) 29 | ) 30 | 31 | Enum.each(test_data_base, fn _test_data -> 32 | assert_sink_buffer(pipeline, :sink, %Buffer{}, @buffer_receive_timeout) 33 | end) 34 | 35 | Pipeline.terminate(pipeline) 36 | end 37 | end 38 | -------------------------------------------------------------------------------- /test/membrane/rtp/rtx_parser_test.exs: -------------------------------------------------------------------------------- 1 | defmodule Membrane.RTP.RTXParserTest do 2 | use ExUnit.Case, async: true 3 | 4 | alias Membrane.Buffer 5 | alias Membrane.RTP.Header.Extension 6 | alias Membrane.RTP.RTXParser 7 | 8 | defp init_state(opts \\ []) do 9 | [original_payload_type: 96] 10 | |> Keyword.merge(opts) 11 | |> then(&struct!(RTXParser, &1)) 12 | |> then(&RTXParser.handle_init(%{}, &1)) 13 | |> then(&elem(&1, 1)) 14 | end 15 | 16 | describe "handle_buffer/4" do 17 | test "RTX buffer" do 18 | state = init_state() 19 | 20 | original_seq_num = 22_406 21 | 22 | packet = %Buffer{ 23 | payload: <>, 24 | pts: nil, 25 | dts: nil, 26 | metadata: %{ 27 | rtp: %{ 28 | csrcs: [], 29 | extensions: [], 30 | marker: true, 31 | padding_size: 0, 32 | payload_type: 97, 33 | sequence_number: 12_923, 34 | ssrc: 1_504_003_399, 35 | timestamp: 3_876_519_202, 36 | total_header_size: 24 37 | } 38 | } 39 | } 40 | 41 | assert {actions, ^state} = RTXParser.handle_buffer(:input, packet, %{}, state) 42 | assert {:output, buffer} = Keyword.fetch!(actions, :buffer) 43 | assert buffer.payload == <<1, 2, 3, 4>> 44 | assert meta = buffer.metadata.rtp 45 | 46 | for key <- [ 47 | :csrcs, 48 | :extensions, 49 | :marker, 50 | :padding_size, 51 | :ssrc, 52 | :timestamp, 53 | :total_header_size 54 | ] do 55 | assert meta[key] == packet.metadata.rtp[key] 56 | end 57 | 58 | assert meta.payload_type == 96 59 | assert meta.sequence_number == original_seq_num 60 | end 61 | 62 | test "ignore padding packet" do 63 | state = init_state() 64 | 65 | padding_packet = %Buffer{ 66 | payload: "", 67 | metadata: %{ 68 | rtp: %{ 69 | csrcs: [], 70 | extensions: [], 71 | marker: false, 72 | padding_size: 224, 73 | payload_type: 97, 74 | sequence_number: 22_802, 75 | ssrc: 2_176_609_592, 76 | timestamp: 2_336_745_526, 77 | total_header_size: 20 78 | } 79 | } 80 | } 81 | 82 | assert RTXParser.handle_buffer(:input, padding_packet, %{}, state) == {[], state} 83 | end 84 | 85 | test "rewrite rid extension" do 86 | state = init_state(rid_id: 10, repaired_rid_id: 11) 87 | 88 | packet = %Buffer{ 89 | payload: <<87, 134, 1, 2, 3, 4>>, 90 | pts: nil, 91 | dts: nil, 92 | metadata: %{ 93 | rtp: %{ 94 | csrcs: [], 95 | extensions: [ 96 | %Extension{identifier: 9, data: <<48>>}, 97 | %Extension{identifier: 11, data: "l"} 98 | ], 99 | marker: true, 100 | padding_size: 0, 101 | payload_type: 97, 102 | sequence_number: 12_923, 103 | ssrc: 1_504_003_399, 104 | timestamp: 3_876_519_202, 105 | total_header_size: 24 106 | } 107 | } 108 | } 109 | 110 | assert {actions, ^state} = RTXParser.handle_buffer(:input, packet, %{}, state) 111 | assert {:output, buffer} = Keyword.fetch!(actions, :buffer) 112 | assert %Extension{identifier: 9, data: <<48>>} in buffer.metadata.rtp.extensions 113 | assert %Extension{identifier: 10, data: "l"} in buffer.metadata.rtp.extensions 114 | end 115 | end 116 | end 117 | -------------------------------------------------------------------------------- /test/membrane/rtp/sender_report_test.exs: -------------------------------------------------------------------------------- 1 | defmodule Membrane.RTP.SenderReportTest do 2 | use ExUnit.Case 3 | 4 | alias Membrane.RTP.Session.SenderReport 5 | alias Membrane.Time 6 | 7 | @ssrc_1 790_688_045 8 | @ssrcs MapSet.new([@ssrc_1]) 9 | @h264_clock_rate 90_000 10 | @packet_count 1038 11 | @octet_count 17_646 12 | @rtp_timestamp 1000 13 | 14 | test "rtp timestamp test" do 15 | {_ssrcs, report_data} = SenderReport.init_report(@ssrcs, %SenderReport.Data{}) 16 | 17 | test_wallclock_time = Time.vm_time() 18 | 19 | mock_serializer_stats = %{ 20 | clock_rate: @h264_clock_rate, 21 | sender_octet_count: @octet_count, 22 | sender_packet_count: @packet_count, 23 | rtp_timestamp: @rtp_timestamp, 24 | timestamp: test_wallclock_time 25 | } 26 | 27 | assert {{:report, [sender_report | _reports]}, _report_data} = 28 | SenderReport.handle_stats(mock_serializer_stats, @ssrc_1, report_data) 29 | 30 | report_wallclock_timestamp = sender_report.sender_info.wallclock_timestamp 31 | report_rtp_timestamp = sender_report.sender_info.rtp_timestamp 32 | 33 | expected_rtp_timestamp = 34 | @rtp_timestamp + 35 | ((@h264_clock_rate * (report_wallclock_timestamp - test_wallclock_time)) 36 | |> Time.as_seconds(:round)) 37 | 38 | assert expected_rtp_timestamp == report_rtp_timestamp 39 | end 40 | end 41 | -------------------------------------------------------------------------------- /test/membrane/rtp/sequence_number_tracker_test.exs: -------------------------------------------------------------------------------- 1 | defmodule Membrane.RTP.SequenceNumberTrackerTest do 2 | use ExUnit.Case, async: true 3 | 4 | alias Membrane.RTP.SequenceNumberTracker, as: Tracker 5 | 6 | describe "track/2" do 7 | test "continuous sequence" do 8 | tracker = Tracker.new() 9 | assert {1, 10, tracker} = Tracker.track(tracker, 10) 10 | assert {1, 11, tracker} = Tracker.track(tracker, 11) 11 | assert {1, 12, _tracker} = Tracker.track(tracker, 12) 12 | end 13 | 14 | test "numbers starting at 0" do 15 | tracker = Tracker.new() 16 | assert {1, 65_536, tracker} = Tracker.track(tracker, 0) 17 | assert {1, 65_537, tracker} = Tracker.track(tracker, 1) 18 | assert {1, 65_538, _tracker} = Tracker.track(tracker, 2) 19 | end 20 | 21 | test "late sequence number when starting at 0" do 22 | tracker = Tracker.new() 23 | assert {1, 65_536, tracker} = Tracker.track(tracker, 0) 24 | assert {-1, 65_535, tracker} = Tracker.track(tracker, 65_535) 25 | assert {1, 65_537, _tracker} = Tracker.track(tracker, 1) 26 | end 27 | 28 | test "numbers starting at 65_535" do 29 | tracker = Tracker.new() 30 | assert {1, 65_535, tracker} = Tracker.track(tracker, 65_535) 31 | assert {1, 65_536, tracker} = Tracker.track(tracker, 0) 32 | assert {1, 65_537, _tracker} = Tracker.track(tracker, 1) 33 | end 34 | 35 | test "rollover" do 36 | tracker = Tracker.new() 37 | assert {1, 65_533, tracker} = Tracker.track(tracker, 65_533) 38 | assert {1, 65_534, tracker} = Tracker.track(tracker, 65_534) 39 | assert {1, 65_535, tracker} = Tracker.track(tracker, 65_535) 40 | assert {1, 65_536, tracker} = Tracker.track(tracker, 0) 41 | assert {1, 65_537, _tracker} = Tracker.track(tracker, 1) 42 | end 43 | 44 | test "gap" do 45 | tracker = Tracker.new() 46 | assert {1, 10, tracker} = Tracker.track(tracker, 10) 47 | assert {3, 13, tracker} = Tracker.track(tracker, 13) 48 | assert {1, 14, _tracker} = Tracker.track(tracker, 14) 49 | end 50 | 51 | test "late sequence number" do 52 | tracker = Tracker.new() 53 | assert {1, 10, tracker} = Tracker.track(tracker, 10) 54 | assert {-1, 9, tracker} = Tracker.track(tracker, 9) 55 | assert {2, 12, tracker} = Tracker.track(tracker, 12) 56 | assert {1, 13, _tracker} = Tracker.track(tracker, 13) 57 | end 58 | 59 | test "gap and late at rollover" do 60 | tracker = Tracker.new() 61 | assert {1, 65_533, tracker} = Tracker.track(tracker, 65_533) 62 | assert {1, 65_534, tracker} = Tracker.track(tracker, 65_534) 63 | assert {3, 65_537, tracker} = Tracker.track(tracker, 1) 64 | assert {-2, 65_535, _tracker} = Tracker.track(tracker, 65_535) 65 | end 66 | 67 | test "repeated number" do 68 | tracker = Tracker.new() 69 | assert {1, 10, tracker} = Tracker.track(tracker, 10) 70 | assert {0, 10, tracker} = Tracker.track(tracker, 10) 71 | assert {1, 11, tracker} = Tracker.track(tracker, 11) 72 | assert {0, 11, _tracker} = Tracker.track(tracker, 11) 73 | end 74 | 75 | test "huge gap" do 76 | tracker = Tracker.new() 77 | assert {1, 1, tracker} = Tracker.track(tracker, 1) 78 | assert {32_767, 32_768, tracker} = Tracker.track(tracker, 32_768) 79 | assert {-32_766, 2, tracker} = Tracker.track(tracker, 2) 80 | assert {-32_765, 3, _tracker} = Tracker.track(tracker, 3) 81 | end 82 | end 83 | end 84 | -------------------------------------------------------------------------------- /test/membrane/rtp/ssrc_router_test.exs: -------------------------------------------------------------------------------- 1 | defmodule Membrane.RTP.SSRCRouterTest do 2 | use ExUnit.Case, async: true 3 | 4 | import Membrane.ChildrenSpec 5 | import Membrane.Testing.Assertions 6 | 7 | alias Membrane.Buffer 8 | alias Membrane.RTP.Header.Extension 9 | alias Membrane.RTP.SSRCRouter 10 | alias Membrane.RTP.Support.TestSource 11 | alias Membrane.Testing.Pipeline 12 | 13 | @rtp_metadata_template %{ 14 | csrcs: [], 15 | extensions: [], 16 | marker: false, 17 | padding_size: 0, 18 | payload_type: 96, 19 | sequence_number: 1, 20 | ssrc: 1, 21 | timestamp: 0, 22 | total_header_size: 20 23 | } 24 | 25 | test "New RTP stream" do 26 | metadata = %{ 27 | @rtp_metadata_template 28 | | extensions: [%Extension{identifier: 4, data: <<0, 2>>}] 29 | } 30 | 31 | pipeline = init_pipeline() 32 | src_send_meta_buffer(pipeline, metadata) 33 | 34 | assert_pipeline_notified(pipeline, :ssrc_router, {:new_rtp_stream, 1, pt, extensions}) 35 | assert pt == metadata.payload_type 36 | assert extensions == metadata.extensions 37 | Pipeline.terminate(pipeline) 38 | end 39 | 40 | test "Wait for required extensions" do 41 | payload_type = 96 42 | 43 | metadata = %{ 44 | @rtp_metadata_template 45 | | payload_type: payload_type, 46 | extensions: [ 47 | %Extension{identifier: 4, data: <<0, 2>>}, 48 | %Extension{identifier: 9, data: "0"} 49 | ] 50 | } 51 | 52 | pipeline = init_pipeline() 53 | 54 | send_router_message(pipeline, %SSRCRouter.StreamsInfo{ 55 | require_extensions: %{payload_type => [9, 10]} 56 | }) 57 | 58 | src_send_meta_buffer(pipeline, metadata) 59 | 60 | metadata = %{ 61 | metadata 62 | | ssrc: 2, 63 | extensions: [ 64 | %Extension{identifier: 4, data: <<0, 3>>}, 65 | %Extension{identifier: 9, data: "0"}, 66 | %Extension{identifier: 10, data: "l"} 67 | ] 68 | } 69 | 70 | src_send_meta_buffer(pipeline, metadata) 71 | 72 | assert_pipeline_notified( 73 | pipeline, 74 | :ssrc_router, 75 | {:new_rtp_stream, 2, ^payload_type, extensions} 76 | ) 77 | 78 | assert extensions == metadata.extensions 79 | 80 | # Ensure the first package was dropped 81 | refute_pipeline_notified(pipeline, :ssrc_router, {:new_rtp_stream, 1, _pt, _extensions}, 200) 82 | 83 | Pipeline.terminate(pipeline) 84 | end 85 | 86 | test "Waiting for extensions doesn't break non-simulcast" do 87 | payload_type = 96 88 | 89 | metadata = %{ 90 | @rtp_metadata_template 91 | | payload_type: payload_type, 92 | extensions: [ 93 | %Extension{identifier: 4, data: <<0, 2>>}, 94 | %Extension{identifier: 9, data: "0"} 95 | ] 96 | } 97 | 98 | pipeline = init_pipeline() 99 | 100 | send_router_message(pipeline, %SSRCRouter.StreamsInfo{ 101 | require_extensions: %{payload_type => [9, 10]}, 102 | accept_ssrcs: [1] 103 | }) 104 | 105 | src_send_meta_buffer(pipeline, metadata) 106 | 107 | assert_pipeline_notified( 108 | pipeline, 109 | :ssrc_router, 110 | {:new_rtp_stream, 1, ^payload_type, extensions} 111 | ) 112 | 113 | assert extensions == metadata.extensions 114 | Pipeline.terminate(pipeline) 115 | end 116 | 117 | defp init_pipeline() do 118 | Pipeline.start_link_supervised!( 119 | spec: 120 | child(:source, %TestSource{output: [], stream_format: %Membrane.RTP{}}) 121 | |> child(:ssrc_router, SSRCRouter) 122 | ) 123 | end 124 | 125 | defp src_send_meta_buffer(pipeline, metadata) do 126 | buffer = %Buffer{payload: "", metadata: %{rtp: metadata}} 127 | 128 | Pipeline.execute_actions(pipeline, 129 | notify_child: {:source, {:execute_actions, [buffer: {:output, buffer}]}} 130 | ) 131 | end 132 | 133 | defp send_router_message(pipeline, msg) do 134 | Pipeline.execute_actions(pipeline, notify_child: {:ssrc_router, msg}) 135 | end 136 | end 137 | -------------------------------------------------------------------------------- /test/membrane/rtp/utils_test.exs: -------------------------------------------------------------------------------- 1 | defmodule Membrane.RTP.UtilsTest do 2 | use ExUnit.Case, async: true 3 | 4 | alias Membrane.RTP.Utils 5 | 6 | test "Utils.generate_padding/1" do 7 | assert <<>> = Utils.generate_padding(0) 8 | assert <<1>> = Utils.generate_padding(1) 9 | assert <<0::size(99)-unit(8), 100>> = Utils.generate_padding(100) 10 | assert <<0::size(254)-unit(8), 255>> = Utils.generate_padding(255) 11 | assert_raise FunctionClauseError, fn -> Utils.generate_padding(-1) end 12 | assert_raise FunctionClauseError, fn -> Utils.generate_padding(256) end 13 | end 14 | end 15 | -------------------------------------------------------------------------------- /test/membrane/rtp/vad/audio_level_queue_test.exs: -------------------------------------------------------------------------------- 1 | defmodule Membrane.RTP.Vad.AudioLevelQueueTest do 2 | @moduledoc """ 3 | AudioLevelQueue api tests. 4 | """ 5 | 6 | use ExUnit.Case 7 | 8 | alias Membrane.RTP.Vad.{AudioLevelQueue, VadParams} 9 | 10 | @target_levels_length VadParams.target_levels_length() 11 | 12 | describe "new" do 13 | test "creates new empty queue" do 14 | queue = AudioLevelQueue.new() 15 | 16 | assert %AudioLevelQueue{levels: levels, length: length} = queue 17 | assert :queue.len(levels) == 0 18 | assert length == 0 19 | end 20 | 21 | test "Creates audio level queue out of full list set" do 22 | expected_list = [2, 1, 3, 7] 23 | actual_queue = AudioLevelQueue.new(expected_list) 24 | 25 | assert actual_queue.length == 4 26 | assert AudioLevelQueue.to_list(actual_queue) == expected_list 27 | end 28 | 29 | test "Creates a queue no longer than the limit" do 30 | an_arbitrary_number = 42 31 | expected_length = @target_levels_length + an_arbitrary_number 32 | expected_levels = Enum.to_list(1..@target_levels_length) 33 | 34 | input_list = 1..expected_length 35 | actual_queue = AudioLevelQueue.new(input_list) 36 | 37 | assert actual_queue.length == @target_levels_length 38 | assert AudioLevelQueue.to_list(actual_queue) == expected_levels 39 | end 40 | end 41 | 42 | describe "add" do 43 | setup do 44 | %{empty_queue: AudioLevelQueue.new()} 45 | end 46 | 47 | test "Adds new element", %{empty_queue: empty_queue} do 48 | level = 90 49 | 50 | queue_with_element = AudioLevelQueue.add(empty_queue, level) 51 | 52 | assert queue_with_element.length == 1 53 | assert AudioLevelQueue.to_list(queue_with_element) == [90] 54 | end 55 | 56 | test "Trims the queue if needed", %{empty_queue: empty_queue} do 57 | an_arbitrary_number = 42 58 | levels = List.duplicate(90, @target_levels_length + an_arbitrary_number) 59 | 60 | queue_with_added_items = 61 | Enum.reduce(levels, empty_queue, fn x, queue -> AudioLevelQueue.add(queue, x) end) 62 | 63 | assert queue_with_added_items.length == @target_levels_length 64 | end 65 | end 66 | 67 | describe "to list" do 68 | test "Creates empty list from an empty queue" do 69 | assert AudioLevelQueue.to_list(AudioLevelQueue.new()) == [] 70 | end 71 | 72 | test "Creates a list from a queue" do 73 | list_from_queue = 74 | AudioLevelQueue.new() 75 | |> AudioLevelQueue.add(2) 76 | |> AudioLevelQueue.add(1) 77 | |> AudioLevelQueue.add(3) 78 | |> AudioLevelQueue.add(7) 79 | |> AudioLevelQueue.to_list() 80 | 81 | assert list_from_queue == [7, 3, 1, 2] 82 | end 83 | end 84 | end 85 | -------------------------------------------------------------------------------- /test/membrane/rtp/vad/is_speaking_estimator_test.exs: -------------------------------------------------------------------------------- 1 | defmodule Membrane.RTP.Vad.IsSpeakingEstimatorTest do 2 | @moduledoc """ 3 | The tests for IsSpeakingEstimator are focused on the estimate_is_speaking function which takes: 4 | - levels: a list of db values inferred from the RTP Header; the values range from 0 (digital silence) to 127 (digital noise) 5 | - threshold: a number which indicates how big the level value must be so the interval is counted as active 6 | For the sake of simplicity in the following tests only 0 and 127 are used as values of the `levels` list and the threshold value is fixed. 7 | 8 | Moreover, the algorithm uses a set of internal parameters. The values are different for the test environment than from other environments. The values set here can be found in the `tests.exs` config file. 9 | 10 | The tests are constructed so the inner workings of the algorithm are checked. They include not only the trivial cases like only silence, only speech or to short input list, but it cover more sophisticated scenarios like: 11 | - alternation every packet - this can mean the person has been alternating on the edge of the threshold which often means sound from background 12 | - alternation every medium packet = since wa assume one medium interval is (roughly) one word, this can mean a person is speaking but with long pauses 13 | - alternation every long packet - we assume a person just finished a sentence, so the algorithm should return speech 14 | - random packet dropped - "one swallow doesn't make a summer" - one packet doesn't make silence... if surrounded by packets with high dB value, it should be counted as speech 15 | """ 16 | 17 | use ExUnit.Case 18 | 19 | alias Membrane.RTP.Vad.{IsSpeakingEstimator, VadParams} 20 | 21 | @immediate_subunits VadParams.immediate().subunits 22 | @medium_subunits VadParams.medium().subunits 23 | @expected_levels_length VadParams.target_levels_length() 24 | 25 | defp silence(n), do: List.duplicate(0, n) 26 | defp noise(n), do: List.duplicate(127, n) 27 | 28 | defp alternating_signal(n, interval) do 29 | repeats = div(n, 2 * interval) 30 | 31 | interval 32 | |> then(&(noise(&1) ++ silence(&1))) 33 | |> List.duplicate(repeats) 34 | |> Enum.concat() 35 | end 36 | 37 | defp noise_with_one_silence(n, low_item_idx) do 38 | n_left = low_item_idx 39 | n_right = n - low_item_idx - 1 40 | 41 | noise(n_left) ++ silence(1) ++ noise(n_right) 42 | end 43 | 44 | describe "estimate is speaking" do 45 | setup do 46 | [threshold: 50] 47 | end 48 | 49 | test "returns :silence when levels length less than target length", %{ 50 | threshold: dummy_threshold 51 | } do 52 | levels = noise(4) 53 | 54 | assert IsSpeakingEstimator.estimate_is_speaking(levels, dummy_threshold) == :silence 55 | end 56 | 57 | test "returns :silence when digital silence", %{threshold: threshold} do 58 | levels = silence(@expected_levels_length) 59 | 60 | assert IsSpeakingEstimator.estimate_is_speaking(levels, threshold) == :silence 61 | end 62 | 63 | test "returns :speech when digital noise", %{threshold: threshold} do 64 | levels = noise(@expected_levels_length) 65 | 66 | assert IsSpeakingEstimator.estimate_is_speaking(levels, threshold) == :speech 67 | end 68 | 69 | test "returns :silence for alternating signal", %{threshold: threshold} do 70 | levels = alternating_signal(@expected_levels_length, 1) 71 | 72 | assert IsSpeakingEstimator.estimate_is_speaking(levels, threshold) == :silence 73 | end 74 | 75 | test "returns :speech for alternating pairs", %{threshold: threshold} do 76 | levels = alternating_signal(@expected_levels_length, @immediate_subunits) 77 | 78 | assert IsSpeakingEstimator.estimate_is_speaking(levels, threshold) == :speech 79 | end 80 | 81 | test "returns :speech for first half above threshold", %{threshold: threshold} do 82 | levels = 83 | alternating_signal( 84 | @expected_levels_length, 85 | @immediate_subunits * @medium_subunits 86 | ) 87 | 88 | assert IsSpeakingEstimator.estimate_is_speaking(levels, threshold) == :speech 89 | end 90 | 91 | test "returns :speech for noise with one 'silent' packet", %{threshold: threshold} do 92 | levels = noise_with_one_silence(@expected_levels_length, 5) 93 | 94 | assert IsSpeakingEstimator.estimate_is_speaking(levels, threshold) 95 | end 96 | end 97 | end 98 | -------------------------------------------------------------------------------- /test/support/buffer_factory.ex: -------------------------------------------------------------------------------- 1 | defmodule Membrane.RTP.BufferFactory do 2 | @moduledoc false 3 | alias Membrane.Buffer 4 | alias Membrane.RTP 5 | 6 | @timestamp_increment 30_000 7 | @clock_rate 10_000 8 | @base_seq_number 50 9 | 10 | @spec timestamp_increment() :: RTP.Header.timestamp_t() 11 | def timestamp_increment(), do: @timestamp_increment 12 | 13 | @spec clock_rate() :: RTP.clock_rate() 14 | def clock_rate(), do: @clock_rate 15 | 16 | @spec base_seq_number() :: RTP.Header.sequence_number_t() 17 | def base_seq_number(), do: @base_seq_number 18 | 19 | @spec sample_buffer(RTP.Header.sequence_number_t()) :: Membrane.Buffer.t() 20 | def sample_buffer(seq_num) do 21 | seq_num_offset = seq_num - @base_seq_number 22 | 23 | %Buffer{ 24 | payload: <<0, 255>>, 25 | pts: div(seq_num_offset * @timestamp_increment * Membrane.Time.second(), @clock_rate), 26 | metadata: %{ 27 | rtp: %{ 28 | timestamp: seq_num_offset * @timestamp_increment, 29 | sequence_number: seq_num 30 | } 31 | } 32 | } 33 | end 34 | end 35 | -------------------------------------------------------------------------------- /test/support/rtp_fixtures.ex: -------------------------------------------------------------------------------- 1 | defmodule Membrane.RTP.Fixtures do 2 | @moduledoc false 3 | alias Membrane.RTP.{Header, Packet} 4 | 5 | @external_resource "test/fixtures/rtp/rtp_packet.bin" 6 | @sample_packet File.read!("test/fixtures/rtp/rtp_packet.bin") 7 | 8 | @external_resource "test/fixtures/rtp/rtp_packet_payload.bin" 9 | @sample_packet_payload File.read!("test/fixtures/rtp/rtp_packet_payload.bin") 10 | 11 | @external_resource "test/fixtures/rtp/rtp_packet_with_padding.bin" 12 | @sample_packet_with_padding File.read!("test/fixtures/rtp/rtp_packet_with_padding.bin") 13 | 14 | @spec sample_packet_binary() :: binary() 15 | def sample_packet_binary, do: @sample_packet 16 | 17 | @spec sample_packet() :: Packet.t() 18 | def sample_packet, do: %Packet{header: sample_header(), payload: sample_packet_payload()} 19 | 20 | @spec sample_packet_payload() :: binary() 21 | def sample_packet_payload, do: @sample_packet_payload 22 | 23 | @spec sample_packet_binary_with_padding() :: binary() 24 | def sample_packet_binary_with_padding, do: @sample_packet_with_padding 25 | 26 | @spec sample_packet_with_padding() :: Packet.t() 27 | def sample_packet_with_padding, 28 | do: %Packet{header: sample_header(), payload: sample_packet_payload()} 29 | 30 | @spec sample_buffer() :: Membrane.Buffer.t() 31 | def sample_buffer, 32 | do: %Membrane.Buffer{ 33 | payload: sample_packet_payload(), 34 | metadata: %{ 35 | rtp_header: sample_header() 36 | } 37 | } 38 | 39 | @spec sample_rtcp_buffer() :: Membrane.Buffer.t() 40 | def sample_rtcp_buffer, 41 | do: %Membrane.Buffer{ 42 | metadata: %{}, 43 | payload: 44 | <<128, 201, 0, 1, 0, 0, 0, 1, 128, 0, 0, 23, 204, 45, 91, 116, 43, 191, 84, 170, 205, 20>> 45 | } 46 | 47 | @spec sample_header() :: Header.t() 48 | def sample_header, 49 | do: %Header{ 50 | payload_type: 14, 51 | sequence_number: 3983, 52 | ssrc: 3_919_876_492, 53 | timestamp: 1_653_702_647, 54 | extensions: [] 55 | } 56 | 57 | @spec fake_packet_list(Range.t()) :: [binary()] 58 | def fake_packet_list(range) do 59 | base_seqnumber = 65_403 60 | base_timestamp = 383_400 61 | ssrc = 562_678_578_632 62 | 63 | Enum.map(range, fn packet_number -> 64 | <<2::2, 0::1, 0::1, 0::4, 0::1, 14::7, base_seqnumber + packet_number::16, 65 | base_timestamp + 30_000 * packet_number::32, ssrc::32, sample_packet_payload()::binary>> 66 | end) 67 | end 68 | end 69 | -------------------------------------------------------------------------------- /test/support/test_source.ex: -------------------------------------------------------------------------------- 1 | defmodule Membrane.RTP.Support.TestSource do 2 | @moduledoc false 3 | # Taken from RTC Engine - should be removed when core supports all features 4 | 5 | use Membrane.Source 6 | 7 | alias Membrane.Buffer 8 | alias Membrane.Element.Action 9 | 10 | @type generator :: 11 | (state :: any(), buffers_cnt :: pos_integer -> {[Action.t()], state :: any()}) 12 | 13 | def_output_pad :output, accepted_format: _any, flow_control: :manual 14 | 15 | def_options output: [ 16 | spec: {initial_state :: any(), generator} | Enum.t(), 17 | default: {0, &__MODULE__.default_buf_gen/2}, 18 | description: """ 19 | If `output` is an enumerable with `Membrane.Payload.t()` then 20 | buffer containing those payloads will be sent through the 21 | `:output` pad and followed by `t:Membrane.Element.Action.end_of_stream_t/0`. 22 | If `output` is a `{initial_state, function}` tuple then the 23 | the function will be invoked each time `handle_demand` is called. 24 | It is an action generator that takes two arguments. 25 | The first argument is the state that is initially set to 26 | `initial_state`. The second one defines the size of the demand. 27 | Such function should return `{actions, next_state}` where 28 | `actions` is a list of actions that will be returned from 29 | `handle_demand/4` and `next_state` is the value that will be 30 | used for the next call. 31 | """ 32 | ], 33 | stream_format: [ 34 | spec: struct(), 35 | default: %Membrane.RemoteStream{}, 36 | description: """ 37 | Stream format struct to be sent before the `output`. 38 | """ 39 | ], 40 | fast_start: [ 41 | spec: boolean(), 42 | default: true, 43 | description: """ 44 | Whether to start sending buffers immediately after going into 45 | playing state. If set to false, source has to be started 46 | manually by sending message `{:set_active, true}`. 47 | """ 48 | ] 49 | 50 | @impl true 51 | def handle_init(_ctx, opts) do 52 | opts = Map.from_struct(opts) 53 | 54 | state = 55 | case opts.output do 56 | {initial_state, generator} when is_function(generator) -> 57 | opts 58 | |> Map.merge(%{ 59 | generator_state: initial_state, 60 | output: generator, 61 | active?: opts.fast_start 62 | }) 63 | 64 | _enumerable_output -> 65 | opts 66 | end 67 | 68 | {[], state} 69 | end 70 | 71 | @impl true 72 | def handle_playing(_ctx, state) do 73 | {[stream_format: {:output, state.stream_format}], state} 74 | end 75 | 76 | @impl true 77 | def handle_demand(:output, size, :buffers, _ctx, %{active?: true} = state) do 78 | get_actions(state, size) 79 | end 80 | 81 | @impl true 82 | def handle_demand(:output, _size, :buffers, _ctx, state) do 83 | {[], state} 84 | end 85 | 86 | @impl true 87 | def handle_event(:output, event, _ctx, state) do 88 | {[notify_parent: %Membrane.Testing.Notification{payload: {:event, event}}], state} 89 | end 90 | 91 | @impl true 92 | def handle_parent_notification({:execute_actions, actions}, _ctx, state) do 93 | {actions, state} 94 | end 95 | 96 | @impl true 97 | def handle_parent_notification({:set_active, active?}, _ctx, state) do 98 | {[redemand: :output], %{state | active?: active?}} 99 | end 100 | 101 | @spec default_buf_gen(integer(), integer()) :: {[Action.t()], integer()} 102 | def default_buf_gen(generator_state, size) do 103 | buffers = 104 | generator_state..(size + generator_state - 1) 105 | |> Enum.map(fn generator_state -> 106 | %Buffer{payload: <>} 107 | end) 108 | 109 | action = [buffer: {:output, buffers}] 110 | {action, generator_state + size} 111 | end 112 | 113 | @doc """ 114 | Creates output with generator function from list of buffers. 115 | """ 116 | @spec output_from_buffers([Buffer.t()]) :: {[Buffer.t()], generator()} 117 | def output_from_buffers(data) do 118 | fun = fn state, size -> 119 | {buffers, leftover} = Enum.split(state, size) 120 | buffer_action = [buffer: {:output, buffers}] 121 | event_action = if leftover == [], do: [end_of_stream: :output], else: [] 122 | to_send = buffer_action ++ event_action 123 | {to_send, leftover} 124 | end 125 | 126 | {data, fun} 127 | end 128 | 129 | defp get_actions(%{generator_state: generator_state, output: actions_generator} = state, size) 130 | when is_function(actions_generator) do 131 | {actions, generator_state} = actions_generator.(generator_state, size) 132 | {actions, %{state | generator_state: generator_state}} 133 | end 134 | 135 | defp get_actions(%{output: output} = state, size) do 136 | {payloads, output} = Enum.split(output, size) 137 | buffers = Enum.map(payloads, &%Buffer{payload: &1}) 138 | 139 | actions = 140 | case output do 141 | [] -> [buffer: {:output, buffers}, end_of_stream: :output] 142 | _non_empty -> [buffer: {:output, buffers}] 143 | end 144 | 145 | {actions, %{state | output: output}} 146 | end 147 | end 148 | -------------------------------------------------------------------------------- /test/test_helper.exs: -------------------------------------------------------------------------------- 1 | ExUnit.start(exclude: [long_running: true], capture_log: true) 2 | --------------------------------------------------------------------------------