├── .circleci └── config.yml ├── .credo.exs ├── .editorconfig ├── .formatter.exs ├── .github ├── CODEOWNERS ├── ISSUE_TEMPLATE │ └── please--open-new-issues-in-membranefranework-membrane_core.md └── workflows │ ├── on_issue_opened.yaml │ └── on_pr_opened.yaml ├── .gitignore ├── LICENSE ├── README.md ├── lib └── membrane_h264_plugin │ ├── h264 │ ├── au_splitter.ex │ ├── au_timestamp_generator.ex │ ├── decoder_configuration_record.ex │ ├── nalu_parser.ex │ ├── nalu_parser │ │ └── schemes │ │ │ ├── nalu_header.ex │ │ │ ├── pps.ex │ │ │ ├── slice.ex │ │ │ └── sps.ex │ └── nalu_types.ex │ ├── h264_parser.ex │ ├── h265 │ ├── au_splitter.ex │ ├── au_timestamp_generator.ex │ ├── decoder_configuration_record.ex │ ├── nalu_parser.ex │ ├── nalu_parser │ │ └── schemes │ │ │ ├── common.ex │ │ │ ├── nalu_header.ex │ │ │ ├── pps.ex │ │ │ ├── slice.ex │ │ │ ├── sps.ex │ │ │ └── vps.ex │ └── nalu_types.ex │ ├── h265_parser.ex │ ├── h26x │ ├── au_splitter.ex │ ├── au_timestamp_generator.ex │ ├── exp_golomb_converter.ex │ ├── nalu.ex │ ├── nalu_parser.ex │ ├── nalu_parser │ │ ├── scheme.ex │ │ └── scheme_parser.ex │ └── nalu_splitter.ex │ └── h26x_parser.ex ├── mix.exs ├── mix.lock └── test ├── exp_golomb_test.exs ├── fixtures ├── h264 │ ├── input-10-320x180.h264 │ ├── input-10-720a.h264 │ ├── input-10-720p-baseline.h264 │ ├── input-10-720p-main.h264 │ ├── input-10-720p-no-b-frames.h264 │ ├── input-10-720p.h264 │ ├── input-10-no-pps-sps.h264 │ ├── input-100-240p-no-b-frames.h264 │ ├── input-100-240p.h264 │ ├── input-20-360p-I422.h264 │ ├── input-30-240p-no-sps-pps.h264 │ ├── input-30-240p-vp-sps-pps.h264 │ ├── input-idr-sps-pps-non-idr.h264 │ ├── input-sps-pps-non-idr-sps-pps-idr.h264 │ ├── input-sps-pps-non-idr.h264 │ ├── mp4 │ │ ├── ref_video.mp4 │ │ ├── ref_video_fast_start.mp4 │ │ └── ref_video_variable_parameters.mp4 │ ├── msr │ │ ├── ref_video-avc1-au.msr │ │ ├── ref_video-avc1-nalu.msr │ │ ├── ref_video-avc3-au.msr │ │ ├── ref_video-avc3-nalu.msr │ │ ├── ref_video_fast_start-avc1-au.msr │ │ ├── ref_video_fast_start-avc1-nalu.msr │ │ ├── ref_video_fast_start-avc3-au.msr │ │ ├── ref_video_fast_start-avc3-nalu.msr │ │ ├── ref_video_variable_parameters-avc3-au.msr │ │ └── ref_video_variable_parameters-avc3-nalu.msr │ ├── reference-30-240p-vp-sps-pps.h264 │ ├── reference-30-240p-with-sps-pps.h264 │ ├── reference-idr-sps-pps-non-idr.h264 │ ├── reference-sps-pps-non-idr-sps-pps-idr.h264 │ └── reference-sps-pps-non-idr.h264 └── h265 │ ├── input-10-1920x1080.h265 │ ├── input-10-480x320-mainstillpicture.h265 │ ├── input-10-640x480-main10.h265 │ ├── input-10-no-vps-sps-pps.h265 │ ├── input-15-1280x720-temporal-id-1.h265 │ ├── input-30-1280x720-rext.h265 │ ├── input-30-640x480-no-bframes.h265 │ ├── input-300-98x58-conformance-window.h265 │ ├── input-32-640x360-main.h265 │ ├── input-60-1920x1080.h265 │ ├── input-60-640x480-no-parameter-sets.h265 │ ├── input-60-640x480-variable-parameters.h265 │ ├── input-60-640x480.h265 │ ├── input-8-2K.h265 │ ├── input-irap-pss-no-irap.h265 │ ├── input-pss-no-irap-pss-irap.h265 │ ├── mp4 │ ├── ref_video.mp4 │ └── ref_video_variable_parameters.mp4 │ ├── msr │ ├── ref_video-hev1-au.msr │ ├── ref_video-hev1-nalu.msr │ ├── ref_video-hvc1-au.msr │ ├── ref_video-hvc1-nalu.msr │ ├── ref_video_variable_parameters-hev1-au.msr │ └── ref_video_variable_parameters-hev1-nalu.msr │ ├── reference-60-640x480-variable-parameters.h265 │ ├── reference-60-640x480-with-parameter-sets.h265 │ ├── reference-irap-pss-no-irap.h265 │ └── reference-pss-no-irap-pss-irap.h265 ├── integration ├── h264 │ ├── modes_test.exs │ └── timestamp_generation_test.exs └── h265 │ ├── modes_test.exs │ └── timestamp_generation_test.exs ├── parser ├── h264 │ ├── au_splitter_test.exs │ ├── process_all_test.exs │ ├── repeat_parameter_sets_test.exs │ ├── skip_until_test.exs │ ├── stream_format_test.exs │ └── stream_structure_conversion_test.exs └── h265 │ ├── au_splitter_test.exs │ ├── process_all_test.exs │ ├── repeat_parameter_sets_test.exs │ ├── skip_until_test.exs │ ├── stream_format_test.exs │ └── stream_structure_conversion_test.exs ├── support ├── common.ex ├── fixture_generator.exs └── test_source.ex └── test_helper.exs /.circleci/config.yml: -------------------------------------------------------------------------------- 1 | version: 2.1 2 | orbs: 3 | elixir: membraneframework/elixir@1 4 | 5 | workflows: 6 | version: 2 7 | build: 8 | jobs: 9 | - elixir/build_test: 10 | filters: &filters 11 | tags: 12 | only: /v.*/ 13 | - elixir/test: 14 | filters: 15 | <<: *filters 16 | - elixir/lint: 17 | filters: 18 | <<: *filters 19 | - elixir/hex_publish: 20 | requires: 21 | - elixir/build_test 22 | - elixir/test 23 | - elixir/lint 24 | context: 25 | - Deployment 26 | filters: 27 | branches: 28 | ignore: /.*/ 29 | tags: 30 | only: /v.*/ 31 | -------------------------------------------------------------------------------- /.credo.exs: -------------------------------------------------------------------------------- 1 | # This file contains the configuration for Credo and you are probably reading 2 | # this after creating it with `mix credo.gen.config`. 3 | # 4 | # If you find anything wrong or unclear in this file, please report an 5 | # issue on GitHub: https://github.com/rrrene/credo/issues 6 | # 7 | %{ 8 | # 9 | # You can have as many configs as you like in the `configs:` field. 10 | configs: [ 11 | %{ 12 | # 13 | # Run any config using `mix credo -C `. If no config name is given 14 | # "default" is used. 15 | # 16 | name: "default", 17 | # 18 | # These are the files included in the analysis: 19 | files: %{ 20 | # 21 | # You can give explicit globs or simply directories. 22 | # In the latter case `**/*.{ex,exs}` will be used. 23 | # 24 | included: [ 25 | "lib/", 26 | "src/", 27 | "test/", 28 | "web/", 29 | "apps/*/lib/", 30 | "apps/*/src/", 31 | "apps/*/test/", 32 | "apps/*/web/" 33 | ], 34 | excluded: [~r"/_build/", ~r"/deps/", ~r"/node_modules/"] 35 | }, 36 | # 37 | # Load and configure plugins here: 38 | # 39 | plugins: [], 40 | # 41 | # If you create your own checks, you must specify the source files for 42 | # them here, so they can be loaded by Credo before running the analysis. 43 | # 44 | requires: [], 45 | # 46 | # If you want to enforce a style guide and need a more traditional linting 47 | # experience, you can change `strict` to `true` below: 48 | # 49 | strict: false, 50 | # 51 | # To modify the timeout for parsing files, change this value: 52 | # 53 | parse_timeout: 5000, 54 | # 55 | # If you want to use uncolored output by default, you can change `color` 56 | # to `false` below: 57 | # 58 | color: true, 59 | # 60 | # You can customize the parameters of any check by adding a second element 61 | # to the tuple. 62 | # 63 | # To disable a check put `false` as second element: 64 | # 65 | # {Credo.Check.Design.DuplicatedCode, false} 66 | # 67 | checks: [ 68 | # 69 | ## Consistency Checks 70 | # 71 | {Credo.Check.Consistency.ExceptionNames, []}, 72 | {Credo.Check.Consistency.LineEndings, []}, 73 | {Credo.Check.Consistency.ParameterPatternMatching, []}, 74 | {Credo.Check.Consistency.SpaceAroundOperators, []}, 75 | {Credo.Check.Consistency.SpaceInParentheses, []}, 76 | {Credo.Check.Consistency.TabsOrSpaces, []}, 77 | 78 | # 79 | ## Design Checks 80 | # 81 | # You can customize the priority of any check 82 | # Priority values are: `low, normal, high, higher` 83 | # 84 | {Credo.Check.Design.AliasUsage, 85 | [priority: :low, if_nested_deeper_than: 2, if_called_more_often_than: 0]}, 86 | # You can also customize the exit_status of each check. 87 | # If you don't want TODO comments to cause `mix credo` to fail, just 88 | # set this value to 0 (zero). 89 | # 90 | {Credo.Check.Design.TagTODO, [exit_status: 0]}, 91 | {Credo.Check.Design.TagFIXME, []}, 92 | 93 | # 94 | ## Readability Checks 95 | # 96 | {Credo.Check.Readability.AliasOrder, [priority: :normal]}, 97 | {Credo.Check.Readability.FunctionNames, []}, 98 | {Credo.Check.Readability.LargeNumbers, []}, 99 | {Credo.Check.Readability.MaxLineLength, [priority: :low, max_length: 120]}, 100 | {Credo.Check.Readability.ModuleAttributeNames, []}, 101 | {Credo.Check.Readability.ModuleDoc, []}, 102 | {Credo.Check.Readability.ModuleNames, []}, 103 | {Credo.Check.Readability.ParenthesesInCondition, []}, 104 | {Credo.Check.Readability.ParenthesesOnZeroArityDefs, parens: true}, 105 | {Credo.Check.Readability.PredicateFunctionNames, []}, 106 | {Credo.Check.Readability.PreferImplicitTry, []}, 107 | {Credo.Check.Readability.RedundantBlankLines, []}, 108 | {Credo.Check.Readability.Semicolons, []}, 109 | {Credo.Check.Readability.SpaceAfterCommas, []}, 110 | {Credo.Check.Readability.StringSigils, []}, 111 | {Credo.Check.Readability.TrailingBlankLine, []}, 112 | {Credo.Check.Readability.TrailingWhiteSpace, []}, 113 | {Credo.Check.Readability.UnnecessaryAliasExpansion, []}, 114 | {Credo.Check.Readability.VariableNames, []}, 115 | {Credo.Check.Readability.WithSingleClause, false}, 116 | 117 | # 118 | ## Refactoring Opportunities 119 | # 120 | {Credo.Check.Refactor.CondStatements, []}, 121 | {Credo.Check.Refactor.CyclomaticComplexity, []}, 122 | {Credo.Check.Refactor.FunctionArity, []}, 123 | {Credo.Check.Refactor.LongQuoteBlocks, []}, 124 | {Credo.Check.Refactor.MapInto, false}, 125 | {Credo.Check.Refactor.MatchInCondition, []}, 126 | {Credo.Check.Refactor.NegatedConditionsInUnless, []}, 127 | {Credo.Check.Refactor.NegatedConditionsWithElse, []}, 128 | {Credo.Check.Refactor.Nesting, []}, 129 | {Credo.Check.Refactor.UnlessWithElse, []}, 130 | {Credo.Check.Refactor.WithClauses, []}, 131 | 132 | # 133 | ## Warnings 134 | # 135 | {Credo.Check.Warning.BoolOperationOnSameValues, []}, 136 | {Credo.Check.Warning.ExpensiveEmptyEnumCheck, []}, 137 | {Credo.Check.Warning.IExPry, []}, 138 | {Credo.Check.Warning.IoInspect, []}, 139 | {Credo.Check.Warning.LazyLogging, false}, 140 | {Credo.Check.Warning.MixEnv, []}, 141 | {Credo.Check.Warning.OperationOnSameValues, []}, 142 | {Credo.Check.Warning.OperationWithConstantResult, []}, 143 | {Credo.Check.Warning.RaiseInsideRescue, []}, 144 | {Credo.Check.Warning.UnusedEnumOperation, []}, 145 | {Credo.Check.Warning.UnusedFileOperation, []}, 146 | {Credo.Check.Warning.UnusedKeywordOperation, []}, 147 | {Credo.Check.Warning.UnusedListOperation, []}, 148 | {Credo.Check.Warning.UnusedPathOperation, []}, 149 | {Credo.Check.Warning.UnusedRegexOperation, []}, 150 | {Credo.Check.Warning.UnusedStringOperation, []}, 151 | {Credo.Check.Warning.UnusedTupleOperation, []}, 152 | {Credo.Check.Warning.UnsafeExec, []}, 153 | 154 | # 155 | # Checks scheduled for next check update (opt-in for now, just replace `false` with `[]`) 156 | 157 | # 158 | # Controversial and experimental checks (opt-in, just replace `false` with `[]`) 159 | # 160 | {Credo.Check.Readability.StrictModuleLayout, 161 | priority: :normal, order: ~w/shortdoc moduledoc behaviour use import require alias/a}, 162 | {Credo.Check.Consistency.MultiAliasImportRequireUse, false}, 163 | {Credo.Check.Consistency.UnusedVariableNames, force: :meaningful}, 164 | {Credo.Check.Design.DuplicatedCode, false}, 165 | {Credo.Check.Readability.AliasAs, false}, 166 | {Credo.Check.Readability.MultiAlias, false}, 167 | {Credo.Check.Readability.Specs, []}, 168 | {Credo.Check.Readability.SinglePipe, false}, 169 | {Credo.Check.Readability.WithCustomTaggedTuple, false}, 170 | {Credo.Check.Refactor.ABCSize, false}, 171 | {Credo.Check.Refactor.AppendSingleItem, false}, 172 | {Credo.Check.Refactor.DoubleBooleanNegation, false}, 173 | {Credo.Check.Refactor.ModuleDependencies, false}, 174 | {Credo.Check.Refactor.NegatedIsNil, false}, 175 | {Credo.Check.Refactor.PipeChainStart, false}, 176 | {Credo.Check.Refactor.VariableRebinding, false}, 177 | {Credo.Check.Warning.LeakyEnvironment, false}, 178 | {Credo.Check.Warning.MapGetUnsafePass, false}, 179 | {Credo.Check.Warning.UnsafeToAtom, false} 180 | 181 | # 182 | # Custom checks can be created using `mix credo.gen.check`. 183 | # 184 | ] 185 | } 186 | ] 187 | } 188 | -------------------------------------------------------------------------------- /.editorconfig: -------------------------------------------------------------------------------- 1 | root = true 2 | 3 | [*] 4 | charset = utf-8 5 | end_of_line = lf 6 | indent_size = 2 7 | indent_style = space 8 | insert_final_newline = true 9 | max_line_length = 100 10 | tab_width = 2 11 | trim_trailing_whitespace = true 12 | -------------------------------------------------------------------------------- /.formatter.exs: -------------------------------------------------------------------------------- 1 | [ 2 | inputs: [ 3 | "{lib,test,config}/**/*.{ex,exs}", 4 | ".formatter.exs", 5 | "*.exs" 6 | ], 7 | import_deps: [:membrane_core] 8 | ] 9 | -------------------------------------------------------------------------------- /.github/CODEOWNERS: -------------------------------------------------------------------------------- 1 | * @varsill 2 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/please--open-new-issues-in-membranefranework-membrane_core.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Please, open new issues in membranefranework/membrane_core 3 | about: New issues related to this repo should be opened there 4 | title: "[DO NOT OPEN]" 5 | labels: '' 6 | assignees: '' 7 | 8 | --- 9 | 10 | Please, do not open this issue here. Open it in the [membrane_core](https://github.com/membraneframework/membrane_core) repository instead. 11 | 12 | Thanks for helping us grow :) 13 | -------------------------------------------------------------------------------- /.github/workflows/on_issue_opened.yaml: -------------------------------------------------------------------------------- 1 | name: 'Close issue when opened' 2 | on: 3 | issues: 4 | types: 5 | - opened 6 | jobs: 7 | close: 8 | runs-on: ubuntu-latest 9 | steps: 10 | - name: Checkout membrane_core 11 | uses: actions/checkout@v3 12 | with: 13 | repository: membraneframework/membrane_core 14 | - name: Close issue 15 | uses: ./.github/actions/close_issue 16 | with: 17 | GITHUB_TOKEN: ${{ secrets.MEMBRANEFRAMEWORKADMIN_TOKEN }} 18 | ISSUE_URL: ${{ github.event.issue.html_url }} 19 | ISSUE_NUMBER: ${{ github.event.issue.number }} 20 | REPOSITORY: ${{ github.repository }} 21 | -------------------------------------------------------------------------------- /.github/workflows/on_pr_opened.yaml: -------------------------------------------------------------------------------- 1 | name: Add PR to Smackore project board, if the author is from outside Membrane Team 2 | on: 3 | pull_request_target: 4 | types: 5 | - opened 6 | jobs: 7 | maybe_add_to_project_board: 8 | runs-on: ubuntu-latest 9 | steps: 10 | - name: Checkout membrane_core 11 | uses: actions/checkout@v3 12 | with: 13 | repository: membraneframework/membrane_core 14 | - name: Puts PR in "New PRs by community" column in the Smackore project, if the author is from outside Membrane Team 15 | uses: ./.github/actions/add_pr_to_smackore_board 16 | with: 17 | GITHUB_TOKEN: ${{ secrets.MEMBRANEFRAMEWORKADMIN_TOKEN }} 18 | AUTHOR_LOGIN: ${{ github.event.pull_request.user.login }} 19 | PR_URL: ${{ github.event.pull_request.html_url }} 20 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | compile_commands.json 2 | .gdb_history 3 | bundlex.sh 4 | bundlex.bat 5 | 6 | # Dir generated by tmp_dir ExUnit tag 7 | /tmp/ 8 | 9 | # Created by https://www.gitignore.io/api/c,vim,linux,macos,elixir,windows,visualstudiocode 10 | # Edit at https://www.gitignore.io/?templates=c,vim,linux,macos,elixir,windows,visualstudiocode 11 | 12 | ### C ### 13 | # Prerequisites 14 | *.d 15 | 16 | # Object files 17 | *.o 18 | *.ko 19 | *.obj 20 | *.elf 21 | 22 | # Linker output 23 | *.ilk 24 | *.map 25 | *.exp 26 | 27 | # Precompiled Headers 28 | *.gch 29 | *.pch 30 | 31 | # Libraries 32 | *.lib 33 | *.a 34 | *.la 35 | *.lo 36 | 37 | # Shared objects (inc. Windows DLLs) 38 | *.dll 39 | *.so 40 | *.so.* 41 | *.dylib 42 | 43 | # Executables 44 | *.exe 45 | *.out 46 | *.app 47 | *.i*86 48 | *.x86_64 49 | *.hex 50 | 51 | # Debug files 52 | *.dSYM/ 53 | *.su 54 | *.idb 55 | *.pdb 56 | 57 | # Kernel Module Compile Results 58 | *.mod* 59 | *.cmd 60 | .tmp_versions/ 61 | modules.order 62 | Module.symvers 63 | Mkfile.old 64 | dkms.conf 65 | 66 | ### Elixir ### 67 | /_build 68 | /cover 69 | /deps 70 | /doc 71 | /.fetch 72 | erl_crash.dump 73 | *.ez 74 | *.beam 75 | /config/*.secret.exs 76 | .elixir_ls/ 77 | 78 | ### Elixir Patch ### 79 | 80 | ### Linux ### 81 | *~ 82 | 83 | # temporary files which can be created if a process still has a handle open of a deleted file 84 | .fuse_hidden* 85 | 86 | # KDE directory preferences 87 | .directory 88 | 89 | # Linux trash folder which might appear on any partition or disk 90 | .Trash-* 91 | 92 | # .nfs files are created when an open file is removed but is still being accessed 93 | .nfs* 94 | 95 | ### macOS ### 96 | # General 97 | .DS_Store 98 | .AppleDouble 99 | .LSOverride 100 | 101 | # Icon must end with two \r 102 | Icon 103 | 104 | # Thumbnails 105 | ._* 106 | 107 | # Files that might appear in the root of a volume 108 | .DocumentRevisions-V100 109 | .fseventsd 110 | .Spotlight-V100 111 | .TemporaryItems 112 | .Trashes 113 | .VolumeIcon.icns 114 | .com.apple.timemachine.donotpresent 115 | 116 | # Directories potentially created on remote AFP share 117 | .AppleDB 118 | .AppleDesktop 119 | Network Trash Folder 120 | Temporary Items 121 | .apdisk 122 | 123 | ### Vim ### 124 | # Swap 125 | [._]*.s[a-v][a-z] 126 | [._]*.sw[a-p] 127 | [._]s[a-rt-v][a-z] 128 | [._]ss[a-gi-z] 129 | [._]sw[a-p] 130 | 131 | # Session 132 | Session.vim 133 | Sessionx.vim 134 | 135 | # Temporary 136 | .netrwhist 137 | # Auto-generated tag files 138 | tags 139 | # Persistent undo 140 | [._]*.un~ 141 | 142 | ### VisualStudioCode ### 143 | .vscode/* 144 | !.vscode/settings.json 145 | !.vscode/tasks.json 146 | !.vscode/launch.json 147 | !.vscode/extensions.json 148 | 149 | ### VisualStudioCode Patch ### 150 | # Ignore all local history of files 151 | .history 152 | 153 | ### Windows ### 154 | # Windows thumbnail cache files 155 | Thumbs.db 156 | Thumbs.db:encryptable 157 | ehthumbs.db 158 | ehthumbs_vista.db 159 | 160 | # Dump file 161 | *.stackdump 162 | 163 | # Folder config file 164 | [Dd]esktop.ini 165 | 166 | # Recycle Bin used on file shares 167 | $RECYCLE.BIN/ 168 | 169 | # Windows Installer files 170 | *.cab 171 | *.msi 172 | *.msix 173 | *.msm 174 | *.msp 175 | 176 | # Windows shortcuts 177 | *.lnk 178 | 179 | # End of https://www.gitignore.io/api/c,vim,linux,macos,elixir,windows,visualstudiocode 180 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Membrane H26x Plugin 2 | 3 | [![Hex.pm](https://img.shields.io/hexpm/v/membrane_h264_plugin.svg)](https://hex.pm/packages/membrane_h26x_plugin) 4 | [![API Docs](https://img.shields.io/badge/api-docs-yellow.svg?style=flat)](https://hexdocs.pm/membrane_h26x_plugin) 5 | [![CircleCI](https://circleci.com/gh/membraneframework/membrane_h264_plugin.svg?style=svg)](https://circleci.com/gh/membraneframework/membrane_h26x_plugin) 6 | 7 | Membrane H.264 and H.265 parsers. 8 | It is a pair of Membrane elements responsible for parsing the incoming H.264 and H.265 streams. The parsing is done as a sequence of the following steps: 9 | * splitting the stream into stream NAL units 10 | * Parsing the NAL unit headers, so that to read the type of the NAL unit 11 | * Parsing the NAL unit body with the appropriate scheme, based on the NAL unit type read in the step before 12 | * Aggregating the NAL units into a stream of *access units* 13 | 14 | The output of the element is the incoming binary payload, enriched with the metadata describing the division of the payload into *access units*. 15 | 16 | It is part of [Membrane Multimedia Framework](https://membraneframework.org). 17 | 18 | ## Installation 19 | 20 | The package can be installed by adding `membrane_h26x_plugin` to your list of dependencies in `mix.exs`: 21 | 22 | ```elixir 23 | def deps do 24 | [ 25 | {:membrane_h26x_plugin, "~> 0.10.5"} 26 | ] 27 | end 28 | ``` 29 | 30 | ## Usage 31 | 32 | The following pipeline takes H264 file, parses it, and then decodes it to the raw video. 33 | 34 | ```elixir 35 | defmodule Decoding.Pipeline do 36 | use Membrane.Pipeline 37 | 38 | alias Membrane.{File, H264} 39 | 40 | @impl true 41 | def handle_init(_ctx, _opts) do 42 | spec = 43 | child(:source, %File.Source{location: "test/fixtures/input-10-720p-main.h264"}) 44 | |> child(:parser, H264.Parser) 45 | |> child(:decoder, H264.FFmpeg.Decoder) 46 | |> child(:sink, %File.Sink{location: "output.raw"}) 47 | 48 | {[spec: spec], nil} 49 | end 50 | 51 | @impl true 52 | def handle_element_end_of_stream(:sink, _ctx_, state) do 53 | {[terminate: :normal], state} 54 | end 55 | end 56 | ``` 57 | 58 | 59 | ## Copyright and License 60 | 61 | Copyright 2022, [Software Mansion](https://swmansion.com/?utm_source=git&utm_medium=readme&utm_campaign=membrane_h26x_plugin) 62 | 63 | [![Software Mansion](https://logo.swmansion.com/logo?color=white&variant=desktop&width=200&tag=membrane-github)](https://swmansion.com/?utm_source=git&utm_medium=readme&utm_campaign=membrane_h26x_plugin) 64 | 65 | Licensed under the [Apache License, Version 2.0](LICENSE) 66 | -------------------------------------------------------------------------------- /lib/membrane_h264_plugin/h264/au_splitter.ex: -------------------------------------------------------------------------------- 1 | defmodule Membrane.H264.AUSplitter do 2 | @moduledoc """ 3 | Module providing functionalities to divide the binary 4 | h264 stream into access units. 5 | 6 | The access unit splitter's behaviour is based on *"7.4.1.2.3 7 | Order of NAL units and coded pictures and association to access units"* 8 | of the *"ITU-T Rec. H.264 (01/2012)"* specification. The most crucial part 9 | of the access unit splitter is the mechanism to detect new primary coded video picture. 10 | 11 | WARNING: Our implementation of that mechanism is based on: 12 | *"7.4.1.2.4 Detection of the first VCL NAL unit of a primary coded picture"* 13 | of the *"ITU-T Rec. H.264 (01/2012)"*, however it adds one more 14 | additional condition which, when satisfied, says that the given 15 | VCL NALu is a new primary coded picture. That condition is whether the picture 16 | is a keyframe or not. 17 | """ 18 | @behaviour Membrane.H26x.AUSplitter 19 | 20 | require Membrane.Logger 21 | 22 | require Membrane.H264.NALuTypes, as: NALuTypes 23 | 24 | alias Membrane.H26x.{AUSplitter, NALu} 25 | 26 | @typedoc """ 27 | A structure holding a state of the access unit splitter. 28 | """ 29 | @opaque t :: %__MODULE__{ 30 | nalus_acc: [NALu.t()], 31 | fsm_state: :first | :second, 32 | previous_primary_coded_picture_nalu: NALu.t() | nil, 33 | access_units_to_output: AUSplitter.access_unit() 34 | } 35 | @enforce_keys [ 36 | :nalus_acc, 37 | :fsm_state, 38 | :previous_primary_coded_picture_nalu, 39 | :access_units_to_output 40 | ] 41 | defstruct @enforce_keys 42 | 43 | @doc """ 44 | Returns a structure holding a clear state of the 45 | access unit splitter. 46 | """ 47 | @spec new() :: t() 48 | def new() do 49 | %__MODULE__{ 50 | nalus_acc: [], 51 | fsm_state: :first, 52 | previous_primary_coded_picture_nalu: nil, 53 | access_units_to_output: [] 54 | } 55 | end 56 | 57 | @non_vcl_nalu_types_at_au_beginning [:sps, :pps, :aud, :sei] 58 | @non_vcl_nalu_types_at_au_end [:end_of_seq, :end_of_stream] 59 | 60 | @doc """ 61 | Splits the given list of NAL units into the access units. 62 | 63 | It can be used for a stream which is not completely available at the time of function invocation, 64 | as the function updates the state of the access unit splitter - the function can 65 | be invoked once more, with new NAL units and the updated state. 66 | Under the hood, `split/2` defines a finite state machine 67 | with two states: `:first` and `:second`. The state `:first` describes the state before 68 | reaching the primary coded picture NALu of a given access unit. The state `:second` 69 | describes the state after processing the primary coded picture NALu of a given 70 | access unit. 71 | 72 | If `assume_au_aligned` flag is set to `true`, input is assumed to form a complete set 73 | of access units and therefore all of them are returned. Otherwise, the last access unit 74 | is not returned until another access unit starts, as it's the only way to prove that 75 | the access unit is complete. 76 | """ 77 | @spec split([NALu.t()], boolean(), t()) :: {[AUSplitter.access_unit()], t()} 78 | def split(nalus, assume_au_aligned \\ false, state) do 79 | state = do_split(nalus, state) 80 | 81 | {aus, state} = 82 | if assume_au_aligned do 83 | {state.access_units_to_output ++ [state.nalus_acc], 84 | %__MODULE__{state | access_units_to_output: [], nalus_acc: []}} 85 | else 86 | {state.access_units_to_output, %__MODULE__{state | access_units_to_output: []}} 87 | end 88 | 89 | {Enum.reject(aus, &Enum.empty?/1), state} 90 | end 91 | 92 | defp do_split([first_nalu | rest_nalus], %{fsm_state: :first} = state) do 93 | cond do 94 | new_primary_coded_vcl_nalu?(first_nalu, state.previous_primary_coded_picture_nalu) -> 95 | do_split( 96 | rest_nalus, 97 | %__MODULE__{ 98 | state 99 | | nalus_acc: state.nalus_acc ++ [first_nalu], 100 | fsm_state: :second, 101 | previous_primary_coded_picture_nalu: first_nalu 102 | } 103 | ) 104 | 105 | first_nalu.type in @non_vcl_nalu_types_at_au_beginning -> 106 | do_split( 107 | rest_nalus, 108 | %__MODULE__{state | nalus_acc: state.nalus_acc ++ [first_nalu]} 109 | ) 110 | 111 | true -> 112 | Membrane.Logger.warning( 113 | "AUSplitter: Improper transition, first_nalu: #{inspect(first_nalu)}" 114 | ) 115 | 116 | state 117 | end 118 | end 119 | 120 | defp do_split([first_nalu | rest_nalus], %{fsm_state: :second} = state) do 121 | cond do 122 | first_nalu.type in @non_vcl_nalu_types_at_au_end -> 123 | do_split( 124 | rest_nalus, 125 | %__MODULE__{ 126 | state 127 | | nalus_acc: state.nalus_acc ++ [first_nalu] 128 | } 129 | ) 130 | 131 | first_nalu.type in @non_vcl_nalu_types_at_au_beginning -> 132 | do_split( 133 | rest_nalus, 134 | %__MODULE__{ 135 | state 136 | | nalus_acc: [first_nalu], 137 | fsm_state: :first, 138 | access_units_to_output: state.access_units_to_output ++ [state.nalus_acc] 139 | } 140 | ) 141 | 142 | new_primary_coded_vcl_nalu?(first_nalu, state.previous_primary_coded_picture_nalu) -> 143 | do_split( 144 | rest_nalus, 145 | %__MODULE__{ 146 | state 147 | | nalus_acc: [first_nalu], 148 | previous_primary_coded_picture_nalu: first_nalu, 149 | access_units_to_output: state.access_units_to_output ++ [state.nalus_acc] 150 | } 151 | ) 152 | 153 | NALuTypes.is_vcl_nalu_type(first_nalu.type) or first_nalu.type == :filler_data -> 154 | do_split( 155 | rest_nalus, 156 | %__MODULE__{state | nalus_acc: state.nalus_acc ++ [first_nalu]} 157 | ) 158 | 159 | true -> 160 | Membrane.Logger.warning( 161 | "AUSplitter: Improper transition, first_nalu: #{inspect(first_nalu)}" 162 | ) 163 | 164 | state 165 | end 166 | end 167 | 168 | defp do_split([], state) do 169 | state 170 | end 171 | 172 | # Reference source for the behaviour below: 173 | # https://github.com/GStreamer/gst-plugins-bad/blob/ca8068c6d793d7aaa6f2e2cc6324fdedfe2f33fa/gst/videoparsers/gsth264parse.c#L1183C45-L1185C49 174 | # 175 | # NOTE: The following check is not a part of the original H264 specification unlike the other checks below. 176 | # 177 | # It happens that some streams have broken frame numbers (that are either non-monotically 178 | # increasing or just reset on a key frame) but the `first_mb_in_slice` set to zero can mean that 179 | # we are dealin with a new AU (given a proper `nal_unit_type`). It seems that it is sufficient 180 | # condition to check for `first_mb_in_slice` set to zero to detect a new AU. 181 | defguardp first_mb_in_slice_zero(a) 182 | when a.first_mb_in_slice == 0 and 183 | a.nal_unit_type in [1, 2, 5] 184 | 185 | defguardp frame_num_differs(a, b) when a.frame_num != b.frame_num 186 | 187 | defguardp pic_parameter_set_id_differs(a, b) 188 | when a.pic_parameter_set_id != b.pic_parameter_set_id 189 | 190 | defguardp field_pic_flag_differs(a, b) when a.field_pic_flag != b.field_pic_flag 191 | 192 | defguardp bottom_field_flag_differs(a, b) when a.bottom_field_flag != b.bottom_field_flag 193 | 194 | defguardp nal_ref_idc_differs_one_zero(a, b) 195 | when (a.nal_ref_idc == 0 or b.nal_ref_idc == 0) and 196 | a.nal_ref_idc != b.nal_ref_idc 197 | 198 | defguardp pic_order_cnt_zero_check(a, b) 199 | when a.pic_order_cnt_type == 0 and b.pic_order_cnt_type == 0 and 200 | (a.pic_order_cnt_lsb != b.pic_order_cnt_lsb or 201 | a.delta_pic_order_cnt_bottom != b.delta_pic_order_cnt_bottom) 202 | 203 | defguardp pic_order_cnt_one_check_zero(a, b) 204 | when a.pic_order_cnt_type == 1 and b.pic_order_cnt_type == 1 and 205 | hd(a.delta_pic_order_cnt) != hd(b.delta_pic_order_cnt) 206 | 207 | defguardp pic_order_cnt_one_check_one(a, b) 208 | when a.pic_order_cnt_type == 1 and b.pic_order_cnt_type == 1 and 209 | hd(hd(a.delta_pic_order_cnt)) != hd(hd(b.delta_pic_order_cnt)) 210 | 211 | defguardp idr_and_non_idr(a, b) 212 | when (a.nal_unit_type == 5 or b.nal_unit_type == 5) and 213 | a.nal_unit_type != b.nal_unit_type 214 | 215 | defguardp idrs_with_idr_pic_id_differ(a, b) 216 | when a.nal_unit_type == 5 and b.nal_unit_type == 5 and a.idr_pic_id != b.idr_pic_id 217 | 218 | defp new_primary_coded_vcl_nalu?(%{type: type}, _last_nalu) 219 | when not NALuTypes.is_vcl_nalu_type(type), 220 | do: false 221 | 222 | defp new_primary_coded_vcl_nalu?(_nalu, nil), do: true 223 | 224 | # Conditions based on 7.4.1.2.4 "Detection of the first VCL NAL unit of a primary coded picture" 225 | # of the "ITU-T Rec. H.264 (01/2012)" 226 | defp new_primary_coded_vcl_nalu?(%{parsed_fields: nalu}, %{parsed_fields: last_nalu}) 227 | when first_mb_in_slice_zero(nalu) 228 | when frame_num_differs(nalu, last_nalu) 229 | when pic_parameter_set_id_differs(nalu, last_nalu) 230 | when field_pic_flag_differs(nalu, last_nalu) 231 | when bottom_field_flag_differs(nalu, last_nalu) 232 | when nal_ref_idc_differs_one_zero(nalu, last_nalu) 233 | when pic_order_cnt_zero_check(nalu, last_nalu) 234 | when pic_order_cnt_one_check_zero(nalu, last_nalu) 235 | when pic_order_cnt_one_check_one(nalu, last_nalu) 236 | when idr_and_non_idr(nalu, last_nalu) 237 | when idrs_with_idr_pic_id_differ(nalu, last_nalu) do 238 | true 239 | end 240 | 241 | defp new_primary_coded_vcl_nalu?(_nalu, _last_nalu) do 242 | false 243 | end 244 | end 245 | -------------------------------------------------------------------------------- /lib/membrane_h264_plugin/h264/au_timestamp_generator.ex: -------------------------------------------------------------------------------- 1 | defmodule Membrane.H264.AUTimestampGenerator do 2 | @moduledoc false 3 | 4 | use Membrane.H26x.AUTimestampGenerator 5 | 6 | require Membrane.H264.NALuTypes, as: NALuTypes 7 | 8 | @impl true 9 | def get_first_vcl_nalu(au) do 10 | Enum.find(au, &NALuTypes.is_vcl_nalu_type(&1.type)) 11 | end 12 | 13 | @impl true 14 | # Calculate picture order count according to section 8.2.1 of the ITU-T H264 specification 15 | def calculate_poc(%{parsed_fields: %{pic_order_cnt_type: 0}} = vcl_nalu, state) do 16 | max_pic_order_cnt_lsb = 2 ** (vcl_nalu.parsed_fields.log2_max_pic_order_cnt_lsb_minus4 + 4) 17 | 18 | {prev_pic_order_cnt_msb, prev_pic_order_cnt_lsb} = 19 | if vcl_nalu.type == :idr do 20 | {0, 0} 21 | else 22 | # As described in the spec, we should check for presence of the 23 | # memory_management_control_operation syntax element equal to 5 24 | # in the previous reference picture and calculate prev_pic_order_cnt_*sb 25 | # values accordingly if it's there. Since getting to that information 26 | # is quite a pain in the ass, we don't do that and assume it's not 27 | # there and it seems to work ¯\_(ツ)_/¯ However, it may happen not to work 28 | # for some streams and we may generate invalid timestamps because of that. 29 | # If that happens, may have to implement the aforementioned lacking part. 30 | 31 | previous_vcl_nalu = state.prev_pic_first_vcl_nalu || vcl_nalu 32 | {state.prev_pic_order_cnt_msb, previous_vcl_nalu.parsed_fields.pic_order_cnt_lsb} 33 | end 34 | 35 | pic_order_cnt_lsb = vcl_nalu.parsed_fields.pic_order_cnt_lsb 36 | 37 | pic_order_cnt_msb = 38 | cond do 39 | pic_order_cnt_lsb < prev_pic_order_cnt_lsb and 40 | prev_pic_order_cnt_lsb - pic_order_cnt_lsb >= max_pic_order_cnt_lsb / 2 -> 41 | prev_pic_order_cnt_msb + max_pic_order_cnt_lsb 42 | 43 | pic_order_cnt_lsb > prev_pic_order_cnt_lsb and 44 | pic_order_cnt_lsb - prev_pic_order_cnt_lsb > max_pic_order_cnt_lsb / 2 -> 45 | prev_pic_order_cnt_msb - max_pic_order_cnt_lsb 46 | 47 | true -> 48 | prev_pic_order_cnt_msb 49 | end 50 | 51 | pic_order_cnt = 52 | if get_slice_type(vcl_nalu) == :frame do 53 | top_field_order_cnt = pic_order_cnt_msb + pic_order_cnt_lsb 54 | 55 | bottom_field_order_cnt = 56 | top_field_order_cnt + vcl_nalu.parsed_fields.delta_pic_order_cnt_bottom 57 | 58 | min(top_field_order_cnt, bottom_field_order_cnt) 59 | else 60 | pic_order_cnt_msb + pic_order_cnt_lsb 61 | end 62 | 63 | {div(pic_order_cnt, 2), 64 | %{state | prev_pic_order_cnt_msb: pic_order_cnt_msb, prev_pic_first_vcl_nalu: vcl_nalu}} 65 | end 66 | 67 | @impl true 68 | def calculate_poc(%{parsed_fields: %{pic_order_cnt_type: 1}}, _state) do 69 | raise "Timestamp generation error: unsupported stream. Unsupported field value pic_order_cnt_type=1" 70 | end 71 | 72 | @impl true 73 | def calculate_poc( 74 | %{parsed_fields: %{pic_order_cnt_type: 2, frame_num: frame_num}} = vcl_nalu, 75 | state 76 | ) do 77 | {frame_num, %{state | prev_pic_first_vcl_nalu: vcl_nalu}} 78 | end 79 | 80 | defp get_slice_type(vcl_nalu) do 81 | case vcl_nalu.parsed_fields do 82 | %{frame_mbs_only_flag: 1} -> :frame 83 | %{field_pic_flag: 0} -> :frame 84 | %{bottom_field_flag: 1} -> :bottom_field 85 | _other -> :top_field 86 | end 87 | end 88 | end 89 | -------------------------------------------------------------------------------- /lib/membrane_h264_plugin/h264/decoder_configuration_record.ex: -------------------------------------------------------------------------------- 1 | defmodule Membrane.H264.DecoderConfigurationRecord do 2 | @moduledoc """ 3 | Utility functions for parsing and generating AVC Configuration Record. 4 | 5 | The structure of the record is described in section 5.2.4.1.1 of MPEG-4 part 15 (ISO/IEC 14496-15). 6 | """ 7 | 8 | @enforce_keys [ 9 | :spss, 10 | :ppss, 11 | :avc_profile_indication, 12 | :avc_level, 13 | :profile_compatibility, 14 | :nalu_length_size 15 | ] 16 | defstruct @enforce_keys 17 | 18 | @typedoc "Structure representing the Decoder Configuartion Record" 19 | @type t() :: %__MODULE__{ 20 | spss: [binary()], 21 | ppss: [binary()], 22 | avc_profile_indication: non_neg_integer(), 23 | profile_compatibility: non_neg_integer(), 24 | avc_level: non_neg_integer(), 25 | nalu_length_size: pos_integer() 26 | } 27 | 28 | @doc """ 29 | Generates a DCR based on given PPSs and SPSs. 30 | """ 31 | @spec generate([binary()], [binary()], Membrane.H264.Parser.stream_structure()) :: 32 | binary() | nil 33 | def generate([], _ppss, _stream_structure) do 34 | nil 35 | end 36 | 37 | def generate(spss, ppss, {avc, nalu_length_size}) do 38 | <<_idc_and_type, profile, compatibility, level, _rest::binary>> = List.last(spss) 39 | 40 | cond do 41 | avc == :avc1 -> 42 | <<1, profile, compatibility, level, 0b111111::6, nalu_length_size - 1::2-integer, 43 | 0b111::3, length(spss)::5-integer, encode_parameter_sets(spss)::binary, 44 | length(ppss)::8-integer, encode_parameter_sets(ppss)::binary>> 45 | 46 | avc == :avc3 -> 47 | <<1, profile, compatibility, level, 0b111111::6, nalu_length_size - 1::2-integer, 48 | 0b111::3, 0::5, 0::8>> 49 | end 50 | end 51 | 52 | defp encode_parameter_sets(pss) do 53 | Enum.map_join(pss, &<>) 54 | end 55 | 56 | @spec remove_parameter_sets(binary()) :: binary() 57 | def remove_parameter_sets(dcr) do 58 | <> = dcr 59 | <> 60 | end 61 | 62 | @doc """ 63 | Parses the DCR. 64 | """ 65 | @spec parse(binary()) :: t() 66 | def parse( 67 | <<1::8, avc_profile_indication::8, profile_compatibility::8, avc_level::8, 0b111111::6, 68 | length_size_minus_one::2, 0b111::3, rest::bitstring>> 69 | ) do 70 | {spss, rest} = parse_spss(rest) 71 | {ppss, _rest} = parse_ppss(rest) 72 | 73 | %__MODULE__{ 74 | spss: spss, 75 | ppss: ppss, 76 | avc_profile_indication: avc_profile_indication, 77 | profile_compatibility: profile_compatibility, 78 | avc_level: avc_level, 79 | nalu_length_size: length_size_minus_one + 1 80 | } 81 | end 82 | 83 | def parse(_data), do: {:error, :unknown_pattern} 84 | 85 | defp parse_spss(<>) do 86 | do_parse_array(num_of_spss, rest) 87 | end 88 | 89 | defp parse_ppss(<>), do: do_parse_array(num_of_ppss, rest) 90 | 91 | defp do_parse_array(amount, rest, acc \\ []) 92 | defp do_parse_array(0, rest, acc), do: {Enum.reverse(acc), rest} 93 | 94 | defp do_parse_array(remaining, <>, acc), 95 | do: do_parse_array(remaining - 1, rest, [data | acc]) 96 | end 97 | -------------------------------------------------------------------------------- /lib/membrane_h264_plugin/h264/nalu_parser.ex: -------------------------------------------------------------------------------- 1 | defmodule Membrane.H264.NALuParser do 2 | @moduledoc """ 3 | This module is an extension to `Membrane.H26x.NALuParser` and contains 4 | H264 specific functions. 5 | """ 6 | 7 | use Membrane.H26x.NALuParser 8 | 9 | alias Membrane.H264.NALuParser.Schemes 10 | alias Membrane.H264.NALuTypes 11 | alias Membrane.H26x.NALuParser.SchemeParser 12 | 13 | @impl true 14 | def get_nalu_header_and_body(<>), 15 | do: {nalu_header, nalu_body} 16 | 17 | @impl true 18 | def parse_nalu_header(nalu_header, state) do 19 | SchemeParser.parse_with_scheme(nalu_header, Schemes.NALuHeader, state) 20 | end 21 | 22 | @impl true 23 | def get_nalu_type(nal_unit_type), do: NALuTypes.get_type(nal_unit_type) 24 | 25 | @impl true 26 | def parse_proper_nalu_type(nalu_body, nalu_type, state) do 27 | try do 28 | {parsed_fields, state} = do_parse_proper_nalu_type(nalu_body, nalu_type, state) 29 | {:ok, parsed_fields, state} 30 | catch 31 | "Cannot load information from SPS" -> 32 | {:error, state} 33 | end 34 | end 35 | 36 | defp do_parse_proper_nalu_type(nalu_body, nalu_type, state) do 37 | case nalu_type do 38 | :sps -> 39 | SchemeParser.parse_with_scheme(nalu_body, Schemes.SPS, state) 40 | 41 | :pps -> 42 | SchemeParser.parse_with_scheme(nalu_body, Schemes.PPS, state) 43 | 44 | :idr -> 45 | SchemeParser.parse_with_scheme(nalu_body, Schemes.Slice, state) 46 | 47 | :non_idr -> 48 | SchemeParser.parse_with_scheme(nalu_body, Schemes.Slice, state) 49 | 50 | _unknown_nalu_type -> 51 | {%{}, state} 52 | end 53 | end 54 | end 55 | -------------------------------------------------------------------------------- /lib/membrane_h264_plugin/h264/nalu_parser/schemes/nalu_header.ex: -------------------------------------------------------------------------------- 1 | defmodule Membrane.H264.NALuParser.Schemes.NALuHeader do 2 | @moduledoc false 3 | @behaviour Membrane.H26x.NALuParser.Scheme 4 | 5 | @impl true 6 | def defaults(), do: [] 7 | 8 | @impl true 9 | def scheme(), 10 | do: [ 11 | field: {:forbidden_zero_bit, :u1}, 12 | field: {:nal_ref_idc, :u2}, 13 | field: {:nal_unit_type, :u5} 14 | ] 15 | end 16 | -------------------------------------------------------------------------------- /lib/membrane_h264_plugin/h264/nalu_parser/schemes/pps.ex: -------------------------------------------------------------------------------- 1 | defmodule Membrane.H264.NALuParser.Schemes.PPS do 2 | @moduledoc false 3 | @behaviour Membrane.H26x.NALuParser.Scheme 4 | 5 | @impl true 6 | def defaults(), do: [] 7 | 8 | @impl true 9 | def scheme(), 10 | do: [ 11 | field: {:pic_parameter_set_id, :ue}, 12 | field: {:seq_parameter_set_id, :ue}, 13 | field: {:entropy_coding_mode_flag, :u1}, 14 | field: {:bottom_field_pic_order_in_frame_present_flag, :u1}, 15 | save_state_as_global_state: {&{:pps, &1}, [:pic_parameter_set_id]} 16 | ] 17 | end 18 | -------------------------------------------------------------------------------- /lib/membrane_h264_plugin/h264/nalu_parser/schemes/slice.ex: -------------------------------------------------------------------------------- 1 | defmodule Membrane.H264.NALuParser.Schemes.Slice do 2 | @moduledoc false 3 | @behaviour Membrane.H26x.NALuParser.Scheme 4 | 5 | @impl true 6 | def defaults(), do: [field_pic_flag: 0, delta_pic_order_cnt_bottom: 0] 7 | 8 | # If we're ever unlucky and have to parse more fields from the slice header 9 | # this implementation may come in handy: 10 | # https://webrtc.googlesource.com/src/webrtc/+/f54860e9ef0b68e182a01edc994626d21961bc4b/common_video/h264/h264_bitstream_parser.cc 11 | @impl true 12 | def scheme(), 13 | do: [ 14 | field: {:first_mb_in_slice, :ue}, 15 | field: {:slice_type, :ue}, 16 | field: {:pic_parameter_set_id, :ue}, 17 | execute: &load_data_from_sps_and_pps(&1, &2, &3), 18 | if: {{&(&1 == 1), [:separate_colour_plane_flag]}, field: {:colour_plane_id, :u2}}, 19 | field: {:frame_num, {:uv, &(&1 + 4), [:log2_max_frame_num_minus4]}}, 20 | if: 21 | {{&(&1 != 1), [:frame_mbs_only_flag]}, 22 | field: {:field_pic_flag, :u1}, 23 | if: {{&(&1 == 1), [:field_pic_flag]}, field: {:bottom_field_flag, :u1}}}, 24 | if: {{&(&1 == 5), [:nal_unit_type]}, field: {:idr_pic_id, :ue}}, 25 | if: 26 | {{&(&1 == 0), [:pic_order_cnt_type]}, 27 | field: {:pic_order_cnt_lsb, {:uv, &(&1 + 4), [:log2_max_pic_order_cnt_lsb_minus4]}}, 28 | if: 29 | {{fn delta_pic_order_cnt_bottom, field_pic_flag -> 30 | delta_pic_order_cnt_bottom == 1 and field_pic_flag == 0 31 | end, [:bottom_field_pic_order_in_frame_present_flag, :field_pic_flag]}, 32 | field: {:delta_pic_order_cnt_bottom, :se}}} 33 | ] 34 | 35 | defp load_data_from_sps_and_pps(payload, state, _iterators) do 36 | with pic_parameter_set_id when pic_parameter_set_id != nil <- 37 | Map.get(state.__local__, :pic_parameter_set_id), 38 | pps when pps != nil <- Map.get(state.__global__, {:pps, pic_parameter_set_id}), 39 | seq_parameter_set_id when seq_parameter_set_id != nil <- 40 | Map.get(pps, :seq_parameter_set_id), 41 | sps <- Map.get(state.__global__, {:sps, seq_parameter_set_id}) do 42 | state = 43 | Bunch.Access.put_in( 44 | state, 45 | [:__local__, :separate_colour_plane_flag], 46 | Map.get(sps, :separate_colour_plane_flag, 0) 47 | ) 48 | 49 | sps_fields = 50 | Map.take(sps, [ 51 | :log2_max_frame_num_minus4, 52 | :frame_mbs_only_flag, 53 | :pic_order_cnt_type, 54 | :log2_max_pic_order_cnt_lsb_minus4 55 | ]) 56 | 57 | pps_fields = 58 | Map.take(pps, [ 59 | :bottom_field_pic_order_in_frame_present_flag 60 | ]) 61 | 62 | state = Map.update(state, :__local__, %{}, &Map.merge(&1, sps_fields)) 63 | state = Map.update(state, :__local__, %{}, &Map.merge(&1, pps_fields)) 64 | 65 | {payload, state} 66 | else 67 | _error -> throw("Cannot load information from SPS") 68 | end 69 | end 70 | end 71 | -------------------------------------------------------------------------------- /lib/membrane_h264_plugin/h264/nalu_types.ex: -------------------------------------------------------------------------------- 1 | defmodule Membrane.H264.NALuTypes do 2 | @moduledoc """ 3 | The module aggregating the mapping of from `nal_unit_type` 4 | fields of the NAL unit to the human-friendly name of a NALu type. 5 | """ 6 | 7 | @nalu_types %{ 8 | 0 => :unspecified, 9 | 1 => :non_idr, 10 | 2 => :part_a, 11 | 3 => :part_b, 12 | 4 => :part_c, 13 | 5 => :idr, 14 | 6 => :sei, 15 | 7 => :sps, 16 | 8 => :pps, 17 | 9 => :aud, 18 | 10 => :end_of_seq, 19 | 11 => :end_of_stream, 20 | 12 => :filler_data, 21 | 13 => :sps_extension, 22 | 14 => :prefix_nal_unit, 23 | 15 => :subset_sps, 24 | (16..18) => :reserved, 25 | 19 => :auxiliary_non_part, 26 | 20 => :extension, 27 | (21..23) => :reserved, 28 | (24..31) => :unspecified 29 | } 30 | |> Enum.flat_map(fn 31 | {k, v} when is_integer(k) -> [{k, v}] 32 | {k, v} -> Enum.map(k, &{&1, v}) 33 | end) 34 | |> Map.new() 35 | 36 | @vcl_nalu_types [:idr, :non_idr, :part_a, :part_b, :part_c] 37 | 38 | @typedoc """ 39 | A type representing all the possible human-friendly names of NAL unit types. 40 | """ 41 | @type nalu_type :: 42 | unquote(Bunch.Typespec.enum_to_alternative(Map.values(@nalu_types) |> Enum.uniq())) 43 | 44 | @type vcl_nalu_type :: unquote(Bunch.Typespec.enum_to_alternative(@vcl_nalu_types)) 45 | 46 | @doc """ 47 | The function which returns the human friendly name of a NALu type 48 | for a given `nal_unit_type`. 49 | 50 | The mapping is based on: "Table 7-1 – NAL unit type codes, syntax element categories, and NAL unit type classes" 51 | of the *"ITU-T Rec. H.264 (01/2012)"* 52 | """ 53 | @spec get_type(non_neg_integer()) :: nalu_type() | nil 54 | def get_type(nal_unit_type_id) do 55 | @nalu_types[nal_unit_type_id] 56 | end 57 | 58 | defguard is_vcl_nalu_type(nalu_type) when nalu_type in @vcl_nalu_types 59 | end 60 | -------------------------------------------------------------------------------- /lib/membrane_h264_plugin/h265/au_splitter.ex: -------------------------------------------------------------------------------- 1 | defmodule Membrane.H265.AUSplitter do 2 | @moduledoc """ 3 | Module providing functionalities to group H265 NAL units 4 | into access units. 5 | 6 | The access unit splitter's behaviour is based on section **7.4.2.4.4** 7 | *"Order of NAL units and coded pictures and association to access units"* 8 | of the *"ITU-T Rec. H.265 (08/2021)"* specification. 9 | """ 10 | @behaviour Membrane.H26x.AUSplitter 11 | 12 | require Logger 13 | require Membrane.H265.NALuTypes, as: NALuTypes 14 | 15 | alias Membrane.H265.NALuTypes 16 | alias Membrane.H26x.{AUSplitter, NALu} 17 | 18 | @typedoc """ 19 | A structure holding a state of the access unit splitter. 20 | """ 21 | @opaque t :: %__MODULE__{ 22 | nalus_acc: [NALu.t()], 23 | fsm_state: :first | :second, 24 | previous_nalu: NALu.t() | nil, 25 | access_units_to_output: [AUSplitter.access_unit()] 26 | } 27 | 28 | @enforce_keys [ 29 | :nalus_acc, 30 | :fsm_state, 31 | :previous_nalu, 32 | :access_units_to_output 33 | ] 34 | defstruct @enforce_keys 35 | 36 | @doc """ 37 | Returns a structure holding a clear state of the 38 | access unit splitter. 39 | """ 40 | @spec new() :: t() 41 | def new() do 42 | %__MODULE__{ 43 | nalus_acc: [], 44 | fsm_state: :first, 45 | previous_nalu: nil, 46 | access_units_to_output: [] 47 | } 48 | end 49 | 50 | @non_vcl_nalus_at_au_beginning [:vps, :sps, :pps, :prefix_sei] 51 | @non_vcl_nalus_at_au_end [:fd, :eos, :eob, :suffix_sei] 52 | 53 | @doc """ 54 | Splits the given list of NAL units into the access units. 55 | 56 | It can be used for a stream which is not completely available at the time of function invocation, 57 | as the function updates the state of the access unit splitter - the function can 58 | be invoked once more, with new NAL units and the updated state. 59 | Under the hood, `split/2` defines a finite state machine 60 | with two states: `:first` and `:second`. The state `:first` describes the state before 61 | reaching the first segment of a coded picture NALu of a given access unit. The state `:second` 62 | describes the state after processing the first segment of the coded picture of a given 63 | access unit. 64 | """ 65 | @spec split([NALu.t()], boolean(), t()) :: {[AUSplitter.access_unit()], t()} 66 | def split(nalus, assume_au_aligned \\ false, state) do 67 | state = do_split(nalus, state) 68 | 69 | {aus, state} = 70 | if assume_au_aligned do 71 | {state.access_units_to_output ++ [state.nalus_acc], 72 | %__MODULE__{state | access_units_to_output: [], nalus_acc: []}} 73 | else 74 | {state.access_units_to_output, %__MODULE__{state | access_units_to_output: []}} 75 | end 76 | 77 | {Enum.reject(aus, &Enum.empty?/1), state} 78 | end 79 | 80 | defp do_split([first_nalu | rest_nalus], %{fsm_state: :first} = state) do 81 | cond do 82 | access_unit_first_slice_segment?(first_nalu) -> 83 | do_split( 84 | rest_nalus, 85 | %__MODULE__{ 86 | state 87 | | nalus_acc: state.nalus_acc ++ [first_nalu], 88 | fsm_state: :second, 89 | previous_nalu: first_nalu 90 | } 91 | ) 92 | 93 | (first_nalu.type == :aud and state.nalus_acc == []) or 94 | first_nalu.type in @non_vcl_nalus_at_au_beginning or 95 | NALu.int_type(first_nalu) in 41..44 or 96 | NALu.int_type(first_nalu) in 48..55 -> 97 | do_split( 98 | rest_nalus, 99 | %__MODULE__{state | nalus_acc: state.nalus_acc ++ [first_nalu]} 100 | ) 101 | 102 | true -> 103 | Logger.warning("AUSplitter: Improper transition") 104 | do_split(rest_nalus, state) 105 | end 106 | end 107 | 108 | defp do_split([first_nalu | rest_nalus], %{fsm_state: :second} = state) do 109 | previous_nalu = state.previous_nalu 110 | 111 | cond do 112 | first_nalu.type == :aud or first_nalu.type in @non_vcl_nalus_at_au_beginning -> 113 | do_split( 114 | rest_nalus, 115 | %__MODULE__{ 116 | state 117 | | nalus_acc: [first_nalu], 118 | fsm_state: :first, 119 | access_units_to_output: state.access_units_to_output ++ [state.nalus_acc] 120 | } 121 | ) 122 | 123 | access_unit_first_slice_segment?(first_nalu) -> 124 | do_split( 125 | rest_nalus, 126 | %__MODULE__{ 127 | state 128 | | nalus_acc: [first_nalu], 129 | previous_nalu: first_nalu, 130 | access_units_to_output: state.access_units_to_output ++ [state.nalus_acc] 131 | } 132 | ) 133 | 134 | first_nalu.type == previous_nalu.type or 135 | first_nalu.type in @non_vcl_nalus_at_au_end or 136 | NALu.int_type(first_nalu) in 45..47 or 137 | NALu.int_type(first_nalu) in 56..63 -> 138 | do_split( 139 | rest_nalus, 140 | %__MODULE__{ 141 | state 142 | | nalus_acc: state.nalus_acc ++ [first_nalu], 143 | previous_nalu: first_nalu 144 | } 145 | ) 146 | 147 | true -> 148 | Logger.warning("AUSplitter: Improper transition") 149 | do_split(rest_nalus, state) 150 | end 151 | end 152 | 153 | defp do_split([], state) do 154 | state 155 | end 156 | 157 | defp access_unit_first_slice_segment?(nalu) do 158 | NALuTypes.is_vcl_nalu_type(nalu.type) and 159 | nalu.parsed_fields[:first_slice_segment_in_pic_flag] == 1 160 | end 161 | end 162 | -------------------------------------------------------------------------------- /lib/membrane_h264_plugin/h265/au_timestamp_generator.ex: -------------------------------------------------------------------------------- 1 | defmodule Membrane.H265.AUTimestampGenerator do 2 | @moduledoc false 3 | 4 | use Membrane.H26x.AUTimestampGenerator 5 | 6 | require Membrane.H265.NALuTypes, as: NALuTypes 7 | 8 | @impl true 9 | def get_first_vcl_nalu(au) do 10 | Enum.find(au, &NALuTypes.is_vcl_nalu_type(&1.type)) 11 | end 12 | 13 | @impl true 14 | # Calculate picture order count according to section 8.3.1 of the ITU-T H265 specification 15 | def calculate_poc(vcl_nalu, state) do 16 | max_pic_order_cnt_lsb = 2 ** (vcl_nalu.parsed_fields.log2_max_pic_order_cnt_lsb_minus4 + 4) 17 | 18 | # We exclude CRA pictures from IRAP pictures since we have no way 19 | # to assert the value of the flag NoRaslOutputFlag. 20 | # If the CRA is the first access unit in the bytestream, the flag would be 21 | # equal to 1 which reset the POC counter, and that condition is 22 | # satisfied here since the initial value for prev_pic_order_cnt_msb and 23 | # prev_pic_order_cnt_lsb are 0 24 | {prev_pic_order_cnt_msb, prev_pic_order_cnt_lsb} = 25 | if vcl_nalu.parsed_fields.nal_unit_type in 16..20 do 26 | {0, 0} 27 | else 28 | {state.prev_pic_order_cnt_msb, 29 | state.prev_pic_first_vcl_nalu.parsed_fields.pic_order_cnt_lsb} 30 | end 31 | 32 | pic_order_cnt_lsb = vcl_nalu.parsed_fields.pic_order_cnt_lsb 33 | 34 | pic_order_cnt_msb = 35 | cond do 36 | pic_order_cnt_lsb < prev_pic_order_cnt_lsb and 37 | prev_pic_order_cnt_lsb - pic_order_cnt_lsb >= div(max_pic_order_cnt_lsb, 2) -> 38 | prev_pic_order_cnt_msb + max_pic_order_cnt_lsb 39 | 40 | pic_order_cnt_lsb > prev_pic_order_cnt_lsb and 41 | pic_order_cnt_lsb - prev_pic_order_cnt_lsb > div(max_pic_order_cnt_lsb, 2) -> 42 | prev_pic_order_cnt_msb - max_pic_order_cnt_lsb 43 | 44 | true -> 45 | prev_pic_order_cnt_msb 46 | end 47 | 48 | {prev_pic_first_vcl_nalu, prev_pic_order_cnt_msb} = 49 | if vcl_nalu.type in [:radl_r, :radl_n, :rasl_r, :rasl_n] or 50 | vcl_nalu.parsed_fields.nal_unit_type in 0..15//2 do 51 | {state.prev_pic_first_vcl_nalu, prev_pic_order_cnt_msb} 52 | else 53 | {vcl_nalu, pic_order_cnt_msb} 54 | end 55 | 56 | {pic_order_cnt_msb + pic_order_cnt_lsb, 57 | %{ 58 | state 59 | | prev_pic_order_cnt_msb: prev_pic_order_cnt_msb, 60 | prev_pic_first_vcl_nalu: prev_pic_first_vcl_nalu 61 | }} 62 | end 63 | end 64 | -------------------------------------------------------------------------------- /lib/membrane_h264_plugin/h265/decoder_configuration_record.ex: -------------------------------------------------------------------------------- 1 | defmodule Membrane.H265.DecoderConfigurationRecord do 2 | @moduledoc """ 3 | Utility functions for parsing and generating HEVC Configuration Record. 4 | 5 | The structure of the record is described in section 8.3.3.1.1 of MPEG-4 part 15 (ISO/IEC 14496-15 Edition 2017-02). 6 | """ 7 | 8 | alias Membrane.H26x.NALu 9 | 10 | @enforce_keys [ 11 | :vpss, 12 | :spss, 13 | :ppss, 14 | :profile_space, 15 | :tier_flag, 16 | :profile_idc, 17 | :profile_compatibility_flags, 18 | :constraint_indicator_flags, 19 | :level_idc, 20 | :temporal_id_nested, 21 | :num_temporal_layers, 22 | :chroma_format_idc, 23 | :bit_depth_luma_minus8, 24 | :bit_depth_chroma_minus8, 25 | :nalu_length_size 26 | ] 27 | defstruct @enforce_keys 28 | 29 | @typedoc "Structure representing the Decoder Configuartion Record" 30 | @type t() :: %__MODULE__{ 31 | vpss: [binary()], 32 | spss: [binary()], 33 | ppss: [binary()], 34 | profile_space: non_neg_integer(), 35 | tier_flag: non_neg_integer(), 36 | profile_idc: non_neg_integer(), 37 | profile_compatibility_flags: non_neg_integer(), 38 | constraint_indicator_flags: non_neg_integer(), 39 | level_idc: non_neg_integer(), 40 | chroma_format_idc: non_neg_integer(), 41 | bit_depth_luma_minus8: non_neg_integer(), 42 | bit_depth_chroma_minus8: non_neg_integer(), 43 | temporal_id_nested: non_neg_integer(), 44 | num_temporal_layers: non_neg_integer(), 45 | nalu_length_size: non_neg_integer() 46 | } 47 | 48 | @doc """ 49 | Generates a DCR based on given PPSs, SPSs and VPSs. 50 | """ 51 | @spec generate([NALu.t()], [NALu.t()], [NALu.t()], Membrane.H265.Parser.stream_structure()) :: 52 | binary() | nil 53 | def generate(_vpss, [], _ppss, _stream_structure) do 54 | nil 55 | end 56 | 57 | def generate(vpss, spss, ppss, {avc, nalu_length_size}) do 58 | %NALu{ 59 | parsed_fields: %{ 60 | profile_space: profile_space, 61 | tier_flag: tier_flag, 62 | profile_idc: profile_idc, 63 | profile_compatibility_flag: profile_compatibility_flag, 64 | progressive_source_flag: progressive_source_flag, 65 | interlaced_source_flag: interlaced_source_flag, 66 | non_packed_constraint_flag: non_packed_constraint_flag, 67 | frame_only_constraint_flag: frame_only_constraint_flag, 68 | reserved_zero_44bits: reserved_zero_44bits, 69 | level_idc: level_idc, 70 | chroma_format_idc: chroma_format_idc, 71 | bit_depth_luma_minus8: bit_depth_luma_minus8, 72 | bit_depth_chroma_minus8: bit_depth_chroma_minus8, 73 | temporal_id_nesting_flag: temporal_id_nested, 74 | max_sub_layers_minus1: num_temporal_layers 75 | } 76 | } = List.last(spss) 77 | 78 | common_config = 79 | <<1, profile_space::2, tier_flag::1, profile_idc::5, profile_compatibility_flag::32, 80 | progressive_source_flag::1, interlaced_source_flag::1, non_packed_constraint_flag::1, 81 | frame_only_constraint_flag::1, reserved_zero_44bits::44, level_idc, 0b1111::4, 82 | _min_spatial_segmentation_idc = 0::12, 0b111111::6, _parallelism_type = 0::2, 0b111111::6, 83 | chroma_format_idc::2, 0b11111::5, bit_depth_luma_minus8::3, 0b11111::5, 84 | bit_depth_chroma_minus8::3, _average_framerate = 0::16, _constant_framerate = 0::2, 85 | num_temporal_layers + 1::3, temporal_id_nested::1, nalu_length_size - 1::2-integer>> 86 | 87 | cond do 88 | avc == :hvc1 -> 89 | <> 91 | 92 | avc == :hev1 -> 93 | <> 94 | end 95 | end 96 | 97 | defp encode_parameter_sets(pss, nalu_type) do 98 | <<2::2, nalu_type::6, length(pss)::16>> <> 99 | Enum.map_join(pss, &<>) 100 | end 101 | 102 | @doc """ 103 | Parses the DCR. 104 | """ 105 | @spec parse(binary()) :: t() 106 | def parse( 107 | <<1::8, profile_space::2, tier_flag::1, profile_idc::5, profile_compatibility_flags::32, 108 | constraint_indicator_flags::48, level_idc::8, 0b1111::4, 109 | _min_spatial_segmentation_idc::12, 0b111111::6, _parallelism_type::2, 0b111111::6, 110 | chroma_format_idc::2, 0b11111::5, bit_depth_luma_minus8::3, 0b11111::5, 111 | bit_depth_chroma_minus8::3, _avg_frame_rate::16, _constant_frame_rate::2, 112 | num_temporal_layers::3, temporal_id_nested::1, length_size_minus_one::2-integer, 113 | num_of_arrays::8, rest::bitstring>> 114 | ) do 115 | {vpss, spss, ppss} = 116 | if num_of_arrays > 0 do 117 | {vpss, rest} = parse_pss(rest, 32) 118 | {spss, rest} = parse_pss(rest, 33) 119 | {ppss, _rest} = parse_pss(rest, 34) 120 | 121 | {vpss, spss, ppss} 122 | else 123 | {[], [], []} 124 | end 125 | 126 | %__MODULE__{ 127 | vpss: vpss, 128 | spss: spss, 129 | ppss: ppss, 130 | profile_space: profile_space, 131 | tier_flag: tier_flag, 132 | profile_idc: profile_idc, 133 | profile_compatibility_flags: profile_compatibility_flags, 134 | constraint_indicator_flags: constraint_indicator_flags, 135 | level_idc: level_idc, 136 | temporal_id_nested: temporal_id_nested, 137 | num_temporal_layers: num_temporal_layers, 138 | chroma_format_idc: chroma_format_idc, 139 | bit_depth_luma_minus8: bit_depth_luma_minus8, 140 | bit_depth_chroma_minus8: bit_depth_chroma_minus8, 141 | nalu_length_size: length_size_minus_one + 1 142 | } 143 | end 144 | 145 | def parse(_data), do: {:error, :unknown_pattern} 146 | 147 | defp parse_pss(<<_reserved::2, type::6, num_of_pss::16, rest::bitstring>>, type) do 148 | do_parse_array(num_of_pss, rest) 149 | end 150 | 151 | defp do_parse_array(amount, rest, acc \\ []) 152 | defp do_parse_array(0, rest, acc), do: {Enum.reverse(acc), rest} 153 | 154 | defp do_parse_array(remaining, <>, acc), 155 | do: do_parse_array(remaining - 1, rest, [data | acc]) 156 | end 157 | -------------------------------------------------------------------------------- /lib/membrane_h264_plugin/h265/nalu_parser.ex: -------------------------------------------------------------------------------- 1 | defmodule Membrane.H265.NALuParser do 2 | @moduledoc """ 3 | This module is an extension to `Membrane.H26x.NALuParser` and contains 4 | H265 specific functions. 5 | """ 6 | 7 | use Membrane.H26x.NALuParser 8 | 9 | require Membrane.H265.NALuTypes 10 | 11 | alias Membrane.H265.NALuParser.Schemes 12 | alias Membrane.H265.NALuTypes 13 | alias Membrane.H26x.NALuParser.SchemeParser 14 | 15 | @impl true 16 | def get_nalu_header_and_body(<>), 17 | do: {nalu_header, nalu_body} 18 | 19 | @impl true 20 | def parse_nalu_header(nalu_header, state) do 21 | SchemeParser.parse_with_scheme(nalu_header, Schemes.NALuHeader, state) 22 | end 23 | 24 | @impl true 25 | def get_nalu_type(nal_unit_type), do: NALuTypes.get_type(nal_unit_type) 26 | 27 | @impl true 28 | def parse_proper_nalu_type(nalu_body, nalu_type, state) do 29 | try do 30 | {parsed_fields, state} = do_parse_proper_nalu_type(nalu_body, nalu_type, state) 31 | {:ok, parsed_fields, state} 32 | catch 33 | # We throw the scheme parser state since we do need the parsed fields for 34 | # acess unit splitter even if the parameter sets are not present 35 | state -> 36 | {:error, state} 37 | end 38 | end 39 | 40 | defp do_parse_proper_nalu_type(nalu_body, nalu_type, state) do 41 | # delete prevention emulation 3 bytes 42 | nalu_body = :binary.split(nalu_body, <<0, 0, 3>>, [:global]) |> Enum.join(<<0, 0>>) 43 | 44 | case nalu_type do 45 | :vps -> 46 | SchemeParser.parse_with_scheme(nalu_body, Schemes.VPS, state) 47 | 48 | :sps -> 49 | SchemeParser.parse_with_scheme(nalu_body, Schemes.SPS, state) 50 | 51 | :pps -> 52 | SchemeParser.parse_with_scheme(nalu_body, Schemes.PPS, state) 53 | 54 | type -> 55 | if NALuTypes.is_vcl_nalu_type(type) do 56 | SchemeParser.parse_with_scheme(nalu_body, Schemes.Slice, state) 57 | else 58 | {SchemeParser.get_local_state(state), state} 59 | end 60 | end 61 | end 62 | end 63 | -------------------------------------------------------------------------------- /lib/membrane_h264_plugin/h265/nalu_parser/schemes/common.ex: -------------------------------------------------------------------------------- 1 | defmodule Membrane.H265.NALuParser.Schemes.Common do 2 | @moduledoc false 3 | 4 | alias Membrane.H26x.NALuParser.Scheme 5 | 6 | @spec profile_tier_level() :: Scheme.t() 7 | def profile_tier_level() do 8 | [ 9 | field: {:profile_space, :u2}, 10 | field: {:tier_flag, :u1}, 11 | field: {:profile_idc, :u5}, 12 | field: {:profile_compatibility_flag, :u32}, 13 | field: {:progressive_source_flag, :u1}, 14 | field: {:interlaced_source_flag, :u1}, 15 | field: {:non_packed_constraint_flag, :u1}, 16 | field: {:frame_only_constraint_flag, :u1}, 17 | field: {:reserved_zero_44bits, :u44} 18 | ] ++ level_fields() 19 | end 20 | 21 | defp level_fields() do 22 | [ 23 | field: {:level_idc, :u8}, 24 | for: { 25 | [iterator: :i, from: 0, to: {&(&1 - 1), [:max_sub_layers_minus1]}], 26 | field: {:sub_layer_profile_present_flag, :u1}, field: {:sub_layer_level_present_flag, :u1} 27 | }, 28 | if: { 29 | {&(&1 > 0), [:max_sub_layers_minus1]}, 30 | for: { 31 | [iterator: :i, from: {& &1, [:max_sub_layers_minus1]}, to: 7], 32 | field: {:reserved_zero_2bits, :u2} 33 | } 34 | }, 35 | for: { 36 | [iterator: :i, from: 0, to: {&(&1 - 1), [:max_sub_layers_minus1]}], 37 | if: { 38 | {&(&1[&2] == 1), [:sub_layer_profile_present_flag, :i]}, 39 | field: {:sub_layer_profile_space, :u2}, 40 | field: {:sub_layer_tier_flag, :u1}, 41 | field: {:sub_layer_profile_idc, :u5}, 42 | for: { 43 | [iterator: :j, from: 0, to: 31], 44 | field: {:sub_layer_profile_compatibility_flag, :u1} 45 | }, 46 | field: {:sub_layer_progressive_source_flag, :u1}, 47 | field: {:sub_layer_interlaced_source_flag, :u1}, 48 | field: {:sub_layer_non_packed_constraint_flag, :u1}, 49 | field: {:sub_layer_frame_only_constraint_flag, :u1}, 50 | field: {:sub_layer_reserved_zero_44bits, :u44} 51 | }, 52 | if: { 53 | {&(&1[&2] == 1), [:sub_layer_level_present_flag, :i]}, 54 | field: {:sub_layer_level_idc, :u8} 55 | } 56 | } 57 | ] 58 | end 59 | end 60 | -------------------------------------------------------------------------------- /lib/membrane_h264_plugin/h265/nalu_parser/schemes/nalu_header.ex: -------------------------------------------------------------------------------- 1 | defmodule Membrane.H265.NALuParser.Schemes.NALuHeader do 2 | @moduledoc false 3 | @behaviour Membrane.H26x.NALuParser.Scheme 4 | 5 | @impl true 6 | def defaults(), do: [] 7 | 8 | @impl true 9 | def scheme(), 10 | do: [ 11 | field: {:forbidden_zero_bit, :u1}, 12 | field: {:nal_unit_type, :u6}, 13 | field: {:nuh_layer_id, :u6}, 14 | field: {:nuh_temporal_id_plus1, :u3} 15 | ] 16 | end 17 | -------------------------------------------------------------------------------- /lib/membrane_h264_plugin/h265/nalu_parser/schemes/pps.ex: -------------------------------------------------------------------------------- 1 | defmodule Membrane.H265.NALuParser.Schemes.PPS do 2 | @moduledoc false 3 | @behaviour Membrane.H26x.NALuParser.Scheme 4 | 5 | @impl true 6 | def defaults(), do: [] 7 | 8 | @impl true 9 | def scheme(), 10 | do: [ 11 | field: {:pic_parameter_set_id, :ue}, 12 | field: {:seq_parameter_set_id, :ue}, 13 | field: {:dependent_slice_segments_enabled_flag, :u1}, 14 | field: {:output_flag_present_flag, :u1}, 15 | field: {:num_extra_slice_header_bits, :u3}, 16 | save_state_as_global_state: {&{:pps, &1}, [:pic_parameter_set_id]} 17 | ] 18 | end 19 | -------------------------------------------------------------------------------- /lib/membrane_h264_plugin/h265/nalu_parser/schemes/slice.ex: -------------------------------------------------------------------------------- 1 | defmodule Membrane.H265.NALuParser.Schemes.Slice do 2 | @moduledoc false 3 | @behaviour Membrane.H26x.NALuParser.Scheme 4 | 5 | @impl true 6 | def defaults(), 7 | do: [ 8 | dependent_slice_segment_flag: 0, 9 | no_output_of_prior_pics_flag: 0, 10 | pic_output_flag: 1, 11 | pic_order_cnt_lsb: 0 12 | ] 13 | 14 | @impl true 15 | def scheme(), 16 | do: [ 17 | field: {:first_slice_segment_in_pic_flag, :u1}, 18 | if: { 19 | {&(&1 >= 16 and &1 <= 23), [:nal_unit_type]}, 20 | field: {:no_output_of_prior_pics_flag, :u1} 21 | }, 22 | field: {:pic_parameter_set_id, :ue}, 23 | execute: &load_data_from_sps/3, 24 | if: { 25 | {&(&1 == 0), [:first_slice_segment_in_pic_flag]}, 26 | if: { 27 | {&(&1 == 1), [:dependent_slice_segments_enabled_flag]}, 28 | field: {:dependent_slice_segment_flag, :u1} 29 | }, 30 | field: {:slice_segment_address, {:uv, & &1, [:segment_addr_length]}} 31 | }, 32 | if: { 33 | {&(&1 == 0), [:dependent_slice_segment_flag]}, 34 | for: { 35 | [iterator: :j, from: 0, to: {&(&1 - 1), [:num_extra_slice_header_bits]}], 36 | field: {:slice_reserved_flag, :u1} 37 | }, 38 | field: {:slice_type, :ue}, 39 | if: { 40 | {&(&1 == 1), [:output_flag_present_flag]}, 41 | field: {:pic_output_flag, :u1} 42 | }, 43 | if: { 44 | {&(&1 == 1), [:separate_colour_plane_flag]}, 45 | field: {:colour_plane_id, :u2} 46 | }, 47 | if: { 48 | {&(&1 != 19 and &1 != 20), [:nal_unit_type]}, 49 | field: {:pic_order_cnt_lsb, {:uv, &(&1 + 4), [:log2_max_pic_order_cnt_lsb_minus4]}} 50 | } 51 | } 52 | ] 53 | 54 | defp load_data_from_sps(payload, state, _iterators) do 55 | with pic_parameter_set_id when pic_parameter_set_id != nil <- 56 | Map.get(state.__local__, :pic_parameter_set_id), 57 | pps when pps != nil <- Map.get(state.__global__, {:pps, pic_parameter_set_id}), 58 | seq_parameter_set_id when seq_parameter_set_id != nil <- 59 | Map.get(pps, :seq_parameter_set_id), 60 | sps <- Map.get(state.__global__, {:sps, seq_parameter_set_id}) do 61 | state = 62 | Bunch.Access.put_in( 63 | state, 64 | [:__local__, :separate_colour_plane_flag], 65 | Map.get(sps, :separate_colour_plane_flag, 0) 66 | ) 67 | 68 | sps_fields = Map.take(sps, [:segment_addr_length, :log2_max_pic_order_cnt_lsb_minus4]) 69 | state = Map.update(state, :__local__, %{}, &Map.merge(&1, sps_fields)) 70 | 71 | # PPS fields 72 | pps_fields = 73 | Map.take(pps, [ 74 | :dependent_slice_segments_enabled_flag, 75 | :output_flag_present_flag, 76 | :num_extra_slice_header_bits 77 | ]) 78 | 79 | state = Map.update(state, :__local__, %{}, &Map.merge(&1, pps_fields)) 80 | 81 | {payload, state} 82 | else 83 | _error -> throw(state) 84 | end 85 | end 86 | end 87 | -------------------------------------------------------------------------------- /lib/membrane_h264_plugin/h265/nalu_parser/schemes/vps.ex: -------------------------------------------------------------------------------- 1 | defmodule Membrane.H265.NALuParser.Schemes.VPS do 2 | @moduledoc false 3 | 4 | @behaviour Membrane.H26x.NALuParser.Scheme 5 | 6 | alias Membrane.H265.NALuParser.Schemes.Common 7 | 8 | @impl true 9 | def defaults(), do: [] 10 | 11 | @impl true 12 | def scheme(), 13 | do: 14 | [ 15 | field: {:video_parameter_set_id, :u4}, 16 | field: {:base_layer_internal_flag, :u1}, 17 | field: {:base_layer_available_flag, :u1}, 18 | field: {:max_layers_minus1, :u6}, 19 | field: {:max_sub_layers_minus1, :u3}, 20 | field: {:temporal_id_nesting_flag, :u1}, 21 | field: {:reserved_0xffff_16bits, :u16} 22 | ] ++ 23 | Common.profile_tier_level() ++ 24 | [ 25 | field: {:sub_layer_ordering_info_present_flag, :u1}, 26 | for: { 27 | [ 28 | iterator: :i, 29 | from: 30 | {&if(&1 == 1, do: 0, else: &2), 31 | [:sub_layer_ordering_info_present_flag, :max_sub_layers_minus1]}, 32 | to: {& &1, [:max_sub_layers_minus1]} 33 | ], 34 | field: {:max_dec_pic_buffering_minus1, :ue}, 35 | field: {:max_num_reorder_pics, :ue}, 36 | field: {:max_latency_increase_plus1, :ue} 37 | }, 38 | field: {:max_layer_id, :u6}, 39 | field: {:num_layer_sets_minus1, :ue}, 40 | for: { 41 | [iterator: :i, from: 1, to: {& &1, [:num_layer_sets_minus1]}], 42 | for: { 43 | [iterator: :j, from: 0, to: {& &1, [:max_layer_id]}], 44 | field: {:layer_id_included_flag, :u1} 45 | } 46 | }, 47 | field: {:timing_info_present_flag, :u1}, 48 | if: { 49 | {&(&1 == 1), [:timing_info_present_flag]}, 50 | field: {:num_units_in_tick, :u32}, 51 | field: {:time_scale, :u32}, 52 | field: {:poc_proportional_to_timing_flag, :u1}, 53 | if: { 54 | {&(&1 == 1), [:poc_proportional_to_timing_flag]}, 55 | field: {:num_ticks_poc_diff_one_minus1, :ue} 56 | } 57 | }, 58 | save_state_as_global_state: {&{:vps, &1}, [:video_parameter_set_id]} 59 | ] 60 | end 61 | -------------------------------------------------------------------------------- /lib/membrane_h264_plugin/h265/nalu_types.ex: -------------------------------------------------------------------------------- 1 | defmodule Membrane.H265.NALuTypes do 2 | @moduledoc """ 3 | The module aggregating the mapping of `nal_unit_type` fields 4 | of the NAL unit to the human-friendly name of a NALu type. 5 | """ 6 | 7 | @nalu_types %{ 8 | 0 => :trail_n, 9 | 1 => :trail_r, 10 | 2 => :tsa_n, 11 | 3 => :tsa_r, 12 | 4 => :stsa_n, 13 | 5 => :stsa_r, 14 | 6 => :radl_n, 15 | 7 => :radl_r, 16 | 8 => :rasl_n, 17 | 9 => :rasl_r, 18 | (10..15) => :reserved_non_irap, 19 | 16 => :bla_w_lp, 20 | 17 => :bla_w_radl, 21 | 18 => :bla_n_lp, 22 | 19 => :idr_w_radl, 23 | 20 => :idr_n_lp, 24 | 21 => :cra, 25 | (22..23) => :reserved_irap, 26 | (24..31) => :reserved_non_irap, 27 | 32 => :vps, 28 | 33 => :sps, 29 | 34 => :pps, 30 | 35 => :aud, 31 | 36 => :eos, 32 | 37 => :eob, 33 | 38 => :fd, 34 | 39 => :prefix_sei, 35 | 40 => :suffix_sei, 36 | (41..47) => :reserved_nvcl, 37 | (48..63) => :unspecified 38 | } 39 | |> Enum.flat_map(fn 40 | {k, v} when is_integer(k) -> [{k, v}] 41 | {k, v} -> Enum.map(k, &{&1, v}) 42 | end) 43 | |> Map.new() 44 | 45 | @irap_nalu_types [:bla_w_lp, :bla_w_radl, :bla_n_lp, :idr_w_radl, :idr_n_lp, :cra] 46 | @vcl_nalu_types [ 47 | :trail_n, 48 | :trail_r, 49 | :tsa_n, 50 | :tsa_r, 51 | :stsa_n, 52 | :stsa_r, 53 | :radl_n, 54 | :radl_r, 55 | :rasl_n, 56 | :rasl_r, 57 | :reserved_non_irap, 58 | :reserved_irap 59 | ] ++ @irap_nalu_types 60 | 61 | @typedoc """ 62 | A type representing all the possible human-friendly names of NAL unit types. 63 | """ 64 | @type nalu_type :: 65 | unquote(Bunch.Typespec.enum_to_alternative(Map.values(@nalu_types) |> Enum.uniq())) 66 | 67 | @doc """ 68 | The function which returns the human friendly name of a NALu type 69 | for a given `nal_unit_type`. 70 | 71 | The mapping is based on: "Table 7-1 – NAL unit type codes, syntax element categories, and NAL unit type classes" 72 | of the *"ITU-T Rec. H.265 (08/2021)"* 73 | """ 74 | @spec get_type(non_neg_integer()) :: atom() 75 | def get_type(nal_unit_type) do 76 | @nalu_types[nal_unit_type] 77 | end 78 | 79 | defguard is_vcl_nalu_type(nalu_type) when nalu_type in @vcl_nalu_types 80 | defguard is_irap_nalu_type(nalu_type) when nalu_type in @irap_nalu_types 81 | end 82 | -------------------------------------------------------------------------------- /lib/membrane_h264_plugin/h26x/au_splitter.ex: -------------------------------------------------------------------------------- 1 | defmodule Membrane.H26x.AUSplitter do 2 | @moduledoc """ 3 | A behaviour module to split NALus into access units 4 | """ 5 | 6 | alias Membrane.H26x.NALu 7 | 8 | @typedoc """ 9 | A type representing an access unit - a list of logically associated NAL units. 10 | """ 11 | @type access_unit() :: list(NALu.t()) 12 | 13 | @type state :: term() 14 | 15 | @callback new() :: state() 16 | @callback split([NALu.t()], boolean(), state()) :: {[access_unit()], state()} 17 | end 18 | -------------------------------------------------------------------------------- /lib/membrane_h264_plugin/h26x/au_timestamp_generator.ex: -------------------------------------------------------------------------------- 1 | defmodule Membrane.H26x.AUTimestampGenerator do 2 | @moduledoc false 3 | 4 | alias Membrane.H26x.{AUSplitter, NALu} 5 | 6 | @type framerate :: {frames :: pos_integer(), seconds :: pos_integer()} 7 | 8 | @type state :: %{ 9 | framerate: framerate, 10 | max_frame_reorder: 0..15, 11 | au_counter: non_neg_integer(), 12 | key_frame_au_idx: non_neg_integer(), 13 | prev_pic_first_vcl_nalu: NALu.t() | nil, 14 | prev_pic_order_cnt_msb: integer() 15 | } 16 | 17 | @callback get_first_vcl_nalu(AUSplitter.access_unit()) :: NALu.t() 18 | @callback calculate_poc(NALu.t(), state()) :: {non_neg_integer(), state()} 19 | 20 | defmacro __using__(_options) do 21 | quote location: :keep do 22 | @behaviour unquote(__MODULE__) 23 | 24 | @typep config :: %{ 25 | :framerate => unquote(__MODULE__).framerate, 26 | optional(:add_dts_offset) => boolean() 27 | } 28 | 29 | @spec new(config()) :: unquote(__MODULE__).state() 30 | def new(config) do 31 | # To make sure that PTS >= DTS at all times, we take maximal possible 32 | # frame reorder (which is 15 according to the spec) and subtract 33 | # `max_frame_reorder * frame_duration` from each frame's DTS. 34 | # This behaviour can be disabled by setting `add_dts_offset: false`. 35 | max_frame_reorder = if Map.get(config, :add_dts_offset, true), do: 15, else: 0 36 | 37 | %{ 38 | framerate: config.framerate, 39 | max_frame_reorder: max_frame_reorder, 40 | au_counter: 0, 41 | key_frame_au_idx: 0, 42 | prev_pic_first_vcl_nalu: nil, 43 | prev_pic_order_cnt_msb: 0 44 | } 45 | end 46 | 47 | @spec generate_ts_with_constant_framerate( 48 | AUSplitter.access_unit(), 49 | unquote(__MODULE__).state() 50 | ) :: 51 | {{pts :: non_neg_integer(), dts :: non_neg_integer()}, unquote(__MODULE__).state()} 52 | def generate_ts_with_constant_framerate(au, state) do 53 | %{ 54 | au_counter: au_counter, 55 | key_frame_au_idx: key_frame_au_idx, 56 | max_frame_reorder: max_frame_reorder, 57 | framerate: {frames, seconds} 58 | } = state 59 | 60 | first_vcl_nalu = get_first_vcl_nalu(au) 61 | {poc, state} = calculate_poc(first_vcl_nalu, state) 62 | key_frame_au_idx = if poc == 0, do: au_counter, else: key_frame_au_idx 63 | pts = div((key_frame_au_idx + poc) * seconds * Membrane.Time.second(), frames) 64 | dts = div((au_counter - max_frame_reorder) * seconds * Membrane.Time.second(), frames) 65 | 66 | state = %{ 67 | state 68 | | au_counter: au_counter + 1, 69 | key_frame_au_idx: key_frame_au_idx 70 | } 71 | 72 | {{pts, dts}, state} 73 | end 74 | end 75 | end 76 | end 77 | -------------------------------------------------------------------------------- /lib/membrane_h264_plugin/h26x/exp_golomb_converter.ex: -------------------------------------------------------------------------------- 1 | defmodule Membrane.H26x.ExpGolombConverter do 2 | @moduledoc """ 3 | This module holds function responsible for converting 4 | from and to Exp-Golomb Notation. 5 | """ 6 | 7 | @doc """ 8 | Reads the appropriate number of bits from the bitstring and decodes an 9 | integer out of these bits. 10 | 11 | Returns the decoded integer and the rest of the bitstring, which wasn't 12 | used for decoding. By default, the decoded integer is an unsigned integer. 13 | If `negatives: true` is passed as an option, the decoded integer will be 14 | a signed integer. 15 | """ 16 | @spec to_integer(bitstring(), keyword()) :: {integer(), bitstring()} 17 | def to_integer(binary, opts \\ [negatives: false]) 18 | 19 | def to_integer(binary, negatives: should_support_negatives) do 20 | zeros_size = cut_zeros(binary) 21 | number_size = zeros_size + 1 22 | <<_zeros::size(zeros_size), number::size(number_size), rest::bitstring>> = binary 23 | number = number - 1 24 | 25 | if should_support_negatives do 26 | if rem(number, 2) == 0, do: {-div(number, 2), rest}, else: {div(number + 1, 2), rest} 27 | else 28 | {number, rest} 29 | end 30 | end 31 | 32 | @doc """ 33 | Returns a bitstring with an Exponential Golomb representation of a integer. 34 | 35 | By default, the function expects the number to be a non-negative integer. 36 | If `negatives: true` option is set, the function can also encode negative 37 | numbers, but number encoded with `negatives: true` option also needs to be 38 | decoded with that option. 39 | """ 40 | @spec to_bitstring(integer(), negatives: boolean()) :: bitstring() 41 | def to_bitstring(integer, opts \\ [negatives: false]) 42 | 43 | def to_bitstring(integer, negatives: false) do 44 | # ceil(log(x)) can be calculated more accuratly and efficiently 45 | number_size = trunc(:math.floor(:math.log2(integer + 1))) + 1 46 | zeros_size = number_size - 1 47 | <<0::size(zeros_size), integer + 1::size(number_size)>> 48 | end 49 | 50 | def to_bitstring(integer, negatives: true) do 51 | if integer < 0, 52 | do: to_bitstring(-2 * integer, negatives: false), 53 | else: to_bitstring(2 * integer - 1, negatives: false) 54 | end 55 | 56 | defp cut_zeros(bitstring, how_many_zeros \\ 0) do 57 | <> = bitstring 58 | 59 | case x do 60 | 0 -> cut_zeros(rest, how_many_zeros + 1) 61 | 1 -> how_many_zeros 62 | end 63 | end 64 | end 65 | -------------------------------------------------------------------------------- /lib/membrane_h264_plugin/h26x/nalu.ex: -------------------------------------------------------------------------------- 1 | defmodule Membrane.H26x.NALu do 2 | @moduledoc """ 3 | A module defining a struct representing a single NAL unit. 4 | """ 5 | 6 | alias Membrane.{H264, H265} 7 | 8 | @typedoc """ 9 | A type defining the structure of a single NAL unit produced by the `Membrane.H26x.NALuParser`. 10 | 11 | In the structure there ardqde following fields: 12 | * `parsed_fields` - the map with keys being the NALu field names and the values being the value fetched from the NALu binary. 13 | They correspond to the NALu schemes defined in the H26x specification documents. 14 | * `stripped_prefix` - prefix that used to split the NAL units in the bytestream and was stripped from the payload. 15 | The prefix is defined as in: *"Annex B"* of the *"ISO/IEC 14496-10"* or in "ISO/IEC 14496-15". 16 | * `type` - an atom representing the type of the NALu. Atom's name is based on the 17 | *"Table 7-1 – NAL unit type codes, syntax element categories, and NAL unit type classes"* of the *"ITU-T Rec. H.264 (01/2012)"*. 18 | * `payload` - the binary, which parsing resulted in that structure being produced stripped of it's prefix 19 | * `status` - `:valid`, if the parsing was successfull, `:error` otherwise 20 | """ 21 | @type t :: %__MODULE__{ 22 | parsed_fields: %{atom() => any()}, 23 | type: H264.NALuTypes.nalu_type() | H265.NALuTypes.nalu_type(), 24 | stripped_prefix: binary(), 25 | payload: binary(), 26 | status: :valid | :error, 27 | timestamps: timestamps() 28 | } 29 | 30 | @type timestamps :: {pts :: integer() | nil, dts :: integer() | nil} 31 | 32 | @enforce_keys [:parsed_fields, :type, :stripped_prefix, :payload, :status] 33 | defstruct @enforce_keys ++ [timestamps: {nil, nil}] 34 | 35 | @spec int_type(t()) :: non_neg_integer() 36 | def int_type(%__MODULE__{parsed_fields: parsed_fields}), do: parsed_fields.nal_unit_type 37 | end 38 | -------------------------------------------------------------------------------- /lib/membrane_h264_plugin/h26x/nalu_parser.ex: -------------------------------------------------------------------------------- 1 | defmodule Membrane.H26x.NALuParser do 2 | @moduledoc """ 3 | A module providing functionality of parsing a stream of binaries, out of which each 4 | is a payload of a single NAL unit. 5 | """ 6 | 7 | alias Membrane.H26x.NALu 8 | alias Membrane.H26x.NALuParser.SchemeParser 9 | 10 | @annexb_prefix_code <<0, 0, 0, 1>> 11 | 12 | @type nalu_type :: atom() 13 | 14 | @callback get_nalu_header_and_body(binary()) :: {binary(), binary()} 15 | @callback parse_nalu_header(binary(), SchemeParser.t()) :: {map(), SchemeParser.t()} 16 | @callback get_nalu_type(non_neg_integer()) :: nalu_type() 17 | @callback parse_proper_nalu_type(binary(), nalu_type(), SchemeParser.t()) :: 18 | {:ok, map(), SchemeParser.t()} | {:error, SchemeParser.t()} 19 | 20 | defmacro __using__(_options) do 21 | quote location: :keep do 22 | @behaviour unquote(__MODULE__) 23 | 24 | alias Membrane.H26x.{NALu, NALuParser} 25 | alias Membrane.H26x.NALuParser.SchemeParser 26 | 27 | defdelegate new(input_stream_structure \\ :annexb), to: NALuParser 28 | 29 | @doc """ 30 | Parses a list of binaries, each representing a single NALu. 31 | 32 | See `parse/4` for details. 33 | """ 34 | @spec parse_nalus([binary()], NALu.timestamps(), boolean(), NALuParser.t()) :: 35 | {[NALu.t()], NALuParser.t()} 36 | def parse_nalus(nalus_payloads, timestamps \\ {nil, nil}, payload_prefixed? \\ true, state) do 37 | Enum.map_reduce(nalus_payloads, state, fn nalu_payload, state -> 38 | parse(nalu_payload, timestamps, payload_prefixed?, state) 39 | end) 40 | end 41 | 42 | @doc """ 43 | Parses a binary representing a single NALu and removes it's prefix (if it exists). 44 | 45 | Returns a structure that 46 | contains parsed fields fetched from that NALu. 47 | When `payload_prefixed?` is true the input binary is expected to contain one of: 48 | * prefix defined as the *"Annex B"* in the H26x recommendation document. 49 | * prefix of size defined in state describing the length of the NALU in bytes, as described in *ISO/IEC 14496-15*. 50 | """ 51 | @spec parse(binary(), NALu.timestamps(), boolean(), NALuParser.t()) :: 52 | {NALu.t(), NALuParser.t()} 53 | def parse(nalu_payload, timestamps \\ {nil, nil}, payload_prefixed? \\ true, state) do 54 | {prefix, unprefixed_nalu_payload} = 55 | if payload_prefixed? do 56 | NALuParser.unprefix_nalu_payload(nalu_payload, state.input_stream_structure) 57 | else 58 | {<<>>, nalu_payload} 59 | end 60 | 61 | {nalu_header, nalu_body} = get_nalu_header_and_body(unprefixed_nalu_payload) 62 | 63 | new_scheme_parser_state = SchemeParser.new(state.scheme_parser_state) 64 | 65 | {parsed_fields, scheme_parser_state} = 66 | parse_nalu_header(nalu_header, new_scheme_parser_state) 67 | 68 | type = get_nalu_type(parsed_fields.nal_unit_type) 69 | 70 | {status, parsed_fields, scheme_parser_state} = 71 | case parse_proper_nalu_type(nalu_body, type, scheme_parser_state) do 72 | {:ok, parsed_fields, state} -> {:valid, parsed_fields, state} 73 | {:error, state} -> {:error, SchemeParser.get_local_state(state), state} 74 | end 75 | 76 | nalu = %NALu{ 77 | parsed_fields: parsed_fields, 78 | type: type, 79 | status: status, 80 | stripped_prefix: prefix, 81 | payload: unprefixed_nalu_payload, 82 | timestamps: timestamps 83 | } 84 | 85 | state = %{state | scheme_parser_state: scheme_parser_state} 86 | 87 | {nalu, state} 88 | end 89 | 90 | defdelegate get_prefixed_nalu_payload( 91 | nalu, 92 | output_stream_structure, 93 | stable_prefixing? \\ true 94 | ), 95 | to: NALuParser 96 | 97 | defdelegate prefix_nalus_payloads(nalus, input_stream_structure), to: NALuParser 98 | end 99 | end 100 | 101 | @typedoc """ 102 | A structure holding the state of the NALu parser. 103 | """ 104 | @type t :: %__MODULE__{ 105 | scheme_parser_state: SchemeParser.t(), 106 | input_stream_structure: Membrane.H264.Parser.stream_structure() 107 | } 108 | @enforce_keys [:input_stream_structure] 109 | defstruct @enforce_keys ++ 110 | [ 111 | scheme_parser_state: SchemeParser.new() 112 | ] 113 | 114 | @doc """ 115 | Returns a structure holding a clear NALu parser state. `input_stream_structure` 116 | determines the prefixes of input NALU payloads. 117 | """ 118 | @spec new(Membrane.H264.Parser.stream_structure()) :: t() 119 | def new(input_stream_structure) do 120 | %__MODULE__{ 121 | input_stream_structure: input_stream_structure 122 | } 123 | end 124 | 125 | @doc """ 126 | Returns payload of the NALu with appropriate prefix generated based on output stream 127 | structure and prefix length. 128 | """ 129 | @spec get_prefixed_nalu_payload(NALu.t(), Membrane.H264.Parser.stream_structure(), boolean()) :: 130 | binary() 131 | def get_prefixed_nalu_payload(nalu, output_stream_structure, stable_prefixing? \\ true) do 132 | case {output_stream_structure, stable_prefixing?} do 133 | {:annexb, true} -> 134 | case nalu.stripped_prefix do 135 | <<0, 0, 1>> -> <<0, 0, 1, nalu.payload::binary>> 136 | <<0, 0, 0, 1>> -> <<0, 0, 0, 1, nalu.payload::binary>> 137 | _prefix -> @annexb_prefix_code <> nalu.payload 138 | end 139 | 140 | {:annexb, false} -> 141 | @annexb_prefix_code <> nalu.payload 142 | 143 | {{_codec_tag, nalu_length_size}, _stable_prefixing?} -> 144 | <> 145 | end 146 | end 147 | 148 | @spec unprefix_nalu_payload(binary(), Membrane.H264.Parser.stream_structure()) :: 149 | {stripped_prefix :: binary(), payload :: binary()} 150 | def unprefix_nalu_payload(nalu_payload, :annexb) do 151 | case nalu_payload do 152 | <<0, 0, 1, rest::binary>> -> {<<0, 0, 1>>, rest} 153 | <<0, 0, 0, 1, rest::binary>> -> {<<0, 0, 0, 1>>, rest} 154 | end 155 | end 156 | 157 | def unprefix_nalu_payload(nalu_payload, {_codec_tag, nalu_length_size}) do 158 | <> = nalu_payload 159 | 160 | {<>, rest} 161 | end 162 | 163 | @spec prefix_nalus_payloads([binary()], Membrane.H264.Parser.stream_structure()) :: binary() 164 | def prefix_nalus_payloads(nalus, :annexb) do 165 | Enum.join([<<>> | nalus], @annexb_prefix_code) 166 | end 167 | 168 | def prefix_nalus_payloads(nalus, {_codec_tag, nalu_length_size}) do 169 | Enum.map_join(nalus, fn nalu -> 170 | <> 171 | end) 172 | end 173 | end 174 | -------------------------------------------------------------------------------- /lib/membrane_h264_plugin/h26x/nalu_parser/scheme.ex: -------------------------------------------------------------------------------- 1 | defmodule Membrane.H26x.NALuParser.Scheme do 2 | @moduledoc false 3 | # A module defining the behaviour which should be implemented 4 | # by each NALu scheme. 5 | 6 | # A NALu scheme is defining the internal structure of the NALu 7 | # and describes the fields which need to by fetched from the binary. 8 | # The `Membrane.H26x.NALuParser.Scheme` behaviour defines a 9 | # callback: `scheme/0`, which returns the description of NALu structure. 10 | # The syntax which can be used to describe the NALu scheme is designed to 11 | # match the syntax forms used in the H26x specification documents. 12 | # The scheme is read by the parser in an imperative manner, line by line. 13 | # The following statements are available: 14 | # * field: reads the appropriate number of bits, decodes the integer based on 15 | # consumed bits and stores that integer under the given key in the parser's state map. 16 | # * if: introduces conditional parsing - if the condition is fulfilled, the 17 | # binary will be parsed with the scheme which is nested inside the statement. 18 | # * for: allows to parse the scheme nested in the statement the desired number of times. 19 | # * calculate: allows to add to the parser's state map the desired value, 20 | # under the given key. 21 | # * execute: allows to perform the custom actions with the payload and state, can be 22 | # used to process the payload in a way which couldn't be acheived with the scheme syntax. 23 | # * save_as_global_state_t: saves the current parser state, concerning the NALu which is 24 | # currently processed, in the map under the `:__global__` key in the state. Information 25 | # from the saved state can be used while parsing the following NALus. 26 | # 27 | # Furthermore, `Scheme` behavior defines `defaults/0` callback, 28 | # which is used to provide the default values of the fields. 29 | alias Membrane.H26x.NALuParser.SchemeParser 30 | 31 | @type field :: {any(), SchemeParser.field()} 32 | @type if_clause :: {SchemeParser.value_provider(boolean()), t()} 33 | @type for_loop :: 34 | {[ 35 | iterator: any(), 36 | from: SchemeParser.value_provider(integer()), 37 | to: SchemeParser.value_provider(integer()) 38 | ], t()} 39 | @type calculate :: {any(), SchemeParser.value_provider(any())} 40 | @type execute :: (binary(), SchemeParser.t(), list(integer()) -> {binary(), SchemeParser.t()}) 41 | @type save_as_global_state :: SchemeParser.value_provider(any()) 42 | 43 | @type directive :: 44 | {:field, field()} 45 | | {:if, if_clause()} 46 | | {:for, for_loop()} 47 | | {:calculate, calculate()} 48 | | {:execute, execute()} 49 | | {:save_as_global_state, save_as_global_state()} 50 | @type t :: list(directive()) 51 | 52 | @callback scheme() :: t() 53 | @callback defaults() :: keyword() 54 | end 55 | -------------------------------------------------------------------------------- /lib/membrane_h264_plugin/h26x/nalu_parser/scheme_parser.ex: -------------------------------------------------------------------------------- 1 | defmodule Membrane.H26x.NALuParser.SchemeParser do 2 | @moduledoc """ 3 | The module providing functions to parse the binary, 4 | based on the given Scheme. 5 | """ 6 | 7 | use Bunch.Access 8 | 9 | alias Membrane.H26x.ExpGolombConverter 10 | 11 | @typedoc """ 12 | A type defining the state of the scheme parser. 13 | 14 | The parser preserves its state in the map, which 15 | consists of two parts: 16 | * a map under the `:__global__` key - it contains information 17 | fetched from a NALu, which might be needed during the parsing 18 | of the following NALus. 19 | * a map under the `:__local__` key - it holds information valid 20 | during a time of a single NALu processing, and it's cleaned 21 | after the NALu is completely parsed. 22 | All information fetched from binary part is put into the 23 | `:__local__` map. If some information needs to be available when 24 | other binary part is parsed, it needs to be stored in the map under 25 | the `:__global__` key of the parser's state, which can be done i.e. 26 | with the `save_as_global_state` statements of the scheme syntax. 27 | """ 28 | @opaque t :: %__MODULE__{__global__: map(), __local__: map()} 29 | 30 | @enforce_keys [:__global__, :__local__] 31 | defstruct @enforce_keys 32 | 33 | @typedoc """ 34 | This type defines a value provider which provides values used in further 35 | processing of a parser. 36 | 37 | A value provider can be either a hardcoded value, known at the compilation 38 | time, or a tuple consisting of a lambda expression and the list of keys 39 | mapping to some values in the parser's state. If the value provider is a tuple, 40 | then it's first element - the lambda expression- is invoked with the arguments 41 | being the values of the fields which are available in the parser's state under 42 | the key names given in the parser's state, and the value used in the further 43 | processing is the value returned by that lambda expression. 44 | """ 45 | @type value_provider(return_type) :: return_type | {(... -> return_type), list(any())} 46 | 47 | @typedoc """ 48 | A type describing the field types which can be used 49 | in NALu scheme definition. 50 | 51 | Defined as in: *"7.2 Specification of syntax functions, categories, and descriptors"* 52 | of the *"ITU-T Rec. H.264 (01/2012)"*. 53 | """ 54 | @type field :: 55 | :u1 56 | | :u2 57 | | :u3 58 | | :u4 59 | | :u5 60 | | :u8 61 | | :u16 62 | | :u16 63 | | {:uv, value_provider(integer())} 64 | | :ue 65 | | :se 66 | 67 | @doc """ 68 | Returns a new `SchemeParser.State` struct instance. 69 | 70 | The new state's `local` state is clear. If the `State` is provided 71 | as an argument, the new state's `__global__` state is copied from 72 | the argument. Otherwise, it is set to the clear state. 73 | """ 74 | @spec new(t()) :: t() 75 | def new(old_state \\ %__MODULE__{__global__: %{}, __local__: %{}}) do 76 | %__MODULE__{__global__: old_state.__global__, __local__: %{}} 77 | end 78 | 79 | @doc """ 80 | Returns the local part of the state. 81 | """ 82 | @spec get_local_state(t()) :: map() 83 | def get_local_state(state) do 84 | state.__local__ 85 | end 86 | 87 | @doc """ 88 | Parses the binary stream representing a NALu, based 89 | on the scheme definition. 90 | 91 | Returns the remaining bitstring and the stated updated 92 | with the information fetched from the NALu. 93 | """ 94 | @spec parse_with_scheme(binary(), module(), t(), list(integer())) :: 95 | {map(), t()} 96 | def parse_with_scheme( 97 | payload, 98 | scheme_module, 99 | state \\ new(), 100 | iterators \\ [] 101 | ) do 102 | scheme = scheme_module.scheme() 103 | defaults_map = Map.new(scheme_module.defaults()) 104 | state = Map.update!(state, :__local__, &Map.merge(defaults_map, &1)) 105 | {_remaining_payload, state} = do_parse_with_scheme(payload, scheme, state, iterators) 106 | {get_local_state(state), state} 107 | end 108 | 109 | defp do_parse_with_scheme( 110 | payload, 111 | scheme, 112 | state, 113 | iterators 114 | ) do 115 | Enum.reduce(scheme, {payload, state}, fn {operator, arguments}, {payload, state} -> 116 | case {operator, arguments} do 117 | {:field, {name, type}} -> 118 | {field_value, payload} = parse_field(payload, state, type) 119 | 120 | {payload, 121 | insert_into_parser_state(state, field_value, [:__local__] ++ [name] ++ iterators)} 122 | 123 | {:if, {condition, scheme}} -> 124 | run_conditionally(payload, state, scheme, condition) 125 | 126 | {:for, {[iterator: iterator_name, from: min_value, to: max_value], scheme}} -> 127 | loop(payload, state, scheme, iterators, iterator_name, min_value, max_value) 128 | 129 | {:calculate, {name, to_calculate}} -> 130 | {function, args_list} = make_function(to_calculate) 131 | 132 | {payload, 133 | Bunch.Access.put_in( 134 | state, 135 | [:__local__, name], 136 | apply(function, get_args(args_list, state.__local__)) 137 | )} 138 | 139 | {:execute, function} -> 140 | function.(payload, state, iterators) 141 | 142 | {:save_state_as_global_state, key_generator} -> 143 | {key_generating_function, args_list} = make_function(key_generator) 144 | key = apply(key_generating_function, get_args(args_list, state.__local__)) 145 | 146 | {payload, Bunch.Access.put_in(state, [:__global__, key], state.__local__)} 147 | end 148 | end) 149 | end 150 | 151 | defp run_conditionally(payload, state, scheme, condition) do 152 | {condition_function, args_list} = make_function(condition) 153 | 154 | if apply(condition_function, get_args(args_list, state.__local__)), 155 | do: do_parse_with_scheme(payload, scheme, state, []), 156 | else: {payload, state} 157 | end 158 | 159 | defp loop(payload, state, scheme, previous_iterators, iterator_name, min_value, max_value) do 160 | {min_value, min_args_list} = make_function(min_value) 161 | {max_value, max_args_list} = make_function(max_value) 162 | 163 | {min_value, max_value} = { 164 | apply(min_value, get_args(min_args_list, state.__local__)), 165 | apply(max_value, get_args(max_args_list, state.__local__)) 166 | } 167 | 168 | {payload, state} = 169 | Enum.reduce( 170 | if(min_value > max_value, do: [], else: min_value..max_value), 171 | {payload, state}, 172 | fn iterator, {payload, state} -> 173 | state = Bunch.Access.put_in(state, [:__local__, iterator_name], iterator) 174 | 175 | do_parse_with_scheme( 176 | payload, 177 | scheme, 178 | state, 179 | previous_iterators ++ [iterator] 180 | ) 181 | end 182 | ) 183 | 184 | state = Bunch.Access.delete_in(state, [:__local__, iterator_name]) 185 | {payload, state} 186 | end 187 | 188 | defp get_args(args_names, state) do 189 | Enum.map(args_names, fn arg_name -> 190 | lexems = Regex.scan(~r"\@.*?\@", Atom.to_string(arg_name)) 191 | 192 | variables = 193 | lexems 194 | |> Enum.map(fn lexem -> 195 | variable_name = String.slice(lexem, 1..-2//-1) 196 | Map.get(state, variable_name) |> Integer.to_string() 197 | end) 198 | 199 | arg_name = Atom.to_string(arg_name) 200 | 201 | full_arg_name = 202 | Enum.zip(lexems, variables) 203 | |> Enum.reduce(arg_name, fn {lexem, variable}, arg_name -> 204 | String.replace(arg_name, lexem, variable) 205 | end) 206 | |> String.to_atom() 207 | 208 | Map.fetch!(state, full_arg_name) 209 | end) 210 | end 211 | 212 | defp parse_field(payload, state, type) do 213 | case type do 214 | {:uv, lambda, args} -> 215 | size = apply(lambda, get_args(args, state.__local__)) 216 | <> = payload 217 | {value, rest} 218 | 219 | :ue -> 220 | ExpGolombConverter.to_integer(payload) 221 | 222 | :se -> 223 | ExpGolombConverter.to_integer(payload, negatives: true) 224 | 225 | unsigned_int -> 226 | how_many_bits = 227 | Atom.to_string(unsigned_int) |> String.slice(1..-1//1) |> String.to_integer() 228 | 229 | <> = payload 230 | {value, rest} 231 | end 232 | end 233 | 234 | defp make_function({function, args}) when is_function(function), do: {function, args} 235 | defp make_function(value), do: {fn -> value end, []} 236 | 237 | defp insert_into_parser_state(state, value, iterators_list, already_consumed_iterators \\ []) 238 | 239 | defp insert_into_parser_state(state, value, [], already_consumed_iterators) do 240 | Bunch.Access.put_in(state, already_consumed_iterators, value) 241 | end 242 | 243 | defp insert_into_parser_state(state, value, iterators_list, already_consumed_iterators) do 244 | [first | rest] = iterators_list 245 | to_insert = Bunch.Access.get_in(state, already_consumed_iterators ++ [first]) 246 | to_insert = if to_insert == nil, do: %{}, else: to_insert 247 | state = Bunch.Access.put_in(state, already_consumed_iterators ++ [first], to_insert) 248 | insert_into_parser_state(state, value, rest, already_consumed_iterators ++ [first]) 249 | end 250 | end 251 | -------------------------------------------------------------------------------- /lib/membrane_h264_plugin/h26x/nalu_splitter.ex: -------------------------------------------------------------------------------- 1 | defmodule Membrane.H26x.NALuSplitter do 2 | @moduledoc """ 3 | A module with functions responsible for splitting the H26x streams into the NAL units. 4 | """ 5 | 6 | alias Membrane.H26x.Parser 7 | 8 | @typedoc """ 9 | A structure holding the state of the NALu splitter. 10 | """ 11 | @opaque t :: %__MODULE__{ 12 | input_stream_structure: Parser.stream_structure(), 13 | unparsed_payload: binary() 14 | } 15 | 16 | @enforce_keys [:input_stream_structure] 17 | defstruct @enforce_keys ++ [unparsed_payload: <<>>] 18 | 19 | @doc """ 20 | Returns a structure holding a NALu splitter state. 21 | 22 | The `input_stream_structure` determines which prefix is considered as delimiting two NALUs. 23 | By default, the inner `unparsed_payload` of the state is clean, but can be set to a given binary. 24 | """ 25 | @spec new(Membrane.H264.Parser.stream_structure(), initial_binary :: binary()) :: t() 26 | def new(input_stream_structure \\ :annexb, initial_binary \\ <<>>) do 27 | %__MODULE__{ 28 | input_stream_structure: input_stream_structure, 29 | unparsed_payload: initial_binary 30 | } 31 | end 32 | 33 | @doc """ 34 | Splits the binary into NALus sequence. 35 | 36 | Takes a binary H264 stream as an input 37 | and produces a list of binaries, where each binary is 38 | a complete NALu that can be passed to the `Membrane.H264.NALuParser.parse/4`. 39 | 40 | If `assume_nalu_aligned` flag is set to `true`, input is assumed to form a complete set 41 | of NAL units and therefore all of them are returned. Otherwise, the NALu is not returned 42 | until another NALu starts, as it's the only way to prove that the NALu is complete. 43 | """ 44 | @spec split(payload :: binary(), assume_nalu_aligned :: boolean, state :: t()) :: 45 | {[binary()], t()} 46 | def split(payload, assume_nalu_aligned \\ false, state) do 47 | total_payload = state.unparsed_payload <> payload 48 | 49 | nalus_payloads_list = get_complete_nalus_list(total_payload, state.input_stream_structure) 50 | 51 | total_nalus_payloads_size = IO.iodata_length(nalus_payloads_list) 52 | 53 | unparsed_payload = 54 | :binary.part( 55 | total_payload, 56 | total_nalus_payloads_size, 57 | byte_size(total_payload) - total_nalus_payloads_size 58 | ) 59 | 60 | cond do 61 | unparsed_payload == <<>> -> 62 | {nalus_payloads_list, %{state | unparsed_payload: <<>>}} 63 | 64 | assume_nalu_aligned -> 65 | {nalus_payloads_list ++ [unparsed_payload], %{state | unparsed_payload: <<>>}} 66 | 67 | true -> 68 | {nalus_payloads_list, %{state | unparsed_payload: unparsed_payload}} 69 | end 70 | end 71 | 72 | defp get_complete_nalus_list(payload, :annexb) do 73 | payload 74 | |> :binary.matches([<<0, 0, 0, 1>>, <<0, 0, 1>>]) 75 | |> Enum.chunk_every(2, 1, [{byte_size(payload), nil}]) 76 | |> then(&Enum.drop(&1, -1)) 77 | |> Enum.map(fn [{from, _from_prefix_len}, {to, _to_prefix_len}] -> 78 | len = to - from 79 | :binary.part(payload, from, len) 80 | end) 81 | end 82 | 83 | defp get_complete_nalus_list(payload, {_codec_tag, nalu_length_size}) 84 | when byte_size(payload) < nalu_length_size do 85 | [] 86 | end 87 | 88 | defp get_complete_nalus_list(payload, {codec_tag, nalu_length_size}) do 89 | <> = payload 90 | 91 | if nalu_length > byte_size(rest) do 92 | [] 93 | else 94 | <> = payload 95 | [nalu | get_complete_nalus_list(rest, {codec_tag, nalu_length_size})] 96 | end 97 | end 98 | end 99 | -------------------------------------------------------------------------------- /mix.exs: -------------------------------------------------------------------------------- 1 | defmodule Membrane.H26x.Plugin.Mixfile do 2 | use Mix.Project 3 | 4 | @version "0.10.5" 5 | @github_url "https://github.com/membraneframework/membrane_h26x_plugin" 6 | 7 | def project do 8 | [ 9 | app: :membrane_h26x_plugin, 10 | version: @version, 11 | elixir: "~> 1.13", 12 | elixirc_paths: elixirc_paths(Mix.env()), 13 | start_permanent: Mix.env() == :prod, 14 | deps: deps(), 15 | dialyzer: dialyzer(), 16 | 17 | # hex 18 | description: "Membrane H.264 and H.265 parser", 19 | package: package(), 20 | 21 | # docs 22 | name: "Membrane H.264 and H.265 plugin", 23 | source_url: @github_url, 24 | homepage_url: "https://membrane.stream", 25 | docs: docs() 26 | ] 27 | end 28 | 29 | def application do 30 | [ 31 | extra_applications: [] 32 | ] 33 | end 34 | 35 | defp elixirc_paths(:test), do: ["lib", "test/support"] 36 | defp elixirc_paths(_env), do: ["lib"] 37 | 38 | defp deps do 39 | [ 40 | {:membrane_core, "~> 1.0"}, 41 | {:membrane_h264_format, "~> 0.6.0"}, 42 | {:membrane_h265_format, "~> 0.2.0"}, 43 | {:bunch, "~> 1.4"}, 44 | {:membrane_stream_plugin, "~> 0.4.0", only: :test}, 45 | {:membrane_file_plugin, "~> 0.16.0", only: :test}, 46 | {:ex_doc, ">= 0.0.0", only: :dev, runtime: false}, 47 | {:dialyxir, ">= 0.0.0", only: :dev, runtime: false}, 48 | {:credo, ">= 0.0.0", only: :dev, runtime: false} 49 | ] 50 | end 51 | 52 | defp dialyzer() do 53 | opts = [ 54 | flags: [:error_handling] 55 | ] 56 | 57 | if System.get_env("CI") == "true" do 58 | # Store PLTs in cacheable directory for CI 59 | [plt_local_path: "priv/plts", plt_core_path: "priv/plts"] ++ opts 60 | else 61 | opts 62 | end 63 | end 64 | 65 | defp package do 66 | [ 67 | maintainers: ["Membrane Team"], 68 | licenses: ["Apache-2.0"], 69 | links: %{ 70 | "GitHub" => @github_url, 71 | "Membrane Framework Homepage" => "https://membrane.stream" 72 | } 73 | ] 74 | end 75 | 76 | defp docs do 77 | [ 78 | main: "readme", 79 | extras: ["README.md", "LICENSE"], 80 | formatters: ["html"], 81 | source_ref: "v#{@version}", 82 | nest_modules_by_prefix: [Membrane.H264, Membrane.H265, Membrane.H26x] 83 | ] 84 | end 85 | end 86 | -------------------------------------------------------------------------------- /mix.lock: -------------------------------------------------------------------------------- 1 | %{ 2 | "bunch": {:hex, :bunch, "1.6.1", "5393d827a64d5f846092703441ea50e65bc09f37fd8e320878f13e63d410aec7", [:mix], [], "hexpm", "286cc3add551628b30605efbe2fca4e38cc1bea89bcd0a1a7226920b3364fe4a"}, 3 | "bunt": {:hex, :bunt, "1.0.0", "081c2c665f086849e6d57900292b3a161727ab40431219529f13c4ddcf3e7a44", [:mix], [], "hexpm", "dc5f86aa08a5f6fa6b8096f0735c4e76d54ae5c9fa2c143e5a1fc7c1cd9bb6b5"}, 4 | "coerce": {:hex, :coerce, "1.0.1", "211c27386315dc2894ac11bc1f413a0e38505d808153367bd5c6e75a4003d096", [:mix], [], "hexpm", "b44a691700f7a1a15b4b7e2ff1fa30bebd669929ac8aa43cffe9e2f8bf051cf1"}, 5 | "credo": {:hex, :credo, "1.7.12", "9e3c20463de4b5f3f23721527fcaf16722ec815e70ff6c60b86412c695d426c1", [:mix], [{:bunt, "~> 0.2.1 or ~> 1.0", [hex: :bunt, repo: "hexpm", optional: false]}, {:file_system, "~> 0.2 or ~> 1.0", [hex: :file_system, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "8493d45c656c5427d9c729235b99d498bd133421f3e0a683e5c1b561471291e5"}, 6 | "dialyxir": {:hex, :dialyxir, "1.4.5", "ca1571ac18e0f88d4ab245f0b60fa31ff1b12cbae2b11bd25d207f865e8ae78a", [:mix], [{:erlex, ">= 0.2.7", [hex: :erlex, repo: "hexpm", optional: false]}], "hexpm", "b0fb08bb8107c750db5c0b324fa2df5ceaa0f9307690ee3c1f6ba5b9eb5d35c3"}, 7 | "earmark_parser": {:hex, :earmark_parser, "1.4.44", "f20830dd6b5c77afe2b063777ddbbff09f9759396500cdbe7523efd58d7a339c", [:mix], [], "hexpm", "4778ac752b4701a5599215f7030989c989ffdc4f6df457c5f36938cc2d2a2750"}, 8 | "erlex": {:hex, :erlex, "0.2.7", "810e8725f96ab74d17aac676e748627a07bc87eb950d2b83acd29dc047a30595", [:mix], [], "hexpm", "3ed95f79d1a844c3f6bf0cea61e0d5612a42ce56da9c03f01df538685365efb0"}, 9 | "ex_doc": {:hex, :ex_doc, "0.38.1", "bae0a0bd5b5925b1caef4987e3470902d072d03347114ffe03a55dbe206dd4c2", [:mix], [{:earmark_parser, "~> 1.4.44", [hex: :earmark_parser, repo: "hexpm", optional: false]}, {:makeup_c, ">= 0.1.0", [hex: :makeup_c, repo: "hexpm", optional: true]}, {:makeup_elixir, "~> 0.14 or ~> 1.0", [hex: :makeup_elixir, repo: "hexpm", optional: false]}, {:makeup_erlang, "~> 0.1 or ~> 1.0", [hex: :makeup_erlang, repo: "hexpm", optional: false]}, {:makeup_html, ">= 0.1.0", [hex: :makeup_html, repo: "hexpm", optional: true]}], "hexpm", "754636236d191b895e1e4de2ebb504c057fe1995fdfdd92e9d75c4b05633008b"}, 10 | "file_system": {:hex, :file_system, "1.1.0", "08d232062284546c6c34426997dd7ef6ec9f8bbd090eb91780283c9016840e8f", [:mix], [], "hexpm", "bfcf81244f416871f2a2e15c1b515287faa5db9c6bcf290222206d120b3d43f6"}, 11 | "jason": {:hex, :jason, "1.4.4", "b9226785a9aa77b6857ca22832cffa5d5011a667207eb2a0ad56adb5db443b8a", [:mix], [{:decimal, "~> 1.0 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "c5eb0cab91f094599f94d55bc63409236a8ec69a21a67814529e8d5f6cc90b3b"}, 12 | "makeup": {:hex, :makeup, "1.2.1", "e90ac1c65589ef354378def3ba19d401e739ee7ee06fb47f94c687016e3713d1", [:mix], [{:nimble_parsec, "~> 1.4", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "d36484867b0bae0fea568d10131197a4c2e47056a6fbe84922bf6ba71c8d17ce"}, 13 | "makeup_elixir": {:hex, :makeup_elixir, "1.0.1", "e928a4f984e795e41e3abd27bfc09f51db16ab8ba1aebdba2b3a575437efafc2", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}, {:nimble_parsec, "~> 1.2.3 or ~> 1.3", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "7284900d412a3e5cfd97fdaed4f5ed389b8f2b4cb49efc0eb3bd10e2febf9507"}, 14 | "makeup_erlang": {:hex, :makeup_erlang, "1.0.2", "03e1804074b3aa64d5fad7aa64601ed0fb395337b982d9bcf04029d68d51b6a7", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}], "hexpm", "af33ff7ef368d5893e4a267933e7744e46ce3cf1f61e2dccf53a111ed3aa3727"}, 15 | "membrane_core": {:hex, :membrane_core, "1.2.3", "0e23f50b2e7dfe95dd6047cc341807991f9d0349cd98455cc5cbfab41ba5233c", [:mix], [{:bunch, "~> 1.6", [hex: :bunch, repo: "hexpm", optional: false]}, {:qex, "~> 0.3", [hex: :qex, repo: "hexpm", optional: false]}, {:ratio, "~> 3.0 or ~> 4.0", [hex: :ratio, repo: "hexpm", optional: false]}, {:telemetry, "~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "6e3099dcb52d136a4aef84b6fbb20905ea55d9f0d2d6726f7b589e8d169a55cd"}, 16 | "membrane_file_plugin": {:hex, :membrane_file_plugin, "0.16.0", "7917f6682c22b9bcfc2ca20ed960eee0f7d03ad31fd5f59ed850f1fe3ddd545a", [:mix], [{:membrane_core, "~> 1.0", [hex: :membrane_core, repo: "hexpm", optional: false]}], "hexpm", "b0727998f75a9b4dab8a2baefdfc13c3eac00a04e061ab1b0e61dc5566927acc"}, 17 | "membrane_h264_format": {:hex, :membrane_h264_format, "0.6.1", "44836cd9de0abe989b146df1e114507787efc0cf0da2368f17a10c47b4e0738c", [:mix], [], "hexpm", "4b79be56465a876d2eac2c3af99e115374bbdc03eb1dea4f696ee9a8033cd4b0"}, 18 | "membrane_h265_format": {:hex, :membrane_h265_format, "0.2.0", "1903c072cf7b0980c4d0c117ab61a2cd33e88782b696290de29570a7fab34819", [:mix], [], "hexpm", "6df418bdf242c0d9f7dbf2e5aea4c2d182e34ac9ad5a8b8cef2610c290002e83"}, 19 | "membrane_stream_plugin": {:hex, :membrane_stream_plugin, "0.4.0", "0c4ab72a4e13bf0faa0f1166fbaf68d2e34167dbec345aedb74ce1eb7497bdda", [:mix], [{:membrane_core, "~> 1.0", [hex: :membrane_core, repo: "hexpm", optional: false]}], "hexpm", "5a9a9c17783e18ad740e6ddfed364581bdb7ebdab8e61ba2c19a1830356f7eb8"}, 20 | "nimble_parsec": {:hex, :nimble_parsec, "1.4.2", "8efba0122db06df95bfaa78f791344a89352ba04baedd3849593bfce4d0dc1c6", [:mix], [], "hexpm", "4b21398942dda052b403bbe1da991ccd03a053668d147d53fb8c4e0efe09c973"}, 21 | "numbers": {:hex, :numbers, "5.2.4", "f123d5bb7f6acc366f8f445e10a32bd403c8469bdbce8ce049e1f0972b607080", [:mix], [{:coerce, "~> 1.0", [hex: :coerce, repo: "hexpm", optional: false]}, {:decimal, "~> 1.9 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "eeccf5c61d5f4922198395bf87a465b6f980b8b862dd22d28198c5e6fab38582"}, 22 | "qex": {:hex, :qex, "0.5.1", "0d82c0f008551d24fffb99d97f8299afcb8ea9cf99582b770bd004ed5af63fd6", [:mix], [], "hexpm", "935a39fdaf2445834b95951456559e9dc2063d0a055742c558a99987b38d6bab"}, 23 | "ratio": {:hex, :ratio, "4.0.1", "3044166f2fc6890aa53d3aef0c336f84b2bebb889dc57d5f95cc540daa1912f8", [:mix], [{:decimal, "~> 1.6 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}, {:numbers, "~> 5.2.0", [hex: :numbers, repo: "hexpm", optional: false]}], "hexpm", "c60cbb3ccdff9ffa56e7d6d1654b5c70d9f90f4d753ab3a43a6bf40855b881ce"}, 24 | "telemetry": {:hex, :telemetry, "1.3.0", "fedebbae410d715cf8e7062c96a1ef32ec22e764197f70cda73d82778d61e7a2", [:rebar3], [], "hexpm", "7015fc8919dbe63764f4b4b87a95b7c0996bd539e0d499be6ec9d7f3875b79e6"}, 25 | } 26 | -------------------------------------------------------------------------------- /test/exp_golomb_test.exs: -------------------------------------------------------------------------------- 1 | defmodule ExpGolombTest do 2 | @moduledoc false 3 | 4 | use ExUnit.Case, async: true 5 | alias Membrane.H26x.ExpGolombConverter 6 | @integers [0, 1, -3, 5, 12, 51, -13_413, 25_542, 2137] 7 | 8 | test "if the decoding the encoded integer results in the original integer" do 9 | for integer <- @integers do 10 | negatives = integer < 0 11 | 12 | assert integer == 13 | integer 14 | |> ExpGolombConverter.to_bitstring(negatives: negatives) 15 | |> ExpGolombConverter.to_integer(negatives: negatives) 16 | |> elem(0) 17 | end 18 | end 19 | end 20 | -------------------------------------------------------------------------------- /test/fixtures/h264/input-10-320x180.h264: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/membraneframework/membrane_h26x_plugin/9ebc5a32c10a29dd56f7bbc0d839b0d37bd68632/test/fixtures/h264/input-10-320x180.h264 -------------------------------------------------------------------------------- /test/fixtures/h264/input-10-720a.h264: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/membraneframework/membrane_h26x_plugin/9ebc5a32c10a29dd56f7bbc0d839b0d37bd68632/test/fixtures/h264/input-10-720a.h264 -------------------------------------------------------------------------------- /test/fixtures/h264/input-10-720p-baseline.h264: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/membraneframework/membrane_h26x_plugin/9ebc5a32c10a29dd56f7bbc0d839b0d37bd68632/test/fixtures/h264/input-10-720p-baseline.h264 -------------------------------------------------------------------------------- /test/fixtures/h264/input-10-720p-main.h264: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/membraneframework/membrane_h26x_plugin/9ebc5a32c10a29dd56f7bbc0d839b0d37bd68632/test/fixtures/h264/input-10-720p-main.h264 -------------------------------------------------------------------------------- /test/fixtures/h264/input-10-720p-no-b-frames.h264: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/membraneframework/membrane_h26x_plugin/9ebc5a32c10a29dd56f7bbc0d839b0d37bd68632/test/fixtures/h264/input-10-720p-no-b-frames.h264 -------------------------------------------------------------------------------- /test/fixtures/h264/input-10-720p.h264: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/membraneframework/membrane_h26x_plugin/9ebc5a32c10a29dd56f7bbc0d839b0d37bd68632/test/fixtures/h264/input-10-720p.h264 -------------------------------------------------------------------------------- /test/fixtures/h264/input-10-no-pps-sps.h264: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/membraneframework/membrane_h26x_plugin/9ebc5a32c10a29dd56f7bbc0d839b0d37bd68632/test/fixtures/h264/input-10-no-pps-sps.h264 -------------------------------------------------------------------------------- /test/fixtures/h264/input-100-240p-no-b-frames.h264: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/membraneframework/membrane_h26x_plugin/9ebc5a32c10a29dd56f7bbc0d839b0d37bd68632/test/fixtures/h264/input-100-240p-no-b-frames.h264 -------------------------------------------------------------------------------- /test/fixtures/h264/input-100-240p.h264: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/membraneframework/membrane_h26x_plugin/9ebc5a32c10a29dd56f7bbc0d839b0d37bd68632/test/fixtures/h264/input-100-240p.h264 -------------------------------------------------------------------------------- /test/fixtures/h264/input-20-360p-I422.h264: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/membraneframework/membrane_h26x_plugin/9ebc5a32c10a29dd56f7bbc0d839b0d37bd68632/test/fixtures/h264/input-20-360p-I422.h264 -------------------------------------------------------------------------------- /test/fixtures/h264/input-30-240p-no-sps-pps.h264: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/membraneframework/membrane_h26x_plugin/9ebc5a32c10a29dd56f7bbc0d839b0d37bd68632/test/fixtures/h264/input-30-240p-no-sps-pps.h264 -------------------------------------------------------------------------------- /test/fixtures/h264/input-30-240p-vp-sps-pps.h264: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/membraneframework/membrane_h26x_plugin/9ebc5a32c10a29dd56f7bbc0d839b0d37bd68632/test/fixtures/h264/input-30-240p-vp-sps-pps.h264 -------------------------------------------------------------------------------- /test/fixtures/h264/input-idr-sps-pps-non-idr.h264: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/membraneframework/membrane_h26x_plugin/9ebc5a32c10a29dd56f7bbc0d839b0d37bd68632/test/fixtures/h264/input-idr-sps-pps-non-idr.h264 -------------------------------------------------------------------------------- /test/fixtures/h264/input-sps-pps-non-idr-sps-pps-idr.h264: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/membraneframework/membrane_h26x_plugin/9ebc5a32c10a29dd56f7bbc0d839b0d37bd68632/test/fixtures/h264/input-sps-pps-non-idr-sps-pps-idr.h264 -------------------------------------------------------------------------------- /test/fixtures/h264/input-sps-pps-non-idr.h264: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/membraneframework/membrane_h26x_plugin/9ebc5a32c10a29dd56f7bbc0d839b0d37bd68632/test/fixtures/h264/input-sps-pps-non-idr.h264 -------------------------------------------------------------------------------- /test/fixtures/h264/mp4/ref_video.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/membraneframework/membrane_h26x_plugin/9ebc5a32c10a29dd56f7bbc0d839b0d37bd68632/test/fixtures/h264/mp4/ref_video.mp4 -------------------------------------------------------------------------------- /test/fixtures/h264/mp4/ref_video_fast_start.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/membraneframework/membrane_h26x_plugin/9ebc5a32c10a29dd56f7bbc0d839b0d37bd68632/test/fixtures/h264/mp4/ref_video_fast_start.mp4 -------------------------------------------------------------------------------- /test/fixtures/h264/mp4/ref_video_variable_parameters.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/membraneframework/membrane_h26x_plugin/9ebc5a32c10a29dd56f7bbc0d839b0d37bd68632/test/fixtures/h264/mp4/ref_video_variable_parameters.mp4 -------------------------------------------------------------------------------- /test/fixtures/h264/msr/ref_video-avc1-au.msr: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/membraneframework/membrane_h26x_plugin/9ebc5a32c10a29dd56f7bbc0d839b0d37bd68632/test/fixtures/h264/msr/ref_video-avc1-au.msr -------------------------------------------------------------------------------- /test/fixtures/h264/msr/ref_video-avc1-nalu.msr: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/membraneframework/membrane_h26x_plugin/9ebc5a32c10a29dd56f7bbc0d839b0d37bd68632/test/fixtures/h264/msr/ref_video-avc1-nalu.msr -------------------------------------------------------------------------------- /test/fixtures/h264/msr/ref_video-avc3-au.msr: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/membraneframework/membrane_h26x_plugin/9ebc5a32c10a29dd56f7bbc0d839b0d37bd68632/test/fixtures/h264/msr/ref_video-avc3-au.msr -------------------------------------------------------------------------------- /test/fixtures/h264/msr/ref_video-avc3-nalu.msr: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/membraneframework/membrane_h26x_plugin/9ebc5a32c10a29dd56f7bbc0d839b0d37bd68632/test/fixtures/h264/msr/ref_video-avc3-nalu.msr -------------------------------------------------------------------------------- /test/fixtures/h264/msr/ref_video_fast_start-avc1-au.msr: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/membraneframework/membrane_h26x_plugin/9ebc5a32c10a29dd56f7bbc0d839b0d37bd68632/test/fixtures/h264/msr/ref_video_fast_start-avc1-au.msr -------------------------------------------------------------------------------- /test/fixtures/h264/msr/ref_video_fast_start-avc1-nalu.msr: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/membraneframework/membrane_h26x_plugin/9ebc5a32c10a29dd56f7bbc0d839b0d37bd68632/test/fixtures/h264/msr/ref_video_fast_start-avc1-nalu.msr -------------------------------------------------------------------------------- /test/fixtures/h264/msr/ref_video_fast_start-avc3-au.msr: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/membraneframework/membrane_h26x_plugin/9ebc5a32c10a29dd56f7bbc0d839b0d37bd68632/test/fixtures/h264/msr/ref_video_fast_start-avc3-au.msr -------------------------------------------------------------------------------- /test/fixtures/h264/msr/ref_video_fast_start-avc3-nalu.msr: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/membraneframework/membrane_h26x_plugin/9ebc5a32c10a29dd56f7bbc0d839b0d37bd68632/test/fixtures/h264/msr/ref_video_fast_start-avc3-nalu.msr -------------------------------------------------------------------------------- /test/fixtures/h264/msr/ref_video_variable_parameters-avc3-au.msr: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/membraneframework/membrane_h26x_plugin/9ebc5a32c10a29dd56f7bbc0d839b0d37bd68632/test/fixtures/h264/msr/ref_video_variable_parameters-avc3-au.msr -------------------------------------------------------------------------------- /test/fixtures/h264/msr/ref_video_variable_parameters-avc3-nalu.msr: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/membraneframework/membrane_h26x_plugin/9ebc5a32c10a29dd56f7bbc0d839b0d37bd68632/test/fixtures/h264/msr/ref_video_variable_parameters-avc3-nalu.msr -------------------------------------------------------------------------------- /test/fixtures/h264/reference-30-240p-vp-sps-pps.h264: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/membraneframework/membrane_h26x_plugin/9ebc5a32c10a29dd56f7bbc0d839b0d37bd68632/test/fixtures/h264/reference-30-240p-vp-sps-pps.h264 -------------------------------------------------------------------------------- /test/fixtures/h264/reference-30-240p-with-sps-pps.h264: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/membraneframework/membrane_h26x_plugin/9ebc5a32c10a29dd56f7bbc0d839b0d37bd68632/test/fixtures/h264/reference-30-240p-with-sps-pps.h264 -------------------------------------------------------------------------------- /test/fixtures/h264/reference-idr-sps-pps-non-idr.h264: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/membraneframework/membrane_h26x_plugin/9ebc5a32c10a29dd56f7bbc0d839b0d37bd68632/test/fixtures/h264/reference-idr-sps-pps-non-idr.h264 -------------------------------------------------------------------------------- /test/fixtures/h264/reference-sps-pps-non-idr-sps-pps-idr.h264: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/membraneframework/membrane_h26x_plugin/9ebc5a32c10a29dd56f7bbc0d839b0d37bd68632/test/fixtures/h264/reference-sps-pps-non-idr-sps-pps-idr.h264 -------------------------------------------------------------------------------- /test/fixtures/h264/reference-sps-pps-non-idr.h264: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/membraneframework/membrane_h26x_plugin/9ebc5a32c10a29dd56f7bbc0d839b0d37bd68632/test/fixtures/h264/reference-sps-pps-non-idr.h264 -------------------------------------------------------------------------------- /test/fixtures/h265/input-10-1920x1080.h265: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/membraneframework/membrane_h26x_plugin/9ebc5a32c10a29dd56f7bbc0d839b0d37bd68632/test/fixtures/h265/input-10-1920x1080.h265 -------------------------------------------------------------------------------- /test/fixtures/h265/input-10-480x320-mainstillpicture.h265: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/membraneframework/membrane_h26x_plugin/9ebc5a32c10a29dd56f7bbc0d839b0d37bd68632/test/fixtures/h265/input-10-480x320-mainstillpicture.h265 -------------------------------------------------------------------------------- /test/fixtures/h265/input-10-640x480-main10.h265: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/membraneframework/membrane_h26x_plugin/9ebc5a32c10a29dd56f7bbc0d839b0d37bd68632/test/fixtures/h265/input-10-640x480-main10.h265 -------------------------------------------------------------------------------- /test/fixtures/h265/input-10-no-vps-sps-pps.h265: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/membraneframework/membrane_h26x_plugin/9ebc5a32c10a29dd56f7bbc0d839b0d37bd68632/test/fixtures/h265/input-10-no-vps-sps-pps.h265 -------------------------------------------------------------------------------- /test/fixtures/h265/input-15-1280x720-temporal-id-1.h265: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/membraneframework/membrane_h26x_plugin/9ebc5a32c10a29dd56f7bbc0d839b0d37bd68632/test/fixtures/h265/input-15-1280x720-temporal-id-1.h265 -------------------------------------------------------------------------------- /test/fixtures/h265/input-30-1280x720-rext.h265: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/membraneframework/membrane_h26x_plugin/9ebc5a32c10a29dd56f7bbc0d839b0d37bd68632/test/fixtures/h265/input-30-1280x720-rext.h265 -------------------------------------------------------------------------------- /test/fixtures/h265/input-30-640x480-no-bframes.h265: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/membraneframework/membrane_h26x_plugin/9ebc5a32c10a29dd56f7bbc0d839b0d37bd68632/test/fixtures/h265/input-30-640x480-no-bframes.h265 -------------------------------------------------------------------------------- /test/fixtures/h265/input-300-98x58-conformance-window.h265: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/membraneframework/membrane_h26x_plugin/9ebc5a32c10a29dd56f7bbc0d839b0d37bd68632/test/fixtures/h265/input-300-98x58-conformance-window.h265 -------------------------------------------------------------------------------- /test/fixtures/h265/input-32-640x360-main.h265: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/membraneframework/membrane_h26x_plugin/9ebc5a32c10a29dd56f7bbc0d839b0d37bd68632/test/fixtures/h265/input-32-640x360-main.h265 -------------------------------------------------------------------------------- /test/fixtures/h265/input-60-1920x1080.h265: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/membraneframework/membrane_h26x_plugin/9ebc5a32c10a29dd56f7bbc0d839b0d37bd68632/test/fixtures/h265/input-60-1920x1080.h265 -------------------------------------------------------------------------------- /test/fixtures/h265/input-60-640x480-no-parameter-sets.h265: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/membraneframework/membrane_h26x_plugin/9ebc5a32c10a29dd56f7bbc0d839b0d37bd68632/test/fixtures/h265/input-60-640x480-no-parameter-sets.h265 -------------------------------------------------------------------------------- /test/fixtures/h265/input-60-640x480-variable-parameters.h265: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/membraneframework/membrane_h26x_plugin/9ebc5a32c10a29dd56f7bbc0d839b0d37bd68632/test/fixtures/h265/input-60-640x480-variable-parameters.h265 -------------------------------------------------------------------------------- /test/fixtures/h265/input-60-640x480.h265: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/membraneframework/membrane_h26x_plugin/9ebc5a32c10a29dd56f7bbc0d839b0d37bd68632/test/fixtures/h265/input-60-640x480.h265 -------------------------------------------------------------------------------- /test/fixtures/h265/input-8-2K.h265: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/membraneframework/membrane_h26x_plugin/9ebc5a32c10a29dd56f7bbc0d839b0d37bd68632/test/fixtures/h265/input-8-2K.h265 -------------------------------------------------------------------------------- /test/fixtures/h265/input-irap-pss-no-irap.h265: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/membraneframework/membrane_h26x_plugin/9ebc5a32c10a29dd56f7bbc0d839b0d37bd68632/test/fixtures/h265/input-irap-pss-no-irap.h265 -------------------------------------------------------------------------------- /test/fixtures/h265/input-pss-no-irap-pss-irap.h265: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/membraneframework/membrane_h26x_plugin/9ebc5a32c10a29dd56f7bbc0d839b0d37bd68632/test/fixtures/h265/input-pss-no-irap-pss-irap.h265 -------------------------------------------------------------------------------- /test/fixtures/h265/mp4/ref_video.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/membraneframework/membrane_h26x_plugin/9ebc5a32c10a29dd56f7bbc0d839b0d37bd68632/test/fixtures/h265/mp4/ref_video.mp4 -------------------------------------------------------------------------------- /test/fixtures/h265/mp4/ref_video_variable_parameters.mp4: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/membraneframework/membrane_h26x_plugin/9ebc5a32c10a29dd56f7bbc0d839b0d37bd68632/test/fixtures/h265/mp4/ref_video_variable_parameters.mp4 -------------------------------------------------------------------------------- /test/fixtures/h265/msr/ref_video-hev1-au.msr: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/membraneframework/membrane_h26x_plugin/9ebc5a32c10a29dd56f7bbc0d839b0d37bd68632/test/fixtures/h265/msr/ref_video-hev1-au.msr -------------------------------------------------------------------------------- /test/fixtures/h265/msr/ref_video-hev1-nalu.msr: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/membraneframework/membrane_h26x_plugin/9ebc5a32c10a29dd56f7bbc0d839b0d37bd68632/test/fixtures/h265/msr/ref_video-hev1-nalu.msr -------------------------------------------------------------------------------- /test/fixtures/h265/msr/ref_video-hvc1-au.msr: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/membraneframework/membrane_h26x_plugin/9ebc5a32c10a29dd56f7bbc0d839b0d37bd68632/test/fixtures/h265/msr/ref_video-hvc1-au.msr -------------------------------------------------------------------------------- /test/fixtures/h265/msr/ref_video-hvc1-nalu.msr: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/membraneframework/membrane_h26x_plugin/9ebc5a32c10a29dd56f7bbc0d839b0d37bd68632/test/fixtures/h265/msr/ref_video-hvc1-nalu.msr -------------------------------------------------------------------------------- /test/fixtures/h265/msr/ref_video_variable_parameters-hev1-au.msr: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/membraneframework/membrane_h26x_plugin/9ebc5a32c10a29dd56f7bbc0d839b0d37bd68632/test/fixtures/h265/msr/ref_video_variable_parameters-hev1-au.msr -------------------------------------------------------------------------------- /test/fixtures/h265/msr/ref_video_variable_parameters-hev1-nalu.msr: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/membraneframework/membrane_h26x_plugin/9ebc5a32c10a29dd56f7bbc0d839b0d37bd68632/test/fixtures/h265/msr/ref_video_variable_parameters-hev1-nalu.msr -------------------------------------------------------------------------------- /test/fixtures/h265/reference-60-640x480-variable-parameters.h265: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/membraneframework/membrane_h26x_plugin/9ebc5a32c10a29dd56f7bbc0d839b0d37bd68632/test/fixtures/h265/reference-60-640x480-variable-parameters.h265 -------------------------------------------------------------------------------- /test/fixtures/h265/reference-60-640x480-with-parameter-sets.h265: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/membraneframework/membrane_h26x_plugin/9ebc5a32c10a29dd56f7bbc0d839b0d37bd68632/test/fixtures/h265/reference-60-640x480-with-parameter-sets.h265 -------------------------------------------------------------------------------- /test/fixtures/h265/reference-irap-pss-no-irap.h265: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/membraneframework/membrane_h26x_plugin/9ebc5a32c10a29dd56f7bbc0d839b0d37bd68632/test/fixtures/h265/reference-irap-pss-no-irap.h265 -------------------------------------------------------------------------------- /test/fixtures/h265/reference-pss-no-irap-pss-irap.h265: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/membraneframework/membrane_h26x_plugin/9ebc5a32c10a29dd56f7bbc0d839b0d37bd68632/test/fixtures/h265/reference-pss-no-irap-pss-irap.h265 -------------------------------------------------------------------------------- /test/integration/h264/modes_test.exs: -------------------------------------------------------------------------------- 1 | defmodule Membrane.H264.ModesTest do 2 | use ExUnit.Case, async: true 3 | 4 | import Membrane.ChildrenSpec 5 | import Membrane.Testing.Assertions 6 | import Membrane.H26x.Support.Common 7 | 8 | alias Membrane.Buffer 9 | alias Membrane.H264.Parser 10 | alias Membrane.H26x.Support.TestSource 11 | alias Membrane.Testing.{Pipeline, Sink} 12 | 13 | @h264_input_file "test/fixtures/h264/input-10-720p.h264" 14 | 15 | test "if the pts and dts are set to nil in :bytestream mode" do 16 | binary = File.read!(@h264_input_file) 17 | mode = :bytestream 18 | input_buffers = prepare_h264_buffers(binary, mode) 19 | 20 | {:ok, _supervisor_pid, pid} = 21 | Pipeline.start_supervised( 22 | spec: [ 23 | child(:source, %TestSource{mode: mode}) 24 | |> child(:parser, Parser) 25 | |> child(:sink, Sink) 26 | ] 27 | ) 28 | 29 | assert_sink_playing(pid, :sink) 30 | send_buffers_actions = for buffer <- input_buffers, do: {:buffer, {:output, buffer}} 31 | Pipeline.notify_child(pid, :source, send_buffers_actions ++ [end_of_stream: :output]) 32 | 33 | output_buffers = prepare_h264_buffers(binary, :au_aligned) 34 | 35 | Enum.each(output_buffers, fn buf -> 36 | payload = buf.payload 37 | assert_sink_buffer(pid, :sink, %Buffer{payload: ^payload, pts: nil, dts: nil}) 38 | end) 39 | 40 | Pipeline.terminate(pid) 41 | end 42 | 43 | test "if the pts and dts are rewritten properly in :nalu_aligned mode" do 44 | binary = File.read!(@h264_input_file) 45 | mode = :nalu_aligned 46 | input_buffers = prepare_h264_buffers(binary, mode) 47 | 48 | {:ok, _supervisor_pid, pid} = 49 | Pipeline.start_supervised( 50 | spec: [ 51 | child(:source, %TestSource{mode: mode}) 52 | |> child(:parser, Parser) 53 | |> child(:sink, Sink) 54 | ] 55 | ) 56 | 57 | assert_sink_playing(pid, :sink) 58 | send_buffers_actions = for buffer <- input_buffers, do: {:buffer, {:output, buffer}} 59 | Pipeline.notify_child(pid, :source, send_buffers_actions ++ [end_of_stream: :output]) 60 | 61 | output_buffers = prepare_h264_buffers(binary, :au_aligned) 62 | 63 | Enum.each(output_buffers, fn buf -> 64 | payload = buf.payload 65 | pts = buf.pts 66 | dts = buf.dts 67 | assert_sink_buffer(pid, :sink, %Buffer{payload: ^payload, pts: ^pts, dts: ^dts}) 68 | end) 69 | 70 | Pipeline.terminate(pid) 71 | end 72 | 73 | test "if the pts and dts are rewritten properly in :au_aligned mode" do 74 | binary = File.read!(@h264_input_file) 75 | mode = :au_aligned 76 | input_buffers = prepare_h264_buffers(binary, mode) 77 | 78 | {:ok, _supervisor_pid, pid} = 79 | Pipeline.start_supervised( 80 | spec: [ 81 | child(:source, %TestSource{mode: mode}) 82 | |> child(:parser, Parser) 83 | |> child(:sink, Sink) 84 | ] 85 | ) 86 | 87 | assert_sink_playing(pid, :sink) 88 | send_buffers_actions = for buffer <- input_buffers, do: {:buffer, {:output, buffer}} 89 | Pipeline.notify_child(pid, :source, send_buffers_actions ++ [end_of_stream: :output]) 90 | 91 | output_buffers = input_buffers 92 | 93 | Enum.each(output_buffers, fn buf -> 94 | assert_sink_buffer(pid, :sink, %Buffer{payload: payload, pts: pts, dts: dts}) 95 | assert payload == buf.payload 96 | assert pts == buf.pts 97 | assert dts == buf.dts 98 | end) 99 | 100 | Pipeline.terminate(pid) 101 | end 102 | 103 | test "if single NAL unit is sent per buffer with `output_alignment: :nalu`" do 104 | {:ok, _supervisor_pid, pid} = 105 | Pipeline.start_supervised( 106 | spec: [ 107 | child(:source, %Membrane.File.Source{location: @h264_input_file}) 108 | |> child(:parser, %Parser{output_alignment: :nalu}) 109 | |> child(:sink, Sink) 110 | ] 111 | ) 112 | 113 | assert_sink_playing(pid, :sink) 114 | assert_sink_stream_format(pid, :sink, %Membrane.H264{alignment: :nalu}) 115 | 116 | binary = File.read!(@h264_input_file) 117 | ref_buffers = prepare_h264_buffers(binary, :nalu_aligned) 118 | 119 | Enum.each(ref_buffers, fn ref_buffer -> 120 | assert_sink_buffer(pid, :sink, buffer) 121 | assert buffer.payload == ref_buffer.payload 122 | assert Map.has_key?(buffer.metadata, :h264) and Map.has_key?(buffer.metadata.h264, :type) 123 | end) 124 | 125 | assert_end_of_stream(pid, :sink) 126 | Pipeline.terminate(pid) 127 | end 128 | end 129 | -------------------------------------------------------------------------------- /test/integration/h264/timestamp_generation_test.exs: -------------------------------------------------------------------------------- 1 | defmodule Membrane.H264.TimestampGenerationTest do 2 | @moduledoc false 3 | 4 | use ExUnit.Case, async: true 5 | 6 | import Membrane.ChildrenSpec 7 | import Membrane.Testing.Assertions 8 | import Membrane.H26x.Support.Common 9 | 10 | alias Membrane.Buffer 11 | alias Membrane.H264.Parser 12 | alias Membrane.H26x.Support.TestSource 13 | alias Membrane.Testing.{Pipeline, Sink} 14 | 15 | @h264_input_file_main "test/fixtures/h264/input-10-720p.h264" 16 | @h264_input_timestamps_main [ 17 | {0, -500}, 18 | {133, -467}, 19 | {67, -433}, 20 | {33, -400}, 21 | {100, -367}, 22 | {267, -333}, 23 | {200, -300}, 24 | {167, -267}, 25 | {233, -233}, 26 | {300, -200} 27 | ] 28 | @h264_input_file_baseline "test/fixtures/h264/input-10-720p-baseline.h264" 29 | @h264_input_timestamps_baseline [0, 33, 67, 100, 133, 167, 200, 233, 267, 300] 30 | |> Enum.map(&{&1, &1 - 500}) 31 | 32 | test "if the pts and dts are set to nil in :bytestream mode when framerate isn't given" do 33 | binary = File.read!(@h264_input_file_baseline) 34 | mode = :bytestream 35 | input_buffers = prepare_h264_buffers(binary, mode) 36 | 37 | {:ok, _supervisor_pid, pid} = 38 | Pipeline.start_supervised( 39 | spec: [ 40 | child(:source, %TestSource{mode: mode}) 41 | |> child(:parser, Parser) 42 | |> child(:sink, Sink) 43 | ] 44 | ) 45 | 46 | assert_sink_playing(pid, :sink) 47 | send_buffers_actions = for buffer <- input_buffers, do: {:buffer, {:output, buffer}} 48 | Pipeline.notify_child(pid, :source, send_buffers_actions ++ [end_of_stream: :output]) 49 | 50 | output_buffers = prepare_h264_buffers(binary, :au_aligned) 51 | 52 | Enum.each(output_buffers, fn buf -> 53 | payload = buf.payload 54 | assert_sink_buffer(pid, :sink, %Buffer{payload: ^payload, pts: nil, dts: nil}) 55 | end) 56 | 57 | Pipeline.terminate(pid) 58 | end 59 | 60 | Enum.map( 61 | [ 62 | {":baseline and :constrained_baseline", @h264_input_file_baseline, 63 | @h264_input_timestamps_baseline}, 64 | {":main and higher", @h264_input_file_main, @h264_input_timestamps_main} 65 | ], 66 | fn {profiles, file, timestamps} -> 67 | test """ 68 | if the pts and dts are generated correctly for profiles #{profiles}\ 69 | in :bytestream mode when framerate is given 70 | """ do 71 | binary = File.read!(unquote(file)) 72 | mode = :bytestream 73 | input_buffers = prepare_h264_buffers(binary, mode) 74 | 75 | framerate = {30, 1} 76 | 77 | {:ok, _supervisor_pid, pid} = 78 | Pipeline.start_supervised( 79 | spec: [ 80 | child(:source, %TestSource{mode: mode}) 81 | |> child(:parser, %Membrane.H264.Parser{ 82 | generate_best_effort_timestamps: %{framerate: framerate} 83 | }) 84 | |> child(:sink, Sink) 85 | ] 86 | ) 87 | 88 | assert_sink_playing(pid, :sink) 89 | send_buffers_actions = for buffer <- input_buffers, do: {:buffer, {:output, buffer}} 90 | Pipeline.notify_child(pid, :source, send_buffers_actions ++ [end_of_stream: :output]) 91 | 92 | output_buffers = prepare_h264_buffers(binary, :au_aligned) 93 | 94 | output_buffers 95 | |> Enum.zip(unquote(timestamps)) 96 | |> Enum.each(fn {%Buffer{payload: ref_payload}, {ref_pts, ref_dts}} -> 97 | assert_sink_buffer(pid, :sink, %Buffer{payload: payload, pts: pts, dts: dts}) 98 | 99 | assert {ref_payload, ref_pts, ref_dts} == 100 | {payload, Membrane.Time.as_milliseconds(pts, :round), 101 | Membrane.Time.as_milliseconds(dts, :round)} 102 | end) 103 | 104 | Pipeline.terminate(pid) 105 | end 106 | end 107 | ) 108 | end 109 | -------------------------------------------------------------------------------- /test/integration/h265/modes_test.exs: -------------------------------------------------------------------------------- 1 | defmodule Membrane.H265.ModesTest do 2 | @moduledoc false 3 | use ExUnit.Case, async: true 4 | 5 | import Membrane.ChildrenSpec 6 | import Membrane.H26x.Support.Common 7 | import Membrane.Testing.Assertions 8 | 9 | alias Membrane.Buffer 10 | alias Membrane.H265.Parser 11 | alias Membrane.H26x.Support.TestSource 12 | alias Membrane.Testing.{Pipeline, Sink} 13 | 14 | @h265_input_file "test/fixtures/h265/input-8-2K.h265" 15 | 16 | test "if the pts and dts are set to nil in :bytestream mode" do 17 | binary = File.read!(@h265_input_file) 18 | mode = :bytestream 19 | input_buffers = prepare_h265_buffers(binary, mode) 20 | 21 | pid = 22 | Pipeline.start_supervised!( 23 | spec: [ 24 | child(:source, %TestSource{mode: mode, codec: :H265}) 25 | |> child(:parser, Parser) 26 | |> child(:sink, Sink) 27 | ] 28 | ) 29 | 30 | assert_sink_playing(pid, :sink) 31 | send_buffers_actions = for buffer <- input_buffers, do: {:buffer, {:output, buffer}} 32 | Pipeline.notify_child(pid, :source, send_buffers_actions ++ [end_of_stream: :output]) 33 | 34 | output_buffers = prepare_h265_buffers(binary, :au_aligned) 35 | 36 | Enum.each(output_buffers, fn buf -> 37 | payload = buf.payload 38 | assert_sink_buffer(pid, :sink, %Buffer{payload: ^payload, pts: nil, dts: nil}) 39 | end) 40 | 41 | Pipeline.terminate(pid) 42 | end 43 | 44 | test "if the pts and dts are rewritten properly in :nalu_aligned mode" do 45 | binary = File.read!(@h265_input_file) 46 | mode = :nalu_aligned 47 | input_buffers = prepare_h265_buffers(binary, mode) 48 | 49 | pid = 50 | Pipeline.start_supervised!( 51 | spec: [ 52 | child(:source, %TestSource{mode: mode, codec: :H265}) 53 | |> child(:parser, Parser) 54 | |> child(:sink, Sink) 55 | ] 56 | ) 57 | 58 | assert_sink_playing(pid, :sink) 59 | send_buffers_actions = for buffer <- input_buffers, do: {:buffer, {:output, buffer}} 60 | Pipeline.notify_child(pid, :source, send_buffers_actions ++ [end_of_stream: :output]) 61 | 62 | output_buffers = prepare_h265_buffers(binary, :au_aligned) 63 | 64 | Enum.each(output_buffers, fn buf -> 65 | payload = buf.payload 66 | pts = buf.pts 67 | dts = buf.dts 68 | assert_sink_buffer(pid, :sink, %Buffer{payload: ^payload, pts: ^pts, dts: ^dts}) 69 | end) 70 | 71 | Pipeline.terminate(pid) 72 | end 73 | 74 | test "if the pts and dts are rewritten properly in :au_aligned mode" do 75 | binary = File.read!(@h265_input_file) 76 | mode = :au_aligned 77 | input_buffers = prepare_h265_buffers(binary, mode) 78 | 79 | pid = 80 | Pipeline.start_supervised!( 81 | spec: [ 82 | child(:source, %TestSource{mode: mode, codec: :H265}) 83 | |> child(:parser, Parser) 84 | |> child(:sink, Sink) 85 | ] 86 | ) 87 | 88 | assert_sink_playing(pid, :sink) 89 | send_buffers_actions = for buffer <- input_buffers, do: {:buffer, {:output, buffer}} 90 | Pipeline.notify_child(pid, :source, send_buffers_actions ++ [end_of_stream: :output]) 91 | 92 | output_buffers = input_buffers 93 | 94 | Enum.each(output_buffers, fn buf -> 95 | payload = buf.payload 96 | pts = buf.pts 97 | dts = buf.dts 98 | assert_sink_buffer(pid, :sink, %Buffer{payload: ^payload, pts: ^pts, dts: ^dts}) 99 | end) 100 | 101 | Pipeline.terminate(pid) 102 | end 103 | 104 | test "if single NAL unit is sent per buffer with `output_alignment: :nalu`" do 105 | pid = 106 | Pipeline.start_supervised!( 107 | spec: [ 108 | child(:source, %Membrane.File.Source{location: @h265_input_file}) 109 | |> child(:parser, %Parser{output_alignment: :nalu}) 110 | |> child(:sink, Sink) 111 | ] 112 | ) 113 | 114 | assert_sink_playing(pid, :sink) 115 | assert_sink_stream_format(pid, :sink, %Membrane.H265{alignment: :nalu}) 116 | 117 | binary = File.read!(@h265_input_file) 118 | ref_buffers = prepare_h265_buffers(binary, :nalu_aligned) 119 | 120 | Enum.each(ref_buffers, fn ref_buffer -> 121 | assert_sink_buffer(pid, :sink, buffer) 122 | assert buffer.payload == ref_buffer.payload 123 | assert Map.has_key?(buffer.metadata, :h265) and Map.has_key?(buffer.metadata.h265, :type) 124 | end) 125 | 126 | assert_end_of_stream(pid, :sink) 127 | Pipeline.terminate(pid) 128 | end 129 | end 130 | -------------------------------------------------------------------------------- /test/integration/h265/timestamp_generation_test.exs: -------------------------------------------------------------------------------- 1 | defmodule Membrane.H265.TimestampGenerationTest do 2 | @moduledoc false 3 | 4 | use ExUnit.Case, async: true 5 | 6 | import Membrane.ChildrenSpec 7 | import Membrane.Testing.Assertions 8 | import Membrane.H26x.Support.Common 9 | 10 | alias Membrane.Buffer 11 | alias Membrane.H265.Parser 12 | alias Membrane.H26x.Support.TestSource 13 | alias Membrane.Testing.{Pipeline, Sink} 14 | 15 | @h265_input_file_main "test/fixtures/h265/input-32-640x360-main.h265" 16 | @h265_input_timestamps_main [ 17 | {0, -500}, 18 | {67, -467}, 19 | {33, -433}, 20 | {100, -400}, 21 | {167, -367}, 22 | {133, -333}, 23 | {233, -300}, 24 | {200, -267}, 25 | {300, -233}, 26 | {267, -200}, 27 | {367, -167}, 28 | {333, -133}, 29 | {400, -100}, 30 | {467, -67}, 31 | {433, -33}, 32 | {533, 0}, 33 | {500, 33}, 34 | {600, 67}, 35 | {567, 100}, 36 | {667, 133}, 37 | {633, 167}, 38 | {700, 200}, 39 | {767, 233}, 40 | {733, 267}, 41 | {833, 300}, 42 | {800, 333}, 43 | {900, 367}, 44 | {867, 400}, 45 | {967, 433}, 46 | {933, 467}, 47 | {1000, 500}, 48 | {1067, 533}, 49 | {1033, 567} 50 | ] 51 | 52 | @h265_input_file_baseline "test/fixtures/h265/input-10-1920x1080.h265" 53 | @h265_input_timestamps_baseline [0, 33, 67, 100, 133, 167, 200, 233, 267, 300] 54 | |> Enum.map(&{&1, &1 - 500}) 55 | 56 | test "if the pts and dts are set to nil in :bytestream mode when framerate isn't given" do 57 | binary = File.read!(@h265_input_file_baseline) 58 | mode = :bytestream 59 | input_buffers = prepare_h265_buffers(binary, mode) 60 | 61 | {:ok, _supervisor_pid, pid} = 62 | Pipeline.start_supervised( 63 | spec: [ 64 | child(:source, %TestSource{mode: mode, codec: :H265}) 65 | |> child(:parser, Parser) 66 | |> child(:sink, Sink) 67 | ] 68 | ) 69 | 70 | assert_sink_playing(pid, :sink) 71 | send_buffers_actions = for buffer <- input_buffers, do: {:buffer, {:output, buffer}} 72 | Pipeline.notify_child(pid, :source, send_buffers_actions ++ [end_of_stream: :output]) 73 | 74 | output_buffers = prepare_h265_buffers(binary, :au_aligned) 75 | 76 | Enum.each(output_buffers, fn buf -> 77 | payload = buf.payload 78 | assert_sink_buffer(pid, :sink, %Buffer{payload: ^payload, pts: nil, dts: nil}) 79 | end) 80 | 81 | Pipeline.terminate(pid) 82 | end 83 | 84 | test "if the pts and dts are generated correctly for stream without frame reorder and no-bframes in :bytestream mode when framerate is given" do 85 | process_test(@h265_input_file_baseline, @h265_input_timestamps_baseline) 86 | end 87 | 88 | test "if the pts and dts are generated correctly in :bytestream mode when framerate is given" do 89 | process_test(@h265_input_file_main, @h265_input_timestamps_main) 90 | end 91 | 92 | test "if the pts and dts are generated correctly without dts offset in :bytestream mode when framerate is given" do 93 | process_test(@h265_input_file_main, @h265_input_timestamps_main, false) 94 | end 95 | 96 | defp process_test(file, timestamps, dts_offset \\ true) do 97 | binary = File.read!(file) 98 | mode = :bytestream 99 | input_buffers = prepare_h265_buffers(binary, mode) 100 | 101 | framerate = {30, 1} 102 | 103 | pid = 104 | Pipeline.start_supervised!( 105 | spec: [ 106 | child(:source, %TestSource{mode: mode, codec: :H265}) 107 | |> child(:parser, %Membrane.H265.Parser{ 108 | generate_best_effort_timestamps: %{framerate: framerate, add_dts_offset: dts_offset} 109 | }) 110 | |> child(:sink, Sink) 111 | ] 112 | ) 113 | 114 | assert_sink_playing(pid, :sink) 115 | send_buffers_actions = for buffer <- input_buffers, do: {:buffer, {:output, buffer}} 116 | Pipeline.notify_child(pid, :source, send_buffers_actions ++ [end_of_stream: :output]) 117 | 118 | output_buffers = prepare_h265_buffers(binary, :au_aligned) 119 | 120 | output_buffers 121 | |> Enum.zip(timestamps) 122 | |> Enum.each(fn {%Buffer{payload: ref_payload}, {ref_pts, ref_dts}} -> 123 | {ref_pts, ref_dts} = if dts_offset, do: {ref_pts, ref_dts}, else: {ref_pts, ref_dts + 500} 124 | assert_sink_buffer(pid, :sink, %Buffer{payload: payload, pts: pts, dts: dts}) 125 | 126 | assert {ref_payload, ref_pts, ref_dts} == 127 | {payload, Membrane.Time.as_milliseconds(pts, :round), 128 | Membrane.Time.as_milliseconds(dts, :round)} 129 | end) 130 | 131 | Pipeline.terminate(pid) 132 | end 133 | end 134 | -------------------------------------------------------------------------------- /test/parser/h264/au_splitter_test.exs: -------------------------------------------------------------------------------- 1 | defmodule Membrane.H264.AUSplitterTest do 2 | @moduledoc false 3 | 4 | use ExUnit.Case, async: true 5 | 6 | @test_files_names ["10-720a", "10-720p"] 7 | 8 | @au_lengths_snapshot %{ 9 | "10-720a" => [777, 146, 93, 136], 10 | "10-720p" => [25_699, 19_043, 14_379, 14_281, 14_761, 18_702, 14_735, 13_602, 12_094, 17_228] 11 | } 12 | 13 | defmodule FullBinaryParser do 14 | @moduledoc false 15 | alias Membrane.H264.{AUSplitter, NALuParser} 16 | alias Membrane.H26x.NALuSplitter 17 | 18 | @spec parse(binary()) :: Membrane.H26x.AUSplitter.access_unit() 19 | def parse(payload) do 20 | {nalus_payloads, _nalu_splitter} = NALuSplitter.split(payload, true, NALuSplitter.new()) 21 | {nalus, _nalu_parser} = NALuParser.parse_nalus(nalus_payloads, NALuParser.new()) 22 | {aus, _au_splitter} = AUSplitter.split(nalus, true, AUSplitter.new()) 23 | aus 24 | end 25 | end 26 | 27 | test "if the access unit lenghts parsed by access unit splitter are the same as access units lengths parsed by FFMPEG" do 28 | for name <- @test_files_names do 29 | full_name = "test/fixtures/h264/input-#{name}.h264" 30 | binary = File.read!(full_name) 31 | 32 | aus = FullBinaryParser.parse(binary) 33 | 34 | au_lengths = 35 | for au <- aus, 36 | do: 37 | Enum.reduce(au, 0, fn %{payload: payload, stripped_prefix: prefix}, acc -> 38 | byte_size(payload) + byte_size(prefix) + acc 39 | end) 40 | 41 | assert au_lengths == @au_lengths_snapshot[name] 42 | end 43 | end 44 | 45 | test "IDR frame split into two NALus" do 46 | # first frame of output of MP4 depayloader from Big Buck Bunny trailer 47 | fixture = 48 | <<0, 0, 0, 1, 39, 66, 224, 21, 169, 24, 60, 17, 253, 96, 13, 65, 128, 65, 173, 183, 160, 15, 49 | 72, 15, 85, 239, 124, 4, 0, 0, 0, 1, 40, 222, 9, 136, 0, 0, 0, 1, 6, 0, 7, 131, 97, 235, 50 | 0, 0, 3, 0, 64, 128, 0, 0, 0, 1, 6, 5, 17, 3, 135, 244, 78, 205, 10, 75, 220, 161, 148, 51 | 58, 195, 212, 155, 23, 31, 3, 128, 0, 0, 0, 1, 37, 184, 32, 32, 255, 255, 252, 61, 20, 0, 52 | 4, 21, 189, 247, 223, 125, 247, 223, 125, 247, 223, 125, 247, 223, 125, 247, 223, 125, 53 | 247, 223, 125, 247, 223, 125, 245, 215, 93, 117, 215, 93, 117, 215, 93, 117, 215, 93, 117, 54 | 215, 93, 117, 215, 93, 117, 215, 93, 117, 215, 93, 117, 215, 93, 117, 215, 93, 117, 215, 55 | 93, 117, 215, 93, 117, 215, 93, 117, 215, 93, 117, 215, 93, 117, 215, 93, 117, 215, 93, 56 | 117, 215, 93, 117, 215, 93, 117, 215, 93, 117, 215, 93, 117, 215, 93, 117, 215, 93, 117, 57 | 215, 93, 117, 215, 93, 117, 215, 93, 117, 215, 93, 117, 215, 93, 117, 215, 93, 117, 215, 58 | 93, 117, 215, 93, 117, 215, 93, 117, 215, 93, 117, 215, 93, 117, 215, 93, 117, 215, 93, 59 | 117, 215, 93, 117, 215, 93, 117, 215, 93, 117, 215, 93, 117, 215, 93, 117, 215, 93, 117, 60 | 215, 93, 117, 215, 93, 117, 215, 93, 117, 215, 93, 117, 215, 93, 117, 215, 93, 117, 215, 61 | 93, 117, 215, 93, 117, 215, 93, 117, 215, 93, 117, 215, 93, 117, 215, 93, 117, 215, 93, 62 | 117, 215, 93, 117, 224, 0, 0, 0, 1, 37, 0, 128, 56, 32, 32, 255, 255, 252, 61, 20, 0, 4, 63 | 21, 189, 247, 223, 125, 247, 223, 125, 247, 223, 125, 247, 255, 255, 240, 244, 80, 0, 16, 64 | 86, 247, 223, 125, 247, 223, 125, 247, 223, 125, 247, 223, 93, 117, 215, 93, 117, 215, 93, 65 | 117, 215, 93, 117, 215, 93, 117, 215, 93, 117, 215, 93, 117, 215, 93, 117, 215, 93, 117, 66 | 215, 93, 117, 215, 93, 117, 215, 93, 117, 215, 93, 117, 215, 93, 117, 215, 93, 117, 215, 67 | 93, 117, 215, 93, 117, 215, 93, 117, 215, 93, 117, 215, 93, 117, 215, 93, 117, 215, 93, 68 | 117, 215, 93, 117, 215, 93, 117, 215, 93, 117, 215, 93, 117, 215, 93, 117, 215, 93, 117, 69 | 215, 93, 117, 215, 93, 117, 215, 93, 117, 215, 93, 117, 215, 93, 117, 215, 93, 117, 215, 70 | 93, 117, 215, 93, 117, 215, 93, 117, 215, 93, 117, 215, 93, 117, 215, 93, 117, 215, 93, 71 | 117, 215, 93, 117, 215, 93, 117, 215, 93, 117, 215, 93, 117, 215, 93, 117, 215, 93, 117, 72 | 215, 93, 117, 215, 93, 117, 255, 252, 126, 8, 2, 152, 28, 64, 32, 172, 183, 223, 125, 247, 73 | 223, 125, 247, 223, 125, 247, 223, 125, 247, 223, 125, 247, 223, 125, 247, 223, 125, 247, 74 | 224>> 75 | 76 | assert [au] = FullBinaryParser.parse(fixture) 77 | 78 | assert au |> Enum.map(&(byte_size(&1.payload) + byte_size(&1.stripped_prefix))) |> Enum.sum() == 79 | byte_size(fixture) 80 | end 81 | end 82 | -------------------------------------------------------------------------------- /test/parser/h264/process_all_test.exs: -------------------------------------------------------------------------------- 1 | defmodule Membrane.H264.ProcessAllTest do 2 | @moduledoc false 3 | 4 | use ExUnit.Case, async: true 5 | import Membrane.ChildrenSpec 6 | import Membrane.Testing.Assertions 7 | alias Membrane.H264 8 | alias Membrane.Testing.Pipeline 9 | 10 | @prefix <<1::32>> 11 | 12 | defp make_pipeline(in_path, out_path, spss, ppss) do 13 | spec = [ 14 | child(:file_src, %Membrane.File.Source{chunk_size: 40_960, location: in_path}) 15 | |> child(:parser, %H264.Parser{spss: spss, ppss: ppss}) 16 | |> child(:sink, %Membrane.File.Sink{location: out_path}) 17 | ] 18 | 19 | Pipeline.start_link_supervised(spec: spec) 20 | end 21 | 22 | defp perform_test(filename, tmp_dir, timeout, spss \\ [], ppss \\ []) do 23 | in_path = "test/fixtures/h264/input-#{filename}.h264" 24 | out_path = Path.join(tmp_dir, "output-all-#{filename}.h264") 25 | 26 | assert {:ok, _supervisor_pid, pid} = make_pipeline(in_path, out_path, spss, ppss) 27 | assert_end_of_stream(pid, :sink, :input, timeout) 28 | 29 | expected = 30 | if length(spss) > 0 and length(ppss) > 0 do 31 | Enum.join([<<>>] ++ spss ++ ppss, @prefix) <> File.read!(in_path) 32 | else 33 | File.read!(in_path) 34 | end 35 | 36 | assert File.read!(out_path) == expected 37 | 38 | Pipeline.terminate(pid) 39 | end 40 | 41 | describe "ProcessAllPipeline should" do 42 | @describetag :tmp_dir 43 | 44 | test "process all 10 720p frames", ctx do 45 | perform_test("10-720p", ctx.tmp_dir, 1000) 46 | end 47 | 48 | test "process all 100 240p frames", ctx do 49 | perform_test("100-240p", ctx.tmp_dir, 1000) 50 | end 51 | 52 | test "process all 20 360p frames with 422 subsampling", ctx do 53 | perform_test("20-360p-I422", ctx.tmp_dir, 1000) 54 | end 55 | 56 | test "process all 10 720p frames with B frames in main profile", ctx do 57 | perform_test("10-720p-main", ctx.tmp_dir, 1000) 58 | end 59 | 60 | test "process all 10 720p frames with no b frames", ctx do 61 | perform_test("10-720p-no-b-frames", ctx.tmp_dir, 1000) 62 | end 63 | 64 | test "process all 100 240p frames with no b frames", ctx do 65 | perform_test("100-240p-no-b-frames", ctx.tmp_dir, 1000) 66 | end 67 | 68 | test "process all 30 240p frames with provided sps and pps", ctx do 69 | sps = 70 | <<103, 100, 0, 50, 172, 114, 132, 64, 80, 5, 187, 1, 16, 0, 0, 3, 0, 16, 0, 0, 3, 3, 192, 71 | 241, 131, 24, 70>> 72 | 73 | pps = <<104, 232, 67, 135, 75, 34, 192>> 74 | 75 | perform_test("30-240p-no-sps-pps", ctx.tmp_dir, 1000, [sps], [pps]) 76 | end 77 | end 78 | end 79 | -------------------------------------------------------------------------------- /test/parser/h264/repeat_parameter_sets_test.exs: -------------------------------------------------------------------------------- 1 | defmodule Membrane.H264.RepeatParameterSetsTest do 2 | @moduledoc false 3 | 4 | use ExUnit.Case, async: true 5 | 6 | import Membrane.ChildrenSpec 7 | import Membrane.Testing.Assertions 8 | import Membrane.H26x.Support.Common 9 | 10 | alias Membrane.{H264, H26x} 11 | alias Membrane.H26x.NALuSplitter 12 | alias Membrane.Testing.{Pipeline, Sink} 13 | 14 | @in_path "test/fixtures/h264/input-30-240p-no-sps-pps.h264" 15 | @ref_path "test/fixtures/h264/reference-30-240p-with-sps-pps.h264" 16 | 17 | @sps <<103, 100, 0, 21, 172, 217, 65, 177, 254, 255, 252, 5, 0, 5, 4, 64, 0, 0, 3, 0, 64, 0, 0, 18 | 15, 3, 197, 139, 101, 128>> 19 | @pps <<104, 235, 227, 203, 34, 192>> 20 | @dcr <<1, 100, 0, 21, 255, 225, 0, 29, 103, 100, 0, 21, 172, 217, 65, 177, 254, 255, 252, 5, 0, 21 | 5, 4, 64, 0, 0, 3, 0, 64, 0, 0, 15, 3, 197, 139, 101, 128, 1, 0, 6, 104, 235, 227, 203, 22 | 34, 192>> 23 | 24 | defp make_pipeline(source, spss \\ [], ppss \\ [], output_stream_structure \\ :annexb) do 25 | spec = 26 | child(:source, source) 27 | |> child(:parser, %H264.Parser{ 28 | spss: spss, 29 | ppss: ppss, 30 | repeat_parameter_sets: true, 31 | output_stream_structure: output_stream_structure 32 | }) 33 | |> child(:sink, Sink) 34 | 35 | Pipeline.start_link_supervised!(spec: spec) 36 | end 37 | 38 | defp perform_test( 39 | pipeline_pid, 40 | data, 41 | mode \\ :bytestream, 42 | parser_input_stream_structure \\ :annexb, 43 | parser_output_stream_structure \\ :annexb 44 | ) do 45 | buffers = prepare_h264_buffers(data, mode, parser_input_stream_structure) 46 | 47 | assert_sink_playing(pipeline_pid, :sink) 48 | actions = for buffer <- buffers, do: {:buffer, {:output, buffer}} 49 | Pipeline.notify_child(pipeline_pid, :source, actions ++ [end_of_stream: :output]) 50 | 51 | output_buffers = 52 | prepare_h264_buffers( 53 | File.read!(@ref_path), 54 | :au_aligned, 55 | parser_output_stream_structure 56 | ) 57 | 58 | Enum.each(output_buffers, fn output_buffer -> 59 | assert_sink_buffer(pipeline_pid, :sink, buffer) 60 | assert buffer.payload == output_buffer.payload 61 | end) 62 | 63 | assert_end_of_stream(pipeline_pid, :sink, :input, 3_000) 64 | Pipeline.terminate(pipeline_pid) 65 | end 66 | 67 | defp split_access_unit(access_unit) do 68 | {nalus, _splitter} = NALuSplitter.split(access_unit, true, NALuSplitter.new()) 69 | Enum.sort(nalus) 70 | end 71 | 72 | describe "Parameter sets should be reapeated on each IDR access unit" do 73 | test "when provided by parser options" do 74 | source = %H26x.Support.TestSource{mode: :bytestream} 75 | pid = make_pipeline(source, [@sps], [@pps]) 76 | perform_test(pid, File.read!(@in_path)) 77 | end 78 | 79 | test "when retrieved from the bytestream" do 80 | source = %H26x.Support.TestSource{mode: :bytestream} 81 | pid = make_pipeline(source) 82 | 83 | data = Enum.join([<<>>, @sps, @pps], <<0, 0, 0, 1>>) <> File.read!(@in_path) 84 | perform_test(pid, data) 85 | end 86 | 87 | test "when provided via DCR" do 88 | source = %H26x.Support.TestSource{ 89 | mode: :au_aligned, 90 | output_raw_stream_structure: {:avc3, @dcr} 91 | } 92 | 93 | pid = make_pipeline(source, [], [], {:avc3, 4}) 94 | perform_test(pid, File.read!(@in_path), :au_aligned, {:avc3, 4}, {:avc3, 4}) 95 | end 96 | 97 | test "when bytestream has variable parameter sets" do 98 | in_path = "./test/fixtures/h264/input-30-240p-vp-sps-pps.h264" 99 | ref_path = "./test/fixtures/h264/reference-30-240p-vp-sps-pps.h264" 100 | 101 | source = %H26x.Support.TestSource{mode: :bytestream} 102 | pid = make_pipeline(source) 103 | 104 | buffers = prepare_h264_buffers(File.read!(in_path), :bytestream) 105 | 106 | assert_sink_playing(pid, :sink) 107 | actions = for buffer <- buffers, do: {:buffer, {:output, buffer}} 108 | Pipeline.notify_child(pid, :source, actions ++ [end_of_stream: :output]) 109 | 110 | File.read!(ref_path) 111 | |> prepare_h264_buffers(:au_aligned) 112 | |> Enum.each(fn output_buffer -> 113 | assert_sink_buffer(pid, :sink, buffer) 114 | assert split_access_unit(output_buffer.payload) == split_access_unit(buffer.payload) 115 | end) 116 | 117 | assert_end_of_stream(pid, :sink, :input, 3_000) 118 | Pipeline.terminate(pid) 119 | end 120 | end 121 | end 122 | -------------------------------------------------------------------------------- /test/parser/h264/skip_until_test.exs: -------------------------------------------------------------------------------- 1 | defmodule Membrane.H264.SkipUntilTest do 2 | @moduledoc false 3 | 4 | use ExUnit.Case, async: true 5 | import Membrane.ChildrenSpec 6 | import Membrane.Testing.Assertions 7 | alias Membrane.H264 8 | alias Membrane.Testing.Pipeline 9 | 10 | defp make_pipeline(in_path, out_path, skip_until_keyframe) do 11 | spec = [ 12 | child(:file_src, %Membrane.File.Source{chunk_size: 40_960, location: in_path}) 13 | |> child(:parser, %H264.Parser{skip_until_keyframe: skip_until_keyframe}) 14 | |> child(:sink, %Membrane.File.Sink{location: out_path}) 15 | ] 16 | 17 | Pipeline.start_link_supervised(spec: spec) 18 | end 19 | 20 | describe "The parser should" do 21 | @describetag :tmp_dir 22 | 23 | test "skip the whole stream if no pps/sps are provided", ctx do 24 | filename = "10-no-pps-sps" 25 | in_path = "test/fixtures/h264/input-#{filename}.h264" 26 | out_path = Path.join(ctx.tmp_dir, "output-all-#{filename}.h264") 27 | 28 | assert {:ok, _supervisor_pid, pid} = make_pipeline(in_path, out_path, false) 29 | assert_end_of_stream(pid, :parser) 30 | refute_sink_buffer(pid, :sink, _, 500) 31 | 32 | Pipeline.terminate(pid) 33 | end 34 | 35 | test "skip until AU with IDR frame is provided, when `skip_until_keyframe: true`", ctx do 36 | filename = "sps-pps-non-idr-sps-pps-idr" 37 | in_path = "test/fixtures/h264/input-#{filename}.h264" 38 | out_path = Path.join(ctx.tmp_dir, "output-#{filename}.h264") 39 | ref_path = "test/fixtures/h264/reference-#{filename}.h264" 40 | assert {:ok, _supervisor_pid, pid} = make_pipeline(in_path, out_path, true) 41 | assert_end_of_stream(pid, :sink) 42 | assert File.read(out_path) == File.read(ref_path) 43 | Pipeline.terminate(pid) 44 | end 45 | 46 | test "skip until AU with parameters is provided, no matter if it contains keyframe, when `skip_until_keyframe: false`", 47 | ctx do 48 | filename = "idr-sps-pps-non-idr" 49 | in_path = "test/fixtures/h264/input-#{filename}.h264" 50 | out_path = Path.join(ctx.tmp_dir, "output-#{filename}.h264") 51 | ref_path = "test/fixtures/h264/reference-#{filename}.h264" 52 | assert {:ok, _supervisor_pid, pid} = make_pipeline(in_path, out_path, false) 53 | assert_end_of_stream(pid, :sink) 54 | assert File.read(out_path) == File.read(ref_path) 55 | Pipeline.terminate(pid) 56 | end 57 | 58 | test "skip until AU with parameters and IDR is provided, when `skip_until_keyframe: true`", 59 | ctx do 60 | filename = "idr-sps-pps-non-idr" 61 | in_path = "test/fixtures/h264/input-#{filename}.h264" 62 | out_path = Path.join(ctx.tmp_dir, "output-#{filename}.h264") 63 | assert {:ok, _supervisor_pid, pid} = make_pipeline(in_path, out_path, true) 64 | assert_end_of_stream(pid, :parser) 65 | refute_sink_buffer(pid, :sink, _, 500) 66 | Pipeline.terminate(pid) 67 | end 68 | end 69 | end 70 | -------------------------------------------------------------------------------- /test/parser/h264/stream_format_test.exs: -------------------------------------------------------------------------------- 1 | defmodule Membrane.H264.StreamFormatTest do 2 | @moduledoc false 3 | 4 | use ExUnit.Case, async: true 5 | import Membrane.ChildrenSpec 6 | import Membrane.Testing.Assertions 7 | alias Membrane.H264 8 | alias Membrane.Testing.Pipeline 9 | 10 | defp make_pipeline(in_path) do 11 | spec = [ 12 | child(:file_src, %Membrane.File.Source{chunk_size: 40_960, location: in_path}) 13 | |> child(:parser, H264.Parser) 14 | |> child(:sink, Membrane.Testing.Sink) 15 | ] 16 | 17 | Pipeline.start_link_supervised(spec: spec) 18 | end 19 | 20 | @video_parameters %{ 21 | "10-720p" => {:high, 1280, 720}, 22 | "100-240p" => {:high, 320, 240}, 23 | "20-360p-I422" => {:high_4_2_2, 480, 360}, 24 | "10-720p-main" => {:main, 1280, 720}, 25 | "10-720p-no-b-frames" => {:high, 1280, 720}, 26 | "100-240p-no-b-frames" => {:high, 320, 240}, 27 | "10-320x180" => {:high, 320, 180} 28 | } 29 | 30 | defp perform_test(filename, timeout) do 31 | in_path = "test/fixtures/h264/input-#{filename}.h264" 32 | assert {:ok, _supervisor_pid, pid} = make_pipeline(in_path) 33 | {profile, width, height} = @video_parameters[filename] 34 | 35 | assert_sink_stream_format(pid, :sink, %H264{profile: ^profile, width: ^width, height: ^height}) 36 | 37 | assert_end_of_stream(pid, :sink, :input, timeout) 38 | 39 | Pipeline.terminate(pid) 40 | end 41 | 42 | describe "Parser should" do 43 | test "read the proper stream format for: 10 720p frames" do 44 | perform_test("10-720p", 1000) 45 | end 46 | 47 | test "read the proper stream format for: 100 240p frames" do 48 | perform_test("100-240p", 1000) 49 | end 50 | 51 | test "read the proper stream format for: 20 360p frames with 422 subsampling" do 52 | perform_test("20-360p-I422", 1000) 53 | end 54 | 55 | test "read the proper stream format for: 10 720p frames with B frames in main profile" do 56 | perform_test("10-720p-main", 1000) 57 | end 58 | 59 | test "read the proper stream format for: 10 720p frames with no b frames" do 60 | perform_test("10-720p-no-b-frames", 10) 61 | end 62 | 63 | test "read the proper stream format for: 100 240p frames with no b frames" do 64 | perform_test("100-240p-no-b-frames", 100) 65 | end 66 | 67 | test "read the proper caps for: 10 320x180 frames (where the cropping is used to describe the resolution)" do 68 | perform_test("10-320x180", 100) 69 | end 70 | end 71 | end 72 | -------------------------------------------------------------------------------- /test/parser/h264/stream_structure_conversion_test.exs: -------------------------------------------------------------------------------- 1 | defmodule Membrane.H264.StreamStructureConversionTest do 2 | @moduledoc false 3 | 4 | use ExUnit.Case 5 | 6 | import Membrane.ChildrenSpec 7 | import Membrane.Testing.Assertions 8 | import Membrane.H26x.Support.Common 9 | 10 | alias Membrane.{H264, H26x} 11 | alias Membrane.Testing.{Pipeline, Sink} 12 | 13 | @annexb_fixtures "test/fixtures/h264/*.h264" 14 | |> Path.wildcard() 15 | |> Enum.reject(&String.contains?(&1, ["no-sps", "no-pps", "sps-pps-non-idr"])) 16 | 17 | @avc1_au_fixtures "test/fixtures/h264/msr/*-avc1-au.msr" |> Path.wildcard() 18 | @avc1_nalu_fixtures "test/fixtures/h264/msr/*-avc1-nalu.msr" |> Path.wildcard() 19 | @avc3_au_fixtures "test/fixtures/h264/msr/*-avc3-au.msr" |> Path.wildcard() 20 | @avc3_nalu_fixtures "test/fixtures/h264/msr/*-avc3-nalu.msr" |> Path.wildcard() 21 | 22 | defp make_annexb_pipeline(alignment, parsers) do 23 | parser_chain = make_parser_chain(parsers) 24 | 25 | mode = get_mode_from_alignment(alignment) 26 | 27 | spec = 28 | child(:source, %H26x.Support.TestSource{ 29 | mode: mode, 30 | output_raw_stream_structure: :annexb 31 | }) 32 | |> parser_chain.() 33 | |> child(:parser_last, %H264.Parser{ 34 | output_alignment: alignment, 35 | output_stream_structure: if(parsers != [], do: :annexb, else: nil) 36 | }) 37 | |> child(:sink, Sink) 38 | 39 | Pipeline.start_link_supervised!(spec: spec) 40 | end 41 | 42 | defp perform_annexb_test(pipeline_pid, data, mode, identical_order?) do 43 | buffers = prepare_h264_buffers(data, mode, :annexb, false) 44 | assert_sink_playing(pipeline_pid, :sink) 45 | actions = for buffer <- buffers, do: {:buffer, {:output, buffer}} 46 | Pipeline.notify_child(pipeline_pid, :source, actions ++ [end_of_stream: :output]) 47 | 48 | assert_end_of_stream(pipeline_pid, :sink, :input, 3_000) 49 | 50 | if identical_order? do 51 | Enum.each(buffers, fn output_buffer -> 52 | assert_sink_buffer(pipeline_pid, :sink, buffer) 53 | assert buffer.payload == output_buffer.payload 54 | end) 55 | else 56 | converted_buffers = receive_buffer_payloads(pipeline_pid) 57 | 58 | fixture_buffers = Enum.map(buffers, & &1.payload) 59 | 60 | {fixture_nalus_set, converted_nalus_set} = 61 | case mode do 62 | :au_aligned -> 63 | { 64 | MapSet.new(split_aus_to_nalus(fixture_buffers, :annexb)), 65 | MapSet.new(split_aus_to_nalus(converted_buffers, :annexb)) 66 | } 67 | 68 | :nalu_aligned -> 69 | {MapSet.new(fixture_buffers), MapSet.new(converted_buffers)} 70 | end 71 | 72 | assert MapSet.equal?(fixture_nalus_set, converted_nalus_set) 73 | end 74 | 75 | Pipeline.terminate(pipeline_pid) 76 | end 77 | 78 | defp make_avc_pipelines(source_file_path, alignment, parsers, avc) do 79 | fixture_pipeline_spec = 80 | child(:source, %Membrane.File.Source{location: source_file_path}) 81 | |> child(:deserializer, Membrane.Stream.Deserializer) 82 | |> child(:sink, Sink) 83 | 84 | parser_chain = make_parser_chain(parsers) 85 | 86 | conversion_pipeline_spec = 87 | child(:source, %Membrane.File.Source{location: source_file_path}) 88 | |> child(:deserializer, Membrane.Stream.Deserializer) 89 | |> parser_chain.() 90 | |> child(:parser_last, %H264.Parser{ 91 | output_alignment: alignment, 92 | output_stream_structure: if(parsers != [], do: {avc, 4}, else: nil) 93 | }) 94 | |> child(:sink, Sink) 95 | 96 | { 97 | Pipeline.start_link_supervised!(spec: fixture_pipeline_spec), 98 | Pipeline.start_link_supervised!(spec: conversion_pipeline_spec) 99 | } 100 | end 101 | 102 | defp perform_avc_test({fixture_pipeline_pid, conversion_pipeline_pid}, avc) do 103 | assert_end_of_stream(fixture_pipeline_pid, :sink, :input, 3_000) 104 | 105 | fixture_buffers = receive_buffer_payloads(fixture_pipeline_pid) 106 | 107 | assert_sink_stream_format(fixture_pipeline_pid, :sink, %H264{ 108 | stream_structure: fixture_stream_structure 109 | }) 110 | 111 | assert_end_of_stream(conversion_pipeline_pid, :sink, :input, 3_000) 112 | 113 | converted_buffers = receive_buffer_payloads(conversion_pipeline_pid) 114 | 115 | case avc do 116 | :avc1 -> 117 | assert fixture_buffers == converted_buffers 118 | 119 | assert_sink_stream_format(conversion_pipeline_pid, :sink, %H264{ 120 | stream_structure: conversion_stream_structure 121 | }) 122 | 123 | assert fixture_stream_structure == conversion_stream_structure 124 | 125 | :avc3 -> 126 | assert_sink_stream_format(conversion_pipeline_pid, :sink, %H264{ 127 | stream_structure: {:avc3, conversion_dcr} 128 | }) 129 | 130 | %{nalu_length_size: converted_nalu_length_size} = 131 | H264.DecoderConfigurationRecord.parse(conversion_dcr) 132 | 133 | {:avc3, fixture_dcr} = fixture_stream_structure 134 | 135 | %{spss: dcr_spss, ppss: dcr_ppss, nalu_length_size: fixture_nalu_length_size} = 136 | H264.DecoderConfigurationRecord.parse(fixture_dcr) 137 | 138 | fixture_nalus = 139 | Enum.map(dcr_spss, &add_length_prefix(&1, converted_nalu_length_size)) ++ 140 | Enum.map(dcr_ppss, &add_length_prefix(&1, converted_nalu_length_size)) ++ 141 | split_aus_to_nalus(fixture_buffers, {:avc3, fixture_nalu_length_size}) 142 | 143 | converted_nalus = 144 | split_aus_to_nalus(converted_buffers, {:avc3, converted_nalu_length_size}) 145 | 146 | assert MapSet.equal?(MapSet.new(fixture_nalus), MapSet.new(converted_nalus)) 147 | end 148 | 149 | Pipeline.terminate(fixture_pipeline_pid) 150 | Pipeline.terminate(conversion_pipeline_pid) 151 | end 152 | 153 | defp perform_test(stream_structure, alignment, parser_stream_structures, identical_order?) do 154 | parsers = 155 | Enum.map(parser_stream_structures, fn stream_structure -> 156 | %H264.Parser{ 157 | output_alignment: alignment, 158 | output_stream_structure: stream_structure 159 | } 160 | end) 161 | 162 | case stream_structure do 163 | :annexb -> 164 | mode = get_mode_from_alignment(alignment) 165 | 166 | Enum.each(@annexb_fixtures, fn path -> 167 | pid = make_annexb_pipeline(alignment, parsers) 168 | perform_annexb_test(pid, File.read!(path), mode, identical_order?) 169 | end) 170 | 171 | avc when avc in [:avc1, :avc3] -> 172 | fixtures = 173 | case {alignment, avc} do 174 | {:au, :avc1} -> @avc1_au_fixtures 175 | {:nalu, :avc1} -> @avc1_nalu_fixtures 176 | {:au, :avc3} -> @avc3_au_fixtures 177 | {:nalu, :avc3} -> @avc3_nalu_fixtures 178 | end 179 | 180 | Enum.each(fixtures, fn path -> 181 | pids = make_avc_pipelines(path, alignment, parsers, avc) 182 | perform_avc_test(pids, avc) 183 | end) 184 | end 185 | end 186 | 187 | defp receive_buffer_payloads(pipeline_pid, acc \\ []) do 188 | receive do 189 | {Membrane.Testing.Pipeline, ^pipeline_pid, 190 | {:handle_child_notification, {{:buffer, buffer}, :sink}}} -> 191 | receive_buffer_payloads(pipeline_pid, acc ++ [buffer.payload]) 192 | after 193 | 0 -> 194 | acc 195 | end 196 | end 197 | 198 | defp make_parser_chain(parsers) do 199 | Enum.reduce(parsers, & &1, fn parser, builder -> &child(builder.(&1), parser) end) 200 | end 201 | 202 | defp split_aus_to_nalus(aus_binaries, stream_structure) do 203 | Enum.map(aus_binaries, fn au_binary -> 204 | {nalus, _splitter} = 205 | H26x.NALuSplitter.split( 206 | au_binary, 207 | true, 208 | H26x.NALuSplitter.new(stream_structure) 209 | ) 210 | 211 | nalus 212 | end) 213 | |> List.flatten() 214 | end 215 | 216 | defp add_length_prefix(nalu_payload, nalu_length_size) do 217 | <> 218 | end 219 | 220 | defp get_mode_from_alignment(alignment) do 221 | case alignment do 222 | :au -> :au_aligned 223 | :nalu -> :nalu_aligned 224 | end 225 | end 226 | 227 | describe "The output stream should be the same as the input stream" do 228 | generate_tests = fn tested_stream_structure_name, parser_chains, name_suffix -> 229 | for parser_types <- parser_chains do 230 | parser_chain_string = 231 | Enum.map_join(parser_types, fn parser_type -> 232 | case parser_type do 233 | :annexb -> "annexb -> " 234 | {:avc1, _prefix_len} -> "avc1 -> " 235 | {:avc3, _prefix_len} -> "avc3 -> " 236 | end 237 | end) 238 | 239 | identical_order? = not Enum.any?(parser_types, &match?({:avc1, _prefix_len}, &1)) 240 | 241 | stream_name = 242 | "stream #{tested_stream_structure_name} -> #{parser_chain_string}#{tested_stream_structure_name}#{name_suffix}" 243 | 244 | @tag String.to_atom("au aligned #{stream_name}") 245 | test "for au aligned #{stream_name}" do 246 | perform_test( 247 | unquote(tested_stream_structure_name), 248 | :au, 249 | unquote(parser_types), 250 | unquote(identical_order?) 251 | ) 252 | end 253 | 254 | @tag String.to_atom("nalu aligned #{stream_name}") 255 | test "for nalu aligned #{stream_name}" do 256 | perform_test( 257 | unquote(tested_stream_structure_name), 258 | :nalu, 259 | unquote(parser_types), 260 | unquote(identical_order?) 261 | ) 262 | end 263 | end 264 | end 265 | 266 | generate_tests.( 267 | :annexb, 268 | [ 269 | [], 270 | [{:avc3, 4}], 271 | [{:avc1, 4}], 272 | [{:avc1, 4}, {:avc3, 4}], 273 | [{:avc3, 4}, {:avc1, 4}] 274 | ], 275 | "" 276 | ) 277 | 278 | generate_tests.( 279 | :avc1, 280 | [[], [:annexb], [{:avc3, 4}], [{:avc3, 4}, :annexb], [:annexb, {:avc3, 4}]], 281 | "" 282 | ) 283 | 284 | generate_tests.( 285 | :avc3, 286 | [[], [:annexb], [{:avc1, 4}], [{:avc1, 4}, :annexb], [:annexb, {:avc1, 4}]], 287 | "" 288 | ) 289 | 290 | generate_tests.(:avc1, [[{:avc3, 2}, {:avc1, 3}, :annexb]], " with varying nalu_length_size") 291 | 292 | generate_tests.( 293 | :avc1, 294 | [ 295 | [ 296 | {:avc3, 4}, 297 | :annexb, 298 | {:avc1, 4}, 299 | :annexb, 300 | {:avc3, 4}, 301 | :annexb, 302 | {:avc1, 4}, 303 | {:avc3, 4}, 304 | :annexb, 305 | {:avc1, 4}, 306 | {:avc3, 4}, 307 | :annexb, 308 | {:avc1, 4} 309 | ] 310 | ], 311 | "" 312 | ) 313 | end 314 | end 315 | -------------------------------------------------------------------------------- /test/parser/h265/au_splitter_test.exs: -------------------------------------------------------------------------------- 1 | defmodule Membrane.H265.AUSplitterTest do 2 | @moduledoc false 3 | 4 | use ExUnit.Case, async: true 5 | 6 | @test_files_names ["10-1920x1080", "10-480x320-mainstillpicture"] 7 | 8 | # These values were obtained with the use of FFmpeg 9 | @au_lengths_ffmpeg %{ 10 | "10-1920x1080" => [10_117, 493, 406, 447, 428, 320, 285, 297, 306, 296], 11 | "10-480x320-mainstillpicture" => [ 12 | 35_114, 13 | 8824, 14 | 8790, 15 | 8762, 16 | 8757, 17 | 8766, 18 | 8731, 19 | 8735, 20 | 8699, 21 | 8710 22 | ] 23 | } 24 | 25 | defmodule FullBinaryParser do 26 | @moduledoc false 27 | alias Membrane.H265.{AUSplitter, NALuParser} 28 | alias Membrane.H26x.NALuSplitter 29 | 30 | @spec parse(binary()) :: AUSplitter.access_unit_t() 31 | def parse(payload) do 32 | {nalus_payloads, _nalu_splitter} = NALuSplitter.split(payload, true, NALuSplitter.new()) 33 | {nalus, _nalu_parser} = NALuParser.parse_nalus(nalus_payloads, NALuParser.new()) 34 | {aus, _au_splitter} = AUSplitter.split(nalus, true, AUSplitter.new()) 35 | aus 36 | end 37 | end 38 | 39 | test "if the access unit lenghts parsed by access unit splitter are the same as access units lengths parsed by FFMPEG" do 40 | for name <- @test_files_names do 41 | full_name = "test/fixtures/h265/input-#{name}.h265" 42 | binary = File.read!(full_name) 43 | 44 | aus = FullBinaryParser.parse(binary) 45 | 46 | au_lengths = 47 | for au <- aus, 48 | do: 49 | Enum.reduce(au, 0, fn %{payload: payload, stripped_prefix: prefix}, acc -> 50 | byte_size(prefix) + byte_size(payload) + acc 51 | end) 52 | 53 | assert au_lengths == @au_lengths_ffmpeg[name] 54 | end 55 | end 56 | end 57 | -------------------------------------------------------------------------------- /test/parser/h265/process_all_test.exs: -------------------------------------------------------------------------------- 1 | defmodule Membrane.H265.ProcessAllTest do 2 | @moduledoc false 3 | 4 | use ExUnit.Case, async: true 5 | 6 | import Membrane.ChildrenSpec 7 | import Membrane.Testing.Assertions 8 | 9 | alias Membrane.H265 10 | alias Membrane.H26x.NALuSplitter 11 | alias Membrane.Testing.Pipeline 12 | 13 | defp make_pipeline(in_path, out_path, parameter_sets) do 14 | spec = [ 15 | child(:file_src, %Membrane.File.Source{chunk_size: 40_960, location: in_path}) 16 | |> child(:parser, %H265.Parser{ 17 | vpss: parameter_sets[:vpss] || [], 18 | spss: parameter_sets[:spss] || [], 19 | ppss: parameter_sets[:ppss] || [] 20 | }) 21 | |> child(:sink, %Membrane.File.Sink{location: out_path}) 22 | ] 23 | 24 | Pipeline.start_link_supervised(spec: spec) 25 | end 26 | 27 | defp strip_parameter_sets(file) do 28 | {nalus, _splitter} = NALuSplitter.split(File.read!(file), true, NALuSplitter.new()) 29 | 30 | nalus 31 | |> Enum.filter(fn 32 | <<0, 0, 1, 0::1, type::6, 0::1, _rest::binary>> when type in [32, 33, 34, 39] -> false 33 | <<0, 0, 0, 1, 0::1, type::6, 0::1, _rest::binary>> when type in [32, 33, 34, 39] -> false 34 | _other -> true 35 | end) 36 | |> Enum.join() 37 | end 38 | 39 | defp perform_test( 40 | filename, 41 | tmp_dir, 42 | timeout, 43 | parameter_sets \\ [], 44 | ignore_parameter_sets \\ false 45 | ) do 46 | in_path = "test/fixtures/h265/input-#{filename}.h265" 47 | out_path = Path.join(tmp_dir, "output-all-#{filename}.h265") 48 | 49 | assert {:ok, _supervisor_pid, pid} = make_pipeline(in_path, out_path, parameter_sets) 50 | assert_end_of_stream(pid, :sink, :input, timeout) 51 | 52 | if ignore_parameter_sets do 53 | assert strip_parameter_sets(out_path) == strip_parameter_sets(in_path) 54 | else 55 | assert File.read(out_path) == File.read(in_path) 56 | end 57 | 58 | Pipeline.terminate(pid) 59 | end 60 | 61 | describe "ProcessAllPipeline should" do 62 | @describetag :tmp_dir 63 | 64 | test "process all 10 320p frames with main still picture profile", ctx do 65 | perform_test("10-480x320-mainstillpicture", ctx.tmp_dir, 1000) 66 | end 67 | 68 | test "process all 10 480p frames with main 10 profile", ctx do 69 | perform_test("10-640x480-main10", ctx.tmp_dir, 1000) 70 | end 71 | 72 | test "process all 10 1080p frames", ctx do 73 | perform_test("10-1920x1080", ctx.tmp_dir, 1000) 74 | end 75 | 76 | test "process all 15 720p frames with more than one temporal sub-layer", ctx do 77 | perform_test("15-1280x720-temporal-id-1", ctx.tmp_dir, 1000) 78 | end 79 | 80 | test "process all 30 480p frames with no b frames", ctx do 81 | perform_test("30-640x480-no-bframes", ctx.tmp_dir, 1000) 82 | end 83 | 84 | test "process all 30 720p frames with rext profile", ctx do 85 | perform_test("30-1280x720-rext", ctx.tmp_dir, 1000) 86 | end 87 | 88 | test "process all 60 1080p frames", ctx do 89 | perform_test("60-1920x1080", ctx.tmp_dir, 1000) 90 | end 91 | 92 | test "process all 300 98p frames with conformance window", ctx do 93 | perform_test("300-98x58-conformance-window", ctx.tmp_dir, 1000) 94 | end 95 | 96 | test "process all 8 2K frames", ctx do 97 | # The bytestream contains AUD nalus and each access unit 98 | # has multiple slices 99 | perform_test("8-2K", ctx.tmp_dir, 1000) 100 | end 101 | 102 | test "process all 60 480p frames with provided parameter sets", ctx do 103 | vps = 104 | <<64, 1, 12, 1, 255, 255, 33, 96, 0, 0, 3, 0, 144, 0, 0, 3, 0, 0, 3, 0, 153, 149, 152, 9>> 105 | 106 | sps = 107 | <<66, 1, 1, 33, 96, 0, 0, 3, 0, 144, 0, 0, 3, 0, 0, 3, 0, 153, 160, 5, 2, 1, 225, 101, 108 | 149, 154, 73, 50, 188, 57, 160, 32, 0, 0, 3, 0, 32, 0, 0, 3, 3, 193>> 109 | 110 | pps = <<68, 1, 193, 114, 180, 98, 64>> 111 | 112 | # Parameter sets without prefix 113 | perform_test( 114 | "60-640x480-no-parameter-sets", 115 | ctx.tmp_dir, 116 | 1000, 117 | [vpss: [vps], spss: [sps], ppss: [pps]], 118 | true 119 | ) 120 | end 121 | end 122 | end 123 | -------------------------------------------------------------------------------- /test/parser/h265/repeat_parameter_sets_test.exs: -------------------------------------------------------------------------------- 1 | defmodule Membrane.H265.RepeatParameterSetsTest do 2 | @moduledoc false 3 | 4 | use ExUnit.Case, async: true 5 | 6 | import Membrane.ChildrenSpec 7 | import Membrane.Testing.Assertions 8 | import Membrane.H26x.Support.Common 9 | 10 | alias Membrane.{H265, H26x} 11 | alias Membrane.H26x.NALuSplitter 12 | alias Membrane.Testing.{Pipeline, Sink} 13 | 14 | @in_path "test/fixtures/h265/input-60-640x480-no-parameter-sets.h265" 15 | @ref_path "test/fixtures/h265/reference-60-640x480-with-parameter-sets.h265" 16 | 17 | @vps <<64, 1, 12, 1, 255, 255, 33, 96, 0, 0, 3, 0, 144, 0, 0, 3, 0, 0, 3, 0, 153, 149, 152, 9>> 18 | @sps <<66, 1, 1, 33, 96, 0, 0, 3, 0, 144, 0, 0, 3, 0, 0, 3, 0, 153, 160, 5, 2, 1, 225, 101, 149, 19 | 154, 73, 50, 188, 57, 160, 32, 0, 0, 3, 0, 32, 0, 0, 3, 3, 193>> 20 | @pps <<68, 1, 193, 114, 180, 98, 64>> 21 | @dcr <<1, 33, 96, 0, 0, 0, 144, 0, 0, 0, 0, 0, 153, 240, 0, 252, 253, 248, 248, 0, 0, 15, 3, 22 | 160, 0, 1, 0, 24, 64, 1, 12, 1, 255, 255, 33, 96, 0, 0, 3, 0, 144, 0, 0, 3, 0, 0, 3, 0, 23 | 153, 149, 152, 9, 161, 0, 1, 0, 42, 66, 1, 1, 33, 96, 0, 0, 3, 0, 144, 0, 0, 3, 0, 0, 3, 24 | 0, 153, 160, 5, 2, 1, 225, 101, 149, 154, 73, 50, 188, 57, 160, 32, 0, 0, 3, 0, 32, 0, 0, 25 | 3, 3, 193, 162, 0, 1, 0, 7, 68, 1, 193, 114, 180, 98, 64>> 26 | 27 | defp make_pipeline( 28 | source, 29 | vpss \\ [], 30 | spss \\ [], 31 | ppss \\ [], 32 | output_stream_structure \\ :annexb 33 | ) do 34 | spec = 35 | child(:source, source) 36 | |> child(:parser, %H265.Parser{ 37 | vpss: vpss, 38 | spss: spss, 39 | ppss: ppss, 40 | repeat_parameter_sets: true, 41 | output_stream_structure: output_stream_structure 42 | }) 43 | |> child(:sink, Sink) 44 | 45 | Pipeline.start_link_supervised!(spec: spec) 46 | end 47 | 48 | defp perform_test( 49 | pipeline_pid, 50 | data, 51 | mode \\ :bytestream, 52 | parser_input_stream_structure \\ :annexb, 53 | parser_output_stream_structure \\ :annexb 54 | ) do 55 | buffers = prepare_h265_buffers(data, mode, parser_input_stream_structure) 56 | 57 | assert_sink_playing(pipeline_pid, :sink) 58 | actions = for buffer <- buffers, do: {:buffer, {:output, buffer}} 59 | Pipeline.notify_child(pipeline_pid, :source, actions ++ [end_of_stream: :output]) 60 | 61 | output_buffers = 62 | prepare_h265_buffers( 63 | File.read!(@ref_path), 64 | :au_aligned, 65 | parser_output_stream_structure 66 | ) 67 | 68 | Enum.each(output_buffers, fn output_buffer -> 69 | assert_sink_buffer(pipeline_pid, :sink, buffer) 70 | assert buffer.payload == output_buffer.payload 71 | end) 72 | 73 | assert_end_of_stream(pipeline_pid, :sink, :input, 3_000) 74 | Pipeline.terminate(pipeline_pid) 75 | end 76 | 77 | defp split_access_unit(access_unit) do 78 | {nalus, _splitter} = NALuSplitter.split(access_unit, true, NALuSplitter.new()) 79 | Enum.sort(nalus) 80 | end 81 | 82 | describe "Parameter sets should be reapeated on each IRAP access unit" do 83 | test "when provided by parser options" do 84 | source = %H26x.Support.TestSource{mode: :bytestream} 85 | pid = make_pipeline(source, [@vps], [@sps], [@pps]) 86 | perform_test(pid, File.read!(@in_path)) 87 | end 88 | 89 | test "when retrieved from the bytestream" do 90 | source = %H26x.Support.TestSource{mode: :bytestream} 91 | pid = make_pipeline(source) 92 | 93 | data = Enum.join([<<>>, @vps, @sps, @pps], <<0, 0, 0, 1>>) <> File.read!(@in_path) 94 | perform_test(pid, data) 95 | end 96 | 97 | test "when provided via DCR" do 98 | source = %H26x.Support.TestSource{ 99 | mode: :au_aligned, 100 | output_raw_stream_structure: {:hev1, @dcr}, 101 | codec: :H265 102 | } 103 | 104 | pid = make_pipeline(source, [], [], [], {:hev1, 4}) 105 | perform_test(pid, File.read!(@in_path), :au_aligned, {:hev1, 4}, {:hev1, 4}) 106 | end 107 | 108 | test "when bytestream has variable parameter sets" do 109 | in_path = "test/fixtures/h265/input-60-640x480-variable-parameters.h265" 110 | ref_path = "test/fixtures/h265/reference-60-640x480-variable-parameters.h265" 111 | 112 | source = %H26x.Support.TestSource{mode: :bytestream} 113 | pid = make_pipeline(source) 114 | 115 | buffers = prepare_h265_buffers(File.read!(in_path), :bytestream) 116 | 117 | assert_sink_playing(pid, :sink) 118 | actions = for buffer <- buffers, do: {:buffer, {:output, buffer}} 119 | Pipeline.notify_child(pid, :source, actions ++ [end_of_stream: :output]) 120 | 121 | File.read!(ref_path) 122 | |> prepare_h265_buffers(:au_aligned) 123 | |> Enum.each(fn output_buffer -> 124 | assert_sink_buffer(pid, :sink, buffer) 125 | assert split_access_unit(output_buffer.payload) == split_access_unit(buffer.payload) 126 | end) 127 | 128 | assert_end_of_stream(pid, :sink, :input, 3_000) 129 | Pipeline.terminate(pid) 130 | end 131 | end 132 | end 133 | -------------------------------------------------------------------------------- /test/parser/h265/skip_until_test.exs: -------------------------------------------------------------------------------- 1 | defmodule Membrane.H265.SkipUntilTest do 2 | @moduledoc false 3 | 4 | use ExUnit.Case, async: true 5 | 6 | import Membrane.ChildrenSpec 7 | import Membrane.Testing.Assertions 8 | 9 | alias Membrane.H265 10 | alias Membrane.Testing.Pipeline 11 | 12 | defp make_pipeline(in_path, out_path, skip_until_keyframe) do 13 | spec = [ 14 | child(:file_src, %Membrane.File.Source{chunk_size: 40_960, location: in_path}) 15 | |> child(:parser, %H265.Parser{skip_until_keyframe: skip_until_keyframe}) 16 | |> child(:sink, %Membrane.File.Sink{location: out_path}) 17 | ] 18 | 19 | Pipeline.start_link_supervised(spec: spec) 20 | end 21 | 22 | describe "The parser should" do 23 | @describetag :tmp_dir 24 | 25 | test "skip the whole stream if no vps/sps/pps are provided", ctx do 26 | filename = "10-no-vps-sps-pps" 27 | in_path = "test/fixtures/h265/input-#{filename}.h265" 28 | out_path = Path.join(ctx.tmp_dir, "output-all-#{filename}.h265") 29 | 30 | assert {:ok, _supervisor_pid, pid} = make_pipeline(in_path, out_path, false) 31 | refute_sink_buffer(pid, :sink, _) 32 | 33 | Pipeline.terminate(pid) 34 | end 35 | 36 | test "skip until IRAP frame is provided when `skip_until_keyframe: true`", ctx do 37 | filename = "pss-no-irap-pss-irap" 38 | in_path = "test/fixtures/h265/input-#{filename}.h265" 39 | out_path = Path.join(ctx.tmp_dir, "output-#{filename}.h265") 40 | ref_path = "test/fixtures/h265/reference-#{filename}.h265" 41 | assert {:ok, _supervisor_pid, pid} = make_pipeline(in_path, out_path, true) 42 | assert_end_of_stream(pid, :sink) 43 | assert File.read(out_path) == File.read(ref_path) 44 | Pipeline.terminate(pid) 45 | end 46 | 47 | test "skip until AU with parameters is provided, no matter if it contains keyframe, when `skip_until_keyframe: false`", 48 | ctx do 49 | filename = "irap-pss-no-irap" 50 | in_path = "test/fixtures/h265/input-#{filename}.h265" 51 | out_path = Path.join(ctx.tmp_dir, "output-#{filename}.h265") 52 | ref_path = "test/fixtures/h265/reference-#{filename}.h265" 53 | assert {:ok, _supervisor_pid, pid} = make_pipeline(in_path, out_path, false) 54 | assert_end_of_stream(pid, :sink) 55 | assert File.read(out_path) == File.read(ref_path) 56 | Pipeline.terminate(pid) 57 | end 58 | 59 | test "skip until AU with parameters and IRAP is provided, when `skip_until_keyframe: true`" do 60 | filename = "irap-pss-no-irap" 61 | in_path = "test/fixtures/h265/input-#{filename}.h265" 62 | ref_path = "test/fixtures/h265/reference-pss-no-irap-pss-irap.h265" 63 | assert {:ok, _supervisor_pid, pid} = make_pipeline(in_path, ref_path, true) 64 | assert_end_of_stream(pid, :parser) 65 | refute_sink_buffer(pid, :sink, _, 500) 66 | Pipeline.terminate(pid) 67 | end 68 | end 69 | end 70 | -------------------------------------------------------------------------------- /test/parser/h265/stream_format_test.exs: -------------------------------------------------------------------------------- 1 | defmodule Membrane.H265.StreamFormatTest do 2 | @moduledoc false 3 | 4 | use ExUnit.Case 5 | import Membrane.ChildrenSpec 6 | import Membrane.Testing.Assertions 7 | alias Membrane.H265 8 | alias Membrane.Testing.Pipeline 9 | 10 | defp make_pipeline(in_path) do 11 | spec = [ 12 | child(:file_src, %Membrane.File.Source{chunk_size: 40_960, location: in_path}) 13 | |> child(:parser, H265.Parser) 14 | |> child(:sink, Membrane.Testing.Sink) 15 | ] 16 | 17 | Pipeline.start_link_supervised(spec: spec) 18 | end 19 | 20 | @video_parameters %{ 21 | "15-1280x720-temporal-id-1" => {:main, 1280, 720}, 22 | "30-640x480-no-bframes" => {:main, 640, 480}, 23 | "30-1280x720-rext" => {:rext, 1280, 720}, 24 | "60-640x480" => {:main, 640, 480}, 25 | "60-1920x1080" => {:main, 1920, 1080}, 26 | "10-640x480-main10" => {:main_10, 640, 480}, 27 | "10-480x320-mainstillpicture" => {:main_still_picture, 480, 320}, 28 | "300-98x58-conformance-window" => {:main, 98, 58} 29 | } 30 | 31 | defp perform_test(filename, timeout) do 32 | in_path = "test/fixtures/h265/input-#{filename}.h265" 33 | assert {:ok, _supervisor_pid, pid} = make_pipeline(in_path) 34 | {profile, width, height} = @video_parameters[filename] 35 | 36 | assert_sink_stream_format(pid, :sink, %H265{profile: ^profile, width: ^width, height: ^height}) 37 | 38 | assert_sink_playing(pid, :sink) 39 | assert_end_of_stream(pid, :sink, :input, timeout) 40 | 41 | Pipeline.terminate(pid) 42 | end 43 | 44 | describe "Parser should" do 45 | test "read the proper stream format for: 15 720p frames with more than one temporal sub-layer" do 46 | perform_test("15-1280x720-temporal-id-1", 1000) 47 | end 48 | 49 | test "read the proper stream format for: 30 480p frames" do 50 | perform_test("30-640x480-no-bframes", 1000) 51 | end 52 | 53 | test "read the proper stream format for: 30 720p with rext profile" do 54 | perform_test("30-1280x720-rext", 1000) 55 | end 56 | 57 | test "read the proper stream format for: 60 480p frames" do 58 | perform_test("60-640x480", 1000) 59 | end 60 | 61 | test "read the proper stream format for: 60 1080p frames" do 62 | perform_test("60-1920x1080", 10) 63 | end 64 | 65 | test "read the proper stream format for: 10 480p frames with main 10 profile" do 66 | perform_test("10-640x480-main10", 100) 67 | end 68 | 69 | test "read the proper stream format for: 10 320p frames with main still picture profile" do 70 | perform_test("10-480x320-mainstillpicture", 100) 71 | end 72 | 73 | test "read the proper stream format for: 300 98p frames with main profile and conformance window" do 74 | perform_test("300-98x58-conformance-window", 100) 75 | end 76 | end 77 | end 78 | -------------------------------------------------------------------------------- /test/parser/h265/stream_structure_conversion_test.exs: -------------------------------------------------------------------------------- 1 | defmodule Membrane.H265.StreamStructureConversionTest do 2 | @moduledoc false 3 | 4 | use ExUnit.Case, async: true 5 | 6 | import Membrane.ChildrenSpec 7 | import Membrane.Testing.Assertions 8 | import Membrane.H26x.Support.Common 9 | 10 | alias Membrane.{H265, H26x} 11 | alias Membrane.Testing.{Pipeline, Sink} 12 | 13 | @annexb_fixtures "test/fixtures/h265/*.h265" 14 | |> Path.wildcard() 15 | |> Enum.reject( 16 | &String.contains?(&1, ["no-parameter-sets", "no-irap", "no-vps"]) 17 | ) 18 | 19 | @hvc1_au_fixtures "test/fixtures/h265/msr/*-hvc1-au.msr" |> Path.wildcard() 20 | @hvc1_nalu_fixtures "test/fixtures/h265/msr/*-hvc1-nalu.msr" |> Path.wildcard() 21 | @hev1_au_fixtures "test/fixtures/h265/msr/*-hev1-au.msr" |> Path.wildcard() 22 | @hev1_nalu_fixtures "test/fixtures/h265/msr/*-hev1-nalu.msr" |> Path.wildcard() 23 | 24 | defp make_annexb_pipeline(alignment, parsers) do 25 | parser_chain = make_parser_chain(parsers) 26 | 27 | mode = get_mode_from_alignment(alignment) 28 | 29 | spec = 30 | child(:source, %H26x.Support.TestSource{ 31 | mode: mode, 32 | output_raw_stream_structure: :annexb, 33 | codec: :H265 34 | }) 35 | |> parser_chain.() 36 | |> child(:parser_last, %H265.Parser{ 37 | output_alignment: alignment, 38 | output_stream_structure: if(parsers != [], do: :annexb, else: nil) 39 | }) 40 | |> child(:sink, Sink) 41 | 42 | Pipeline.start_link_supervised!(spec: spec) 43 | end 44 | 45 | defp perform_annexb_test(pipeline_pid, data, mode, identical_order?) do 46 | buffers = prepare_h265_buffers(data, mode, :annexb, false) 47 | assert_sink_playing(pipeline_pid, :sink) 48 | actions = for buffer <- buffers, do: {:buffer, {:output, buffer}} 49 | Pipeline.notify_child(pipeline_pid, :source, actions ++ [end_of_stream: :output]) 50 | 51 | assert_end_of_stream(pipeline_pid, :sink, :input, 3_000) 52 | 53 | if identical_order? do 54 | Enum.each(buffers, fn output_buffer -> 55 | assert_sink_buffer(pipeline_pid, :sink, buffer) 56 | assert buffer.payload == output_buffer.payload 57 | end) 58 | else 59 | converted_buffers = receive_buffer_payloads(pipeline_pid) 60 | 61 | fixture_buffers = Enum.map(buffers, & &1.payload) 62 | 63 | {fixture_nalus_set, converted_nalus_set} = 64 | case mode do 65 | :au_aligned -> 66 | { 67 | MapSet.new(split_aus_to_nalus(fixture_buffers, :annexb)), 68 | MapSet.new(split_aus_to_nalus(converted_buffers, :annexb)) 69 | } 70 | 71 | :nalu_aligned -> 72 | {MapSet.new(fixture_buffers), MapSet.new(converted_buffers)} 73 | end 74 | 75 | assert MapSet.equal?(fixture_nalus_set, converted_nalus_set) 76 | end 77 | 78 | Pipeline.terminate(pipeline_pid) 79 | end 80 | 81 | defp make_hvc_pipelines(source_file_path, alignment, parsers, hvc) do 82 | fixture_pipeline_structure = 83 | child(:source, %Membrane.File.Source{location: source_file_path}) 84 | |> child(:deserializer, Membrane.Stream.Deserializer) 85 | |> child(:sink, Sink) 86 | 87 | parser_chain = make_parser_chain(parsers) 88 | 89 | conversion_pipeline_structure = 90 | child(:source, %Membrane.File.Source{location: source_file_path}) 91 | |> child(:deserializer, Membrane.Stream.Deserializer) 92 | |> parser_chain.() 93 | |> child(:parser_last, %H265.Parser{ 94 | output_alignment: alignment, 95 | output_stream_structure: if(parsers != [], do: {hvc, 4}, else: nil) 96 | }) 97 | |> child(:sink, Sink) 98 | 99 | { 100 | Pipeline.start_link_supervised!(spec: fixture_pipeline_structure), 101 | Pipeline.start_link_supervised!(spec: conversion_pipeline_structure) 102 | } 103 | end 104 | 105 | defp perform_hvc_test({fixture_pipeline_pid, conversion_pipeline_pid}, hvc) do 106 | assert_sink_playing(fixture_pipeline_pid, :sink) 107 | assert_end_of_stream(fixture_pipeline_pid, :sink, :input, 3_000) 108 | 109 | fixture_buffers = receive_buffer_payloads(fixture_pipeline_pid) 110 | 111 | assert_sink_stream_format(fixture_pipeline_pid, :sink, %H265{ 112 | stream_structure: fixture_stream_structure 113 | }) 114 | 115 | assert_sink_playing(conversion_pipeline_pid, :sink) 116 | assert_end_of_stream(conversion_pipeline_pid, :sink, :input, 3_000) 117 | 118 | converted_buffers = receive_buffer_payloads(conversion_pipeline_pid) 119 | 120 | case hvc do 121 | :hvc1 -> 122 | assert fixture_buffers == converted_buffers 123 | 124 | assert_sink_stream_format(conversion_pipeline_pid, :sink, %H265{ 125 | stream_structure: conversion_stream_structure 126 | }) 127 | 128 | assert fixture_stream_structure == conversion_stream_structure 129 | 130 | :hev1 -> 131 | assert_sink_stream_format(conversion_pipeline_pid, :sink, %H265{ 132 | stream_structure: {:hev1, conversion_dcr} 133 | }) 134 | 135 | %{nalu_length_size: converted_nalu_length_size} = 136 | H265.DecoderConfigurationRecord.parse(conversion_dcr) 137 | 138 | {:hev1, fixture_dcr} = fixture_stream_structure 139 | 140 | %{ 141 | vpss: dcr_vpss, 142 | spss: dcr_spss, 143 | ppss: dcr_ppss, 144 | nalu_length_size: fixture_nalu_length_size 145 | } = H265.DecoderConfigurationRecord.parse(fixture_dcr) 146 | 147 | fixture_nalus = 148 | Enum.map(dcr_vpss, &add_length_prefix(&1, converted_nalu_length_size)) ++ 149 | Enum.map(dcr_spss, &add_length_prefix(&1, converted_nalu_length_size)) ++ 150 | Enum.map(dcr_ppss, &add_length_prefix(&1, converted_nalu_length_size)) ++ 151 | split_aus_to_nalus(fixture_buffers, {:hev1, fixture_nalu_length_size}) 152 | 153 | converted_nalus = 154 | split_aus_to_nalus(converted_buffers, {:hev1, converted_nalu_length_size}) 155 | 156 | assert MapSet.equal?(MapSet.new(fixture_nalus), MapSet.new(converted_nalus)) 157 | end 158 | 159 | Pipeline.terminate(fixture_pipeline_pid) 160 | Pipeline.terminate(conversion_pipeline_pid) 161 | end 162 | 163 | defp perform_test(stream_structure, alignment, parser_stream_structures, identical_order?) do 164 | parsers = 165 | Enum.map(parser_stream_structures, fn stream_structure -> 166 | %H265.Parser{ 167 | output_alignment: alignment, 168 | output_stream_structure: stream_structure 169 | } 170 | end) 171 | 172 | case stream_structure do 173 | :annexb -> 174 | mode = get_mode_from_alignment(alignment) 175 | 176 | Enum.each(@annexb_fixtures, fn path -> 177 | pid = make_annexb_pipeline(alignment, parsers) 178 | perform_annexb_test(pid, File.read!(path), mode, identical_order?) 179 | end) 180 | 181 | hvc when hvc in [:hvc1, :hev1] -> 182 | fixtures = 183 | case {alignment, hvc} do 184 | {:au, :hvc1} -> @hvc1_au_fixtures 185 | {:nalu, :hvc1} -> @hvc1_nalu_fixtures 186 | {:au, :hev1} -> @hev1_au_fixtures 187 | {:nalu, :hev1} -> @hev1_nalu_fixtures 188 | end 189 | 190 | Enum.each(fixtures, fn path -> 191 | pids = make_hvc_pipelines(path, alignment, parsers, hvc) 192 | perform_hvc_test(pids, hvc) 193 | end) 194 | end 195 | end 196 | 197 | defp receive_buffer_payloads(pipeline_pid, acc \\ []) do 198 | receive do 199 | {Membrane.Testing.Pipeline, ^pipeline_pid, 200 | {:handle_child_notification, {{:buffer, buffer}, :sink}}} -> 201 | receive_buffer_payloads(pipeline_pid, acc ++ [buffer.payload]) 202 | after 203 | 0 -> 204 | acc 205 | end 206 | end 207 | 208 | defp make_parser_chain(parsers) do 209 | Enum.reduce(parsers, & &1, fn parser, builder -> &child(builder.(&1), parser) end) 210 | end 211 | 212 | defp split_aus_to_nalus(aus_binaries, stream_structure) do 213 | Enum.map(aus_binaries, fn au_binary -> 214 | {nalus, _splitter} = 215 | H26x.NALuSplitter.split( 216 | au_binary, 217 | true, 218 | H26x.NALuSplitter.new(stream_structure) 219 | ) 220 | 221 | nalus 222 | end) 223 | |> List.flatten() 224 | end 225 | 226 | defp add_length_prefix(nalu_payload, nalu_length_size) do 227 | <> 228 | end 229 | 230 | defp get_mode_from_alignment(alignment) do 231 | case alignment do 232 | :au -> :au_aligned 233 | :nalu -> :nalu_aligned 234 | end 235 | end 236 | 237 | describe "The output stream should be the same as the input stream" do 238 | generate_tests = fn tested_stream_structure_name, parser_chains, name_suffix -> 239 | for parser_types <- parser_chains do 240 | parser_chain_string = 241 | Enum.map_join(parser_types, fn parser_type -> 242 | case parser_type do 243 | :annexb -> "annexb -> " 244 | {:hvc1, _prefix_len} -> "hvc1 -> " 245 | {:hev1, _prefix_len} -> "hev1 -> " 246 | end 247 | end) 248 | 249 | identical_order? = not Enum.any?(parser_types, &match?({:hvc1, _prefix_len}, &1)) 250 | 251 | stream_name = 252 | "stream #{tested_stream_structure_name} -> #{parser_chain_string}#{tested_stream_structure_name}#{name_suffix}" 253 | 254 | @tag String.to_atom("au aligned #{stream_name}") 255 | test "for au aligned #{stream_name}" do 256 | perform_test( 257 | unquote(tested_stream_structure_name), 258 | :au, 259 | unquote(parser_types), 260 | unquote(identical_order?) 261 | ) 262 | end 263 | 264 | @tag String.to_atom("nalu aligned #{stream_name}") 265 | test "for nalu aligned #{stream_name}" do 266 | perform_test( 267 | unquote(tested_stream_structure_name), 268 | :nalu, 269 | unquote(parser_types), 270 | unquote(identical_order?) 271 | ) 272 | end 273 | end 274 | end 275 | 276 | generate_tests.( 277 | :annexb, 278 | [ 279 | [], 280 | [{:hev1, 4}], 281 | [{:hvc1, 4}], 282 | [{:hvc1, 4}, {:hev1, 4}], 283 | [{:hev1, 4}, {:hvc1, 4}] 284 | ], 285 | "" 286 | ) 287 | 288 | generate_tests.( 289 | :hvc1, 290 | [[], [:annexb], [{:hev1, 4}], [{:hev1, 4}, :annexb], [:annexb, {:hev1, 4}]], 291 | "" 292 | ) 293 | 294 | generate_tests.( 295 | :hev1, 296 | [[], [:annexb], [{:hvc1, 4}], [{:hvc1, 4}, :annexb], [:annexb, {:hvc1, 4}]], 297 | "" 298 | ) 299 | 300 | generate_tests.(:hvc1, [[{:hev1, 2}, {:hvc1, 3}, :annexb]], " with varying nalu_length_size") 301 | 302 | generate_tests.( 303 | :hvc1, 304 | [ 305 | [ 306 | {:hev1, 4}, 307 | :annexb, 308 | {:hvc1, 4}, 309 | :annexb, 310 | {:hev1, 4}, 311 | :annexb, 312 | {:hvc1, 4}, 313 | {:hev1, 4}, 314 | :annexb, 315 | {:hvc1, 4}, 316 | {:hev1, 4}, 317 | :annexb, 318 | {:hvc1, 4} 319 | ] 320 | ], 321 | "" 322 | ) 323 | end 324 | end 325 | -------------------------------------------------------------------------------- /test/support/common.ex: -------------------------------------------------------------------------------- 1 | defmodule Membrane.H26x.Support.Common do 2 | @moduledoc false 3 | 4 | alias Membrane.{H264, H265} 5 | alias Membrane.H26x.NALuSplitter 6 | 7 | @spec prepare_h264_buffers( 8 | binary, 9 | :au | :bytestream | :nalu, 10 | Membrane.H26x.Parser.stream_structure(), 11 | boolean() 12 | ) :: list 13 | 14 | def prepare_h264_buffers( 15 | binary, 16 | alignment, 17 | output_stream_structure \\ :annexb, 18 | stable_reprefixing? \\ true 19 | ) 20 | 21 | def prepare_h264_buffers(binary, :bytestream, _output_stream_structure, _stable_reprefixing?) do 22 | do_prepare_bytestream_buffers(binary) 23 | end 24 | 25 | def prepare_h264_buffers(binary, mode, output_stream_structure, stable_reprefixing?) do 26 | do_prepare_buffers( 27 | binary, 28 | mode, 29 | output_stream_structure, 30 | stable_reprefixing?, 31 | H264.NALuParser, 32 | H264.AUSplitter 33 | ) 34 | end 35 | 36 | @spec prepare_h265_buffers( 37 | binary, 38 | :au | :bytestream | :nalu, 39 | Membrane.H26x.Parser.stream_structure(), 40 | boolean() 41 | ) :: list 42 | 43 | def prepare_h265_buffers( 44 | binary, 45 | alignment, 46 | output_stream_structure \\ :annexb, 47 | stable_reprefixing? \\ true 48 | ) 49 | 50 | def prepare_h265_buffers(binary, :bytestream, _output_stream_structure, _stable_reprefixing?) do 51 | do_prepare_bytestream_buffers(binary) 52 | end 53 | 54 | def prepare_h265_buffers(binary, mode, output_stream_structure, stable_reprefixing?) do 55 | do_prepare_buffers( 56 | binary, 57 | mode, 58 | output_stream_structure, 59 | stable_reprefixing?, 60 | H265.NALuParser, 61 | H265.AUSplitter 62 | ) 63 | end 64 | 65 | defp do_prepare_bytestream_buffers(binary) do 66 | buffers = 67 | :binary.bin_to_list(binary) |> Enum.chunk_every(400) |> Enum.map(&:binary.list_to_bin(&1)) 68 | 69 | Enum.map(buffers, &%Membrane.Buffer{payload: &1}) 70 | end 71 | 72 | defp do_prepare_buffers( 73 | binary, 74 | mode, 75 | output_stream_structure, 76 | stable_reprefixing?, 77 | nalu_parser_mod, 78 | au_splitter_mod 79 | ) do 80 | {nalus_payloads, _nalu_splitter} = NALuSplitter.split(binary, true, NALuSplitter.new(:annexb)) 81 | {nalus, _nalu_parser} = nalu_parser_mod.parse_nalus(nalus_payloads, nalu_parser_mod.new()) 82 | {aus, _au_splitter} = au_splitter_mod.split(nalus, true, au_splitter_mod.new()) 83 | 84 | case mode do 85 | :nalu_aligned -> 86 | Enum.map_reduce(aus, 0, fn au, ts -> 87 | {Enum.map(au, fn nalu -> 88 | nalu_payload = 89 | nalu_parser_mod.get_prefixed_nalu_payload( 90 | nalu, 91 | output_stream_structure, 92 | stable_reprefixing? 93 | ) 94 | 95 | %Membrane.Buffer{payload: nalu_payload, pts: ts, dts: ts} 96 | end), ts + 1} 97 | end) 98 | |> elem(0) 99 | |> List.flatten() 100 | 101 | :au_aligned -> 102 | Enum.map_reduce(aus, 0, fn au, ts -> 103 | {%Membrane.Buffer{ 104 | payload: 105 | Enum.map_join(au, fn nalu -> 106 | nalu_parser_mod.get_prefixed_nalu_payload( 107 | nalu, 108 | output_stream_structure, 109 | stable_reprefixing? 110 | ) 111 | end), 112 | pts: ts, 113 | dts: ts 114 | }, ts + 1} 115 | end) 116 | |> elem(0) 117 | end 118 | end 119 | end 120 | -------------------------------------------------------------------------------- /test/support/fixture_generator.exs: -------------------------------------------------------------------------------- 1 | Mix.install([ 2 | {:membrane_file_plugin, "~> 0.15.0"}, 3 | {:membrane_mp4_plugin, "~> 0.29.0"}, 4 | {:membrane_stream_plugin, "~> 0.3.1"} 5 | ]) 6 | 7 | alias Membrane.H264.DecoderConfigurationRecord 8 | alias Membrane.H26x.NALuSplitter 9 | 10 | defmodule Aligner do 11 | @moduledoc false 12 | 13 | use Membrane.Filter 14 | 15 | def_input_pad :input, 16 | demand_unit: :buffers, 17 | flow_control: :auto, 18 | accepted_format: Membrane.H264 19 | 20 | def_output_pad :output, 21 | flow_control: :auto, 22 | accepted_format: Membrane.H264 23 | 24 | def_options output_alignment: [ 25 | spec: :au | :nalu, 26 | default: :au 27 | ], 28 | output_stream_structure: [ 29 | spec: {:avc1 | :avc3, pos_integer()} 30 | ] 31 | 32 | @impl true 33 | def handle_stream_format( 34 | :input, 35 | %Membrane.H264{stream_structure: {:avc1, dcr}} = stream_format, 36 | _ctx, 37 | %{output_stream_structure: {avc, nalu_length_size}} = state 38 | ) do 39 | %{nalu_length_size: dcr_nalu_length_size} = DecoderConfigurationRecord.parse(dcr) 40 | 41 | if dcr_nalu_length_size != nalu_length_size do 42 | raise "incoming NALu length size must be equal to the one provided via options" 43 | end 44 | 45 | {[ 46 | stream_format: 47 | {:output, 48 | %Membrane.H264{ 49 | stream_format 50 | | alignment: state.output_alignment, 51 | stream_structure: {avc, dcr} 52 | }} 53 | ], state} 54 | end 55 | 56 | @impl true 57 | def handle_buffer(:input, buffer, _ctx, state) do 58 | buffers = 59 | case state.output_alignment do 60 | :au -> 61 | buffer 62 | 63 | :nalu -> 64 | splitter = NALuSplitter.new(state.output_stream_structure) 65 | {nalus, splitter} = NALuSplitter.split(buffer.payload, splitter) 66 | 67 | Enum.map(nalus, fn nalu -> %Membrane.Buffer{payload: nalu} end) 68 | end 69 | 70 | {[buffer: {:output, buffers}], state} 71 | end 72 | 73 | @impl true 74 | def handle_end_of_stream(:input, _ctx, state) do 75 | {[end_of_stream: :output], state} 76 | end 77 | end 78 | 79 | defmodule FixtureGeneratorPipeline do 80 | @moduledoc false 81 | 82 | use Membrane.Pipeline 83 | 84 | import Membrane.ChildrenSpec 85 | 86 | @impl true 87 | def handle_init(_ctx, options) do 88 | spec = [ 89 | child(:video_source, %Membrane.File.Source{location: options.input_location}) 90 | |> child(:demuxer, Membrane.MP4.Demuxer.ISOM) 91 | |> via_out(Pad.ref(:output, 1)) 92 | |> child(:filter, %Aligner{ 93 | output_alignment: options.output_alignment, 94 | output_stream_structure: options.stream_structure 95 | }) 96 | |> child(:serializer, Membrane.Stream.Serializer) 97 | |> child(:sink, %Membrane.File.Sink{location: options.output_location}) 98 | ] 99 | 100 | {[spec: spec], %{children_with_eos: MapSet.new()}} 101 | end 102 | 103 | @impl true 104 | def handle_element_end_of_stream(element, _pad, _ctx, state) do 105 | state = %{state | children_with_eos: MapSet.put(state.children_with_eos, element)} 106 | 107 | actions = 108 | if :sink in state.children_with_eos, 109 | do: [terminate: :shutdown], 110 | else: [] 111 | 112 | {actions, state} 113 | end 114 | end 115 | 116 | defmodule AVCFixtureGenerator do 117 | @moduledoc false 118 | 119 | @mp4_avc1_fixtures [ 120 | "test/fixtures/mp4/ref_video.mp4", 121 | "test/fixtures/mp4/ref_video_fast_start.mp4" 122 | ] 123 | 124 | @mp4_avc3_fixtures [ 125 | "test/fixtures/mp4/ref_video.mp4", 126 | "test/fixtures/mp4/ref_video_fast_start.mp4", 127 | "test/fixtures/mp4/ref_video_variable_parameters.mp4" 128 | ] 129 | 130 | @spec generate_avc_fixtures() :: :ok 131 | def generate_avc_fixtures() do 132 | Enum.each(@mp4_avc1_fixtures, fn input_location -> 133 | generate_fixture(input_location, :au, {:avc1, 4}) 134 | generate_fixture(input_location, :nalu, {:avc1, 4}) 135 | end) 136 | 137 | Enum.each(@mp4_avc3_fixtures, fn input_location -> 138 | generate_fixture(input_location, :au, {:avc3, 4}) 139 | generate_fixture(input_location, :nalu, {:avc3, 4}) 140 | end) 141 | end 142 | 143 | defp generate_fixture(input_location, output_alignment, {avc, _prefix_len} = stream_structure) do 144 | output_location = 145 | input_location 146 | |> Path.split() 147 | |> List.replace_at(-2, "msr") 148 | |> List.update_at(-1, fn file -> 149 | [name, "mp4"] = String.split(file, ".") 150 | "#{name}-#{avc}-#{output_alignment}.msr" 151 | end) 152 | |> Path.join() 153 | 154 | options = %{ 155 | input_location: input_location, 156 | output_location: output_location, 157 | output_alignment: output_alignment, 158 | stream_structure: stream_structure 159 | } 160 | 161 | {:ok, _supervisor_pid, pipeline_pid} = FixtureGeneratorPipeline.start(options) 162 | ref = Process.monitor(pipeline_pid) 163 | 164 | receive do 165 | {:DOWN, ^ref, :process, _pipeline_pid, _reason} -> 166 | :ok 167 | end 168 | 169 | Membrane.Pipeline.terminate(pipeline_pid) 170 | end 171 | end 172 | 173 | AVCFixtureGenerator.generate_avc_fixtures() 174 | -------------------------------------------------------------------------------- /test/support/test_source.ex: -------------------------------------------------------------------------------- 1 | defmodule Membrane.H26x.Support.TestSource do 2 | @moduledoc false 3 | 4 | use Membrane.Source 5 | 6 | def_options mode: [], 7 | output_raw_stream_structure: [default: :annexb], 8 | codec: [default: :H264] 9 | 10 | def_output_pad :output, 11 | flow_control: :push, 12 | accepted_format: 13 | any_of( 14 | %Membrane.RemoteStream{type: :packetized}, 15 | %Membrane.H264{alignment: alignment} when alignment in [:au, :nalu], 16 | %Membrane.H265{alignment: alignment} when alignment in [:au, :nalu] 17 | ) 18 | 19 | @impl true 20 | def handle_init(_ctx, opts) do 21 | {[], 22 | %{ 23 | mode: opts.mode, 24 | codec: opts.codec, 25 | output_raw_stream_structure: opts.output_raw_stream_structure 26 | }} 27 | end 28 | 29 | @impl true 30 | def handle_parent_notification(actions, _ctx, state) do 31 | {actions, state} 32 | end 33 | 34 | @impl true 35 | def handle_playing(_ctx, state) do 36 | stream_format = 37 | case {state.codec, state.mode} do 38 | {_codec, :bytestream} -> 39 | %Membrane.RemoteStream{type: :packetized} 40 | 41 | {:H264, :nalu_aligned} -> 42 | %Membrane.H264{alignment: :nalu, stream_structure: state.output_raw_stream_structure} 43 | 44 | {:H264, :au_aligned} -> 45 | %Membrane.H264{alignment: :au, stream_structure: state.output_raw_stream_structure} 46 | 47 | {:H265, :nalu_aligned} -> 48 | %Membrane.H265{alignment: :nalu, stream_structure: state.output_raw_stream_structure} 49 | 50 | {:H265, :au_aligned} -> 51 | %Membrane.H265{alignment: :au, stream_structure: state.output_raw_stream_structure} 52 | end 53 | 54 | {[stream_format: {:output, stream_format}], state} 55 | end 56 | end 57 | -------------------------------------------------------------------------------- /test/test_helper.exs: -------------------------------------------------------------------------------- 1 | ExUnit.start(capture_log: true) 2 | --------------------------------------------------------------------------------