├── .dockerignore ├── .github └── workflows │ └── ruff.yml ├── .gitignore ├── Dockerfile.rocm ├── LICENSE ├── README.md ├── docker-compose.yaml ├── pyproject.toml ├── scripts ├── __init__.py ├── encode.py ├── metrics.py ├── plot.py ├── scores.py └── stats.py └── static ├── butter_distance_plot.svg ├── ssimu2_hmean_plot.svg └── wxpsnr_plot.svg /.dockerignore: -------------------------------------------------------------------------------- 1 | 2 | # Byte-compiled / optimized / DLL files 3 | __pycache__/ 4 | *.py[cod] 5 | *$py.class 6 | 7 | # C extensions 8 | *.so 9 | 10 | # Distribution / packaging 11 | .Python 12 | build/ 13 | build/** 14 | develop-eggs/ 15 | dist/ 16 | downloads/ 17 | eggs/ 18 | .eggs/ 19 | lib/ 20 | lib64/ 21 | parts/ 22 | sdist/ 23 | var/ 24 | wheels/ 25 | share/python-wheels/ 26 | *.egg-info/ 27 | .installed.cfg 28 | *.egg 29 | MANIFEST 30 | 31 | # PyInstaller 32 | # Usually these files are written by a python script from a template 33 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 34 | *.manifest 35 | *.spec 36 | 37 | # Installer logs 38 | pip-log.txt 39 | pip-delete-this-directory.txt 40 | 41 | # Unit test / coverage reports 42 | htmlcov/ 43 | .tox/ 44 | .nox/ 45 | .coverage 46 | .coverage.* 47 | .cache 48 | nosetests.xml 49 | coverage.xml 50 | *.cover 51 | *.py,cover 52 | .hypothesis/ 53 | .pytest_cache/ 54 | cover/ 55 | 56 | # Translations 57 | *.mo 58 | *.pot 59 | 60 | # Django stuff: 61 | *.log 62 | local_settings.py 63 | db.sqlite3 64 | db.sqlite3-journal 65 | 66 | # Flask stuff: 67 | instance/ 68 | .webassets-cache 69 | 70 | # Scrapy stuff: 71 | .scrapy 72 | 73 | # Sphinx documentation 74 | docs/_build/ 75 | 76 | # PyBuilder 77 | .pybuilder/ 78 | target/ 79 | 80 | # Jupyter Notebook 81 | .ipynb_checkpoints 82 | 83 | # IPython 84 | profile_default/ 85 | ipython_config.py 86 | 87 | # pyenv 88 | # For a library or package, you might want to ignore these files since the code is 89 | # intended to run in multiple environments; otherwise, check them in: 90 | # .python-version 91 | 92 | # pipenv 93 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 94 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 95 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 96 | # install all needed dependencies. 97 | #Pipfile.lock 98 | 99 | # UV 100 | # Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control. 101 | # This is especially recommended for binary packages to ensure reproducibility, and is more 102 | # commonly ignored for libraries. 103 | #uv.lock 104 | 105 | # poetry 106 | # Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. 107 | # This is especially recommended for binary packages to ensure reproducibility, and is more 108 | # commonly ignored for libraries. 109 | # https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control 110 | #poetry.lock 111 | 112 | # pdm 113 | # Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. 114 | #pdm.lock 115 | # pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it 116 | # in version control. 117 | # https://pdm.fming.dev/latest/usage/project/#working-with-version-control 118 | .pdm.toml 119 | .pdm-python 120 | .pdm-build/ 121 | 122 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm 123 | __pypackages__/ 124 | 125 | # Celery stuff 126 | celerybeat-schedule 127 | celerybeat.pid 128 | 129 | # SageMath parsed files 130 | *.sage.py 131 | 132 | # Environments 133 | .env 134 | .venv 135 | env/ 136 | venv/ 137 | ENV/ 138 | env.bak/ 139 | venv.bak/ 140 | 141 | # Spyder project settings 142 | .spyderproject 143 | .spyproject 144 | 145 | # Rope project settings 146 | .ropeproject 147 | 148 | # mkdocs documentation 149 | /site 150 | 151 | # mypy 152 | .mypy_cache/ 153 | .dmypy.json 154 | dmypy.json 155 | 156 | # Pyre type checker 157 | .pyre/ 158 | 159 | # pytype static type analyzer 160 | .pytype/ 161 | 162 | # Cython debug symbols 163 | cython_debug/ 164 | 165 | # PyCharm 166 | # JetBrains specific template is maintained in a separate JetBrains.gitignore that can 167 | # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore 168 | # and can be added to the global gitignore or merged into this file. For a more nuclear 169 | # option (not recommended) you can uncomment the following to ignore the entire idea folder. 170 | #.idea/ 171 | .DS_Store 172 | 173 | # PyPI configuration file 174 | .pypirc 175 | 176 | # uv 177 | uv.lock 178 | .ruff_cache/ 179 | 180 | # Project-specific 181 | *.csv 182 | *.ivf 183 | *.266 184 | *.265 185 | *.264 186 | *.png 187 | *.sh 188 | scripts/*.svg 189 | -------------------------------------------------------------------------------- /.github/workflows/ruff.yml: -------------------------------------------------------------------------------- 1 | name: Ruff Lint 2 | 3 | on: 4 | push: 5 | branches: 6 | - main 7 | pull_request: 8 | branches: 9 | - main 10 | 11 | jobs: 12 | build: 13 | runs-on: ubuntu-latest 14 | strategy: 15 | matrix: 16 | python-version: ["3.13.3"] 17 | steps: 18 | - uses: actions/checkout@v4 19 | - uses: astral-sh/ruff-action@v3 20 | with: 21 | version: "latest" 22 | src: >- 23 | ./scripts/encode.py 24 | ./scripts/metrics.py 25 | ./scripts/plot.py 26 | ./scripts/scores.py 27 | ./scripts/stats.py 28 | - run: ruff check --fix 29 | - run: ruff format 30 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | share/python-wheels/ 24 | *.egg-info/ 25 | .installed.cfg 26 | *.egg 27 | MANIFEST 28 | 29 | # PyInstaller 30 | # Usually these files are written by a python script from a template 31 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 32 | *.manifest 33 | *.spec 34 | 35 | # Installer logs 36 | pip-log.txt 37 | pip-delete-this-directory.txt 38 | 39 | # Unit test / coverage reports 40 | htmlcov/ 41 | .tox/ 42 | .nox/ 43 | .coverage 44 | .coverage.* 45 | .cache 46 | nosetests.xml 47 | coverage.xml 48 | *.cover 49 | *.py,cover 50 | .hypothesis/ 51 | .pytest_cache/ 52 | cover/ 53 | 54 | # Translations 55 | *.mo 56 | *.pot 57 | 58 | # Django stuff: 59 | *.log 60 | local_settings.py 61 | db.sqlite3 62 | db.sqlite3-journal 63 | 64 | # Flask stuff: 65 | instance/ 66 | .webassets-cache 67 | 68 | # Scrapy stuff: 69 | .scrapy 70 | 71 | # Sphinx documentation 72 | docs/_build/ 73 | 74 | # PyBuilder 75 | .pybuilder/ 76 | target/ 77 | 78 | # Jupyter Notebook 79 | .ipynb_checkpoints 80 | 81 | # IPython 82 | profile_default/ 83 | ipython_config.py 84 | 85 | # pyenv 86 | # For a library or package, you might want to ignore these files since the code is 87 | # intended to run in multiple environments; otherwise, check them in: 88 | # .python-version 89 | 90 | # pipenv 91 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 92 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 93 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 94 | # install all needed dependencies. 95 | #Pipfile.lock 96 | 97 | # UV 98 | # Similar to Pipfile.lock, it is generally recommended to include uv.lock in version control. 99 | # This is especially recommended for binary packages to ensure reproducibility, and is more 100 | # commonly ignored for libraries. 101 | #uv.lock 102 | 103 | # poetry 104 | # Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control. 105 | # This is especially recommended for binary packages to ensure reproducibility, and is more 106 | # commonly ignored for libraries. 107 | # https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control 108 | #poetry.lock 109 | 110 | # pdm 111 | # Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control. 112 | #pdm.lock 113 | # pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it 114 | # in version control. 115 | # https://pdm.fming.dev/latest/usage/project/#working-with-version-control 116 | .pdm.toml 117 | .pdm-python 118 | .pdm-build/ 119 | 120 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm 121 | __pypackages__/ 122 | 123 | # Celery stuff 124 | celerybeat-schedule 125 | celerybeat.pid 126 | 127 | # SageMath parsed files 128 | *.sage.py 129 | 130 | # Environments 131 | .env 132 | .venv 133 | env/ 134 | venv/ 135 | ENV/ 136 | env.bak/ 137 | venv.bak/ 138 | 139 | # Spyder project settings 140 | .spyderproject 141 | .spyproject 142 | 143 | # Rope project settings 144 | .ropeproject 145 | 146 | # mkdocs documentation 147 | /site 148 | 149 | # mypy 150 | .mypy_cache/ 151 | .dmypy.json 152 | dmypy.json 153 | 154 | # Pyre type checker 155 | .pyre/ 156 | 157 | # pytype static type analyzer 158 | .pytype/ 159 | 160 | # Cython debug symbols 161 | cython_debug/ 162 | 163 | # PyCharm 164 | # JetBrains specific template is maintained in a separate JetBrains.gitignore that can 165 | # be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore 166 | # and can be added to the global gitignore or merged into this file. For a more nuclear 167 | # option (not recommended) you can uncomment the following to ignore the entire idea folder. 168 | #.idea/ 169 | .DS_Store 170 | 171 | # PyPI configuration file 172 | .pypirc 173 | 174 | # uv 175 | uv.lock 176 | .ruff_cache/ 177 | 178 | # Project-specific 179 | *.csv 180 | *.ivf 181 | *.266 182 | *.265 183 | *.264 184 | *.png 185 | *.sh 186 | scripts/*.svg 187 | -------------------------------------------------------------------------------- /Dockerfile.rocm: -------------------------------------------------------------------------------- 1 | FROM archlinux:latest AS base 2 | # pass GPU_ARCH=$(gpu-arch) to build against your specific GPU 3 | ARG GPU_ARCH 4 | 5 | RUN pacman -Syu --noconfirm 6 | RUN pacman -S --needed base-devel ffms2 wget make curl vapoursynth git \ 7 | python jq rocm-hip-sdk rocm-opencl-sdk --noconfirm 8 | 9 | ENV PATH="${PATH}:/opt/rocm/bin" 10 | ENV ROCM_PATH=/opt/rocm 11 | 12 | RUN git clone https://github.com/dnjulek/vapoursynth-zip.git --depth 1 --branch R6 && \ 13 | cd vapoursynth-zip/build-help && \ 14 | chmod +x build.sh && ./build.sh 15 | 16 | # offload-arch=native build wont work; need to pass in specific arch of host machine to compile. 17 | RUN git clone https://github.com/Line-fr/Vship.git && \ 18 | cd Vship && \ 19 | sed -i "s/--offload-arch=native/--offload-arch=${GPU_ARCH}/g" Makefile && \ 20 | make build && make install 21 | 22 | # VENV Setup 23 | ENV VIRTUAL_ENV=/opt/venv 24 | RUN python -m venv $VIRTUAL_ENV 25 | # Enable venv 26 | ENV PATH="$VIRTUAL_ENV/bin:$PATH" 27 | FROM base AS app 28 | 29 | WORKDIR /metrics 30 | COPY . /metrics 31 | RUN pip install uv && \ 32 | uv pip install -e . 33 | VOLUME ["/videos"] 34 | WORKDIR /videos 35 | 36 | ENTRYPOINT ["uv", "run", "--active"] 37 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # PSY-EX Metrics 2 | 3 | The Psychovisual Experts group presents `metrics`, a video quality assessment 4 | toolkit that provides a suite of scripts for measuring and comparing video 5 | codecs using metrics such as: 6 | 7 | - Average SSIMULACRA2 across frames 8 | - Average Butteraugli (3pnorm) across frames 9 | - Weighted XPSNR 10 | - VMAF NEG (Harmonic Mean) 11 | - VMAF 12 | - SSIM 13 | - PSNR 14 | 15 | We include modules for processing video files, running video encoding, and 16 | generating both numerical and visual reports from quality metrics. 17 | 18 | The four main scripts are: 19 | 20 | - `scores.py`: Compute detailed quality scores for a given source/distorted 21 | video pair. 22 | - `encode.py`: Encode videos using various codecs (e.g., x264, x265, svtav1, 23 | aomenc, libvpx/VP9) and print metrics. 24 | - `stats.py`: Encode videos at various quality settings and log metric 25 | statistics to a CSV file. 26 | - `plot.py`: Generate visual plots from CSV metric data for side-by-side codec 27 | comparison. 28 | 29 | Read more below on how to install and use these utilities. 30 | 31 | ## Sample Graphs 32 | 33 | ![SSIMULACRA2](./static/ssimu2_hmean_plot.svg) 34 | 35 | ![Butteraugli](./static/butter_distance_plot.svg) 36 | 37 | ![W-XPSNR](./static/wxpsnr_plot.svg) 38 | 39 | ## Overview 40 | 41 | PSY-EX Metrics enables you to: 42 | 43 | - Encode videos using various codecs (e.g., x264, x265, svtav1, aomenc, 44 | libvpx/VP9). 45 | - Calculate various metrics. 46 | - Generate CSV files with computed metric statistics for further analysis. 47 | - Visualize the metrics side-by-side, comparing codec results through 48 | customizable plots. 49 | 50 | ## Installation 51 | 52 | ### Dependencies 53 | 54 | - [uv](https://github.com/astral-sh/uv/blob/main/README.md), a Python project 55 | manager 56 | - FFmpeg >= 7.1 (required for XPSNR) 57 | - VapourSynth, and required plugins: 58 | - ffms2 59 | - [vszip](https://github.com/dnjulek/vapoursynth-zip) 60 | - [vship](https://github.com/Line-fr/Vship) [Optional: provides GPU support] 61 | 62 | ### Install Steps 63 | 64 | 0. Install required dependencies outlined in the previous section 65 | 66 | 1. Clone the repository 67 | 68 | ```bash 69 | git clone https://github.com/psy-ex/metrics.git 70 | cd metrics/ 71 | ``` 72 | 73 | 2. Enter the scripts directory & mark the scripts as executable 74 | 75 | ```bash 76 | cd scripts/ 77 | chmod a+x stats.py scores.py plot.py encode.py 78 | ``` 79 | 80 | 3. Run a script, no Python package installation required 81 | 82 | ```bash 83 | ./scores.py source.mkv distorted.mkv 84 | ``` 85 | 86 | ## Usage 87 | 88 | ### scores.py 89 | 90 | ```bash 91 | % ./scores.py --help 92 | usage: scores.py [-h] [-e EVERY] [-g GPU_STREAMS] [-t THREADS] source distorted 93 | 94 | Run metrics given a source video & a distorted video. 95 | 96 | positional arguments: 97 | source Source video path 98 | distorted Distorted video path 99 | 100 | options: 101 | -h, --help show this help message and exit 102 | -e, --every EVERY Only score every nth frame. Default 1 (every frame) 103 | -g, --gpu-streams GPU_STREAMS 104 | Number of GPU streams for SSIMULACRA2/Butteraugli 105 | -t, --threads THREADS 106 | Number of threads for SSIMULACRA2/Butteraugli 107 | ``` 108 | 109 | Example: 110 | 111 | ```bash 112 | ./scores.py source.mkv distorted.mkv -e 3 113 | ``` 114 | 115 | This command compares a reference `source.mkv` with `distorted.mkv`, scoring 116 | every 3rd frame. 117 | 118 | ### encode.py 119 | 120 | ```bash 121 | % ./encode.py --help 122 | usage: encode.py [-h] -i INPUT -q QUALITY [-b KEEP] [-e EVERY] [-g GPU_STREAMS] [-t THREADS] [-n] {x264,x265,svtav1,aomenc,vpxenc} ... 123 | 124 | Generate statistics for a single video encode. 125 | 126 | positional arguments: 127 | {x264,x265,svtav1,aomenc,vpxenc} 128 | Which video encoder to use 129 | encoder_args Additional encoder arguments (pass these after a '--' delimiter) 130 | 131 | options: 132 | -h, --help show this help message and exit 133 | -i, --input INPUT Path to source video file 134 | -q, --quality QUALITY 135 | Desired CRF value for the encoder 136 | -b, --keep KEEP Output video file name 137 | -e, --every EVERY Only score every nth frame. Default 1 (every frame) 138 | -g, --gpu-streams GPU_STREAMS 139 | Number of GPU streams for SSIMULACRA2/Butteraugli 140 | -t, --threads THREADS 141 | Number of threads for SSIMULACRA2/Butteraugli 142 | -n, --no-metrics Skip metrics calculations 143 | ``` 144 | 145 | Examples: 146 | 147 | ```bash 148 | ./encode.py -i source.mkv --keep video.ivf -q 29 svtav1 -- --preset 2 149 | ``` 150 | 151 | This command encodes `source.mkv` at a CRF of 29 using the SVT-AV1 encoder with 152 | the `--preset 2` argument. It will print metrics after encoding. 153 | 154 | ```bash 155 | ./encode.py -i source.mkv --keep video.ivf -q 29 -g 4 svtav1 -- --preset 8 156 | ``` 157 | 158 | This command does the same as the previous command, but uses 4 GPU streams 159 | (instead of using the CPU) as well as passing a higher preset value to SVT-AV1. 160 | 161 | ### stats.py 162 | 163 | ```bash 164 | usage: stats.py [-h] -i INPUTS [INPUTS ...] -q QUALITY -o OUTPUT [-e EVERY] [-g GPU_STREAMS] [-t THREADS] [-k] {x264,x265,svtav1,aomenc,vpxenc} ... 165 | 166 | Generate statistics for a series of video encodes. 167 | 168 | positional arguments: 169 | {x264,x265,svtav1,aomenc,vpxenc} 170 | Which video encoder to use 171 | encoder_args Additional encoder arguments (pass these after a '--' delimiter) 172 | 173 | options: 174 | -h, --help show this help message and exit 175 | -i, --inputs INPUTS [INPUTS ...] 176 | Path(s) to source video file(s) 177 | -q, --quality QUALITY 178 | List of quality values to test (e.g. 20 30 40 50) 179 | -o, --output OUTPUT Path to output CSV file 180 | -e, --every EVERY Only score every nth frame. Default 1 (every frame) 181 | -g, --gpu-streams GPU_STREAMS 182 | Number of GPU streams for SSIMULACRA2/Butteraugli 183 | -t, --threads THREADS 184 | Number of threads for SSIMULACRA2/Butteraugli 185 | -k, --keep Keep output video files 186 | ``` 187 | 188 | Example: 189 | 190 | ```bash 191 | ./stats.py \ 192 | -i source.mkv \ 193 | -q "20 25 30 35 40" \ 194 | -o ./svtav1_2.3.0-B_p8.csv \ 195 | -e 3 \ 196 | svtav1 -- --preset 8 --tune 2 197 | ``` 198 | 199 | This command processes `source.mkv` at quality levels 20, 25, 30, 35, & 40 using 200 | the SVT-AV1 encoder, scoring every 3rd frame, and writes the results to an 201 | output `svtav1_2.3.0-B_p8.csv`. 202 | 203 | You can also script this to run across multiple speed presets: 204 | 205 | ```bash 206 | #!/bin/bash -eu 207 | 208 | for speed in {2..6}; do 209 | ./stats.py \ 210 | -i ~/Videos/reference/*.y4m \ 211 | -q "20 25 30 35 40" \ 212 | -o ./svtav1_2.3.0-B_p${speed}.csv \ 213 | -e 3 \ 214 | svtav1 -- --preset $speed --tune 2 215 | done 216 | ``` 217 | 218 | This snippet here will run the same command as before, but across all speed 219 | presets from 2 to 6 (naming the output CSV files accordingly). It will also 220 | encode all of the files ending in `.y4m` in the `~/Videos/reference` directory. 221 | 222 | You can also use libvpx for VP9 benchmarking: 223 | 224 | ```bash 225 | ./stats.py \ 226 | -i source.mkv \ 227 | -q "20 25 30 35 40" \ 228 | -o ./libvpx_vp9.csv \ 229 | -g 4 \ 230 | vpxenc -- --cpu-used=4 231 | ``` 232 | 233 | ### plot.py 234 | 235 | ```bash 236 | ./plot.py --help 237 | usage: plot.py [-h] -i INPUT [INPUT ...] [-f FORMAT] 238 | 239 | Plot codec metrics from one or more CSV files (one per codec) for side-by-side comparison. 240 | 241 | options: 242 | -h, --help show this help message and exit 243 | -i, --input INPUT [INPUT ...] 244 | Path(s) to CSV file(s). Each CSV file should have the same columns. 245 | -f, --format FORMAT Save the plot as 'svg', 'png', or 'webp' 246 | ``` 247 | 248 | Example: 249 | 250 | ```bash 251 | ./plot.py -i codec1_results.csv codec2_results.csv -f webp 252 | ``` 253 | 254 | This command reads `codec1_results.csv` and `codec2_results.csv`, generating 255 | separate plots (one for each metric) as WebP images. 256 | 257 | It will also print BD-rate statistics for each metric, comparing the two results 258 | from the CSV files. 259 | 260 | `plot.py` also outputs a CSV file containing the average encode time for an 261 | input file accompanied by the corresponding average BD-rate. These statistics 262 | can assist in looking at overall encoder efficiency across multiple speed 263 | presets or configurations. 264 | 265 | ### Run via Docker 266 | 267 | See the pre-requisites for host machine: 268 | https://rocm.docs.amd.com/projects/install-on-linux/en/latest/how-to/docker.html 269 | 270 | 271 | 1. Build image against your GPU architechture: 272 | ```bash 273 | GPU_ARCH=$(amdgpu-arch) docker-compose --build 274 | ``` 275 | 276 | 2. Run `docker-compose`: 277 | ```bash 278 | docker-compose -f run -v :/videos metrics-rocm <..args> 279 | ``` 280 | 281 | ## License 282 | 283 | This project was originally authored by @gianni-rosato & is provided as FOSS 284 | through the Psychovisual Experts Group. 285 | 286 | Usage is subject to the terms of the [LICENSE](LICENSE). Please refer to the 287 | linked file for more information. 288 | -------------------------------------------------------------------------------- /docker-compose.yaml: -------------------------------------------------------------------------------- 1 | version: "3.9" 2 | services: 3 | # rocm config 4 | metrics-rocm: 5 | image: metrics 6 | build: 7 | context: . 8 | dockerfile: Dockerfile.rocm 9 | args: 10 | - GPU_ARCH=${GPU_ARCH:-native} 11 | devices: 12 | - /dev/kfd 13 | - /dev/dri 14 | security_opt: 15 | - seccomp:unconfined 16 | group_add: 17 | - video 18 | - render 19 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [project] 2 | name = "metrics" 3 | version = "0.1.0" 4 | description = "Run metrics given a source video & a distorted video." 5 | readme = "README.md" 6 | requires-python = ">=3.13.1" 7 | dependencies = [ 8 | "argparse>=1.4.0", 9 | "matplotlib>=3.10.0", 10 | "numpy>=2.2.2", 11 | "scipy>=1.15.1", 12 | "statistics>=1.0.3.5", 13 | "tqdm>=4.67.1", 14 | "vapoursynth>=70", 15 | "vstools>=3.3.4", 16 | ] 17 | 18 | [build-system] 19 | requires = ["setuptools >= 77.0.3"] 20 | build-backend = "setuptools.build_meta" 21 | 22 | [tool.setuptools] 23 | # specify root package dir 24 | package-dir = { "" = "scripts" } 25 | 26 | 27 | [project.scripts] 28 | stats = "stats:main" 29 | plot = "plot:main" 30 | scores = "scores:main" 31 | encode = "encode:main" 32 | -------------------------------------------------------------------------------- /scripts/__init__.py: -------------------------------------------------------------------------------- 1 | from .metrics import CoreVideo, DstVideo 2 | 3 | __all__ = ["DstVideo", "CoreVideo"] 4 | -------------------------------------------------------------------------------- /scripts/encode.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env -S uv run --script 2 | # /// script 3 | # requires-python = ">=3.13.1" 4 | # dependencies = [ 5 | # "argparse>=1.4.0", 6 | # "statistics>=1.0.3.5", 7 | # "tqdm>=4.67.1", 8 | # "vapoursynth>=70", 9 | # "vstools>=3.3.4", 10 | # ] 11 | # /// 12 | 13 | import argparse 14 | from argparse import Namespace 15 | 16 | from metrics import CoreVideo, DstVideo, VideoEnc 17 | 18 | 19 | def main(): 20 | parser = argparse.ArgumentParser( 21 | description="Generate statistics for a single video encode." 22 | ) 23 | parser.add_argument( 24 | "-i", "--input", required=True, type=str, help="Path to source video file" 25 | ) 26 | parser.add_argument( 27 | "-q", 28 | "--quality", 29 | required=True, 30 | type=int, 31 | help="Desired CRF value for the encoder", 32 | ) 33 | parser.add_argument( 34 | "encoder", 35 | choices=["x264", "x265", "svtav1", "aomenc", "vpxenc"], 36 | type=str, 37 | help="Which video encoder to use", 38 | ) 39 | parser.add_argument("-b", "--keep", type=str, help="Output video file name") 40 | parser.add_argument( 41 | "-e", 42 | "--every", 43 | type=int, 44 | default=1, 45 | help="Only score every nth frame. Default 1 (every frame)", 46 | ) 47 | parser.add_argument( 48 | "encoder_args", 49 | nargs=argparse.REMAINDER, 50 | type=str, 51 | help="Additional encoder arguments (pass these after a '--' delimiter)", 52 | ) 53 | parser.add_argument( 54 | "-g", 55 | "--gpu-streams", 56 | type=int, 57 | default=0, 58 | help="Number of GPU streams for SSIMULACRA2/Butteraugli", 59 | ) 60 | parser.add_argument( 61 | "-t", 62 | "--threads", 63 | type=int, 64 | default=0, 65 | help="Number of threads for SSIMULACRA2/Butteraugli", 66 | ) 67 | parser.add_argument( 68 | "-n", "--no-metrics", action="store_true", help="Skip metrics calculations" 69 | ) 70 | 71 | args: Namespace = parser.parse_args() 72 | src_pth: str = args.input 73 | dst_pth: str = args.keep 74 | q: int = args.quality 75 | enc: str = args.encoder 76 | every: int = args.every 77 | enc_args: list[str] = args.encoder_args 78 | threads: int = args.threads 79 | gpu_streams: int = args.gpu_streams 80 | run_metrics: bool = not args.no_metrics 81 | 82 | s: CoreVideo = CoreVideo(src_pth, every, threads, gpu_streams) 83 | 84 | if dst_pth: 85 | e: VideoEnc = VideoEnc(s, q, enc, enc_args, dst_pth) 86 | else: 87 | e: VideoEnc = VideoEnc(s, q, enc, enc_args) 88 | v: DstVideo = e.encode(every, threads, gpu_streams) 89 | print(f"Encoded {s.name} --> {e.dst_pth} (took {e.time:.2f} seconds)") 90 | 91 | if run_metrics: 92 | v.calculate_ssimulacra2(s) 93 | v.print_ssimulacra2() 94 | v.calculate_butteraugli(s) 95 | if gpu_streams: 96 | v.print_butteraugli() 97 | v.calculate_ffmpeg_metrics(s) 98 | v.print_ffmpeg_metrics() 99 | 100 | if not dst_pth: 101 | e.remove_output() 102 | print(f"Discarded encode {e.dst_pth}") 103 | 104 | 105 | if __name__ == "__main__": 106 | main() 107 | -------------------------------------------------------------------------------- /scripts/metrics.py: -------------------------------------------------------------------------------- 1 | import math 2 | import os 3 | import re 4 | import statistics 5 | import subprocess 6 | import time 7 | from subprocess import Popen 8 | 9 | import vapoursynth as vs 10 | from tqdm import tqdm 11 | from vapoursynth import VideoNode 12 | from vstools import initialize_clip 13 | 14 | 15 | class CoreVideo: 16 | """ 17 | Source video class. 18 | """ 19 | 20 | path: str 21 | name: str 22 | size: int 23 | e: int 24 | threads: int 25 | gpu_streams: int 26 | 27 | video_width: int 28 | video_height: int 29 | 30 | # VapourSynth video object 31 | video: VideoNode 32 | 33 | def __init__(self, pth: str, e: int, t: int, g: int) -> None: 34 | self.path = pth 35 | self.name = os.path.basename(pth) 36 | self.size = self.get_input_filesize() 37 | self.e = e 38 | self.threads = t 39 | self.gpu_streams = g 40 | self.video = self.vapoursynth_init() 41 | self.video_width, self.video_height = self.get_video_dimensions() 42 | 43 | def get_input_filesize(self) -> int: 44 | """ 45 | Get the input file size of the distorted video. 46 | """ 47 | return os.path.getsize(self.path) 48 | 49 | def vapoursynth_init(self) -> VideoNode: 50 | """ 51 | Initialize VapourSynth video object for the distorted video. 52 | """ 53 | core = vs.core 54 | print( 55 | f"Using {self.threads} {'GPU' if self.gpu_streams else 'CPU'} threads for {'SSIMULACRA2 & Butteraugli' if self.gpu_streams else 'SSIMULACRA2'}" 56 | ) 57 | if self.gpu_streams: 58 | print(f"Using {self.gpu_streams} GPU streams for SSIMULACRA2 & Butteraugli") 59 | core.num_threads = self.threads 60 | video = core.ffms2.Source(source=self.path, cache=False, threads=self.threads) 61 | video = initialize_clip(video, bits=0) 62 | video = video.resize.Bicubic(format=vs.RGBS) 63 | if self.e > 1: 64 | video = video.std.SelectEvery(cycle=self.e, offsets=0) 65 | return video 66 | 67 | def get_video_dimensions(self) -> tuple[int, int]: 68 | """ 69 | Get the width & height of the distorted video. 70 | """ 71 | core = vs.core 72 | src_data = core.ffms2.Source(source=self.path, cache=False, threads=int(-1)) 73 | return (src_data.width, src_data.height) 74 | 75 | 76 | class DstVideo(CoreVideo): 77 | """ 78 | Distorted video class containing metric scores. 79 | """ 80 | 81 | # SSIMULACRA2 scores 82 | ssimu2_avg: float 83 | ssimu2_sdv: float 84 | ssimu2_p10: float 85 | 86 | # Butteraugli scores 87 | butter_dis: float 88 | butter_mds: float 89 | 90 | # XPSNR scores 91 | xpsnr_y: float 92 | xpsnr_u: float 93 | xpsnr_v: float 94 | w_xpsnr: float 95 | 96 | # VMAF scores 97 | vmaf: float 98 | vmaf_neg_hmn: float 99 | 100 | # SSIM score 101 | ssim: float 102 | 103 | # PSNR score 104 | psnr: float 105 | 106 | # Average VMAF, SSIM, PSNR 107 | svt_triple: float 108 | 109 | def __init__(self, pth: str, e: int, t: int, g: int) -> None: 110 | self.path = pth 111 | self.name = os.path.basename(pth) 112 | self.size = self.get_input_filesize() 113 | self.e = e 114 | self.video_width, self.video_height = self.get_video_dimensions() 115 | self.threads = t 116 | self.gpu_streams = g 117 | self.video = self.vapoursynth_init() 118 | self.ssimu2_avg = 0.0 119 | self.ssimu2_sdv = 0.0 120 | self.ssimu2_p10 = 0.0 121 | self.butter_dis = 0.0 122 | self.butter_mds = 0.0 123 | self.xpsnr_y = 0.0 124 | self.xpsnr_u = 0.0 125 | self.xpsnr_v = 0.0 126 | self.w_xpsnr = 0.0 127 | self.vmaf = 0.0 128 | self.vmaf_neg_hmn = 0.0 129 | self.ssim = 0.0 130 | self.psnr = 0.0 131 | self.svt_triple = 0.0 132 | 133 | def calculate_ssimulacra2(self, src: CoreVideo) -> None: 134 | """ 135 | Calculate SSIMULACRA2 score between a source video & a distorted video. 136 | """ 137 | 138 | if self.gpu_streams: 139 | ssimu2_obj = src.video.vship.SSIMULACRA2( 140 | self.video, numStream=self.gpu_streams 141 | ) 142 | else: 143 | ssimu2_obj = src.video.vszip.Metrics(self.video, [0]) 144 | 145 | ssimu2_list: list[float] = [] 146 | with tqdm( 147 | total=ssimu2_obj.num_frames, 148 | desc="Calculating SSIMULACRA2 scores", 149 | unit=" frame", 150 | colour="blue", 151 | ) as pbar: 152 | for i, f in enumerate(ssimu2_obj.frames()): 153 | ssimu2_list.append(f.props["_SSIMULACRA2"]) 154 | pbar.update(1) 155 | if not i % 24: 156 | avg: float = sum(ssimu2_list) / len(ssimu2_list) 157 | pbar.set_postfix( 158 | { 159 | "avg": f"{avg:.2f}", 160 | } 161 | ) 162 | 163 | self.ssimu2_avg, self.ssimu2_sdv, self.ssimu2_p10 = calc_some_scores( 164 | ssimu2_list 165 | ) 166 | 167 | def calculate_butteraugli( 168 | self, 169 | src: CoreVideo, 170 | ) -> None: 171 | """ 172 | Calculate Butteraugli score between a source video & a distorted video. 173 | """ 174 | 175 | if self.gpu_streams: 176 | butter_obj = src.video.vship.BUTTERAUGLI( 177 | self.video, numStream=self.gpu_streams 178 | ) 179 | else: 180 | print("Skipping Butteraugli, no GPU threads available (set with -g)") 181 | self.butter_dis = 0.0 182 | self.butter_mds = 0.0 183 | return 184 | 185 | butter_distance_list: list[float] = [] 186 | with tqdm( 187 | total=butter_obj.num_frames, 188 | desc="Calculating Butteraugli scores", 189 | unit=" frame", 190 | colour="yellow", 191 | ) as pbar: 192 | for i, f in enumerate(butter_obj.frames()): 193 | d: float = f.props["_BUTTERAUGLI_3Norm"] 194 | butter_distance_list.append(d) 195 | pbar.update(1) 196 | if not i % 24: 197 | dis: float = sum(butter_distance_list) / len(butter_distance_list) 198 | pbar.set_postfix( 199 | { 200 | "dis": f"{dis:.2f}", 201 | } 202 | ) 203 | 204 | self.butter_dis = sum(butter_distance_list) / len(butter_distance_list) 205 | self.butter_mds = max(butter_distance_list) 206 | 207 | def calculate_ffmpeg_metrics(self, src: CoreVideo) -> None: 208 | """ 209 | Calculate XPSNR, SSIM, PSNR, VMAF & VMAF-NEG scores between a source 210 | video & a distorted video with FFmpeg. 211 | """ 212 | 213 | filtergraph: str = ( 214 | "[0:v]split=5[dst0][dst1][dst2][dst3][dst4];" 215 | "[1:v]split=5[src0][src1][src2][src3][src4];" 216 | "[dst0][src0]ssim;" 217 | "[dst1][src1]psnr;" 218 | "[dst2][src2]xpsnr=shortest=1;" 219 | f"[dst3][src3]libvmaf=model='version=vmaf_v0.6.1':n_threads={self.threads}:n_subsample={self.e};" 220 | f"[dst4][src4]libvmaf=model='version=vmaf_v0.6.1neg':n_threads={self.threads}:n_subsample={self.e}:pool=harmonic_mean" 221 | ) 222 | 223 | cmd: list[str] = [ 224 | "ffmpeg", 225 | "-threads", 226 | str(self.threads), 227 | "-hide_banner", 228 | "-i", 229 | self.path, 230 | "-i", 231 | src.path, 232 | "-lavfi", 233 | filtergraph, 234 | "-f", 235 | "null", 236 | "-", 237 | ] 238 | 239 | print("Calculating XPSNR, SSIM, PSNR, VMAF & VMAF-NEG scores...") 240 | process: Popen[str] = subprocess.Popen( 241 | cmd, 242 | stdout=subprocess.PIPE, 243 | stderr=subprocess.PIPE, 244 | universal_newlines=True, 245 | text=True, 246 | ) 247 | _, stderr_output = process.communicate() 248 | 249 | ssim_match = re.search( 250 | r"\[Parsed_ssim_2\s*@\s*.*?\]\s*SSIM.*?All:(\d+\.\d+)", 251 | stderr_output, 252 | ) 253 | if ssim_match: 254 | self.ssim = float(ssim_match.group(1)) * 100 255 | 256 | psnr_match = re.search( 257 | r"\[Parsed_psnr_3\s*@\s*.*?\]\s*PSNR\s+y:([\d\.]+)\s+u:([\d\.]+)\s+v:([\d\.]+)\s+average:([\d\.]+)\s+min:([\d\.]+)\s+max:([\d\.]+)", 258 | stderr_output, 259 | ) 260 | if psnr_match: 261 | self.psnr = float(psnr_match.group(4)) 262 | 263 | xpsnr_match = re.search( 264 | r"\[Parsed_xpsnr_4\s*@\s*.*?\]\s*XPSNR\s+y:\s*(\d+\.\d+)\s+u:\s*(\d+\.\d+)\s+v:\s*(\d+\.\d+)", 265 | stderr_output, 266 | ) 267 | if xpsnr_match: 268 | self.xpsnr_y = float(xpsnr_match.group(1)) 269 | self.xpsnr_u = float(xpsnr_match.group(2)) 270 | self.xpsnr_v = float(xpsnr_match.group(3)) 271 | 272 | maxval: int = 255 273 | xpsnr_mse_y: float = psnr_to_mse(self.xpsnr_y, maxval) 274 | xpsnr_mse_u: float = psnr_to_mse(self.xpsnr_u, maxval) 275 | xpsnr_mse_v: float = psnr_to_mse(self.xpsnr_v, maxval) 276 | w_xpsnr_mse: float = ((4.0 * xpsnr_mse_y) + xpsnr_mse_u + xpsnr_mse_v) / 6.0 277 | self.w_xpsnr = 10.0 * math.log10((maxval**2) / w_xpsnr_mse) 278 | 279 | vmaf_score_pattern = ( 280 | r"\[Parsed_libvmaf_([56])\s*@\s*.*?\]\s*VMAF score:\s*(\d+\.\d+)" 281 | ) 282 | all_vmaf_matches: list[str] = re.findall(vmaf_score_pattern, stderr_output) 283 | for filter_idx, score_str in all_vmaf_matches: 284 | score = float(score_str) 285 | match filter_idx: 286 | case "5": 287 | self.vmaf = score 288 | case "6": 289 | self.vmaf_neg_hmn = score 290 | 291 | self.svt_triple = statistics.mean([self.vmaf, self.ssim, self.psnr]) 292 | 293 | def print_ssimulacra2(self) -> None: 294 | """ 295 | Print SSIMULACRA2 scores. 296 | """ 297 | print( 298 | f"\033[94mSSIMULACRA2\033[0m scores for every \033[95m{self.e}\033[0m frame:" 299 | ) 300 | print(f" Average: \033[1m{self.ssimu2_avg:.5f}\033[0m") 301 | print(f" Std Deviation: \033[1m{self.ssimu2_sdv:.5f}\033[0m") 302 | print(f" 10th Pctile: \033[1m{self.ssimu2_p10:.5f}\033[0m") 303 | 304 | def print_butteraugli(self) -> None: 305 | """ 306 | Print Butteraugli scores. 307 | """ 308 | print( 309 | f"\033[93mButteraugli\033[0m scores for every \033[95m{self.e}\033[0m frame:" 310 | ) 311 | print(f" Distance: \033[1m{self.butter_dis:.5f}\033[0m") 312 | print(f" Max Distance: \033[1m{self.butter_mds:.5f}\033[0m") 313 | 314 | def print_ffmpeg_metrics(self) -> None: 315 | """ 316 | Print XPSNR, SSIM, PSNR, VMAF & VMAF-NEG scores. 317 | """ 318 | print("\033[91mXPSNR\033[0m scores:") 319 | print(f" XPSNR: \033[1m{self.xpsnr_y:.5f}\033[0m") 320 | print(f" W-XPSNR: \033[1m{self.w_xpsnr:.5f}\033[0m") 321 | print( 322 | f"\033[38;5;208mVMAF\033[0m scores for every \033[95m{self.e}\033[0m frame:" 323 | ) 324 | print(f" VMAF NEG: \033[1m{self.vmaf_neg_hmn:.5f}\033[0m") 325 | print(f" VMAF: \033[1m{self.vmaf:.5f}\033[0m") 326 | print(f"SSIM Score: \033[1m{self.ssim:.5f}\033[0m") 327 | print(f"PSNR Score: \033[1m{self.psnr:.5f}\033[0m") 328 | print("AVG VMAF/SSIM/PSNR score:") 329 | print(f" \033[1m{self.svt_triple:.5f}\033[0m") 330 | 331 | 332 | class VideoEnc: 333 | """ 334 | Video encoding class, containing encoder commands. 335 | """ 336 | 337 | src: CoreVideo 338 | dst_pth: str 339 | q: int 340 | encoder: str 341 | encoder_args: list[str] 342 | enc_cmd: list[str] 343 | time: float 344 | 345 | def __init__( 346 | self, 347 | src: CoreVideo, 348 | q: int, 349 | encoder: str, 350 | encoder_args: list[str], 351 | dst_pth: str = "", 352 | ) -> None: 353 | self.src = src 354 | self.dst_pth = dst_pth 355 | self.q = q 356 | self.encoder = encoder 357 | self.encoder_args = encoder_args if encoder_args else [""] 358 | self.enc_cmd = self.set_enc_cmd() 359 | self.time = 0 360 | 361 | def set_enc_cmd(self) -> list[str]: 362 | p: str = os.path.splitext(os.path.basename(self.src.path))[0] 363 | if not self.dst_pth: 364 | match self.encoder: 365 | case "x264": 366 | self.dst_pth = f"./{p}_{self.encoder}_q{self.q}.264" 367 | case "x265": 368 | self.dst_pth = f"./{p}_{self.encoder}_q{self.q}.265" 369 | case "vvenc": 370 | self.dst_pth = f"./{p}_{self.encoder}_q{self.q}.266" 371 | case "vpxenc": 372 | self.dst_pth = f"./{p}_{self.encoder}_q{self.q}.ivf" 373 | case _: 374 | self.dst_pth = f"./{p}_{self.encoder}_q{self.q}.ivf" 375 | else: 376 | match self.encoder: 377 | case "x264": 378 | self.dst_pth = self.dst_pth + ".264" 379 | case "x265": 380 | self.dst_pth = self.dst_pth + ".265" 381 | case "vvenc": 382 | self.dst_pth = self.dst_pth + ".266" 383 | case "vpxenc": 384 | self.dst_pth = self.dst_pth + ".ivf" 385 | case _: 386 | self.dst_pth = self.dst_pth + ".ivf" 387 | 388 | match self.encoder: 389 | case "x264": 390 | cmd: list[str] = [ 391 | "x264", 392 | "--demuxer", 393 | "y4m", 394 | "--crf", 395 | f"{self.q}", 396 | "-o", 397 | f"{self.dst_pth}", 398 | "-", 399 | ] 400 | case "x265": 401 | cmd: list[str] = [ 402 | "x265", 403 | "--y4m", 404 | "-", 405 | "--crf", 406 | f"{self.q}", 407 | "-o", 408 | f"{self.dst_pth}", 409 | ] 410 | case "vvenc": 411 | cmd: list[str] = [ 412 | "vvencapp", 413 | "--y4m", 414 | "-i", 415 | "-", 416 | "--qp", 417 | f"{self.q}", 418 | "-o", 419 | f"{self.dst_pth}", 420 | ] 421 | case "svtav1": 422 | cmd: list[str] = [ 423 | "SvtAv1EncApp", 424 | "-i", 425 | "-", 426 | "-b", 427 | f"{self.dst_pth}", 428 | "--crf", 429 | f"{self.q}", 430 | ] 431 | case "vpxenc": 432 | cmd: list[str] = [ 433 | "vpxenc", 434 | "--codec=vp9", 435 | "--ivf", 436 | "--end-usage=q", 437 | "--bit-depth=10", 438 | "--input-bit-depth=10", 439 | "--profile=2", 440 | "--passes=1", 441 | f"--cq-level={self.q}", 442 | ] 443 | case _: # "aomenc": 444 | cmd: list[str] = [ 445 | "aomenc", 446 | "--ivf", 447 | "--end-usage=q", 448 | f"--cq-level={self.q}", 449 | "--passes=1", 450 | "-y", 451 | "-", 452 | "-o", 453 | f"{self.dst_pth}", 454 | ] 455 | 456 | if self.encoder_args != [""]: 457 | cmd.extend(self.encoder_args) 458 | if self.encoder == "vpxenc": 459 | extra_args: list[str] = ["-o", f"{self.dst_pth}", "-"] 460 | cmd.extend(extra_args) 461 | print(" ".join(cmd)) 462 | return cmd 463 | 464 | def encode(self, e: int, t: int, g: int) -> DstVideo: 465 | """ 466 | Encode the video using FFmpeg piped to your chosen encoder. 467 | """ 468 | ff_cmd: list[str] = [ 469 | "ffmpeg", 470 | "-hide_banner", 471 | "-y", 472 | "-loglevel", 473 | "error", 474 | "-i", 475 | f"{self.src.path}", 476 | "-pix_fmt", 477 | "yuv420p10le", 478 | "-strict", 479 | "-2", 480 | "-f", 481 | "yuv4mpegpipe", 482 | "-", 483 | ] 484 | print( 485 | f"Encoding {self.src.name} at Q{self.q} with {self.encoder} ({self.src.path} --> {self.dst_pth})" 486 | ) 487 | ff_proc: Popen[bytes] = subprocess.Popen( 488 | ff_cmd, 489 | stdout=subprocess.PIPE, 490 | stderr=subprocess.PIPE, 491 | ) 492 | start_time: float = time.time() 493 | enc_proc: Popen[str] = subprocess.Popen( 494 | self.enc_cmd, 495 | stdin=ff_proc.stdout, 496 | stdout=subprocess.PIPE, 497 | stderr=subprocess.PIPE, 498 | universal_newlines=True, 499 | ) 500 | _, stderr = enc_proc.communicate() 501 | encode_time: float = time.time() - start_time 502 | self.time = encode_time 503 | print(stderr) 504 | return DstVideo(self.dst_pth, e, t, g) 505 | 506 | def remove_output(self) -> None: 507 | """ 508 | Remove the output file. 509 | """ 510 | os.remove(self.dst_pth) 511 | 512 | 513 | def psnr_to_mse(p: float, m: int) -> float: 514 | """ 515 | Convert PSNR to MSE (Mean Squared Error). Used in weighted XPSNR calculation. 516 | """ 517 | return (m**2) / (10 ** (p / 10)) 518 | 519 | 520 | def calc_some_scores(score_list: list[float]) -> tuple[float, float, float]: 521 | """ 522 | Calculate the average, standard deviation, & 10th percentile of a list of scores. 523 | """ 524 | average: float = statistics.mean(score_list) 525 | std_dev: float = statistics.stdev(score_list) 526 | percentile_10th: float = statistics.quantiles(score_list, n=100)[10] 527 | return (average, std_dev, percentile_10th) 528 | -------------------------------------------------------------------------------- /scripts/plot.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env -S uv run --script 2 | # /// script 3 | # requires-python = ">=3.13.1" 4 | # dependencies = [ 5 | # "argparse>=1.4.0", 6 | # "matplotlib>=3.10.0", 7 | # "numpy>=2.2.2", 8 | # "scipy>=1.15.1", 9 | # ] 10 | # /// 11 | 12 | import argparse 13 | import csv 14 | import math 15 | import os 16 | from argparse import Namespace 17 | 18 | import matplotlib.pyplot as plt 19 | import numpy as np 20 | from scipy.integrate import simpson 21 | from scipy.interpolate import pchip_interpolate 22 | 23 | 24 | def read_csv(filename): 25 | """ 26 | Read CSV file and return data as dictionary of lists. 27 | Assumes CSV columns: 'q', 'encode_time', 'output_filesize', 28 | 'ssimu2_mean', 'butter_distance', 'wxpsnr', 'vmaf_neg', 29 | 'vmaf', 'ssim', and 'psnr'. 30 | """ 31 | data = { 32 | "q": [], 33 | "encode_time": [], 34 | "output_filesize": [], 35 | "ssimu2_mean": [], 36 | "butter_distance": [], 37 | "wxpsnr": [], 38 | "vmaf_neg": [], 39 | "vmaf": [], 40 | "ssim": [], 41 | "psnr": [], 42 | } 43 | 44 | with open(filename, "r") as csvfile: 45 | reader = csv.DictReader(csvfile) 46 | for row in reader: 47 | for key in data: 48 | data[key].append(float(row[key])) 49 | return data 50 | 51 | 52 | def bdrate_vs_time_csv( 53 | name: str, 54 | time: float, 55 | bd_rates: dict, 56 | ) -> None: 57 | """ 58 | Write BD-rate vs encode time to a CSV file. 59 | """ 60 | csv_file: str = "bd_vs_time.csv" 61 | headers = [ 62 | "name", 63 | "avg_encode_time", 64 | "ssimu2_mean_bd", 65 | "butter_distance_bd", 66 | "wxpsnr_bd", 67 | "vmaf_neg_bd", 68 | "vmaf_bd", 69 | "ssim_bd", 70 | "psnr_bd", 71 | ] 72 | 73 | if os.path.exists(csv_file): 74 | # Append to existing file 75 | with open(csv_file, "a") as f: 76 | f.write( 77 | f"{name},{time:.5f},{bd_rates.get('ssimu2_mean', 0):.5f},{bd_rates.get('butter_distance', 0):.5f},{bd_rates.get('wxpsnr', 0):.5f},{bd_rates.get('vmaf_neg', 0):.5f},{bd_rates.get('vmaf', 0):.5f},{bd_rates.get('ssim', 0):.5f},{bd_rates.get('psnr', 0):.5f}\n" 78 | ) 79 | else: 80 | # Create new file with headers 81 | with open(csv_file, "w") as f: 82 | f.write(",".join(headers) + "\n") 83 | f.write( 84 | f"{name},{time:.5f},{bd_rates.get('ssimu2_mean', 0):.5f},{bd_rates.get('butter_distance', 0):.5f},{bd_rates.get('wxpsnr', 0):.5f},{bd_rates.get('vmaf_neg', 0):.5f},{bd_rates.get('vmaf', 0):.5f},{bd_rates.get('ssim', 0):.5f},{bd_rates.get('psnr', 0):.5f}\n" 85 | ) 86 | 87 | 88 | def create_metric_plot(datasets, metric_name: str, fmt: str): 89 | """ 90 | Create a plot for the specified metric for all given datasets. 91 | The datasets argument should be a list of tuples (label, data). 92 | Each dataset's data is a dictionary (from read_csv), and label is the dataset identifier. 93 | """ 94 | plt.figure(figsize=(10, 6)) 95 | plt.style.use("dark_background") 96 | 97 | # For SSIMULACRA2, set background colors to black. 98 | if metric_name == "ssimu2_mean": 99 | plt.gca().set_facecolor("black") 100 | plt.gcf().set_facecolor("black") 101 | 102 | # For each metric, define a colormap so that multiple CSV files 103 | # use different shades of the same base hue. 104 | colormaps = { 105 | "ssimu2_mean": plt.colormaps.get_cmap("Blues"), 106 | "butter_distance": plt.colormaps.get_cmap("YlOrBr"), 107 | "wxpsnr": plt.colormaps.get_cmap("Reds"), 108 | "vmaf_neg": plt.colormaps.get_cmap("RdGy"), 109 | "vmaf": plt.colormaps.get_cmap("autumn"), 110 | "ssim": plt.colormaps.get_cmap("BuGn"), 111 | "psnr": plt.colormaps.get_cmap("Greys"), 112 | } 113 | cmap = colormaps.get(metric_name, plt.colormaps.get_cmap("viridis")) 114 | 115 | n_datasets = len(datasets) 116 | # Iterate over datasets, computing a shade for each from the colormap. 117 | for idx, (label, df) in enumerate(datasets): 118 | # Normalize the index value for the colormap (between 0 and 1) 119 | color_value = (idx + 1) / (n_datasets + 1) 120 | line_color = cmap(color_value) 121 | filesize = df["output_filesize"] 122 | ydata = df[metric_name] 123 | plt.plot( 124 | filesize, ydata, marker="o", linestyle="-.", color=line_color, label=label 125 | ) 126 | # Annotate each data point with its Q value. To avoid annotation overlap among datasets, 127 | # a small vertical offset is added depending on the dataset index. 128 | for i, (x, y) in enumerate(zip(filesize, ydata)): 129 | # Offset increases with dataset index 130 | offset = 10 + idx * 5 131 | plt.annotate( 132 | f"Q{df['q'][i]}", 133 | (x, y), 134 | textcoords="offset points", 135 | xytext=(0, offset), 136 | ha="center", 137 | color="white", 138 | fontsize=8, 139 | ) 140 | 141 | # Set grid and axes formatting. 142 | plt.grid(True, color="grey", linestyle="--", linewidth=0.5, alpha=0.5) 143 | for spine in plt.gca().spines.values(): 144 | spine.set_color("grey") 145 | 146 | # Set labels. 147 | plt.xlabel("Output Filesize (MB)", color="gainsboro", family="monospace") 148 | metric_labels = { 149 | "ssimu2_mean": "Average SSIMULACRA2", 150 | "butter_distance": "Butteraugli Distance", 151 | "wxpsnr": "W-XPSNR", 152 | "vmaf_neg": "VMAF NEG (Harmonic Mean)", 153 | "vmaf": "VMAF", 154 | "ssim": "SSIM", 155 | "psnr": "PSNR", 156 | } 157 | plt.ylabel( 158 | metric_labels.get(metric_name, metric_name), 159 | color="gainsboro", 160 | family="monospace", 161 | ) 162 | 163 | # Set title. 164 | plt.title( 165 | f"Codec Comparison: {metric_labels.get(metric_name, metric_name)} vs Filesize", 166 | color="white", 167 | family="monospace", 168 | pad=12, 169 | ) 170 | 171 | plt.tick_params(axis="both", colors="grey") 172 | plt.legend() 173 | plt.tight_layout() 174 | 175 | # Save the plot to file. 176 | output_filename = f"{metric_name}_plot.{fmt}" 177 | plt.savefig(output_filename, format=fmt) 178 | plt.close() 179 | 180 | 181 | def bd_rate_simpson(metric_set1, metric_set2): 182 | """ 183 | Calculate BD-rate (Bjontegaard Delta Rate) difference between two rate-distortion curves, 184 | using a Piecewise Cubic Hermite Interpolating Polynomial (PCHIP) and Simpson integration. 185 | 186 | Each metric_set is a list of tuples (bitrate, metric_value). The bitrate is first logged. 187 | The function computes the average percentage difference in bitrate over the overlapping interval 188 | of the metric curves. 189 | 190 | Returns BD-rate % as a float. 191 | """ 192 | if not metric_set1 or not metric_set2: 193 | return 0.0 194 | try: 195 | # Sort each metric set by metric value. 196 | metric_set1.sort(key=lambda tup: tup[1]) 197 | metric_set2.sort(key=lambda tup: tup[1]) 198 | 199 | # For each set, take the logarithm of bitrate values and fix any infinite metric values. 200 | log_rate1 = [math.log(x[0]) for x in metric_set1] 201 | metric1 = [100.0 if x[1] == float("inf") else x[1] for x in metric_set1] 202 | log_rate2 = [math.log(x[0]) for x in metric_set2] 203 | metric2 = [100.0 if x[1] == float("inf") else x[1] for x in metric_set2] 204 | 205 | # Define the overlapping integration interval on the metric axis. 206 | min_int = max(min(metric1), min(metric2)) 207 | max_int = min(max(metric1), max(metric2)) 208 | if max_int <= min_int: 209 | return 0.0 210 | 211 | # Create 100 sample points between min_int and max_int. 212 | samples, interval = np.linspace(min_int, max_int, num=100, retstep=True) 213 | 214 | # Interpolate the log-rate values at the sample metric values. 215 | v1 = pchip_interpolate(metric1, log_rate1, samples) 216 | v2 = pchip_interpolate(metric2, log_rate2, samples) 217 | 218 | # Integrate the curves using Simpson's rule. 219 | int_v1 = simpson(v1, dx=float(interval)) 220 | int_v2 = simpson(v2, dx=float(interval)) 221 | 222 | # Compute the average difference in the log domain. 223 | avg_exp_diff = (int_v2 - int_v1) / (max_int - min_int) 224 | except (TypeError, ZeroDivisionError, ValueError): 225 | return 0.0 226 | 227 | # Convert the averaged log difference back to a percentage difference. 228 | bd_rate_percent = (math.exp(avg_exp_diff) - 1) * 100 229 | return bd_rate_percent 230 | 231 | 232 | def calculate_average_encode_time(data): 233 | """ 234 | Calculate the average encode time from the data. 235 | """ 236 | encode_times = data["encode_time"] 237 | return sum(encode_times) / len(encode_times) 238 | 239 | 240 | def main(): 241 | parser = argparse.ArgumentParser( 242 | description="Plot codec metrics from one or more CSV files (one per codec) for side-by-side comparison." 243 | ) 244 | parser.add_argument( 245 | "-i", 246 | "--input", 247 | required=True, 248 | nargs="+", 249 | type=str, 250 | help="Path(s) to CSV file(s). Each CSV file should have the same columns.", 251 | ) 252 | parser.add_argument( 253 | "-f", 254 | "--format", 255 | default="svg", 256 | type=str, 257 | help="Save the plot as 'svg', 'png', or 'webp'", 258 | ) 259 | args: Namespace = parser.parse_args() 260 | csv_files = args.input 261 | fmt: str = args.format 262 | 263 | # Read all CSV files. 264 | datasets = [] 265 | for filename in csv_files: 266 | data = read_csv(filename) 267 | label = os.path.basename(filename) 268 | datasets.append((label, data)) 269 | 270 | # Create plots for each metric. 271 | metrics = [ 272 | "ssimu2_mean", 273 | "butter_distance", 274 | "wxpsnr", 275 | "vmaf_neg", 276 | "vmaf", 277 | "ssim", 278 | "psnr", 279 | ] 280 | for metric in metrics: 281 | create_metric_plot(datasets, metric, fmt) 282 | 283 | # Calculate and output the average encode time for each CSV file 284 | for label, data in datasets: 285 | avg_time = calculate_average_encode_time(data) 286 | print(f"Average encode time for {label}: {avg_time:.5f} seconds") 287 | 288 | # If there are at least two datasets, compute and print the BD-rate for each metric. 289 | if len(datasets) >= 2: 290 | metric_labels = { 291 | "ssimu2_mean": "\033[94mSSIMULACRA2\033[0m Average: ", 292 | "butter_distance": "\033[93mButteraugli\033[0m Distance: ", 293 | "wxpsnr": "W-\033[91mXPSNR\033[0m: ", 294 | "vmaf_neg": "\033[38;5;208mVMAF NEG\033[0m (Harmonic Mean): ", 295 | "vmaf": "\033[38;5;208mVMAF\033[0m: ", 296 | "ssim": "\033[94mSSIM\033[0m: ", 297 | "psnr": "PSNR: ", 298 | } 299 | 300 | # Compare the first CSV file with each of the other CSV files. 301 | for idx in range(1, len(datasets)): 302 | file1_label = datasets[0][0] 303 | filecmp_label = datasets[idx][0] 304 | print( 305 | "BD-rate values between '{}' & '{}'".format(file1_label, filecmp_label) 306 | ) 307 | 308 | # Store BD-rate values for this comparison 309 | bd_rates = {} 310 | 311 | for metric in metrics: 312 | # Create lists of tuples (output_filesize, metric_value) for the two files. 313 | data1 = datasets[0][1] 314 | data2 = datasets[idx][1] 315 | metric_set1 = list(zip(data1["output_filesize"], data1[metric])) 316 | metric_set2 = list(zip(data2["output_filesize"], data2[metric])) 317 | 318 | bd_rate = bd_rate_simpson(metric_set1, metric_set2) 319 | bd_rates[metric] = bd_rate 320 | 321 | # Decide which file is better based on the sign of the bd_rate value. 322 | if bd_rate < 0: 323 | # Negative BD-rate indicates that the second file (the one it is compared with) 324 | # is better (i.e., lower bitrate for the same quality). 325 | winner_msg = f"{filecmp_label} is better" 326 | elif bd_rate > 0: 327 | winner_msg = f"{file1_label} is better" 328 | else: 329 | winner_msg = "No difference" 330 | 331 | print( 332 | f"{metric_labels.get(metric, metric)}\033[1m{bd_rate:7.2f}%\033[0m -> {winner_msg}" 333 | ) 334 | 335 | # Calculate average encode times 336 | avg_time1 = calculate_average_encode_time(data1) 337 | avg_time2 = calculate_average_encode_time(data2) 338 | 339 | # Write BD-rate vs time data to CSV 340 | bdrate_vs_time_csv( 341 | file1_label, avg_time1, {metric: 0.0 for metric in metrics} 342 | ) # Reference file 343 | bdrate_vs_time_csv(filecmp_label, avg_time2, bd_rates) 344 | 345 | if idx < len(datasets) - 1: 346 | print() # Empty line for readability 347 | else: 348 | print("Need at least two CSV files to compute BD-rate values.") 349 | 350 | # Still write the encode time for a single file 351 | if len(datasets) == 1: 352 | label, data = datasets[0] 353 | avg_time = calculate_average_encode_time(data) 354 | bdrate_vs_time_csv(label, avg_time, {metric: 0.0 for metric in metrics}) 355 | 356 | 357 | if __name__ == "__main__": 358 | main() 359 | -------------------------------------------------------------------------------- /scripts/scores.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env -S uv run --script 2 | # /// script 3 | # requires-python = ">=3.13.1" 4 | # dependencies = [ 5 | # "argparse>=1.4.0", 6 | # "statistics>=1.0.3.5", 7 | # "tqdm>=4.67.1", 8 | # "vapoursynth>=70", 9 | # "vstools>=3.3.4", 10 | # ] 11 | # /// 12 | 13 | import argparse 14 | from argparse import Namespace 15 | 16 | from metrics import CoreVideo, DstVideo 17 | 18 | 19 | def main() -> None: 20 | parser = argparse.ArgumentParser( 21 | description="Run metrics given a source video & a distorted video." 22 | ) 23 | parser.add_argument("source", help="Source video path") 24 | parser.add_argument("distorted", help="Distorted video path") 25 | parser.add_argument( 26 | "-e", 27 | "--every", 28 | type=int, 29 | default=1, 30 | help="Only score every nth frame. Default 1 (every frame)", 31 | ) 32 | parser.add_argument( 33 | "-g", 34 | "--gpu-streams", 35 | type=int, 36 | default=0, 37 | help="Number of GPU streams for SSIMULACRA2/Butteraugli", 38 | ) 39 | parser.add_argument( 40 | "-t", 41 | "--threads", 42 | type=int, 43 | default=0, 44 | help="Number of threads for SSIMULACRA2/Butteraugli", 45 | ) 46 | 47 | args: Namespace = parser.parse_args() 48 | every: int = args.every 49 | threads: int = args.threads 50 | gpu_streams: int = args.gpu_streams 51 | 52 | s: CoreVideo = CoreVideo(args.source, every, threads, gpu_streams) 53 | v: DstVideo = DstVideo(args.distorted, every, threads, gpu_streams) 54 | 55 | print(f"Source video: \033[4m{s.name}\033[0m") 56 | print(f"Distorted video: \033[4m{v.name}\033[0m") 57 | 58 | # Calculate SSIMULACRA2 scores 59 | v.calculate_ssimulacra2(s) 60 | v.print_ssimulacra2() 61 | 62 | # Calculate Butteraugli scores 63 | if gpu_streams: 64 | v.calculate_butteraugli(s) 65 | v.print_butteraugli() 66 | else: 67 | v.calculate_butteraugli(s) 68 | 69 | # Calculate XPSNR scores 70 | v.calculate_ffmpeg_metrics(s) 71 | v.print_ffmpeg_metrics() 72 | 73 | 74 | if __name__ == "__main__": 75 | main() 76 | -------------------------------------------------------------------------------- /scripts/stats.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env -S uv run --script 2 | # /// script 3 | # requires-python = ">=3.13.1" 4 | # dependencies = [ 5 | # "argparse>=1.4.0", 6 | # "statistics>=1.0.3.5", 7 | # "tqdm>=4.67.1", 8 | # "vapoursynth>=70", 9 | # "vstools>=3.3.4", 10 | # ] 11 | # /// 12 | 13 | import argparse 14 | import os 15 | from argparse import Namespace 16 | 17 | from metrics import CoreVideo, DstVideo, VideoEnc 18 | 19 | 20 | def write_stats( 21 | name: str, 22 | q: int, 23 | encode_time: float, 24 | size: int, 25 | ssimu2_mean: float, 26 | butter_distance: float, 27 | w_xpsnr: float, 28 | vmaf_neg: float, 29 | vmaf: float, 30 | ssim: float, 31 | psnr: float, 32 | ) -> None: 33 | """ 34 | Write metric stats to a CSV file. 35 | """ 36 | 37 | csv: str = name if name.endswith(".csv") else f"{name}.csv" 38 | if not os.path.exists(csv): 39 | with open(csv, "w") as f: 40 | f.write( 41 | "q,encode_time,output_filesize,ssimu2_mean,butter_distance,wxpsnr,vmaf_neg,vmaf,ssim,psnr\n" 42 | ) 43 | f.write( 44 | f"{q},{encode_time:.5f},{size},{ssimu2_mean:.5f},{butter_distance:.5f},{w_xpsnr:.5f},{vmaf_neg:.5f},{vmaf:.5f},{ssim:.5f},{psnr:.5f}\n" 45 | ) 46 | else: 47 | with open(csv, "a") as f: 48 | f.write( 49 | f"{q},{encode_time:.5f},{size},{ssimu2_mean:.5f},{butter_distance:.5f},{w_xpsnr:.5f},{vmaf_neg:.5f},{vmaf:.5f},{ssim:.5f},{psnr:.5f}\n" 50 | ) 51 | 52 | 53 | def main(): 54 | parser = argparse.ArgumentParser( 55 | description="Generate statistics for a series of video encodes." 56 | ) 57 | parser.add_argument( 58 | "-i", 59 | "--inputs", 60 | required=True, 61 | type=str, 62 | nargs="+", 63 | help="Path(s) to source video file(s)", 64 | ) 65 | parser.add_argument( 66 | "-q", 67 | "--quality", 68 | required=True, 69 | type=str, 70 | help="List of quality values to test (e.g. 20 30 40 50)", 71 | ) 72 | parser.add_argument( 73 | "encoder", 74 | choices=["x264", "x265", "svtav1", "aomenc", "vpxenc"], 75 | type=str, 76 | help="Which video encoder to use", 77 | ) 78 | parser.add_argument( 79 | "-o", "--output", required=True, type=str, help="Path to output CSV file" 80 | ) 81 | parser.add_argument( 82 | "-e", 83 | "--every", 84 | type=int, 85 | default=1, 86 | help="Only score every nth frame. Default 1 (every frame)", 87 | ) 88 | parser.add_argument( 89 | "-g", 90 | "--gpu-streams", 91 | type=int, 92 | default=0, 93 | help="Number of GPU streams for SSIMULACRA2/Butteraugli", 94 | ) 95 | parser.add_argument( 96 | "-t", 97 | "--threads", 98 | type=int, 99 | default=0, 100 | help="Number of threads for SSIMULACRA2/Butteraugli", 101 | ) 102 | parser.add_argument( 103 | "-k", 104 | "--keep", 105 | default=True, 106 | action="store_false", 107 | help="Keep output video files", 108 | ) 109 | parser.add_argument( 110 | "encoder_args", 111 | nargs=argparse.REMAINDER, 112 | type=str, 113 | help="Additional encoder arguments (pass these after a '--' delimiter)", 114 | ) 115 | 116 | args: Namespace = parser.parse_args() 117 | src_pth: list[str] = [p for p in args.inputs] 118 | quality_list: list[int] = [int(q) for q in args.quality.split()] 119 | enc: str = args.encoder 120 | csv_out: str = args.output 121 | every: int = args.every 122 | threads: int = args.threads 123 | gpu_streams: bool = args.gpu_streams 124 | clean: bool = args.keep 125 | enc_args: list[str] = args.encoder_args 126 | 127 | cumulative_sizes: list[dict[int, int]] = [ 128 | {q: 0 for q in quality_list} for _ in range(len(src_pth)) 129 | ] 130 | cumulative_times: list[dict[int, float]] = [ 131 | {q: 0.0 for q in quality_list} for _ in range(len(src_pth)) 132 | ] 133 | cumulative_ssimu2: list[dict[int, float]] = [ 134 | {q: 0.0 for q in quality_list} for _ in range(len(src_pth)) 135 | ] 136 | cumulative_butter: list[dict[int, float]] = [ 137 | {q: 0.0 for q in quality_list} for _ in range(len(src_pth)) 138 | ] 139 | cumulative_wxpsnr: list[dict[int, float]] = [ 140 | {q: 0.0 for q in quality_list} for _ in range(len(src_pth)) 141 | ] 142 | cumulative_vmafneg: list[dict[int, float]] = [ 143 | {q: 0.0 for q in quality_list} for _ in range(len(src_pth)) 144 | ] 145 | cumulative_vmaf: list[dict[int, float]] = [ 146 | {q: 0.0 for q in quality_list} for _ in range(len(src_pth)) 147 | ] 148 | cumulative_ssim: list[dict[int, float]] = [ 149 | {q: 0.0 for q in quality_list} for _ in range(len(src_pth)) 150 | ] 151 | cumulative_psnr: list[dict[int, float]] = [ 152 | {q: 0.0 for q in quality_list} for _ in range(len(src_pth)) 153 | ] 154 | i: int = 0 155 | 156 | for src in src_pth: 157 | s: CoreVideo = CoreVideo(src, every, threads, gpu_streams) 158 | 159 | print(f"Running encoder at qualities: {quality_list}") 160 | for q in quality_list: 161 | print(f"Quality: {q}") 162 | 163 | e: VideoEnc = VideoEnc(s, q, enc, enc_args) 164 | v: DstVideo = e.encode(every, threads, gpu_streams) 165 | print(f"Encoded {s.name} --> {e.dst_pth} (took {e.time:.2f} seconds)") 166 | 167 | v.calculate_ssimulacra2(s) 168 | v.calculate_butteraugli(s) 169 | v.calculate_ffmpeg_metrics(s) 170 | 171 | cumulative_times[i][q] = e.time 172 | cumulative_sizes[i][q] = v.size 173 | cumulative_ssimu2[i][q] = v.ssimu2_avg 174 | cumulative_butter[i][q] = v.butter_dis 175 | cumulative_wxpsnr[i][q] = v.w_xpsnr 176 | cumulative_vmafneg[i][q] = v.vmaf_neg_hmn 177 | cumulative_vmaf[i][q] = v.vmaf 178 | cumulative_ssim[i][q] = v.ssim 179 | cumulative_psnr[i][q] = v.psnr 180 | 181 | if clean: 182 | e.remove_output() 183 | i += 1 184 | 185 | avg_time: dict[int, float] = {} 186 | avg_size: dict[int, int] = {} 187 | avg_ssimu2: dict[int, float] = {} 188 | avg_butter: dict[int, float] = {} 189 | avg_wxpsnr: dict[int, float] = {} 190 | avg_vmafneg: dict[int, float] = {} 191 | avg_vmaf: dict[int, float] = {} 192 | avg_ssim: dict[int, float] = {} 193 | avg_psnr: dict[int, float] = {} 194 | 195 | for q in quality_list: 196 | avg_time[q] = sum(cumulative_times[j][q] for j in range(i)) / i 197 | avg_size[q] = int(sum(cumulative_sizes[j][q] for j in range(i)) / i) 198 | avg_ssimu2[q] = sum(cumulative_ssimu2[j][q] for j in range(i)) / i 199 | avg_butter[q] = sum(cumulative_butter[j][q] for j in range(i)) / i 200 | avg_wxpsnr[q] = sum(cumulative_wxpsnr[j][q] for j in range(i)) / i 201 | avg_vmafneg[q] = sum(cumulative_vmafneg[j][q] for j in range(i)) / i 202 | avg_vmaf[q] = sum(cumulative_vmaf[j][q] for j in range(i)) / i 203 | avg_ssim[q] = sum(cumulative_ssim[j][q] for j in range(i)) / i 204 | avg_psnr[q] = sum(cumulative_psnr[j][q] for j in range(i)) / i 205 | write_stats( 206 | csv_out, 207 | q, 208 | avg_time[q], 209 | avg_size[q], 210 | avg_ssimu2[q], 211 | avg_butter[q], 212 | avg_wxpsnr[q], 213 | avg_vmafneg[q], 214 | avg_vmaf[q], 215 | avg_ssim[q], 216 | avg_psnr[q], 217 | ) 218 | 219 | 220 | if __name__ == "__main__": 221 | main() 222 | -------------------------------------------------------------------------------- /static/butter_distance_plot.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /static/ssimu2_hmean_plot.svg: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /static/wxpsnr_plot.svg: -------------------------------------------------------------------------------- 1 | --------------------------------------------------------------------------------