├── .github ├── dependabot.yml └── workflows │ ├── build.yml │ ├── codecov.yml │ ├── codeql-analysis.yml │ ├── documentation.yml │ ├── mypy.yml │ └── pythonpublish.yml ├── .gitignore ├── .readthedocs.yml ├── LICENSE ├── MANIFEST.in ├── README.md ├── SECURITY.md ├── deploy └── dvt │ ├── Dockerfile │ ├── start-celery-worker.sh │ └── start-django-dev.sh ├── docker-compose.yml ├── docs ├── Makefile ├── requirements.txt └── source │ ├── application.md │ ├── architecture.md │ ├── conf.py │ ├── development.md │ ├── index.rst │ ├── operation.md │ └── quickstart.md ├── mypy.ini ├── pyproject.toml ├── requirements.txt ├── src ├── __init__.py ├── dvt │ ├── __init__.py │ ├── asgi.py │ ├── settings.py │ ├── urls.py │ └── wsgi.py ├── manage.py └── video_transcoding │ ├── __init__.py │ ├── admin.py │ ├── apps.py │ ├── celery.py │ ├── defaults.py │ ├── forms.py │ ├── helpers.py │ ├── locale │ └── ru │ │ └── LC_MESSAGES │ │ └── django.po │ ├── migrations │ ├── 0001_initial.py │ ├── 0002_auto_20200226_0919.py │ ├── 0003_auto_20200525_1130.py │ ├── 0004_audioprofile_audiotrack_preset_audioprofiletracks_and_more.py │ ├── 0005_video_metadata.py │ ├── 0006_video_duration.py │ ├── 0007_videoprofile_segment_duration.py │ └── __init__.py │ ├── models.py │ ├── signals.py │ ├── strategy.py │ ├── tasks.py │ ├── tests │ ├── __init__.py │ ├── base.py │ ├── test_analysis.py │ ├── test_celery.py │ ├── test_extract.py │ ├── test_metadata.py │ ├── test_models.py │ ├── test_strategy.py │ ├── test_tasks.py │ ├── test_transcoder.py │ ├── test_transcoding.py │ ├── test_workspace.py │ └── test_wrappers.py │ ├── transcoding │ ├── __init__.py │ ├── analysis.py │ ├── codecs.py │ ├── extract.py │ ├── ffprobe.py │ ├── inputs.py │ ├── metadata.py │ ├── outputs.py │ ├── profiles.py │ ├── transcoder.py │ └── workspace.py │ └── utils.py └── tox.ini /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | # To get started with Dependabot version updates, you'll need to specify which 2 | # package ecosystems to update and where the package manifests are located. 3 | # Please see the documentation for all configuration options: 4 | # https://docs.github.com/github/administering-a-repository/configuration-options-for-dependency-updates 5 | 6 | version: 2 7 | updates: 8 | - package-ecosystem: "pip" # See documentation for possible values 9 | directory: "/requirements.txt" # Location of package manifests 10 | schedule: 11 | interval: "daily" 12 | -------------------------------------------------------------------------------- /.github/workflows/build.yml: -------------------------------------------------------------------------------- 1 | name: build 2 | 3 | on: [push, pull_request] 4 | 5 | jobs: 6 | build: 7 | runs-on: ubuntu-latest 8 | strategy: 9 | max-parallel: 4 10 | matrix: 11 | python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"] 12 | 13 | steps: 14 | - uses: actions/checkout@v1 15 | - name: Set up Python ${{ matrix.python-version }} 16 | uses: actions/setup-python@v2 17 | with: 18 | python-version: ${{ matrix.python-version }} 19 | - name: Install python dependencies 20 | run: | 21 | python -m pip install --upgrade pip 22 | pip install tox tox-gh-actions build wheel 23 | sed -i '/Django==.*/d' ./requirements.txt # delete django dependency 24 | - name: Test with tox 25 | run: | 26 | tox 27 | python -m build 28 | -------------------------------------------------------------------------------- /.github/workflows/codecov.yml: -------------------------------------------------------------------------------- 1 | name: codecov 2 | 3 | on: [push, pull_request] 4 | 5 | jobs: 6 | run: 7 | runs-on: ubuntu-latest 8 | steps: 9 | - uses: actions/checkout@master 10 | - name: Setup Python 11 | uses: actions/setup-python@v2 12 | with: 13 | python-version: '3.10' 14 | - name: Generate coverage report 15 | run: | 16 | pip install -r requirements.txt 17 | pip install coverage 18 | pip install -q -e . 19 | coverage run src/manage.py test src/ 20 | coverage xml 21 | - name: Upload coverage to Codecov 22 | uses: codecov/codecov-action@v2 23 | with: 24 | token: ${{ secrets.CODECOV_TOKEN }} 25 | file: ./coverage.xml 26 | flags: unittests 27 | name: codecov-umbrella 28 | fail_ci_if_error: true -------------------------------------------------------------------------------- /.github/workflows/codeql-analysis.yml: -------------------------------------------------------------------------------- 1 | name: "Code scanning - action" 2 | 3 | on: 4 | pull_request: 5 | schedule: 6 | - cron: '0 2 * * 2' 7 | 8 | jobs: 9 | CodeQL-Build: 10 | 11 | runs-on: ubuntu-latest 12 | 13 | steps: 14 | - name: Checkout repository 15 | uses: actions/checkout@v2 16 | with: 17 | # We must fetch at least the immediate parents so that if this is 18 | # a pull request then we can checkout the head. 19 | fetch-depth: 2 20 | 21 | # If this run was triggered by a pull request event, then checkout 22 | # the head of the pull request instead of the merge commit. 23 | - run: git checkout HEAD^2 24 | if: ${{ github.event_name == 'pull_request' }} 25 | 26 | # Initializes the CodeQL tools for scanning. 27 | - name: Initialize CodeQL 28 | uses: github/codeql-action/init@v1 29 | # Override language selection by uncommenting this and choosing your languages 30 | # with: 31 | # languages: go, javascript, csharp, python, cpp, java 32 | 33 | # Autobuild attempts to build any compiled languages (C/C++, C#, or Java). 34 | # If this step fails, then you should remove it and run the build manually (see below) 35 | - name: Autobuild 36 | uses: github/codeql-action/autobuild@v1 37 | 38 | # ℹ️ Command-line programs to run using the OS shell. 39 | # 📚 https://git.io/JvXDl 40 | 41 | # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines 42 | # and modify them (or add more) to build your code if your project 43 | # uses a compiled language 44 | 45 | #- run: | 46 | # make bootstrap 47 | # make release 48 | 49 | - name: Perform CodeQL Analysis 50 | uses: github/codeql-action/analyze@v1 51 | -------------------------------------------------------------------------------- /.github/workflows/documentation.yml: -------------------------------------------------------------------------------- 1 | name: documentation 2 | 3 | on: [push, pull_request] 4 | 5 | jobs: 6 | run: 7 | runs-on: ubuntu-latest 8 | steps: 9 | - uses: actions/checkout@master 10 | - name: Setup Python 11 | uses: actions/setup-python@v2 12 | with: 13 | python-version: '3.11' # Pillow-9.5.0 14 | - name: Generate documentation 15 | run: | 16 | pip install -r docs/requirements.txt 17 | cd docs && make autodoc html 18 | -------------------------------------------------------------------------------- /.github/workflows/mypy.yml: -------------------------------------------------------------------------------- 1 | # This workflow will run MyPy static typing analysis. 2 | 3 | name: mypy testing 4 | 5 | on: [push, pull_request] 6 | 7 | jobs: 8 | deploy: 9 | 10 | runs-on: ubuntu-latest 11 | 12 | steps: 13 | - uses: actions/checkout@v2 14 | - name: Set up Python 15 | uses: actions/setup-python@v2 16 | with: 17 | python-version: '3.10' 18 | - name: Install dependencies 19 | run: | 20 | python -m pip install --upgrade pip 21 | pip install -r requirements.txt 22 | - name: Run Mypy tests 23 | run: | 24 | cd src && mypy --config-file=../mypy.ini -p video_transcoding -p django_stubs_ext 25 | -------------------------------------------------------------------------------- /.github/workflows/pythonpublish.yml: -------------------------------------------------------------------------------- 1 | # This workflows will upload a Python Package using Twine when a release is created 2 | 3 | name: Upload Python Package 4 | 5 | on: 6 | release: 7 | types: [published] 8 | 9 | jobs: 10 | deploy: 11 | 12 | runs-on: ubuntu-latest 13 | 14 | steps: 15 | - uses: actions/checkout@v2 16 | - name: Set up Python 17 | uses: actions/setup-python@v2 18 | with: 19 | python-version: '3.x' 20 | - name: Install system requirements 21 | run: | 22 | sudo apt-get install -y gettext 23 | - name: Install python dependencies 24 | run: | 25 | python -m pip install --upgrade pip 26 | pip install -r requirements.txt 27 | pip install build wheel twine 28 | - name: Build and publish 29 | env: 30 | TWINE_USERNAME: __token__ 31 | TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }} 32 | run: | 33 | src/manage.py compilemessages 34 | python -m build 35 | twine upload dist/* 36 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | *$py.class 5 | 6 | # C extensions 7 | *.so 8 | 9 | # Distribution / packaging 10 | .Python 11 | build/ 12 | develop-eggs/ 13 | dist/ 14 | downloads/ 15 | eggs/ 16 | .eggs/ 17 | lib/ 18 | lib64/ 19 | parts/ 20 | sdist/ 21 | var/ 22 | wheels/ 23 | pip-wheel-metadata/ 24 | share/python-wheels/ 25 | *.egg-info/ 26 | .installed.cfg 27 | *.egg 28 | MANIFEST 29 | 30 | # PyInstaller 31 | # Usually these files are written by a python script from a template 32 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 33 | *.manifest 34 | *.spec 35 | 36 | # Installer logs 37 | pip-log.txt 38 | pip-delete-this-directory.txt 39 | 40 | # Unit test / coverage reports 41 | htmlcov/ 42 | .tox/ 43 | .nox/ 44 | .coverage 45 | .coverage.* 46 | .cache 47 | nosetests.xml 48 | coverage.xml 49 | *.cover 50 | *.py,cover 51 | .hypothesis/ 52 | .pytest_cache/ 53 | 54 | # Translations 55 | *.mo 56 | *.pot 57 | 58 | # Django stuff: 59 | *.log 60 | local_settings.py 61 | db.sqlite3 62 | db.sqlite3-journal 63 | 64 | # Flask stuff: 65 | instance/ 66 | .webassets-cache 67 | 68 | # Scrapy stuff: 69 | .scrapy 70 | 71 | # Sphinx documentation 72 | docs/_build/ 73 | 74 | # PyBuilder 75 | target/ 76 | 77 | # Jupyter Notebook 78 | .ipynb_checkpoints 79 | 80 | # IPython 81 | profile_default/ 82 | ipython_config.py 83 | 84 | # pyenv 85 | .python-version 86 | 87 | # pipenv 88 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 89 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 90 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 91 | # install all needed dependencies. 92 | #Pipfile.lock 93 | 94 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow 95 | __pypackages__/ 96 | 97 | # Celery stuff 98 | celerybeat-schedule 99 | celerybeat.pid 100 | 101 | # SageMath parsed files 102 | *.sage.py 103 | 104 | # Environments 105 | .env 106 | .venv 107 | env/ 108 | venv/ 109 | ENV/ 110 | env.bak/ 111 | venv.bak/ 112 | 113 | # Spyder project settings 114 | .spyderproject 115 | .spyproject 116 | 117 | # Rope project settings 118 | .ropeproject 119 | 120 | # mkdocs documentation 121 | /site 122 | 123 | # mypy 124 | .mypy_cache/ 125 | .dmypy.json 126 | dmypy.json 127 | 128 | # Pyre type checker 129 | .pyre/ 130 | 131 | # PyCharm 132 | /.idea/ 133 | 134 | # Internal 135 | db/ 136 | /docs/source/modules/ 137 | -------------------------------------------------------------------------------- /.readthedocs.yml: -------------------------------------------------------------------------------- 1 | # Read the Docs configuration file 2 | # See https://docs.readthedocs.io/en/stable/config-file/v2.html for details 3 | 4 | # Required 5 | version: 2 6 | build: 7 | os: "ubuntu-22.04" 8 | tools: 9 | python: "3.11" # For Pillow-9.5.0 10 | jobs: 11 | pre_build: 12 | - cd docs && make autodoc 13 | 14 | # Build documentation in the docs/ directory with Sphinx 15 | sphinx: 16 | configuration: docs/source/conf.py 17 | 18 | # Build documentation with MkDocs 19 | #mkdocs: 20 | # configuration: mkdocs.yml 21 | 22 | # Optionally build your docs in additional formats such as PDF 23 | formats: 24 | - pdf 25 | 26 | # Optionally set the version of Python and requirements required to build your docs 27 | python: 28 | install: 29 | - requirements: docs/requirements.txt 30 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2020 Just Work 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | recursive-include src/video_transcoding/locale * -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # django-video-transcoding 2 | 3 | Simple video transcoding application for Django Framework 4 | 5 | [![build](https://github.com/just-work/django-video-transcoding/workflows/build/badge.svg?branch=master)](https://github.com/just-work/django-video-transcoding/actions?query=event%3Apush+branch%3Amaster+workflow%3Abuild) 6 | [![codecov](https://codecov.io/gh/just-work/django-video-transcoding/branch/master/graph/badge.svg)](https://codecov.io/gh/just-work/django-video-transcoding) 7 | [![Updates](https://pyup.io/repos/github/just-work/django-video-transcoding/shield.svg)](https://pyup.io/repos/github/just-work/django-video-transcoding/) 8 | [![PyPI version](https://badge.fury.io/py/django-video-transcoding.svg)](http://badge.fury.io/py/django-video-transcoding) 9 | [![Documentation Status](https://readthedocs.org/projects/django-video-transcoding/badge/?version=latest)](https://django-video-transcoding.readthedocs.io/en/latest/?badge=latest) 10 | 11 | ## Use as a service 12 | 13 | Use `docker-compose.yml` as a source of inspiration. 14 | 15 | See [quickstart.md](docs/source/quickstart.md) for details. 16 | 17 | ## Install a Django app 18 | 19 | Use `src/dvt/settings.py` as a source of inspiration. 20 | 21 | See [application.md](docs/source/application.md) for details. 22 | 23 | ## Develop and extend 24 | 25 | See [development.md](docs/source/development.md) for details. 26 | -------------------------------------------------------------------------------- /SECURITY.md: -------------------------------------------------------------------------------- 1 | # Security Policy 2 | 3 | ## Supported Versions 4 | 5 | | Version | Supported | 6 | |---------|--------------------| 7 | | 0.x.0 | :x: | 8 | | 1.x.0 | :white_check_mark: | 9 | | 2.x.0 | :white_check_mark: | 10 | 11 | ## Reporting a Vulnerability 12 | 13 | Just create an issue on github. We'll try to respond in two weeks. 14 | -------------------------------------------------------------------------------- /deploy/dvt/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM ubuntu:24.04 2 | 3 | RUN apt-get update && apt-get install -y libmediainfo-dev ffmpeg python3-dev python3-pip python-is-python3 4 | 5 | WORKDIR /app/src/ 6 | ADD ./requirements.txt /app/ 7 | RUN cd /app/ && pip install --break-system-packages --no-cache-dir -r requirements.txt 8 | 9 | ADD ./deploy/dvt/start-django-dev.sh /app/ 10 | ADD ./deploy/dvt/start-celery-worker.sh /app/ 11 | RUN chmod +x /app/*.sh 12 | EXPOSE 8000 13 | ADD ./src/ /app/src/ 14 | -------------------------------------------------------------------------------- /deploy/dvt/start-celery-worker.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | exec celery --app video_transcoding.celery worker --loglevel=DEBUG -c 1 3 | 4 | -------------------------------------------------------------------------------- /deploy/dvt/start-django-dev.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | python /app/src/manage.py migrate --noinput && \ 4 | python /app/src/manage.py createsuperuser --noinput 2>/dev/null || true && \ 5 | 6 | exec python /app/src/manage.py runserver --noreload 0.0.0.0:8000 7 | -------------------------------------------------------------------------------- /docker-compose.yml: -------------------------------------------------------------------------------- 1 | services: 2 | rabbitmq: 3 | image: rabbitmq:management 4 | container_name: dvt-rabbitmq 5 | hostname: rabbitmq 6 | ports: 7 | - "15672:15672" 8 | expose: 9 | - 5672 10 | - 15672 11 | environment: 12 | - RABBITMQ_ERLANG_COOKIE=ASDFCASDFASDFSHFGH 13 | - RABBITMQ_DEFAULT_USER=guest 14 | - RABBITMQ_DEFAULT_PASS=guest 15 | - RABBITMQ_DEFAULT_VHOST=/ 16 | admin: 17 | build: 18 | context: . 19 | dockerfile: deploy/dvt/Dockerfile 20 | command: bash -c /app/start-django-dev.sh 21 | container_name: dvt-admin 22 | restart: unless-stopped 23 | stop_signal: SIGINT 24 | volumes: 25 | - "database:/app/db/" 26 | - "results:/data/results/" 27 | ports: 28 | - "8000:8000" 29 | expose: 30 | - "8000" 31 | depends_on: 32 | - rabbitmq 33 | environment: 34 | - VIDEO_TRANSCODING_CELERY_BROKER_URL 35 | - VIDEO_EDGES 36 | - VIDEO_URL 37 | - DJANGO_SUPERUSER_USERNAME=admin 38 | - DJANGO_SUPERUSER_PASSWORD=admin 39 | - DJANGO_SUPERUSER_EMAIL=admin@dvt.localhost 40 | 41 | celery: 42 | build: 43 | context: . 44 | dockerfile: deploy/dvt/Dockerfile 45 | command: bash -c /app/start-celery-worker.sh 46 | container_name: dvt-celery 47 | restart: unless-stopped 48 | volumes: 49 | - "database:/app/db/" 50 | - "tmp:/data/tmp/" 51 | - "results:/data/results/" 52 | depends_on: 53 | - rabbitmq 54 | links: 55 | - "sources:sources.local" 56 | environment: 57 | - DJANGO_SETTINGS_MODULE=dvt.settings 58 | - VIDEO_TRANSCODING_CELERY_BROKER_URL 59 | - VIDEO_TEMP_URI 60 | - VIDEO_RESULTS_URI 61 | stop_signal: SIGTERM 62 | 63 | sources: 64 | image: clickhouse/nginx-dav:latest 65 | hostname: sources.local 66 | networks: 67 | default: 68 | aliases: 69 | - sources.local 70 | ports: 71 | - "80:80" 72 | volumes: 73 | - "sources:/usr/share/nginx/" 74 | 75 | volumes: 76 | database: 77 | sources: 78 | tmp: 79 | results: 80 | -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Minimal makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line, and also 5 | # from the environment for the first two. 6 | SPHINXOPTS ?= 7 | SPHINXBUILD ?= sphinx-build 8 | SOURCEDIR = source 9 | BUILDDIR = build 10 | 11 | # Put it first so that "make" without argument is like "make help". 12 | help: 13 | @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 14 | 15 | .PHONY: help Makefile 16 | 17 | # Catch-all target: route all unknown targets to Sphinx using the new 18 | # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). 19 | %: Makefile 20 | @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 21 | 22 | autodoc: 23 | sphinx-apidoc -o source/modules ../src/video_transcoding -d 0 24 | -------------------------------------------------------------------------------- /docs/requirements.txt: -------------------------------------------------------------------------------- 1 | -r ../requirements.txt 2 | 3 | sphinx==8.1.3 4 | sphinxcontrib-blockdiag==3.0.0 5 | sphinxcontrib-seqdiag==3.0.0 6 | sphinx-markdown-tables==0.0.17 7 | recommonmark==0.7.1 8 | funcparserlib==1.0.1 9 | Pillow==9.5.0 # For blockdiag to work 10 | -------------------------------------------------------------------------------- /docs/source/application.md: -------------------------------------------------------------------------------- 1 | Application 2 | ============ 3 | 4 | This page describes `django-video-transcoding` integration into existing 5 | Django project. 6 | 7 | Installation 8 | ------------ 9 | 10 | ### Infrastructure requirements 11 | 12 | 1. [Compatible](https://docs.celeryq.dev/en/stable/getting-started/backends-and-brokers/index.html#broker-instructions) 13 | message broker for Celery 14 | 2. Persistent storage for temporary files 15 | * i.e. local FS for a single transcoding server 16 | * S3 persistent volume if transcoding container can move between hosts 17 | 3. Persistent storage with HTTP server for transcoded HLS streams 18 | * i.e. `nginx` or `S3` 19 | 20 | ### System requirements 21 | 22 | 1. [ffmpeg-6.1](http://ffmpeg.org/) or later 23 | 2. [libmediainfo](https://mediaarea.net/en/MediaInfo) 24 | 25 | ```shell 26 | apt-get install ffmpeg libmediainfo-dev 27 | ``` 28 | 29 | ### Python requirements 30 | 31 | ```shell 32 | pip install django-video-transcoding 33 | ``` 34 | 35 | ### Django integration 36 | 37 | Add `video_transcoding` to project settings 38 | 39 | ```python 40 | INSTALLED_APPS.append("video_transcoding") 41 | ``` 42 | 43 | ### Celery configuration 44 | 45 | `video_transcoding.celery` contains Celery application that can use environment 46 | variables for configuration. This application can be used as a starting point 47 | for configuring own app. 48 | 49 | | env | description | 50 | |---------------------------------------|--------------------------| 51 | | `VIDEO_TRANSCODING_CELERY_BROKER_URL` | celery broker | 52 | | `VIDEO_TEMP_URI` | URI for temporary files | 53 | | `VIDEO_RESULTS_URI` | URI for transcoded files | 54 | 55 | ### Serving HLS streams 56 | 57 | Demo uses Django Static Files for serving transcoded files, but this is 58 | inacceptable for production usage. Please, configure HTTP server to serve files. 59 | It may be `nginx` to serve static files and/or any `CDN` solution. 60 | 61 | | env | description | 62 | |---------------|------------------------------| 63 | | `VIDEO_EDGES` | URIs for serving HLS streams | 64 | | `VIDEO_URL` | HLS stream url template | 65 | 66 | Environment variables 67 | --------------------- 68 | 69 | * `VIDEO_TRANSCODING_CELERY_BROKER_URL` (`amqp://guest:guest@rabbitmq:5672/`) - 70 | Celery broker url for django-video-transcoding 71 | * `VIDEO_TRANSCODING_CELERY_RESULT_BACKEND` (not set) - Celery result backend 72 | (not used) 73 | * `VIDEO_TRANSCODING_CELERY_CONCURRENCY` (not set) - Celery concurrency 74 | * `VIDEO_TRANSCODING_TIMEOUT` - task acknowledge timeout for AMQP backend 75 | (see `x-consumer-timeout` for [RabbitMQ](https://www.rabbitmq.com/docs/consumers#per-queue-delivery-timeouts-using-an-optional-queue-argument)) 76 | * `VIDEO_TRANSCODING_COUNTDOWN` (10) - transcoding task delay in seconds 77 | * `VIDEO_TRANSCODING_WAIT` (0) - transcoding start delay in seconds (used if 78 | task delay is not supported by Celery broker) 79 | * `VIDEO_TEMP_URI` - URI for temporary files (`file:///data/tmp/`). 80 | Supports `file`, `http` and `https`. For HTTP uses `PUT` **and** `POST` 81 | requests to store files. 82 | * `VIDEO_RESULTS_URI` - URI for transcoded files (`file:///data/results/`). 83 | Supports `file`, `http` and `https`. 84 | * `VIDEO_EDGES` - comma-separated list of public endpoints for transcoded files. 85 | By default uses Django static files (`http://localhost:8000/media/`). 86 | * `VIDEO_URL` - public HLS stream template (`{edge}/results/{filename}/index.m3u8`). 87 | `edge` is one of `VIDEO_EDGES` and `filename` is `Video.basename` value. 88 | * `VIDEO_CONNECT_TIMEOUT` (1) - connect timeout for HTTP requests in seconds. 89 | * `VIDEO_REQUEST_TIMEOUT` (1) - request timeout for HTTP requests in seconds. 90 | * `VIDEO_CHUNK_DURATION` (60) - chunk duration in seconds. Transcoder splits 91 | source file into chunks and then transcodes them one-by-one to handle 92 | container restarts. It's recommended to align this value with 93 | `VideoProfile.segment_duration` to prevent short HLS fragments every N seconds. 94 | 95 | ### Generating streaming links 96 | 97 | `django-video-transcoding` supports edge-server load balancing by generating 98 | multiple links to video streams. Video player should support choosing single 99 | server from multiple links. 100 | 101 | ```bash 102 | export VIDEO_EDGES=http://edge-1/streaming/,http://edge-1/streaming/ 103 | ``` 104 | 105 | ### Generating manifest links 106 | 107 | By default, `django-video-transcoding` generates links to HLS manifests 108 | accessible via HTTP server, but this can be customized. 109 | 110 | ```bash 111 | export VIDEO_URL={edge}/results/{filename}/index.m3u8 112 | ``` 113 | 114 | See `video_transcoding.models.Video.format_video_url`. 115 | -------------------------------------------------------------------------------- /docs/source/architecture.md: -------------------------------------------------------------------------------- 1 | Architecture 2 | ============ 3 | 4 | This document describes a typical architecture of Video-On-Demand website. 5 | 6 | Components 7 | ---------- 8 | 9 | ``` blockdiag:: 10 | 11 | blockdiag { 12 | node_width = 128; 13 | node_height = 40; 14 | span_width=128; 15 | span_height=60; 16 | 17 | CMS -> Broker [label="tasks", style=dotted]; 18 | Transcoder <- Sources [label="src", thick, folded]; 19 | Broker -> Transcoder [label="tasks", style=dotted]; 20 | Transcoder -> Storage [label="HLS", folded, thick]; 21 | Transcoder <-> TMP [label="files", thick]; 22 | Storage -> CDN [label="HLS chunks"]; 23 | CDN -> Player [label="cached chunks"]; 24 | CMS [color=lightblue]; 25 | Sources [shape = flowchart.database]; 26 | Transcoder [color=lightblue]; 27 | Broker [shape=flowchart.terminator]; 28 | Storage [shape = flowchart.database]; 29 | TMP [shape = flowchart.database]; 30 | Player [shape=actor]; 31 | } 32 | 33 | ``` 34 | 35 | * `Sources` storage contains original media for video content 36 | * `CMS` stores video content list. 37 | * When a new video is created in `CMS`, it sends transcoding task via `Broker` 38 | to `Transcoder` 39 | * `Transcoder` downloads source media from `Sources` storage 40 | * `Transcoder` stores intermediate files at `TMP` storage 41 | for resumable processing support 42 | * `Transcoder` stores final HLS segments at `Storage` 43 | * `Player` requests HLS segments from `CDN` 44 | * `CDN` requests segments from `Storage` and caches them for scalability. 45 | 46 | Video processing steps 47 | ---------------------- 48 | 49 | ``` seqdiag:: 50 | 51 | seqdiag { 52 | CMS;Broker;DB;Transcoder; 53 | 54 | CMS => DB [label = "new video created"]; 55 | CMS => Broker [label = "sends a task", return="ACK"]; 56 | Broker -->> Transcoder [label = "receives a task"] { 57 | Transcoder => DB [label = "marks session started"]; 58 | Transcoder -> Transcoder [label = "transcode video"]; 59 | Transcoder => DB [label = "store video metadata"]; 60 | Transcoder => DB [label = "marks session done"]; 61 | } 62 | Broker <<-- Transcoder [label = "ACK"]; 63 | } 64 | ``` 65 | 66 | 1. A new video created in `CMS` 67 | 2. `CMS` puts a new task to a Celery task `Broker` 68 | 3. Celery worker at `Transcoder` node changes video status and transcodes video 69 | 4. Celery worker changes video status and saves result metadata (filename, 70 | video characteristics etc...). 71 | 72 | Transcoding steps 73 | ----------------- 74 | 75 | ``` seqdiag:: 76 | 77 | seqdiag { 78 | Transcoder;Sources;TMP;Storage; 79 | === Split source === 80 | Transcoder -->> Sources [label = "requests source file"]{ 81 | Transcoder -->> TMP [label = "splits source to chunks", note="source\nchunks"]; 82 | Transcoder <<-- TMP; 83 | } 84 | Transcoder <<-- Sources [label = "source file"] 85 | === Chunk transcode loop === 86 | Transcoder -->> TMP [label = "read src chunk"] { 87 | Transcoder -->> TMP [label = "write transcoded chunk", note="transcoded\nchunk"]; 88 | Transcoder <<-- TMP; 89 | } 90 | Transcoder <<-- TMP [label = "source chunk"]; 91 | === Segment result === 92 | Transcoder -->> TMP [label = "read transcoded chunks"] { 93 | Transcoder -->> Storage [label = "segment to HLS", note="HLS\nsegments"]; 94 | Transcoder <<-- Storage; 95 | } 96 | Transcoder <<-- TMP "chunks"; 97 | } 98 | ``` 99 | 100 | 1. `Transcoder` downloads and splits source file into chunks 101 | 2. Source chunks are stored at `TMP` storage to support resumable processing 102 | 3. `Transcoder` processes source chunks one-by-one and stores results at `TMP` 103 | 4. `Transcoder` concatenates all resulting chunks from `TMP` storage, 104 | segments them to HLS and saves at `Storage`. 105 | 106 | ## Fault tolerance 107 | 108 | Transcoding video files is a long-time operation, anything can happen while 109 | transcoder is active: 110 | 111 | * Hardware failure 112 | * Container failure 113 | * Storage failure 114 | * Network failure 115 | * New release deployment 116 | * Container/VM eviction (i.e. for preemptible VMs) 117 | 118 | Django-video-transcoder addresses some of this failures: 119 | 120 | * It relies on a fault-tolerant distributed storage for temporary files 121 | * This allows to resume video processing from the checkpoint 122 | (last successfully transcoded chunk), even if transcoding is continued at 123 | another host 124 | * Transcoder tracks processing state at the database to prevent multiple 125 | worker to process same video 126 | * Automatic task retry feature allows to handle temporary failures without 127 | manual steps 128 | 129 | ## Load balancing 130 | 131 | ### Transcoding 132 | 133 | Transcoding video files requires lot's of CPU power, or even GPU. `ffmpeg` 134 | under the hood of `django-video-transcoding` utilizes all CPU cores, so every 135 | physical host should launch single celery worker. When high transcoding 136 | throughput is required, new physical hosts should be added. Load balancing is 137 | done transparently as Celery task broker clients handle messages independently. 138 | 139 | ### Storage 140 | 141 | Storing video files has some performance concerns: 142 | 143 | 1. Videos are large, all content may not fit to a single server. 144 | 2. Lot's of disk IO is needed to handle multiple clients accessing different 145 | video files. 146 | 3. Files could be damaged or disappear because of disk failures. 147 | 148 | It's recommended to use production-ready distributed storage solution, the 149 | easiest option is `S3`-compatible service from cloud provider. 150 | 151 | ### Serving video 152 | 153 | Sending video to multiple clients is limited with: 154 | 155 | * Network bandwidth 156 | * Disk iops 157 | * CPU resources for HTTPS encryption 158 | * Fault tolerance 159 | 160 | The easiest way to handle all these problems is to use `CDN` in front of 161 | your video storage. 162 | 163 | ### Conclusion 164 | 165 | Video-on-demand performance is a large and exciting topic; in some cases it 166 | could be addressed with simple approaches, in another lot's of work need to be 167 | done. Despite these advices above, `django-video-transcoding` does not provide 168 | universal high-performance solution; it's purpose is simplicity and 169 | extensibility. -------------------------------------------------------------------------------- /docs/source/conf.py: -------------------------------------------------------------------------------- 1 | # Configuration file for the Sphinx documentation builder. 2 | # 3 | # This file only contains a selection of the most common options. For a full 4 | # list see the documentation: 5 | # https://www.sphinx-doc.org/en/master/usage/configuration.html 6 | 7 | # -- Path setup -------------------------------------------------------------- 8 | 9 | # If extensions (or modules to document with autodoc) are in another directory, 10 | # add these directories to sys.path here. If the directory is relative to the 11 | # documentation root, use os.path.abspath to make it absolute, like shown here. 12 | # 13 | import os 14 | import sys 15 | import django 16 | 17 | # noinspection PyPackageRequirements 18 | from recommonmark.transform import AutoStructify 19 | 20 | sys.path.insert(0, os.path.abspath('../../src/')) 21 | os.environ.setdefault("DJANGO_SETTINGS_MODULE", "dvt.settings") 22 | django.setup() 23 | 24 | # -- Project information ----------------------------------------------------- 25 | 26 | project = 'Django Video Transcoding' 27 | copyright = '2020, Sergey Tikhonov' 28 | author = 'Sergey Tikhonov' 29 | 30 | # The full version, including alpha/beta/rc tags 31 | release = '0.1.0' 32 | 33 | # -- General configuration --------------------------------------------------- 34 | 35 | # Add any Sphinx extension module names here, as strings. They can be 36 | # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom 37 | # ones. 38 | extensions = [ 39 | 'sphinx.ext.autodoc', 40 | 'sphinx.ext.doctest', 41 | 'sphinx.ext.napoleon', 42 | 'sphinxcontrib.blockdiag', 43 | 'sphinxcontrib.seqdiag', 44 | 'recommonmark', 45 | 'sphinx_markdown_tables', 46 | ] 47 | 48 | source_suffix = { 49 | '.rst': 'restructuredtext', 50 | '.md': 'markdown', 51 | } 52 | 53 | # Add any paths that contain templates here, relative to this directory. 54 | templates_path = ['_templates'] 55 | 56 | # List of patterns, relative to source directory, that match files and 57 | # directories to ignore when looking for source files. 58 | # This pattern also affects html_static_path and html_extra_path. 59 | exclude_patterns = ['**/*.migrations.rst', '**/*.tests.rst'] 60 | 61 | master_doc = 'index' 62 | 63 | # -- Options for HTML output ------------------------------------------------- 64 | 65 | # The theme to use for HTML and HTML Help pages. See the documentation for 66 | # a list of builtin themes. 67 | # 68 | html_theme = 'alabaster' 69 | 70 | # Add any paths that contain custom static files (such as style sheets) here, 71 | # relative to this directory. They are copied after the builtin static files, 72 | # so a file named "default.css" will overwrite the builtin "default.css". 73 | html_static_path = ['_static'] 74 | 75 | # -- sphinx.ext.autodoc 76 | 77 | autodoc_member_order = 'groupwise' 78 | 79 | # -- modindex -- 80 | 81 | modindex_common_prefix = ['video_transcoding'] 82 | 83 | 84 | def setup(app): 85 | app.add_config_value('recommonmark_config', { 86 | # 'url_resolver': lambda url: github_doc_root + url, 87 | 'auto_toc_tree_section': 'Contents', 88 | 'enable_eval_rst': True, 89 | }, True) 90 | app.add_transform(AutoStructify) 91 | -------------------------------------------------------------------------------- /docs/source/development.md: -------------------------------------------------------------------------------- 1 | Development 2 | =========== 3 | 4 | This document describes how to develop and extend 5 | `django-video-transcoding`. 6 | 7 | Developing 8 | ---------- 9 | 10 | ### Running tests 11 | 12 | ```shell 13 | $> src/manage.py test 14 | ``` 15 | 16 | ### Type checking 17 | 18 | ```shell 19 | $> pip install mypy django-stubs 20 | $> cd src && mypy \ 21 | --config-file=../mypy.ini \ 22 | -p video_transcoding \ 23 | -p django_stubs_ext 24 | ``` 25 | 26 | Configuration 27 | ------------- 28 | 29 | * All configuration is stored in `video_transcoding.defaults` module. 30 | * Most important variables are configured from ENV 31 | * `settings.VIDEO_TRANSCODING_CONFIG` may be used for overriding defaults. 32 | 33 | 34 | Extending 35 | --------- 36 | 37 | ### Celery application 38 | 39 | * if you are running transcoder in docker, make sure that celery master process 40 | has pid 1 (docker will send SIGTERM to it by default) 41 | * when using separate celery app, send SIGUSR1 from master to workers to trigger 42 | soft shutdown handling 43 | (see `video_transcoding.celery.send_term_to_children`) 44 | * celery master should set process group in order to send SIGUSR1 to workeres 45 | (see `video_transcoding.celery.set_same_process_group`) 46 | 47 | ### Transcoding implementation 48 | 49 | * extend `video_transcoding.tasks.TranscodeVideo` to change task behavior 50 | * top-level transcoding strategy is selected in `TranscodeVideo.init_strategy`, 51 | see `video_transcoding.strategy.ResumableStrategy` as an example 52 | * see `video_transcoding.transcoding.transcoder` module for low-level 53 | transcoding steps examples 54 | * dealing with different intermediate files requires metadata extraction and 55 | specific logic for this process is implemented in 56 | `video_transcoding.transcoding.extract.Extractor` subclasses. 57 | * missing metadata is restored by different analyzers in 58 | `video_transcoding.transcoding.analysis` module. 59 | 60 | ### Model inheritance 61 | 62 | * For preset-related models use `Base` abstract models defined in 63 | `video_transcoding.models`. 64 | * For overriding `Video` model set `VIDEO_TRANSCODING_CONFIG["VIDEO_MODEL"]` 65 | key to `app_label.ModelName` in `settings`. 66 | * Connect other django models to `Video` using 67 | `video_transcoding.models.get_video_model()`. 68 | * When `Video` is overridden, video model admin is not registered automatically. 69 | As with migrations, this should be done manually. 70 | -------------------------------------------------------------------------------- /docs/source/index.rst: -------------------------------------------------------------------------------- 1 | Django Video Transcoding 2 | ======================== 3 | 4 | `django-video-transcoding 5 | `_ is a 6 | `Django `_ application that helps to prepare 7 | video files for video-on-demand services. 8 | 9 | .. toctree:: 10 | :maxdepth: 2 11 | :caption: Contents: 12 | 13 | quickstart 14 | application 15 | architecture 16 | development 17 | operation 18 | modules/modules 19 | 20 | 21 | Indices and tables 22 | ================== 23 | 24 | * :ref:`genindex` 25 | * :ref:`modindex` 26 | * :ref:`search` 27 | -------------------------------------------------------------------------------- /docs/source/operation.md: -------------------------------------------------------------------------------- 1 | Operation 2 | ========= 3 | 4 | This document describes `django-video-transcoding` operation details. 5 | 6 | ### Proper shutdown 7 | 8 | Processing video files is a very long operation, so waiting for celery task 9 | complition while shutting down is inacceptable. On the other hand, if celery 10 | worker processes are killed, lost tasks an zombie ffmpeg processes may appear. 11 | 12 | * if you are running transcoder in a container, make sure that celery master 13 | process has pid 1 (docker will send SIGTERM to it by default) 14 | * for separate celery app make sure that it sets process group for all workers 15 | and propagates SIGUSR1 to trigger graceful worker shutdown 16 | 17 | ### Resumable workers 18 | 19 | For modern cloud-based installations an optimal solution is to use 20 | [Preemptible VM instances](https://cloud.google.com/compute/docs/instances/preemptible). 21 | On the one hand they are inexpensive, but on the another these VMs can be 22 | shut down at any time. 23 | 24 | Transcoding VOD files is a very time and CPU consuming operation, 25 | so the implemetation should support resumable transcoding. 26 | 27 | For large container-based deployments (docker, K8s) resumable transcoding 28 | simplifies hardware failure handling and release management. 29 | 30 | Django-video-transcoding implements resumable strategy for transcoding: 31 | 32 | * It splits source file into 1-minute chunks while downloading 33 | * Each chunk is transcoded independently 34 | * At the end all chunks are concatenated and segmented to HLS streams 35 | * After restart each step can be skipped if it's result already exists 36 | 37 | So, temporary storage should be persistent and host-independent. We recommend 38 | mounting `S3` bucket as a file system. 39 | 40 | ### Getting sources 41 | 42 | `ffmpeg` supports a large number of ingest protocols, such as `http` and `ftp`. 43 | This allows downloading source video and splitting it to chunks in a single pass. 44 | The main drawback is inability to tune some protocol options, especially for 45 | `ftp`. We recommend to use `http` for source videos. 46 | 47 | ### Storing temporary files 48 | 49 | For some reasons the `segment` muxer is used to split source video into chunks, 50 | and this muxer does not support setting http method. If temporary storage 51 | is accessed via HTTP, it must support storing files via `POST` requests. 52 | 53 | ### Serving HLS streams 54 | 55 | Serving video requires high network bandwidth and fast drives, 56 | especially if number of users is large. It's recommended to use CDN to protect 57 | video storage from high repetitive load. 58 | 59 | * Use `CloudFront` with `S3` or there analogues from other cloud providers 60 | * Or use `CDN` provider in front of HTTP server 61 | * For self-hosted solutions distribute network load across multiple edge servers 62 | (round robin is supported by multiple hosts in `VIDEO_EDGES` env variable). 63 | -------------------------------------------------------------------------------- /docs/source/quickstart.md: -------------------------------------------------------------------------------- 1 | Quick Start 2 | =========== 3 | 4 | This document describes how to start django-video-transcoding as a service 5 | with demo docker-compose config. 6 | 7 | ## Code checkout 8 | 9 | ```sh 10 | git clone git@github.com:just-work/django-video-transcoding.git 11 | cd django-video-transcoding 12 | ``` 13 | 14 | ## Run admin, webdav and celery worker 15 | 16 | ```sh 17 | docker-compose up 18 | ``` 19 | 20 | * - Django admin (credentials are `admin:admin`) 21 | * - Transcoded HLS streams served by Django 22 | * - WebDAV for sources 23 | 24 | ### Transcode something 25 | 26 | * Add `sources.local` to hosts file 27 | * `curl -T cat.mp4 http://sources.local/` 28 | * Create new video with link above 29 | * Wait till video will change status to DONE. 30 | * On video change form admin page there is a sample video player. 31 | 32 | ## Development environment 33 | 34 | Development environment is deployed with `docker-compose`. It contains several 35 | containers: 36 | 37 | 1. `rabbitmq` - celery task broker container 38 | 2. `admin` - django admin container 39 | 3. `celery` - transcoder worker container 40 | 4. `sources` - `WebDAV` write-enabled server for source files 41 | 42 | * `SQLite` database file is used for simplicity, it is shared via `database` 43 | volume between `admin` and `celery` containers 44 | * `sources` volume is used by `sources` container for sources video 45 | * `tmp` volume is used by `celery` container for temporary files 46 | * `results` volume is used by `celery` container for saving transcoded HLS 47 | streams which are then served by `admin` container for CORS bypass 48 | -------------------------------------------------------------------------------- /mypy.ini: -------------------------------------------------------------------------------- 1 | [mypy] 2 | follow_imports = normal 3 | plugins = 4 | mypy_django_plugin.main 5 | [mypy.plugins.django-stubs] 6 | django_settings_module = "dvt.settings" 7 | 8 | [mypy-billiard.*] 9 | ignore_missing_imports = true 10 | 11 | [mypy-celery.*] 12 | ignore_missing_imports = true 13 | 14 | [mypy-kombu.*] 15 | ignore_missing_imports = true 16 | 17 | [mypy-pymediainfo.*] 18 | ignore_missing_imports = true 19 | 20 | [mypy-fffw.*] 21 | ignore_missing_imports = true 22 | 23 | [mypy-model_utils.*] 24 | ignore_missing_imports = true 25 | 26 | [mypy-video_transcoding.*] 27 | disallow_untyped_calls = true 28 | disallow_untyped_defs = true 29 | disallow_incomplete_defs = true 30 | 31 | [mypy-video_transcoding.migrations.*] 32 | ignore_errors = true 33 | 34 | [mypy-video_transcoding.tests.*] 35 | ignore_errors = true 36 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [build-system] 2 | requires = ["setuptools>=61.0", "setuptools-git-versioning"] 3 | build-backend = "setuptools.build_meta" 4 | 5 | [tool.setuptools-git-versioning] 6 | enabled = true 7 | 8 | 9 | [project] 10 | name = "django_video_transcoding" 11 | dynamic = ["version"] 12 | description = "Simple video transcoding application for Django framework" 13 | readme = "README.md" 14 | authors = [ 15 | { name = "Sergey Tikhonov", email = "zimbler@gmail.com" }, 16 | ] 17 | 18 | dependencies = [ 19 | # Django Framework and batteries 20 | "Django>=3.2,<5.2", 21 | "django-model-utils>=4.2.0,<5.2.0", 22 | # Background processing 23 | "Celery>=5.0.5,<5.5.0", 24 | "kombu>=5.0.2,<5.4.0", 25 | "billiard>=3.6.4.0,<4.3.0", 26 | # Uploading 27 | "requests>=2.25,<2.33", 28 | # Video processing 29 | "pymediainfo>=5.0.3,<6.2", 30 | "fffw~=7.0.0", 31 | ] 32 | license = { text = "MIT" } 33 | keywords = [ 34 | "django", 35 | "video", 36 | "media", 37 | "transcoding", 38 | "processing", 39 | ] 40 | classifiers = [ 41 | 'Development Status :: 4 - Beta', 42 | 'Environment :: Console', 43 | 'Framework :: Django :: 3.2', 44 | 'Framework :: Django :: 4.0', 45 | 'Framework :: Django :: 4.1', 46 | 'Framework :: Django :: 4.2', 47 | 'Framework :: Django :: 5.0', 48 | 'Framework :: Django :: 5.1', 49 | 'Operating System :: POSIX', 50 | 'Programming Language :: Python :: 3.9', 51 | 'Programming Language :: Python :: 3.10', 52 | 'Programming Language :: Python :: 3.11', 53 | 'Programming Language :: Python :: 3.12', 54 | 'Topic :: Multimedia :: Video :: Conversion', 55 | ] 56 | 57 | [project.urls] 58 | homepage = "https://github.com/just-work/django-video-transcoding" 59 | documentation = "https://django-video-transcoding.readthedocs.io/en/latest/" 60 | repository = "https://github.com/just-work/django-video-transcoding.git" 61 | issues = "https://github.com/just-work/django-video-transcoding/issues" 62 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | Django==5.1.5 2 | django-model-utils==5.0.0 3 | requests==2.32.3 4 | Celery==5.4.0 5 | pymediainfo==6.1.0 6 | fffw==7.0.0 7 | mypy==1.10.1 8 | types-requests==2.32.0.20240622 9 | django-stubs==5.0.2 10 | billiard==4.2.1 11 | kombu==5.4.2 12 | importlib-metadata==8.0.0 13 | -------------------------------------------------------------------------------- /src/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/just-work/django-video-transcoding/aa4a945a319bb4bd540c2acf4fcc3cbd6e3fad26/src/__init__.py -------------------------------------------------------------------------------- /src/dvt/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/just-work/django-video-transcoding/aa4a945a319bb4bd540c2acf4fcc3cbd6e3fad26/src/dvt/__init__.py -------------------------------------------------------------------------------- /src/dvt/asgi.py: -------------------------------------------------------------------------------- 1 | """ 2 | ASGI config for dvt project. 3 | 4 | It exposes the ASGI callable as a module-level variable named ``application``. 5 | 6 | For more information on this file, see 7 | https://docs.djangoproject.com/en/3.0/howto/deployment/asgi/ 8 | """ 9 | 10 | import os 11 | 12 | from django.core.asgi import get_asgi_application # type: ignore 13 | 14 | os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'dvt.settings') 15 | 16 | application = get_asgi_application() 17 | -------------------------------------------------------------------------------- /src/dvt/settings.py: -------------------------------------------------------------------------------- 1 | """ 2 | Django settings for dvt project. 3 | 4 | Generated by 'django-admin startproject' using Django 3.0.3. 5 | 6 | For more information on this file, see 7 | https://docs.djangoproject.com/en/3.0/topics/settings/ 8 | 9 | For the full list of settings and their values, see 10 | https://docs.djangoproject.com/en/3.0/ref/settings/ 11 | """ 12 | 13 | import os 14 | 15 | # Build paths inside the project like this: os.path.join(BASE_DIR, ...) 16 | from typing import List, Optional 17 | 18 | BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) 19 | 20 | 21 | # Quick-start development settings - unsuitable for production 22 | # See https://docs.djangoproject.com/en/3.0/howto/deployment/checklist/ 23 | 24 | # SECURITY WARNING: keep the secret key used in production secret! 25 | SECRET_KEY = '4@+rlnsb=3@%@c_%di5o$-rdw(b5tswcmo8r+hd9t^_8+uixjk' 26 | 27 | # SECURITY WARNING: don't run with debug turned on in production! 28 | DEBUG = True 29 | 30 | ALLOWED_HOSTS: List[str] = [] 31 | 32 | 33 | # Application definition 34 | 35 | INSTALLED_APPS = [ 36 | 'django.contrib.admin', 37 | 'django.contrib.auth', 38 | 'django.contrib.contenttypes', 39 | 'django.contrib.sessions', 40 | 'django.contrib.messages', 41 | 'django.contrib.staticfiles', 42 | 43 | 'video_transcoding', 44 | ] 45 | 46 | MIDDLEWARE = [ 47 | 'django.middleware.security.SecurityMiddleware', 48 | 'django.contrib.sessions.middleware.SessionMiddleware', 49 | 'django.middleware.common.CommonMiddleware', 50 | 'django.middleware.csrf.CsrfViewMiddleware', 51 | 'django.contrib.auth.middleware.AuthenticationMiddleware', 52 | 'django.contrib.messages.middleware.MessageMiddleware', 53 | 'django.middleware.clickjacking.XFrameOptionsMiddleware', 54 | ] 55 | 56 | ROOT_URLCONF = "dvt.urls" 57 | 58 | TEMPLATES = [ 59 | { 60 | 'BACKEND': 'django.template.backends.django.DjangoTemplates', 61 | 'APP_DIRS': True, 62 | 'OPTIONS': { 63 | 'context_processors': [ 64 | 'django.template.context_processors.debug', 65 | 'django.template.context_processors.request', 66 | 'django.contrib.auth.context_processors.auth', 67 | 'django.contrib.messages.context_processors.messages', 68 | ], 69 | }, 70 | }, 71 | ] 72 | 73 | WSGI_APPLICATION = 'dvt.wsgi.application' 74 | 75 | 76 | # Database 77 | # https://docs.djangoproject.com/en/3.0/ref/settings/#databases 78 | 79 | DATABASES = { 80 | 'default': { 81 | 'ENGINE': 'django.db.backends.sqlite3', 82 | 'NAME': os.getenv("DATABASE_FILE", 83 | os.path.join(BASE_DIR, '../db/db.sqlite3')), 84 | } 85 | } 86 | 87 | DEFAULT_AUTO_FIELD = 'django.db.models.AutoField' 88 | 89 | # Password validation 90 | # https://docs.djangoproject.com/en/3.0/ref/settings/#auth-password-validators 91 | 92 | AUTH_PASSWORD_VALIDATORS = [ 93 | { 94 | 'NAME': 'django.contrib.auth.password_validation.UserAttributeSimilarityValidator', 95 | }, 96 | { 97 | 'NAME': 'django.contrib.auth.password_validation.MinimumLengthValidator', 98 | }, 99 | { 100 | 'NAME': 'django.contrib.auth.password_validation.CommonPasswordValidator', 101 | }, 102 | { 103 | 'NAME': 'django.contrib.auth.password_validation.NumericPasswordValidator', 104 | }, 105 | ] 106 | 107 | 108 | # Internationalization 109 | # https://docs.djangoproject.com/en/3.0/topics/i18n/ 110 | 111 | LANGUAGE_CODE = 'en-us' 112 | 113 | TIME_ZONE = 'UTC' 114 | 115 | USE_I18N = True 116 | 117 | USE_L10N = True 118 | 119 | USE_TZ = True 120 | 121 | 122 | # Static files (CSS, JavaScript, Images) 123 | # https://docs.djangoproject.com/en/3.0/howto/static-files/ 124 | 125 | STATIC_URL = '/static/' 126 | 127 | MEDIA_ROOT = '/data/' 128 | MEDIA_URL = '/media/' 129 | 130 | VIDEO_TRANSCODING_CONFIG = { 131 | 'VIDEO_MODEL': "video_transcoding.Video", 132 | } 133 | -------------------------------------------------------------------------------- /src/dvt/urls.py: -------------------------------------------------------------------------------- 1 | """dvt URL Configuration 2 | 3 | The `urlpatterns` list routes URLs to views. For more information please see: 4 | https://docs.djangoproject.com/en/3.0/topics/http/urls/ 5 | Examples: 6 | Function views 7 | 1. Add an import: from my_app import views 8 | 2. Add a URL to urlpatterns: path('', views.home, name='home') 9 | Class-based views 10 | 1. Add an import: from other_app.views import Home 11 | 2. Add a URL to urlpatterns: path('', Home.as_view(), name='home') 12 | Including another URLconf 13 | 1. Import the include() function: from django.urls import include, path 14 | 2. Add a URL to urlpatterns: path('blog/', include('blog.urls')) 15 | """ 16 | from django.conf import settings 17 | from django.conf.urls.static import static 18 | from django.contrib import admin 19 | from django.urls import path 20 | 21 | urlpatterns = [ 22 | path('admin/', admin.site.urls), 23 | ] + static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT) 24 | -------------------------------------------------------------------------------- /src/dvt/wsgi.py: -------------------------------------------------------------------------------- 1 | """ 2 | WSGI config for dvt project. 3 | 4 | It exposes the WSGI callable as a module-level variable named ``application``. 5 | 6 | For more information on this file, see 7 | https://docs.djangoproject.com/en/3.0/howto/deployment/wsgi/ 8 | """ 9 | 10 | import os 11 | 12 | from django.core.wsgi import get_wsgi_application 13 | 14 | os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'dvt.settings') 15 | 16 | application = get_wsgi_application() 17 | -------------------------------------------------------------------------------- /src/manage.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | """Django's command-line utility for administrative tasks.""" 3 | import os 4 | import sys 5 | 6 | 7 | def main(): 8 | os.environ.setdefault('DJANGO_SETTINGS_MODULE', 'dvt.settings') 9 | try: 10 | from django.core.management import execute_from_command_line 11 | except ImportError as exc: 12 | raise ImportError( 13 | "Couldn't import Django. Are you sure it's installed and " 14 | "available on your PYTHONPATH environment variable? Did you " 15 | "forget to activate a virtual environment?" 16 | ) from exc 17 | execute_from_command_line(sys.argv) 18 | 19 | 20 | if __name__ == '__main__': 21 | main() 22 | -------------------------------------------------------------------------------- /src/video_transcoding/__init__.py: -------------------------------------------------------------------------------- 1 | default_app_config = 'video_transcoding.apps.VideoTranscodingConfig' 2 | -------------------------------------------------------------------------------- /src/video_transcoding/admin.py: -------------------------------------------------------------------------------- 1 | import random 2 | from typing import Any, TypeVar, Callable, Union 3 | 4 | from django.contrib import admin 5 | from django.db.models import QuerySet 6 | from django.http import HttpRequest, HttpResponse 7 | from django.utils.functional import Promise 8 | from django.utils.safestring import mark_safe 9 | from django.utils.translation import gettext_lazy as _ 10 | 11 | from video_transcoding import helpers, models, defaults, forms 12 | 13 | C = TypeVar("C", bound=Callable) 14 | 15 | 16 | def short_description(name: Union[str, Promise]) -> Callable[[C], C]: 17 | """ Sets short description for function.""" 18 | 19 | def inner(func: C) -> C: 20 | setattr(func, 'short_description', name) 21 | return func 22 | 23 | return inner 24 | 25 | 26 | # noinspection PyUnresolvedReferences 27 | class VideoAdmin(admin.ModelAdmin): 28 | list_display = ('basename', 'source', 'status_display') 29 | list_filter = ('status',) 30 | search_fields = ('source', '=basename') 31 | actions = ['transcode'] 32 | readonly_fields = ('created', 'modified', 'video_player') 33 | 34 | class Media: 35 | js = ('https://cdn.jsdelivr.net/npm/hls.js@1',) 36 | 37 | @short_description(_("Status")) 38 | def status_display(self, obj: models.Video) -> str: 39 | return obj.get_status_display() 40 | 41 | # noinspection PyUnusedLocal 42 | @short_description(_('Send transcode task')) 43 | def transcode(self, 44 | request: HttpRequest, 45 | queryset: "QuerySet[models.Video]") -> None: 46 | for video in queryset: 47 | helpers.send_transcode_task(video) 48 | 49 | @short_description(_('Video player')) 50 | def video_player(self, obj: models.Video) -> str: 51 | if obj.basename is None: 52 | return "" 53 | edge = random.choice(defaults.VIDEO_EDGES) 54 | source = obj.format_video_url(edge) 55 | return mark_safe(''' 56 | 57 | 68 | ''' % (source,)) 69 | 70 | def add_view(self, 71 | request: HttpRequest, 72 | form_url: str = '', 73 | extra_context: Any = None 74 | ) -> HttpResponse: 75 | fields, self.fields = self.fields, ('source', 'preset') 76 | try: 77 | return super().add_view(request, form_url, extra_context) 78 | finally: 79 | self.fields = fields 80 | 81 | 82 | if defaults.VIDEO_MODEL == 'video_transcoding.Video': 83 | admin.register(models.Video)(VideoAdmin) 84 | 85 | 86 | # noinspection PyUnresolvedReferences 87 | class TrackAdmin(admin.ModelAdmin): 88 | list_display = ('name', 'preset', 'created', 'modified') 89 | list_filter = ('preset',) 90 | readonly_fields = ('created', 'modified') 91 | search_fields = ('=name',) 92 | 93 | 94 | @admin.register(models.VideoTrack) 95 | class VideoTrackAdmin(TrackAdmin): 96 | form = forms.VideoTrackForm 97 | 98 | 99 | @admin.register(models.AudioTrack) 100 | class AudioTrackAdmin(TrackAdmin): 101 | form = forms.AudioTrackForm 102 | 103 | 104 | class ProfileTracksInline(admin.TabularInline): 105 | list_display = ('track', 'order_number') 106 | extra = 0 107 | autocomplete_fields = ('track',) 108 | 109 | 110 | class VideoProfileTracksInline(ProfileTracksInline): 111 | model = models.VideoProfileTracks 112 | 113 | 114 | class AudioProfileTracksInline(ProfileTracksInline): 115 | model = models.AudioProfileTracks 116 | 117 | 118 | # noinspection PyUnresolvedReferences 119 | class ProfileAdmin(admin.ModelAdmin): 120 | list_display = ('name', 'preset', 'order_number', 'created', 'modified') 121 | list_filter = ('preset',) 122 | readonly_fields = ('created', 'modified') 123 | search_fields = ('=name',) 124 | ordering = ('preset', 'order_number',) 125 | 126 | 127 | @admin.register(models.VideoProfile) 128 | class VideoProfileAdmin(ProfileAdmin): 129 | inlines = [VideoProfileTracksInline] 130 | form = forms.VideoProfileForm 131 | 132 | 133 | @admin.register(models.AudioProfile) 134 | class AudioProfileAdmin(ProfileAdmin): 135 | inlines = [AudioProfileTracksInline] 136 | form = forms.AudioProfileForm 137 | 138 | 139 | class ProfileInline(admin.TabularInline): 140 | list_display = ('name',) 141 | extra = 0 142 | readonly_fields = ('condition',) 143 | 144 | 145 | class VideoProfileInline(ProfileInline): 146 | model = models.VideoProfile 147 | 148 | 149 | class AudioProfileInline(ProfileInline): 150 | model = models.AudioProfile 151 | 152 | 153 | # noinspection PyUnresolvedReferences 154 | @admin.register(models.Preset) 155 | class PresetAdmin(admin.ModelAdmin): 156 | list_display = ('name', 'created', 'modified') 157 | readonly_fields = ('created', 'modified') 158 | search_fields = ('=name',) 159 | 160 | inlines = [VideoProfileInline, AudioProfileInline] 161 | -------------------------------------------------------------------------------- /src/video_transcoding/apps.py: -------------------------------------------------------------------------------- 1 | from django.apps import AppConfig 2 | from django.utils.translation import gettext_lazy as _ 3 | 4 | 5 | class VideoTranscodingConfig(AppConfig): 6 | name = 'video_transcoding' 7 | verbose_name = _('Video Transcoding') 8 | 9 | def ready(self) -> None: 10 | __import__('video_transcoding.signals') 11 | -------------------------------------------------------------------------------- /src/video_transcoding/celery.py: -------------------------------------------------------------------------------- 1 | import os 2 | import signal 3 | from typing import Any 4 | 5 | from celery import Celery 6 | from celery import signals 7 | from celery.utils.log import get_logger 8 | from django.conf import settings 9 | 10 | from video_transcoding import defaults 11 | 12 | app = Celery(defaults.CELERY_APP_NAME) 13 | app.config_from_object(defaults.VIDEO_TRANSCODING_CELERY_CONF) 14 | app.autodiscover_tasks(lambda: settings.INSTALLED_APPS) 15 | 16 | 17 | # noinspection PyUnusedLocal 18 | @signals.worker_init.connect 19 | def set_same_process_group(**kwargs: Any) -> None: 20 | logger = get_logger(app.__module__) 21 | os.setpgrp() 22 | logger.info("Set process group to %s for %s", 23 | os.getpgid(os.getpid()), os.getpid()) 24 | 25 | 26 | # noinspection PyUnusedLocal 27 | @signals.worker_shutting_down.connect 28 | def send_term_to_children(**kwargs: Any) -> None: 29 | logger = get_logger(app.__module__) 30 | logger.warning( 31 | "Received shutdown signal, sending SIGUSR1 to worker process group") 32 | # raises SoftTimeLimitExceeded in worker processes 33 | try: 34 | os.killpg(os.getpid(), signal.SIGUSR1) 35 | except ProcessLookupError: 36 | logger.error("failed to send SIGUSR1 to %s", os.getpid()) 37 | -------------------------------------------------------------------------------- /src/video_transcoding/defaults.py: -------------------------------------------------------------------------------- 1 | from os import getenv as e 2 | 3 | from django.conf import settings 4 | from kombu import Queue 5 | 6 | CELERY_APP_NAME = 'video_transcoding' 7 | 8 | try: 9 | VIDEO_TRANSCODING_CELERY_CONF = getattr( 10 | settings, 'VIDEO_TRANSCODING_CELERY_CONF', 11 | ) 12 | except AttributeError: 13 | video_transcoding_timeout = int(e('VIDEO_TRANSCODING_TIMEOUT', 0)) 14 | if video_transcoding_timeout: # pragma: no cover 15 | queue_arguments = { 16 | # Prevent RabbitMQ closing broker connection while running 17 | # a long transcoding task 18 | 'x-consumer-timeout': video_transcoding_timeout * 1000 19 | } 20 | else: 21 | queue_arguments = {} 22 | VIDEO_TRANSCODING_CELERY_CONF = { 23 | 'broker_url': e('VIDEO_TRANSCODING_CELERY_BROKER_URL', 24 | 'amqp://guest:guest@rabbitmq:5672/'), 25 | 'result_backend': e('VIDEO_TRANSCODING_CELERY_RESULT_BACKEND', None), 26 | 'task_default_exchange': CELERY_APP_NAME, 27 | 'task_default_exchange_type': 'topic', 28 | 'task_default_queue': CELERY_APP_NAME, 29 | 'worker_prefetch_multiplier': 1, 30 | 'worker_concurrency': e('VIDEO_TRANSCODING_CELERY_CONCURRENCY'), 31 | 'task_acks_late': True, 32 | 'task_reject_on_worker_lost': True, 33 | 'task_queues': [ 34 | Queue( 35 | CELERY_APP_NAME, 36 | routing_key=CELERY_APP_NAME, 37 | queue_arguments=queue_arguments 38 | ), 39 | ] 40 | } 41 | 42 | # delay between sending celery task and applying it 43 | VIDEO_TRANSCODING_COUNTDOWN = int(e('VIDEO_TRANSCODING_COUNTDOWN', 10)) 44 | # delay between applying celery task and locking video 45 | VIDEO_TRANSCODING_WAIT = int(e('VIDEO_TRANSCODING_WAIT', 0)) 46 | 47 | # URI for shared files 48 | VIDEO_TEMP_URI = e('VIDEO_TEMP_URI', 'file:///data/tmp/') 49 | # URI for result files 50 | VIDEO_RESULTS_URI = e('VIDEO_RESULTS_URI', 'file:///data/results/') 51 | 52 | # Video streamer public urls (comma-separated) 53 | VIDEO_EDGES = e('VIDEO_EDGES', 'http://localhost:8000/media/').split(',') 54 | 55 | # Edge video manifest url template 56 | VIDEO_URL = e('VIDEO_URL', '{edge}/results/{filename}/index.m3u8') 57 | 58 | # HTTP Request timeouts 59 | VIDEO_CONNECT_TIMEOUT = float(e('VIDEO_CONNECT_TIMEOUT', 1)) 60 | VIDEO_REQUEST_TIMEOUT = float(e('VIDEO_REQUEST_TIMEOUT', 1)) 61 | 62 | # Processing segment duration 63 | VIDEO_CHUNK_DURATION = int(e('VIDEO_CHUNK_DURATION', 60)) 64 | 65 | VIDEO_MODEL = 'video_transcoding.Video' 66 | 67 | _default_config = locals() 68 | _local_config = getattr(settings, 'VIDEO_TRANSCODING_CONFIG', {}) 69 | for k, v in _local_config.items(): 70 | if k not in _default_config: # pragma: no cover 71 | raise KeyError(k) 72 | _default_config[k] = v 73 | -------------------------------------------------------------------------------- /src/video_transcoding/forms.py: -------------------------------------------------------------------------------- 1 | from typing import List, Any, Dict 2 | 3 | from django import forms 4 | from django.utils.translation import gettext_lazy as _ 5 | 6 | from video_transcoding import models 7 | 8 | FORCE_KEY_FRAMES = "expr:if(isnan(prev_forced_t),1,gte(t,prev_forced_t+4))" 9 | 10 | 11 | class NestedJSONForm(forms.ModelForm): 12 | json_field: str 13 | nested_fields: List[str] 14 | 15 | def __init__(self, *args: Any, **kwargs: Any) -> None: 16 | super().__init__(*args, **kwargs) 17 | try: 18 | data = self.initial[self.json_field] 19 | except KeyError: 20 | return 21 | for k in self.nested_fields: 22 | try: 23 | self.fields[f'_{k}'].initial = data[k] 24 | except KeyError: 25 | pass 26 | 27 | def clean(self) -> Dict[str, Any]: 28 | cd = self.cleaned_data 29 | missing = set() 30 | for k in self.nested_fields: 31 | if f'_{k}' not in cd: 32 | missing.add(f'_{k}') 33 | if not missing: 34 | cd[self.json_field] = {k: cd[f'_{k}'] for k in self.nested_fields} 35 | return cd 36 | 37 | 38 | class VideoProfileForm(NestedJSONForm): 39 | json_field = 'condition' 40 | nested_fields = ['min_bitrate', 'min_width', 'min_height', 'min_frame_rate', 41 | 'min_dar', 'max_dar'] 42 | 43 | class Meta: 44 | model = models.VideoProfile 45 | fields = '__all__' 46 | 47 | condition = forms.JSONField(disabled=True, 48 | required=False, 49 | widget=forms.HiddenInput(), 50 | initial={}) 51 | _min_bitrate = forms.IntegerField(label=_('Min bitrate')) 52 | _min_width = forms.IntegerField(label=_('Min width')) 53 | _min_height = forms.IntegerField(label=_('Min height')) 54 | _min_frame_rate = forms.FloatField(label=_('Min frame rate')) 55 | _min_dar = forms.FloatField(label=_('Min aspect')) 56 | _max_dar = forms.FloatField(label=_('Max aspect')) 57 | 58 | 59 | class AudioProfileForm(NestedJSONForm): 60 | json_field = 'condition' 61 | nested_fields = ['min_bitrate', 'min_sample_rate'] 62 | 63 | class Meta: 64 | model = models.AudioProfile 65 | fields = '__all__' 66 | 67 | condition = forms.JSONField(disabled=True, 68 | required=False, 69 | widget=forms.HiddenInput(), 70 | initial={}) 71 | _min_bitrate = forms.IntegerField(label=_('Min bitrate')) 72 | _min_sample_rate = forms.IntegerField(label=_('Min sample rate')) 73 | 74 | 75 | class VideoTrackForm(NestedJSONForm): 76 | json_field = 'params' 77 | nested_fields = [ 78 | 'codec', 79 | 'constant_rate_factor', 80 | 'preset', 81 | 'max_rate', 82 | 'buf_size', 83 | 'profile', 84 | 'pix_fmt', 85 | 'width', 86 | 'height', 87 | 'frame_rate', 88 | 'gop_size', 89 | 'force_key_frames', 90 | ] 91 | 92 | class Meta: 93 | model = models.VideoTrack 94 | fields = '__all__' 95 | 96 | params = forms.JSONField(disabled=True, 97 | required=False, 98 | widget=forms.HiddenInput(), 99 | initial={}) 100 | _codec = forms.CharField(label=_('Codec'), initial='libx264') 101 | _constant_rate_factor = forms.IntegerField( 102 | label=_('CRF'), 103 | help_text=_('Constant rate factor or CRF value for ffmpeg'), 104 | initial=23) 105 | _preset = forms.CharField(label=_('Preset'), initial='slow') 106 | _max_rate = forms.IntegerField(label=_('Max rate')) 107 | _buf_size = forms.IntegerField(label=_('Buf size')) 108 | _profile = forms.CharField(label=_('Profile'), initial='main') 109 | _pix_fmt = forms.CharField(label=_('Pix fmt'), initial='yuv420p') 110 | _width = forms.IntegerField(label=_('Width')) 111 | _height = forms.IntegerField(label=_('Height')) 112 | _frame_rate = forms.FloatField(label=_('Frame rate'), initial=30.0) 113 | _gop_size = forms.IntegerField( 114 | label=_('GOP size'), 115 | help_text=_('Group of pictures size'), 116 | initial=30) 117 | _force_key_frames = forms.CharField( 118 | label=_('Force key frames'), 119 | help_text=_('ffmpeg -force_key_frames option value'), 120 | initial=FORCE_KEY_FRAMES) 121 | 122 | 123 | class AudioTrackForm(NestedJSONForm): 124 | json_field = 'params' 125 | nested_fields = [ 126 | 'codec', 127 | 'bitrate', 128 | 'channels', 129 | 'sample_rate', 130 | ] 131 | 132 | class Meta: 133 | model = models.AudioTrack 134 | fields = '__all__' 135 | 136 | params = forms.JSONField(disabled=True, 137 | required=False, 138 | widget=forms.HiddenInput(), 139 | initial={}) 140 | _codec = forms.CharField(label=_('Codec'), initial='libfdk_aac') 141 | _bitrate = forms.IntegerField(label=_('Bitrate')) 142 | _channels = forms.IntegerField(label=_('Channels'), initial=2) 143 | _sample_rate = forms.IntegerField(label=_('Sample rate'), initial=48000) 144 | -------------------------------------------------------------------------------- /src/video_transcoding/helpers.py: -------------------------------------------------------------------------------- 1 | from celery.result import AsyncResult 2 | 3 | from video_transcoding import models, defaults 4 | from video_transcoding import tasks 5 | 6 | 7 | def send_transcode_task(video: models.Video) -> AsyncResult: 8 | """ 9 | Send a video transcoding task. 10 | 11 | If task is successfully sent to broker, Video status is changed to QUEUED 12 | and Celery task identifier is saved. 13 | 14 | :param video: video object 15 | :type video: video.models.Video 16 | :returns: Celery task result 17 | :rtype: celery.result.AsyncResult 18 | """ 19 | result = tasks.transcode_video.apply_async( 20 | args=(video.pk,), 21 | countdown=defaults.VIDEO_TRANSCODING_COUNTDOWN) 22 | video.change_status(video.QUEUED, task_id=result.task_id) 23 | return result 24 | -------------------------------------------------------------------------------- /src/video_transcoding/locale/ru/LC_MESSAGES/django.po: -------------------------------------------------------------------------------- 1 | # SOME DESCRIPTIVE TITLE. 2 | # Copyright (C) YEAR THE PACKAGE'S COPYRIGHT HOLDER 3 | # This file is distributed under the same license as the PACKAGE package. 4 | # FIRST AUTHOR , YEAR. 5 | # 6 | #, fuzzy 7 | msgid "" 8 | msgstr "" 9 | "Project-Id-Version: PACKAGE VERSION\n" 10 | "Report-Msgid-Bugs-To: \n" 11 | "POT-Creation-Date: 2024-12-27 14:16+0000\n" 12 | "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" 13 | "Last-Translator: FULL NAME \n" 14 | "Language-Team: LANGUAGE \n" 15 | "Language: \n" 16 | "MIME-Version: 1.0\n" 17 | "Content-Type: text/plain; charset=UTF-8\n" 18 | "Content-Transfer-Encoding: 8bit\n" 19 | "Plural-Forms: nplurals=4; plural=(n%10==1 && n%100!=11 ? 0 : n%10>=2 && " 20 | "n%10<=4 && (n%100<12 || n%100>14) ? 1 : n%10==0 || (n%10>=5 && n%10<=9) || " 21 | "(n%100>=11 && n%100<=14)? 2 : 3);\n" 22 | 23 | #: video_transcoding/admin.py:37 video_transcoding/models.py:193 24 | msgid "Status" 25 | msgstr "Статус" 26 | 27 | #: video_transcoding/admin.py:42 28 | msgid "Send transcode task" 29 | msgstr "Отправить на перекодировку" 30 | 31 | #: video_transcoding/admin.py:49 32 | msgid "Video player" 33 | msgstr "Видеоплеер" 34 | 35 | #: video_transcoding/apps.py:7 36 | msgid "Video Transcoding" 37 | msgstr "Перекодировка видео" 38 | 39 | #: video_transcoding/forms.py:51 video_transcoding/forms.py:71 40 | msgid "Min bitrate" 41 | msgstr "Минимальный битрейт" 42 | 43 | #: video_transcoding/forms.py:52 44 | msgid "Min width" 45 | msgstr "Минимальная ширина" 46 | 47 | #: video_transcoding/forms.py:53 48 | msgid "Min height" 49 | msgstr "Минимальная высота" 50 | 51 | #: video_transcoding/forms.py:54 52 | msgid "Min frame rate" 53 | msgstr "Минимальная частота кадров" 54 | 55 | #: video_transcoding/forms.py:55 56 | msgid "Min aspect" 57 | msgstr "Минимальное соотношение сторон" 58 | 59 | #: video_transcoding/forms.py:56 60 | msgid "Max aspect" 61 | msgstr "Максимальное соотношение сторон" 62 | 63 | #: video_transcoding/forms.py:72 64 | msgid "Min sample rate" 65 | msgstr "Минимальная частота дискретизации звука" 66 | 67 | #: video_transcoding/forms.py:100 video_transcoding/forms.py:140 68 | msgid "Codec" 69 | msgstr "Кодек" 70 | 71 | #: video_transcoding/forms.py:102 72 | msgid "CRF" 73 | msgstr "CRF" 74 | 75 | #: video_transcoding/forms.py:103 76 | msgid "Constant rate factor or CRF value for ffmpeg" 77 | msgstr "Коэффициент постоянного качества или значение CRF для ffmpeg" 78 | 79 | #: video_transcoding/forms.py:105 video_transcoding/models.py:25 80 | msgid "Preset" 81 | msgstr "Пресет" 82 | 83 | #: video_transcoding/forms.py:106 84 | msgid "Max rate" 85 | msgstr "Максимальный битрейт" 86 | 87 | #: video_transcoding/forms.py:107 88 | msgid "Buf size" 89 | msgstr "Размер буфера" 90 | 91 | #: video_transcoding/forms.py:108 92 | msgid "Profile" 93 | msgstr "Профиль" 94 | 95 | #: video_transcoding/forms.py:109 96 | msgid "Pix fmt" 97 | msgstr "Формат пикселей" 98 | 99 | #: video_transcoding/forms.py:110 100 | msgid "Width" 101 | msgstr "Ширина" 102 | 103 | #: video_transcoding/forms.py:111 104 | msgid "Height" 105 | msgstr "Высота" 106 | 107 | #: video_transcoding/forms.py:112 108 | msgid "Frame rate" 109 | msgstr "Частота кадров" 110 | 111 | #: video_transcoding/forms.py:114 112 | msgid "GOP size" 113 | msgstr "Размер GOP" 114 | 115 | #: video_transcoding/forms.py:115 116 | msgid "Group of pictures size" 117 | msgstr "Размер группы кадров" 118 | 119 | #: video_transcoding/forms.py:118 120 | msgid "Force key frames" 121 | msgstr "Вставка ключевых кадров" 122 | 123 | #: video_transcoding/forms.py:119 124 | msgid "ffmpeg -force_key_frames option value" 125 | msgstr "значение параметра ffmpeg -force_key_frames" 126 | 127 | #: video_transcoding/forms.py:141 128 | msgid "Bitrate" 129 | msgstr "Битрейт" 130 | 131 | #: video_transcoding/forms.py:142 132 | msgid "Channels" 133 | msgstr "Количество каналов" 134 | 135 | #: video_transcoding/forms.py:143 136 | msgid "Sample rate" 137 | msgstr "Частота дискретизации звука" 138 | 139 | #: video_transcoding/models.py:21 video_transcoding/models.py:38 140 | #: video_transcoding/models.py:60 video_transcoding/models.py:82 141 | #: video_transcoding/models.py:133 142 | msgid "name" 143 | msgstr "Идентификатор результата" 144 | 145 | #: video_transcoding/models.py:26 146 | msgid "Presets" 147 | msgstr "Пресеты" 148 | 149 | #: video_transcoding/models.py:39 video_transcoding/models.py:61 150 | msgid "params" 151 | msgstr "Параметры" 152 | 153 | #: video_transcoding/models.py:42 video_transcoding/models.py:64 154 | #: video_transcoding/models.py:87 video_transcoding/models.py:138 155 | #: video_transcoding/models.py:204 156 | msgid "preset" 157 | msgstr "пресет" 158 | 159 | #: video_transcoding/models.py:47 160 | msgid "Video track" 161 | msgstr "Видеодорожка" 162 | 163 | #: video_transcoding/models.py:48 video_transcoding/models.py:93 164 | msgid "Video tracks" 165 | msgstr "Видеодорожки" 166 | 167 | #: video_transcoding/models.py:69 168 | msgid "Audio track" 169 | msgstr "Аудиодорожка" 170 | 171 | #: video_transcoding/models.py:70 video_transcoding/models.py:143 172 | msgid "Audio tracks" 173 | msgstr "Аудиодорожки" 174 | 175 | #: video_transcoding/models.py:83 video_transcoding/models.py:114 176 | #: video_transcoding/models.py:134 video_transcoding/models.py:164 177 | msgid "order number" 178 | msgstr "порядковый номер" 179 | 180 | #: video_transcoding/models.py:84 video_transcoding/models.py:135 181 | msgid "condition" 182 | msgstr "условие" 183 | 184 | #: video_transcoding/models.py:88 185 | msgid "segment duration" 186 | msgstr "длительность сегмента" 187 | 188 | #: video_transcoding/models.py:100 189 | msgid "Video profile" 190 | msgstr "Видео профиль" 191 | 192 | #: video_transcoding/models.py:101 193 | msgid "Video profiles" 194 | msgstr "Видео профили" 195 | 196 | #: video_transcoding/models.py:112 video_transcoding/models.py:162 197 | msgid "profile" 198 | msgstr "профиль" 199 | 200 | #: video_transcoding/models.py:113 video_transcoding/models.py:163 201 | msgid "track" 202 | msgstr "дорожка" 203 | 204 | #: video_transcoding/models.py:120 205 | msgid "Video profile track" 206 | msgstr "Видеодорожка профиля" 207 | 208 | #: video_transcoding/models.py:121 209 | msgid "Video profile tracks" 210 | msgstr "Видеодорожки профилей" 211 | 212 | #: video_transcoding/models.py:150 213 | msgid "Audio profile" 214 | msgstr "Аудио профиль" 215 | 216 | #: video_transcoding/models.py:151 217 | msgid "Audio profiles" 218 | msgstr "Аудио профили" 219 | 220 | #: video_transcoding/models.py:170 221 | msgid "Audio profile track" 222 | msgstr "Аудиодорожка профиля" 223 | 224 | #: video_transcoding/models.py:171 225 | msgid "Audio profile tracks" 226 | msgstr "Аудиодорожки профилей" 227 | 228 | #: video_transcoding/models.py:185 229 | msgid "new" 230 | msgstr "создано" 231 | 232 | #: video_transcoding/models.py:186 233 | msgid "queued" 234 | msgstr "в очереди" 235 | 236 | #: video_transcoding/models.py:187 237 | msgid "process" 238 | msgstr "обрабатывается" 239 | 240 | #: video_transcoding/models.py:188 241 | msgid "done" 242 | msgstr "завершено" 243 | 244 | #: video_transcoding/models.py:189 245 | msgid "error" 246 | msgstr "ошибка" 247 | 248 | #: video_transcoding/models.py:194 249 | msgid "Error" 250 | msgstr "Ошибка" 251 | 252 | #: video_transcoding/models.py:196 253 | msgid "Task ID" 254 | msgstr "Id задачи" 255 | 256 | #: video_transcoding/models.py:198 257 | msgid "Source" 258 | msgstr "Источник" 259 | 260 | #: video_transcoding/models.py:201 261 | msgid "Basename" 262 | msgstr "Идентификатор результата" 263 | 264 | #: video_transcoding/models.py:207 265 | msgid "metadata" 266 | msgstr "метаданные" 267 | 268 | #: video_transcoding/models.py:208 269 | msgid "duration" 270 | msgstr "длительность" 271 | 272 | #: video_transcoding/models.py:212 video_transcoding/models.py:213 273 | msgid "Video" 274 | msgstr "Видео" 275 | 276 | #: video_transcoding/utils.py:21 277 | msgid "created" 278 | msgstr "Дата создания" 279 | 280 | #: video_transcoding/utils.py:22 281 | msgid "modified" 282 | msgstr "Дата изменения" 283 | -------------------------------------------------------------------------------- /src/video_transcoding/migrations/0001_initial.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 3.0.3 on 2020-02-10 05:42 2 | 3 | import django.core.validators 4 | from django.db import migrations, models 5 | import django.utils.timezone 6 | import model_utils.fields 7 | 8 | from video_transcoding import defaults 9 | 10 | 11 | class Migration(migrations.Migration): 12 | 13 | initial = True 14 | 15 | dependencies = [ 16 | ] 17 | operations = [] 18 | if defaults.VIDEO_MODEL == 'video_transcoding.Video': 19 | operations.extend([ 20 | migrations.CreateModel( 21 | name='Video', 22 | fields=[ 23 | ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), 24 | ('created', model_utils.fields.AutoCreatedField(default=django.utils.timezone.now, editable=False, verbose_name='created')), 25 | ('modified', model_utils.fields.AutoLastModifiedField(default=django.utils.timezone.now, editable=False, verbose_name='modified')), 26 | ('status', models.SmallIntegerField(choices=[(0, 'created'), (1, 'queued'), (2, 'process'), (3, 'done'), (4, 'error')], default=0)), 27 | ('error', models.TextField(blank=True, null=True)), 28 | ('task_id', models.UUIDField(blank=True, null=True)), 29 | ('source', models.URLField(validators=[django.core.validators.URLValidator(schemes=('ftp', 'http'))])), 30 | ('basename', models.UUIDField(blank=True, null=True)), 31 | ], 32 | options={ 33 | 'verbose_name': 'Video', 34 | 'verbose_name_plural': 'Video', 35 | }, 36 | ), 37 | ]) 38 | -------------------------------------------------------------------------------- /src/video_transcoding/migrations/0002_auto_20200226_0919.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 3.0.3 on 2020-02-26 09:19 2 | 3 | import django.core.validators 4 | from django.db import migrations, models 5 | 6 | from video_transcoding import defaults 7 | 8 | 9 | class Migration(migrations.Migration): 10 | dependencies = [ 11 | ('video_transcoding', '0001_initial'), 12 | ] 13 | 14 | operations = [] 15 | if defaults.VIDEO_MODEL == 'video_transcoding.Video': 16 | operations.extend([ 17 | migrations.AlterField( 18 | model_name='video', 19 | name='source', 20 | field=models.URLField(validators=[ 21 | django.core.validators.URLValidator( 22 | schemes=('http', 'https'))]), 23 | ), 24 | ]) 25 | -------------------------------------------------------------------------------- /src/video_transcoding/migrations/0003_auto_20200525_1130.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 3.0.6 on 2020-05-25 11:30 2 | 3 | import django.core.validators 4 | from django.db import migrations, models 5 | 6 | from video_transcoding import defaults 7 | 8 | 9 | class Migration(migrations.Migration): 10 | dependencies = [ 11 | ('video_transcoding', '0002_auto_20200226_0919'), 12 | ] 13 | operations = [] 14 | if defaults.VIDEO_MODEL == 'video_transcoding.Video': 15 | operations.extend([ 16 | migrations.AlterField( 17 | model_name='video', 18 | name='basename', 19 | field=models.UUIDField(blank=True, null=True, 20 | verbose_name='Basename'), 21 | ), 22 | migrations.AlterField( 23 | model_name='video', 24 | name='error', 25 | field=models.TextField(blank=True, null=True, 26 | verbose_name='Error'), 27 | ), 28 | migrations.AlterField( 29 | model_name='video', 30 | name='source', 31 | field=models.URLField(validators=[ 32 | django.core.validators.URLValidator( 33 | schemes=('ftp', 'http', 'https'))], 34 | verbose_name='Source'), 35 | ), 36 | migrations.AlterField( 37 | model_name='video', 38 | name='status', 39 | field=models.SmallIntegerField( 40 | choices=[(0, 'created'), (1, 'queued'), (2, 'process'), 41 | (3, 'done'), (4, 'error')], default=0, 42 | verbose_name='Status'), 43 | ), 44 | migrations.AlterField( 45 | model_name='video', 46 | name='task_id', 47 | field=models.UUIDField(blank=True, null=True, 48 | verbose_name='Task ID'), 49 | ), 50 | ]) 51 | -------------------------------------------------------------------------------- /src/video_transcoding/migrations/0004_audioprofile_audiotrack_preset_audioprofiletracks_and_more.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 5.0.9 on 2024-10-14 11:15 2 | 3 | import django.db.models.deletion 4 | import django.utils.timezone 5 | import model_utils.fields 6 | from django.db import migrations, models 7 | 8 | from video_transcoding import defaults 9 | 10 | 11 | class Migration(migrations.Migration): 12 | 13 | dependencies = [ 14 | ('video_transcoding', '0003_auto_20200525_1130'), 15 | ] 16 | 17 | operations = [ 18 | migrations.CreateModel( 19 | name='AudioProfile', 20 | fields=[ 21 | ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), 22 | ('created', model_utils.fields.AutoCreatedField(default=django.utils.timezone.now, editable=False, verbose_name='created')), 23 | ('modified', model_utils.fields.AutoLastModifiedField(default=django.utils.timezone.now, editable=False, verbose_name='modified')), 24 | ('name', models.SlugField(max_length=255, verbose_name='name')), 25 | ('order_number', models.SmallIntegerField(default=0, verbose_name='order number')), 26 | ('condition', models.JSONField(default=dict, verbose_name='condition')), 27 | ], 28 | options={ 29 | 'verbose_name': 'Audio profile', 30 | 'verbose_name_plural': 'Audio profiles', 31 | 'ordering': ['order_number'], 32 | }, 33 | ), 34 | migrations.CreateModel( 35 | name='AudioTrack', 36 | fields=[ 37 | ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), 38 | ('created', model_utils.fields.AutoCreatedField(default=django.utils.timezone.now, editable=False, verbose_name='created')), 39 | ('modified', model_utils.fields.AutoLastModifiedField(default=django.utils.timezone.now, editable=False, verbose_name='modified')), 40 | ('name', models.SlugField(max_length=255, verbose_name='name')), 41 | ('params', models.JSONField(default=dict, verbose_name='params')), 42 | ], 43 | options={ 44 | 'verbose_name': 'Audio track', 45 | 'verbose_name_plural': 'Audio tracks', 46 | }, 47 | ), 48 | migrations.CreateModel( 49 | name='Preset', 50 | fields=[ 51 | ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), 52 | ('created', model_utils.fields.AutoCreatedField(default=django.utils.timezone.now, editable=False, verbose_name='created')), 53 | ('modified', model_utils.fields.AutoLastModifiedField(default=django.utils.timezone.now, editable=False, verbose_name='modified')), 54 | ('name', models.SlugField(max_length=255, unique=True, verbose_name='name')), 55 | ], 56 | options={ 57 | 'verbose_name': 'Preset', 58 | 'verbose_name_plural': 'Presets', 59 | }, 60 | ), 61 | migrations.CreateModel( 62 | name='AudioProfileTracks', 63 | fields=[ 64 | ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), 65 | ('order_number', models.SmallIntegerField(default=0, verbose_name='order number')), 66 | ('profile', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='video_transcoding.audioprofile', verbose_name='profile')), 67 | ('track', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='video_transcoding.audiotrack', verbose_name='track')), 68 | ], 69 | options={ 70 | 'verbose_name': 'Audio profile track', 71 | 'verbose_name_plural': 'Audio profile tracks', 72 | 'ordering': ['order_number'], 73 | 'unique_together': {('profile', 'track')}, 74 | }, 75 | ), 76 | migrations.AddField( 77 | model_name='audioprofile', 78 | name='audio', 79 | field=models.ManyToManyField(through='video_transcoding.AudioProfileTracks', to='video_transcoding.audiotrack', verbose_name='Audio tracks'), 80 | ), 81 | migrations.AddField( 82 | model_name='audiotrack', 83 | name='preset', 84 | field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='audio_tracks', to='video_transcoding.preset', verbose_name='preset'), 85 | ), 86 | migrations.AddField( 87 | model_name='audioprofile', 88 | name='preset', 89 | field=models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='audio_profiles', to='video_transcoding.preset', verbose_name='preset'), 90 | ), 91 | migrations.CreateModel( 92 | name='VideoProfile', 93 | fields=[ 94 | ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), 95 | ('created', model_utils.fields.AutoCreatedField(default=django.utils.timezone.now, editable=False, verbose_name='created')), 96 | ('modified', model_utils.fields.AutoLastModifiedField(default=django.utils.timezone.now, editable=False, verbose_name='modified')), 97 | ('name', models.SlugField(max_length=255, verbose_name='name')), 98 | ('order_number', models.SmallIntegerField(default=0, verbose_name='order number')), 99 | ('condition', models.JSONField(default=dict, verbose_name='condition')), 100 | ('preset', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='video_profiles', to='video_transcoding.preset', verbose_name='preset')), 101 | ], 102 | options={ 103 | 'verbose_name': 'Video profile', 104 | 'verbose_name_plural': 'Video profiles', 105 | 'ordering': ['order_number'], 106 | }, 107 | ), 108 | migrations.CreateModel( 109 | name='VideoTrack', 110 | fields=[ 111 | ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), 112 | ('created', model_utils.fields.AutoCreatedField(default=django.utils.timezone.now, editable=False, verbose_name='created')), 113 | ('modified', model_utils.fields.AutoLastModifiedField(default=django.utils.timezone.now, editable=False, verbose_name='modified')), 114 | ('name', models.SlugField(max_length=255, verbose_name='name')), 115 | ('params', models.JSONField(default=dict, verbose_name='params')), 116 | ('preset', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, related_name='video_tracks', to='video_transcoding.preset', verbose_name='preset')), 117 | ], 118 | options={ 119 | 'verbose_name': 'Video track', 120 | 'verbose_name_plural': 'Video tracks', 121 | 'unique_together': {('name', 'preset')}, 122 | }, 123 | ), 124 | migrations.CreateModel( 125 | name='VideoProfileTracks', 126 | fields=[ 127 | ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')), 128 | ('order_number', models.SmallIntegerField(default=0, verbose_name='order number')), 129 | ('profile', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='video_transcoding.videoprofile', verbose_name='profile')), 130 | ('track', models.ForeignKey(on_delete=django.db.models.deletion.CASCADE, to='video_transcoding.videotrack', verbose_name='track')), 131 | ], 132 | options={ 133 | 'verbose_name': 'Video profile track', 134 | 'verbose_name_plural': 'Video profile tracks', 135 | 'ordering': ['order_number'], 136 | 'unique_together': {('profile', 'track')}, 137 | }, 138 | ), 139 | migrations.AddField( 140 | model_name='videoprofile', 141 | name='video', 142 | field=models.ManyToManyField(through='video_transcoding.VideoProfileTracks', to='video_transcoding.videotrack', verbose_name='Video tracks'), 143 | ), 144 | migrations.AlterUniqueTogether( 145 | name='audiotrack', 146 | unique_together={('name', 'preset')}, 147 | ), 148 | migrations.AlterUniqueTogether( 149 | name='audioprofile', 150 | unique_together={('name', 'preset')}, 151 | ), 152 | migrations.AlterUniqueTogether( 153 | name='videoprofile', 154 | unique_together={('name', 'preset')}, 155 | ), 156 | ] 157 | 158 | if defaults.VIDEO_MODEL == 'video_transcoding.Video': 159 | operations.extend([ 160 | migrations.AddField( 161 | model_name='video', 162 | name='preset', 163 | field=models.ForeignKey(blank=True, null=True, 164 | on_delete=django.db.models.deletion.SET_NULL, 165 | to='video_transcoding.preset', 166 | verbose_name='preset'), 167 | 168 | ), 169 | ]) 170 | -------------------------------------------------------------------------------- /src/video_transcoding/migrations/0005_video_metadata.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 5.0.6 on 2024-06-26 07:24 2 | 3 | from django.db import migrations, models 4 | 5 | from video_transcoding import defaults 6 | 7 | 8 | class Migration(migrations.Migration): 9 | dependencies = [ 10 | ('video_transcoding', 11 | '0004_audioprofile_audiotrack_preset_audioprofiletracks_and_more'), 12 | ] 13 | 14 | operations = [] 15 | if defaults.VIDEO_MODEL == 'video_transcoding.Video': 16 | operations.extend([ 17 | migrations.AddField( 18 | model_name='video', 19 | name='metadata', 20 | field=models.JSONField(blank=True, null=True, verbose_name='metadata'), 21 | ), 22 | ]) 23 | -------------------------------------------------------------------------------- /src/video_transcoding/migrations/0006_video_duration.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 5.0.7 on 2024-07-15 08:37 2 | 3 | from django.db import migrations, models 4 | 5 | from video_transcoding import defaults 6 | 7 | 8 | class Migration(migrations.Migration): 9 | dependencies = [ 10 | ('video_transcoding', '0005_video_metadata'), 11 | ] 12 | 13 | operations = [] 14 | if defaults.VIDEO_MODEL == 'video_transcoding.Video': 15 | operations.extend([ 16 | migrations.AddField( 17 | model_name='video', 18 | name='duration', 19 | field=models.DurationField(blank=True, null=True, verbose_name='duration'), 20 | ), 21 | ]) 22 | -------------------------------------------------------------------------------- /src/video_transcoding/migrations/0007_videoprofile_segment_duration.py: -------------------------------------------------------------------------------- 1 | # Generated by Django 5.0.7 on 2024-07-24 08:45 2 | 3 | import datetime 4 | from django.db import migrations, models 5 | 6 | 7 | class Migration(migrations.Migration): 8 | 9 | dependencies = [ 10 | ('video_transcoding', '0006_video_duration'), 11 | ] 12 | 13 | operations = [ 14 | migrations.AddField( 15 | model_name='videoprofile', 16 | name='segment_duration', 17 | field=models.DurationField(default=datetime.timedelta(seconds=4), verbose_name='segment duration'), 18 | preserve_default=False, 19 | ), 20 | ] 21 | -------------------------------------------------------------------------------- /src/video_transcoding/migrations/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/just-work/django-video-transcoding/aa4a945a319bb4bd540c2acf4fcc3cbd6e3fad26/src/video_transcoding/migrations/__init__.py -------------------------------------------------------------------------------- /src/video_transcoding/models.py: -------------------------------------------------------------------------------- 1 | import os 2 | from typing import Any, cast, Type 3 | from uuid import UUID 4 | 5 | from django.apps import apps 6 | from django.core.validators import URLValidator 7 | from django.db import models 8 | from django.db.models.fields import related_descriptors 9 | from django.utils.translation import gettext_lazy as _ 10 | from model_utils.models import TimeStampedModel 11 | 12 | from video_transcoding import defaults 13 | 14 | 15 | class PresetBase(TimeStampedModel): 16 | """ Transcoding preset.""" 17 | video_tracks: related_descriptors.ReverseManyToOneDescriptor 18 | audio_tracks: related_descriptors.ReverseManyToOneDescriptor 19 | video_profiles: related_descriptors.ReverseManyToOneDescriptor 20 | audio_profiles: related_descriptors.ReverseManyToOneDescriptor 21 | name = models.SlugField(verbose_name=_('name'), max_length=255, unique=True) 22 | 23 | class Meta: 24 | abstract = True 25 | verbose_name = _('Preset') 26 | verbose_name_plural = _('Presets') 27 | 28 | def __str__(self) -> str: 29 | return self.name 30 | 31 | 32 | class Preset(PresetBase): 33 | pass 34 | 35 | 36 | class VideoTrackBase(TimeStampedModel): 37 | """ Video stream transcoding parameters.""" 38 | name = models.SlugField(verbose_name=_('name'), max_length=255, unique=False) 39 | params = models.JSONField(verbose_name=_('params'), default=dict) 40 | preset = models.ForeignKey(Preset, models.CASCADE, 41 | related_name='video_tracks', 42 | verbose_name=_('preset')) 43 | 44 | class Meta: 45 | abstract = True 46 | unique_together = (('name', 'preset'),) 47 | verbose_name = _('Video track') 48 | verbose_name_plural = _('Video tracks') 49 | 50 | def __str__(self) -> str: 51 | return f'{self.name}@{self.preset}' 52 | 53 | 54 | class VideoTrack(VideoTrackBase): 55 | pass 56 | 57 | 58 | class AudioTrackBase(TimeStampedModel): 59 | """ Audio stream transcoding parameters.""" 60 | name = models.SlugField(verbose_name=_('name'), max_length=255, unique=False) 61 | params = models.JSONField(verbose_name=_('params'), default=dict) 62 | preset = models.ForeignKey(Preset, models.CASCADE, 63 | related_name='audio_tracks', 64 | verbose_name=_('preset')) 65 | 66 | class Meta: 67 | abstract = True 68 | unique_together = (('name', 'preset'),) 69 | verbose_name = _('Audio track') 70 | verbose_name_plural = _('Audio tracks') 71 | 72 | def __str__(self) -> str: 73 | return f'{self.name}@{self.preset}' 74 | 75 | 76 | class AudioTrack(AudioTrackBase): 77 | pass 78 | 79 | 80 | class VideoProfileBase(TimeStampedModel): 81 | """ Video transcoding profile.""" 82 | name = models.SlugField(verbose_name=_('name'), max_length=255, unique=False) 83 | order_number = models.SmallIntegerField(verbose_name=_('order number'), default=0) 84 | condition = models.JSONField(verbose_name=_('condition'), default=dict) 85 | preset = models.ForeignKey(Preset, models.CASCADE, 86 | related_name='video_profiles', 87 | verbose_name=_('preset')) 88 | segment_duration = models.DurationField(verbose_name=_('segment duration')) 89 | 90 | video = cast( 91 | related_descriptors.ManyToManyDescriptor, 92 | models.ManyToManyField(VideoTrack, 93 | verbose_name=_('Video tracks'), 94 | through='VideoProfileTracks')) 95 | 96 | class Meta: 97 | abstract = True 98 | unique_together = (('name', 'preset'),) 99 | ordering = ['order_number'] 100 | verbose_name = _('Video profile') 101 | verbose_name_plural = _('Video profiles') 102 | 103 | def __str__(self) -> str: 104 | return f'{self.name}@{self.preset}' 105 | 106 | 107 | class VideoProfile(VideoProfileBase): 108 | pass 109 | 110 | 111 | class VideoProfileTracksBase(models.Model): 112 | profile = models.ForeignKey(VideoProfile, models.CASCADE, verbose_name=_('profile')) 113 | track = models.ForeignKey(VideoTrack, models.CASCADE, verbose_name=_('track')) 114 | order_number = models.SmallIntegerField(default=0, verbose_name=_('order number')) 115 | 116 | class Meta: 117 | abstract = True 118 | unique_together = (('profile', 'track'),) 119 | ordering = ['order_number'] 120 | verbose_name = _('Video profile track') 121 | verbose_name_plural = _('Video profile tracks') 122 | 123 | def __str__(self) -> str: 124 | return f'{self.track.name}/{self.profile.name}@{self.profile.preset}' 125 | 126 | 127 | class VideoProfileTracks(VideoProfileTracksBase): 128 | pass 129 | 130 | 131 | class AudioProfileBase(TimeStampedModel): 132 | """ Audio transcoding profile.""" 133 | name = models.SlugField(verbose_name=_('name'), max_length=255, unique=False) 134 | order_number = models.SmallIntegerField(verbose_name=_('order number'), default=0) 135 | condition = models.JSONField(verbose_name=_('condition'), default=dict) 136 | preset = models.ForeignKey(Preset, models.CASCADE, 137 | related_name='audio_profiles', 138 | verbose_name=_('preset')) 139 | 140 | audio = cast( 141 | related_descriptors.ManyToManyDescriptor, 142 | models.ManyToManyField(AudioTrack, 143 | verbose_name=_('Audio tracks'), 144 | through='AudioProfileTracks')) 145 | 146 | class Meta: 147 | abstract = True 148 | unique_together = (('name', 'preset'),) 149 | ordering = ['order_number'] 150 | verbose_name = _('Audio profile') 151 | verbose_name_plural = _('Audio profiles') 152 | 153 | def __str__(self) -> str: 154 | return f'{self.name}@{self.preset}' 155 | 156 | 157 | class AudioProfile(AudioProfileBase): 158 | pass 159 | 160 | 161 | class AudioProfileTracksBase(models.Model): 162 | profile = models.ForeignKey(AudioProfile, models.CASCADE, verbose_name=_('profile')) 163 | track = models.ForeignKey(AudioTrack, models.CASCADE, verbose_name=_('track')) 164 | order_number = models.SmallIntegerField(default=0, verbose_name=_('order number')) 165 | 166 | class Meta: 167 | abstract = True 168 | unique_together = (('profile', 'track'),) 169 | ordering = ['order_number'] 170 | verbose_name = _('Audio profile track') 171 | verbose_name_plural = _('Audio profile tracks') 172 | 173 | def __str__(self) -> str: 174 | return f'{self.track.name}/{self.profile.name}@{self.profile.preset}' 175 | 176 | 177 | class AudioProfileTracks(AudioProfileTracksBase): 178 | pass 179 | 180 | 181 | class Video(TimeStampedModel): 182 | """ Video model.""" 183 | CREATED, QUEUED, PROCESS, DONE, ERROR = range(5) 184 | STATUS_CHOICES = ( 185 | (CREATED, _('new')), # And editor created video in db 186 | (QUEUED, _('queued')), # Transcoding task is sent to broker 187 | (PROCESS, _('process')), # Celery worker started video processing 188 | (DONE, _('done')), # Video processing is done successfully 189 | (ERROR, _('error')), # Video processing error 190 | ) 191 | 192 | status = models.SmallIntegerField(default=CREATED, choices=STATUS_CHOICES, 193 | verbose_name=_('Status')) 194 | error = models.TextField(blank=True, null=True, verbose_name=_('Error')) 195 | task_id = models.UUIDField(blank=True, null=True, 196 | verbose_name=_('Task ID')) 197 | source = models.URLField( 198 | verbose_name=_('Source'), 199 | validators=[URLValidator(schemes=('ftp', 'http', 'https'))]) 200 | basename = models.UUIDField(blank=True, null=True, verbose_name=_('Basename')) 201 | preset = models.ForeignKey(Preset, 202 | models.SET_NULL, 203 | verbose_name=_('preset'), 204 | blank=True, 205 | null=True) 206 | metadata = models.JSONField(verbose_name=_('metadata'), blank=True, null=True) 207 | duration = models.DurationField(verbose_name=_('duration'), blank=True, null=True) 208 | 209 | class Meta: 210 | abstract = defaults.VIDEO_MODEL != 'video_transcoding.Video' 211 | verbose_name = _('Video') 212 | verbose_name_plural = _('Video') 213 | 214 | def __str__(self) -> str: 215 | basename = os.path.basename(self.source) 216 | return f'{basename} ({self.get_status_display()})' 217 | 218 | def format_video_url(self, edge: str) -> str: 219 | """ 220 | Returns a link to m3u8 playlist on one of randomly chosen edges. 221 | """ 222 | if self.basename is None: 223 | raise RuntimeError("Video has no files") 224 | basename = cast(UUID, self.basename) 225 | return defaults.VIDEO_URL.format( 226 | edge=edge.rstrip('/'), 227 | filename=basename.hex) 228 | 229 | def change_status(self, status: int, **fields: Any) -> None: 230 | """ 231 | Changes video status. 232 | 233 | Also saves another model fields and always updates `modified` value. 234 | 235 | :param status: one of statuses for Video.status (see STATUS_CHOICES) 236 | :param fields: dict with model field values. 237 | """ 238 | self.status = status 239 | update_fields = {'status', 'modified'} 240 | for k, v in fields.items(): 241 | setattr(self, k, v) 242 | update_fields.add(k) 243 | # suppress mypy [no-untyped-calls] 244 | self.save(update_fields=tuple(update_fields)) # type: ignore 245 | 246 | 247 | def get_video_model() -> Type[Video]: 248 | app_label, model_name = defaults.VIDEO_MODEL.split('.') 249 | return cast(Type[Video], apps.get_registered_model(app_label, model_name)) 250 | -------------------------------------------------------------------------------- /src/video_transcoding/signals.py: -------------------------------------------------------------------------------- 1 | from typing import Any 2 | 3 | import celery 4 | from django.core.signals import request_started, request_finished 5 | from django.db import transaction 6 | from django.db.models.signals import post_save 7 | from django.dispatch import receiver 8 | 9 | from video_transcoding import helpers, models 10 | from celery.signals import task_prerun, task_postrun 11 | 12 | 13 | # noinspection PyUnusedLocal 14 | @receiver(post_save, sender=models.get_video_model()) 15 | def send_transcode_task(sender: Any, *, instance: models.Video, created: bool, 16 | **kw: Any) -> None: 17 | if not created: 18 | return 19 | transaction.on_commit(lambda: helpers.send_transcode_task(instance)) 20 | 21 | 22 | # noinspection PyUnusedLocal 23 | @task_prerun.connect 24 | def send_request_started(task: celery.Task, **kwargs: Any) -> None: 25 | """ 26 | Send request_started signal to launch django life cycle handlers. 27 | """ 28 | request_started.send(sender=task.__class__, request=task.request) 29 | 30 | 31 | # noinspection PyUnusedLocal 32 | @task_postrun.connect 33 | def send_request_finished(task: celery.Task, **kwargs: Any) -> None: 34 | """ 35 | Send request_finished signal to launch django life cycle handlers. 36 | """ 37 | request_finished.send(sender=task.__class__, request=task.request) 38 | -------------------------------------------------------------------------------- /src/video_transcoding/tasks.py: -------------------------------------------------------------------------------- 1 | import dataclasses 2 | import time 3 | from datetime import timedelta, datetime 4 | from typing import Optional, List, Iterable, Any, Dict, Union 5 | from uuid import UUID, uuid4 6 | 7 | import celery 8 | from billiard.exceptions import SoftTimeLimitExceeded 9 | from django.db import close_old_connections 10 | from django.db.transaction import atomic 11 | from django.db.utils import OperationalError 12 | 13 | from video_transcoding import models, strategy, defaults 14 | from video_transcoding.celery import app 15 | from video_transcoding.transcoding import profiles 16 | from video_transcoding.utils import LoggerMixin 17 | 18 | Video = models.get_video_model() 19 | 20 | DESTINATION_FILENAME = '{basename}.mp4' 21 | 22 | CONNECT_TIMEOUT = 1 23 | DOWNLOAD_TIMEOUT = 60 * 60 24 | UPLOAD_TIMEOUT = 60 * 60 25 | 26 | 27 | class TranscodeVideo(LoggerMixin, celery.Task): 28 | """ Video processing task.""" 29 | routing_key = 'video_transcoding' 30 | autoretry_for = (OperationalError,) 31 | inifinite_retry_for = (OperationalError,) 32 | retry_backoff = True 33 | 34 | def retry(self, 35 | args: Optional[Iterable[Any]] = None, 36 | kwargs: Optional[Dict[str, Any]] = None, 37 | exc: Optional[Exception] = None, 38 | throw: bool = True, 39 | eta: Optional[datetime] = None, 40 | countdown: Optional[Union[float, int]] = None, 41 | max_retries: Optional[int] = None, 42 | **options: Any) -> Any: 43 | if isinstance(exc, self.inifinite_retry_for): 44 | # increment max_retries by one to achieve unlimited retries 45 | # for infrastructure errors 46 | max_retries = (max_retries or self.max_retries) + 1 47 | return super().retry(args, kwargs, exc, throw, eta, countdown, 48 | max_retries, **options) 49 | 50 | def run(self, video_id: int) -> Optional[str]: 51 | """ 52 | Process video. 53 | 54 | 1. Locks video changing status from QUEUED to PROCESS 55 | 2. Transcodes video and stores result to origins 56 | 3. Changes video status to DONE, stores result basename 57 | 4. On errors changes video status ERROR, stores error message 58 | 59 | :param video_id: Video id. 60 | """ 61 | status = Video.DONE 62 | error = meta = duration = None 63 | video = self.lock_video(video_id) 64 | try: 65 | meta = self.process_video(video) 66 | duration = timedelta(seconds=meta['duration']) 67 | except SoftTimeLimitExceeded as e: 68 | self.logger.debug("Received SIGUSR1, return video to queue") 69 | # celery graceful shutdown 70 | status = Video.QUEUED 71 | error = repr(e) 72 | raise self.retry(countdown=10) 73 | except Exception as e: 74 | status = Video.ERROR 75 | error = repr(e) 76 | self.logger.exception("Processing error %s", error) 77 | finally: 78 | # Close possible stale connections after long operation 79 | close_old_connections() 80 | self.unlock_video(video_id, status, error, meta, duration) 81 | return error 82 | 83 | def select_for_update(self, video_id: int, status: int) -> models.Video: 84 | """ Lock video in DB for current task. 85 | 86 | :param video_id: Video primary key 87 | :param status: expected video status 88 | :returns: Video object from db 89 | 90 | :raises models.Video.DoesNotExist: in case of missing or locked 91 | Video for primary key 92 | :raises ValueError: in case of unexpected Video status or task_id 93 | 94 | """ 95 | try: 96 | video = Video.objects.select_for_update( 97 | skip_locked=True, of=('self',)).get(pk=video_id) 98 | except Video.DoesNotExist: 99 | self.logger.error("Can't lock video %s", video_id) 100 | raise 101 | 102 | if video.task_id != UUID(self.request.id): 103 | self.logger.error("Unexpected video %s task_id %s", 104 | video.id, video.task_id) 105 | raise ValueError(video.task_id) 106 | 107 | if video.status != status: 108 | self.logger.error("Unexpected video %s status %s", 109 | video.id, video.get_status_display()) 110 | raise ValueError(video.status) 111 | return video 112 | 113 | @atomic 114 | def lock_video(self, video_id: int) -> models.Video: 115 | """ 116 | Gets video in QUEUED status from DB and changes status to PROCESS. 117 | 118 | :param video_id: Video primary key 119 | :returns: Video object 120 | :raises Retry: in case of unexpected video status or task_id 121 | """ 122 | if defaults.VIDEO_TRANSCODING_WAIT: # pragma: no cover 123 | # Handle database replication and transaction commit related delay 124 | time.sleep(defaults.VIDEO_TRANSCODING_WAIT) 125 | try: 126 | video = self.select_for_update(video_id, Video.QUEUED) 127 | except (Video.DoesNotExist, ValueError) as e: 128 | # if video is locked or task_id is not equal to current task, retry. 129 | raise self.retry(exc=e) 130 | if video.basename is None: 131 | video.basename = uuid4() 132 | video.change_status(Video.PROCESS, basename=video.basename) 133 | return video 134 | 135 | @atomic 136 | def unlock_video(self, video_id: int, status: int, error: Optional[str], 137 | meta: Optional[dict], duration: Optional[timedelta], 138 | ) -> None: 139 | """ 140 | Marks video with final status. 141 | 142 | :param video_id: Video primary key 143 | :param status: final video status (Video.DONE, Video.ERROR) 144 | :param error: error message 145 | :param meta: resulting media metadata 146 | :param duration: media duration 147 | :raises RuntimeError: in case of unexpected video status or task id 148 | """ 149 | try: 150 | video = self.select_for_update(video_id, Video.PROCESS) 151 | except (Video.DoesNotExist, ValueError) as e: 152 | # if video is locked or task_id differs from current task, do 153 | # nothing because video is modified somewhere else. 154 | raise RuntimeError("Can't unlock locked video %s: %s", 155 | video_id, repr(e)) 156 | 157 | video.change_status(status, 158 | error=error, 159 | metadata=meta, 160 | duration=duration) 161 | 162 | def process_video(self, video: models.Video) -> dict: 163 | """ 164 | Makes an HLS adaptation set from video source. 165 | """ 166 | preset = self.init_preset(video.preset) 167 | basename = video.basename 168 | if basename is None: # pragma: no cover 169 | raise RuntimeError("basename not set") 170 | s = self.init_strategy( 171 | source_uri=video.source, 172 | basename=basename.hex, 173 | preset=preset, 174 | ) 175 | output_meta = s() 176 | 177 | # noinspection PyTypeChecker 178 | data = dataclasses.asdict(output_meta) 179 | duration = None 180 | # cleanup internal metadata and compute duration 181 | for stream in data['audios'] + data['videos']: 182 | for f in ('scenes', 'streams', 'start', 'device'): 183 | stream.pop(f, None) 184 | if duration is None: 185 | duration = stream['duration'] 186 | else: 187 | duration = min(duration, stream['duration']) 188 | data['duration'] = duration 189 | 190 | return data 191 | 192 | @staticmethod 193 | def init_strategy( 194 | source_uri: str, 195 | basename: str, 196 | preset: profiles.Preset 197 | ) -> strategy.Strategy: 198 | return strategy.ResumableStrategy( 199 | source_uri=source_uri, 200 | basename=basename, 201 | preset=preset, 202 | ) 203 | 204 | @staticmethod 205 | def init_preset(preset: Optional[models.Preset]) -> profiles.Preset: 206 | """ 207 | Initializes preset entity from database objects. 208 | """ 209 | if preset is None: 210 | return profiles.DEFAULT_PRESET 211 | video_tracks: List[profiles.VideoTrack] = [] 212 | for vt in preset.video_tracks.all(): # type: models.VideoTrack 213 | kwargs = dict(**vt.params) 214 | kwargs['id'] = vt.name 215 | video_tracks.append(profiles.VideoTrack(**kwargs)) 216 | 217 | audio_tracks: List[profiles.AudioTrack] = [] 218 | for at in preset.audio_tracks.all(): # type: models.AudioTrack 219 | kwargs = dict(**at.params) 220 | kwargs['id'] = at.name 221 | audio_tracks.append(profiles.AudioTrack(**kwargs)) 222 | 223 | video_profiles: List[profiles.VideoProfile] = [] 224 | for vp in preset.video_profiles.all(): # type: models.VideoProfile 225 | vc = profiles.VideoCondition(**vp.condition) 226 | 227 | vqs = vp.videoprofiletracks_set.select_related('track') 228 | tracks = [vpt.track.name for vpt in vqs] 229 | video_profiles.append(profiles.VideoProfile( 230 | condition=vc, 231 | video=tracks, 232 | segment_duration=vp.segment_duration.total_seconds(), 233 | )) 234 | 235 | audio_profiles: List[profiles.AudioProfile] = [] 236 | for ap in preset.audio_profiles.all(): # type: models.AudioProfile 237 | ac = profiles.AudioCondition(**ap.condition) 238 | 239 | aqs = ap.audioprofiletracks_set.select_related('track') 240 | tracks = [apt.track.name for apt in aqs] 241 | audio_profiles.append(profiles.AudioProfile( 242 | condition=ac, 243 | audio=tracks, 244 | )) 245 | 246 | return profiles.Preset( 247 | video_profiles=video_profiles, 248 | audio_profiles=audio_profiles, 249 | video=video_tracks, 250 | audio=audio_tracks, 251 | ) 252 | 253 | 254 | transcode_video: TranscodeVideo = app.register_task( 255 | TranscodeVideo()) # type: ignore 256 | -------------------------------------------------------------------------------- /src/video_transcoding/tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/just-work/django-video-transcoding/aa4a945a319bb4bd540c2acf4fcc3cbd6e3fad26/src/video_transcoding/tests/__init__.py -------------------------------------------------------------------------------- /src/video_transcoding/tests/base.py: -------------------------------------------------------------------------------- 1 | from unittest import mock 2 | from urllib.parse import ParseResult, urlparse, urlunparse 3 | from uuid import uuid4 4 | 5 | from celery.result import AsyncResult 6 | from django.test import TestCase 7 | from fffw.graph import VideoMeta, TS, Scene, AudioMeta 8 | 9 | from video_transcoding.transcoding import profiles, metadata, workspace 10 | 11 | 12 | class BaseTestCase(TestCase): 13 | 14 | def setUp(self): 15 | super().setUp() 16 | self.apply_async_patcher = mock.patch( 17 | 'video_transcoding.tasks.transcode_video.apply_async', 18 | return_value=AsyncResult(str(uuid4()))) 19 | self.apply_async_mock = self.apply_async_patcher.start() 20 | 21 | def tearDown(self): 22 | super().tearDown() 23 | self.apply_async_patcher.stop() 24 | 25 | 26 | class ProfileMixin: 27 | @staticmethod 28 | def default_profile() -> profiles.Profile: 29 | return profiles.Profile( 30 | video=[profiles.VideoTrack( 31 | frame_rate=30, 32 | width=1920, 33 | height=1080, 34 | profile='main', 35 | pix_fmt='yuv420p', 36 | buf_size=3_000_000, 37 | gop_size=30, 38 | max_rate=1_500_000, 39 | id='v', 40 | force_key_frames='1.0', 41 | codec='libx264', 42 | preset='slow', 43 | constant_rate_factor=23, 44 | )], 45 | audio=[profiles.AudioTrack( 46 | codec='libfdk_aac', 47 | id='a', 48 | bitrate=128_000, 49 | channels=2, 50 | sample_rate=48000, 51 | )], 52 | container=profiles.Container(segment_duration=1.0) 53 | ) 54 | 55 | 56 | class MetadataMixin: 57 | 58 | @staticmethod 59 | def make_meta(*scenes: float, uri='uri') -> metadata.Metadata: 60 | duration = sum(scenes) 61 | # technically, merged scenes are incorrect because start value is 62 | # always zero, but we don't care as we don't use them. 63 | return metadata.Metadata( 64 | uri=uri, 65 | videos=[ 66 | VideoMeta( 67 | bitrate=100500, 68 | frame_rate=30.0, 69 | dar=1.778, 70 | par=1.0, 71 | width=1920, 72 | height=1080, 73 | frames=int(duration * 30.0), 74 | streams=['v'], 75 | start=TS(0), 76 | duration=TS(duration), 77 | device=None, 78 | scenes=[Scene(stream='v', 79 | duration=TS(s), 80 | start=TS(0), 81 | position=TS(0)) 82 | for s in scenes] 83 | ), 84 | ], 85 | audios=[ 86 | AudioMeta( 87 | bitrate=100500, 88 | sampling_rate=48000, 89 | channels=2, 90 | samples=int(duration * 48000), 91 | streams=['a'], 92 | start=TS(0), 93 | duration=TS(duration), 94 | scenes=[Scene(stream='a', 95 | duration=TS(s), 96 | start=TS(0), 97 | position=TS(0)) 98 | for s in scenes] 99 | ), 100 | ] 101 | ) 102 | 103 | 104 | class MemoryWorkspace: 105 | def __init__(self, basename: str): 106 | self.tree = {} 107 | self.root = workspace.Collection(basename) 108 | 109 | def ensure_collection(self, path: str) -> workspace.Collection: 110 | parts = self.root.parts + tuple(path.strip('/').split('/')) 111 | t = self.tree 112 | for p in parts: 113 | t = t.setdefault(p, {}) 114 | return workspace.Collection(*parts) 115 | 116 | def create_collection(self, c: workspace.Collection) -> None: 117 | t = self.tree 118 | for p in c.parts: 119 | t = t.setdefault(p, {}) 120 | 121 | def delete_collection(self, c: workspace.Collection) -> None: 122 | t = self.tree 123 | parent = p = None 124 | for p in c.parts: 125 | parent = t 126 | try: 127 | t = t[p] 128 | except KeyError: # pragma: no cover 129 | break 130 | else: 131 | del parent[p] 132 | 133 | def exists(self, r: workspace.Resource) -> bool: 134 | t = self.tree 135 | for p in r.parts: 136 | try: 137 | t = t[p] 138 | except KeyError: 139 | return False 140 | else: 141 | return True 142 | 143 | def read(self, f: workspace.File) -> str: 144 | t = self.tree 145 | for p in f.parts: 146 | t = t[p] 147 | return t 148 | 149 | def write(self, f: workspace.File, content: str) -> None: 150 | t = self.tree 151 | for p in f.parts[:-1]: 152 | t = t[p] 153 | t[f.parts[-1]] = content 154 | 155 | @staticmethod 156 | def get_absolute_uri(r: workspace.Resource) -> ParseResult: 157 | path = '/'.join(r.parts) 158 | # noinspection PyArgumentList 159 | return urlparse(urlunparse(('memory', '', path, '', '', ''))) 160 | -------------------------------------------------------------------------------- /src/video_transcoding/tests/test_analysis.py: -------------------------------------------------------------------------------- 1 | from django.test import TestCase 2 | from fffw.analysis.ffprobe import ProbeInfo 3 | from fffw.graph import VideoMeta, AudioMeta 4 | 5 | from video_transcoding.tests import base 6 | from video_transcoding.transcoding import analysis 7 | 8 | 9 | class MKVPlaylistAnalyzerTestCase(TestCase): 10 | def setUp(self): 11 | super().setUp() 12 | self.info = ProbeInfo( 13 | streams=[ 14 | {'duration': 30.0}, 15 | {'duration': 45.0}, 16 | ], 17 | format={ 18 | 'duration': 60.0, 19 | } 20 | ) 21 | self.analyzer = analysis.MKVPlaylistAnalyzer(self.info) 22 | 23 | def test_multiple_streams_duration_normalize(self): 24 | d = self.analyzer.get_duration(self.info.streams[0]) 25 | 26 | self.assertEqual(d, 30.0) 27 | 28 | self.info.streams[0]['duration'] = 0.0 29 | 30 | d = self.analyzer.get_duration(self.info.streams[0]) 31 | 32 | self.assertEqual(d, 0.0) 33 | 34 | del self.info.streams[1:] 35 | 36 | d = self.analyzer.get_duration(self.info.streams[0]) 37 | 38 | self.assertEqual(d, 60.0) 39 | 40 | 41 | class MKVSegmentAnalyzerTestCase(TestCase): 42 | def setUp(self): 43 | super().setUp() 44 | self.info = ProbeInfo( 45 | streams=[ 46 | {'duration': 30.0, 'bit_rate': 3_000_000}, 47 | {'duration': 45.0, 'bit_rate': 128_000}, 48 | ], 49 | format={ 50 | 'duration': 60.0, 'bit_rate': 3_500_000, 51 | } 52 | ) 53 | self.analyzer = analysis.MKVSegmentAnalyzer(self.info) 54 | 55 | def test_multiple_streams_duration_normalize(self): 56 | d = self.analyzer.get_duration(self.info.streams[0]) 57 | 58 | self.assertEqual(d, 30.0) 59 | 60 | self.info.streams[0]['duration'] = 0.0 61 | 62 | d = self.analyzer.get_duration(self.info.streams[0]) 63 | 64 | self.assertEqual(d, 0.0) 65 | 66 | del self.info.streams[1:] 67 | 68 | d = self.analyzer.get_duration(self.info.streams[0]) 69 | 70 | self.assertEqual(d, 60.0) 71 | 72 | def test_multiple_streams_bitrate_normalize(self): 73 | b = self.analyzer.get_bitrate(self.info.streams[0]) 74 | 75 | self.assertEqual(b, 3_000_000) 76 | 77 | self.info.streams[0]['bit_rate'] = 0 78 | 79 | b = self.analyzer.get_bitrate(self.info.streams[0]) 80 | 81 | self.assertEqual(b, 0) 82 | 83 | del self.info.streams[1:] 84 | 85 | d = self.analyzer.get_bitrate(self.info.streams[0]) 86 | 87 | self.assertEqual(d, 3_500_000) 88 | 89 | 90 | class FFprobeHLSAnalyzerTestCase(TestCase): 91 | def setUp(self): 92 | super().setUp() 93 | self.info = ProbeInfo( 94 | streams=[ 95 | { 96 | 'duration': 30.0, 97 | 'bit_rate': 3_000_000, 98 | 'codec_type': 'video', 99 | 'tags': {'variant_bitrate': 2_200_000} 100 | }, 101 | { 102 | 'duration': 45.0, 103 | 'bit_rate': 128_000, 104 | 'codec_type': 'audio', 105 | }, 106 | ], 107 | format={ 108 | 'duration': 60.0, 'bit_rate': 3_500_000, 109 | } 110 | ) 111 | self.analyzer = analysis.FFProbeHLSAnalyzer(self.info) 112 | 113 | def test_container_duration_normalize(self): 114 | d = self.analyzer.get_duration(self.info.streams[0]) 115 | 116 | self.assertEqual(d, 30.0) 117 | 118 | self.info.streams[0]['duration'] = 0.0 119 | 120 | d = self.analyzer.get_duration(self.info.streams[0]) 121 | self.assertEqual(d, 60.0) 122 | 123 | def test_variant_bitrate_normalize(self): 124 | b = self.analyzer.get_bitrate(self.info.streams[0]) 125 | 126 | self.assertEqual(b, 3_000_000) 127 | 128 | self.info.streams[0]['bit_rate'] = 0 129 | 130 | b = self.analyzer.get_bitrate(self.info.streams[0]) 131 | 132 | self.assertEqual(b, 2_000_000) 133 | 134 | del self.info.streams[0]['tags']['variant_bitrate'] 135 | 136 | b = self.analyzer.get_bitrate(self.info.streams[0]) 137 | 138 | self.assertEqual(b, 0) 139 | 140 | del self.info.streams[0]['tags'] 141 | 142 | b = self.analyzer.get_bitrate(self.info.streams[0]) 143 | 144 | self.assertEqual(b, 0) 145 | 146 | def test_skip_unrelated_streams(self): 147 | streams = self.analyzer.analyze() 148 | 149 | self.assertEqual(len(streams), 2) 150 | self.assertIsInstance(streams[0], VideoMeta) 151 | self.assertIsInstance(streams[1], AudioMeta) 152 | 153 | self.info.streams[1]['tags'] = {'comment': 'a:group0'} 154 | 155 | streams = self.analyzer.analyze() 156 | 157 | self.assertEqual(len(streams), 1) 158 | self.assertIsInstance(streams[0], VideoMeta) 159 | 160 | self.info.streams[0]['codec_type'] = 'side_data' 161 | 162 | streams = self.analyzer.analyze() 163 | 164 | self.assertEqual(len(streams), 0) 165 | -------------------------------------------------------------------------------- /src/video_transcoding/tests/test_celery.py: -------------------------------------------------------------------------------- 1 | import os 2 | import signal 3 | from unittest import mock 4 | 5 | from celery import signals 6 | from django.test import TestCase 7 | 8 | 9 | class CelerySignalsTestCase(TestCase): 10 | 11 | @classmethod 12 | def setUpClass(cls): 13 | super().setUpClass() 14 | __import__('video_transcoding.celery') 15 | 16 | @mock.patch('os.setpgrp') 17 | def test_worker_init_signal(self, m: mock.Mock): 18 | """ 19 | Celery master process creates a process group to send signals to all 20 | children via killpg. 21 | """ 22 | signals.worker_init.send(None) 23 | m.assert_called_once_with() 24 | 25 | @mock.patch('os.killpg') 26 | def test_worker_shutting_down_signal(self, m: mock.Mock): 27 | """ 28 | After receiving TERM signal celery master process propagates it to 29 | all worker processes via process group. 30 | """ 31 | signals.worker_shutting_down.send(None) 32 | m.assert_called_once_with(os.getpid(), signal.SIGUSR1) 33 | 34 | m.side_effect = ProcessLookupError() 35 | 36 | try: 37 | signals.worker_shutting_down.send(None) 38 | except ProcessLookupError: # pragma: no cover 39 | self.fail("exception not handled") 40 | -------------------------------------------------------------------------------- /src/video_transcoding/tests/test_extract.py: -------------------------------------------------------------------------------- 1 | import json 2 | from dataclasses import asdict 3 | from typing import Type, TYPE_CHECKING 4 | from unittest import mock 5 | 6 | from django.test import TestCase 7 | from fffw.analysis import ffprobe 8 | from fffw.graph import VIDEO, AUDIO 9 | 10 | from video_transcoding.tests import base 11 | from video_transcoding.transcoding import extract 12 | 13 | 14 | class ExtractorBaseTestCase(base.MetadataMixin, TestCase): 15 | analyzer: str 16 | extractor_class: Type[extract.Extractor] 17 | 18 | def setUp(self): 19 | super().setUp() 20 | self.meta = self.make_meta(30.0) 21 | self.analyzer_instance = mock.MagicMock() 22 | meta = [s.meta for s in self.meta.streams] 23 | self.analyze_mock = self.analyzer_instance.analyze 24 | self.analyze_mock.return_value = meta 25 | self.analyzer_patcher = mock.patch( 26 | f'video_transcoding.transcoding.analysis.{self.analyzer}', 27 | return_value=self.analyzer_instance 28 | ) 29 | self.analyzer_mock = self.analyzer_patcher.start() 30 | self.mediainfo_pacher = mock.patch( 31 | 'video_transcoding.transcoding.extract.Extractor.mediainfo', 32 | return_value=mock.sentinel.mediainfo) 33 | self.mediainfo_mock = self.mediainfo_pacher.start() 34 | self.ffprobe_patcher = mock.patch( 35 | 'video_transcoding.transcoding.extract.Extractor.ffprobe', 36 | return_value=mock.sentinel.ffprobe 37 | ) 38 | self.ffprobe_mock = self.ffprobe_patcher.start() 39 | self.extractor = self.extractor_class() 40 | 41 | def tearDown(self): 42 | super().tearDown() 43 | self.analyzer_patcher.stop() 44 | self.mediainfo_pacher.stop() 45 | self.ffprobe_patcher.stop() 46 | 47 | 48 | class SourceExtractorTestCase(ExtractorBaseTestCase): 49 | analyzer = 'SourceAnalyzer' 50 | extractor_class = extract.SourceExtractor 51 | 52 | def test_extract(self): 53 | meta = self.extractor.get_meta_data('uri') 54 | 55 | self.analyzer_mock.assert_called_once_with(mock.sentinel.mediainfo) 56 | self.analyze_mock.assert_called_once_with() 57 | self.assertEqual(meta, self.meta) 58 | 59 | def test_mediainfo(self): 60 | try: 61 | self.mediainfo_pacher.stop() 62 | with mock.patch('pymediainfo.MediaInfo.parse', 63 | return_value=mock.sentinel.mi) as m: 64 | result = self.extractor.mediainfo('uri') 65 | 66 | self.assertEqual(result, mock.sentinel.mi) 67 | m.assert_called_once_with('uri') 68 | finally: 69 | self.mediainfo_pacher.start() 70 | 71 | 72 | if TYPE_CHECKING: # pragma: no cover 73 | MKVVideoSegmentTestsMixinTarget = ExtractorBaseTestCase 74 | else: 75 | MKVVideoSegmentTestsMixinTarget = object 76 | 77 | 78 | class MKVVideoSegmentTestsMixin(MKVVideoSegmentTestsMixinTarget): 79 | 80 | def test_extract(self): 81 | # video segments don't contain audio streams 82 | self.meta.audios.clear() 83 | self.analyze_mock.return_value = [s.meta for s in self.meta.streams] 84 | 85 | meta = self.extractor.get_meta_data('uri') 86 | 87 | self.analyzer_mock.assert_called_once_with(mock.sentinel.ffprobe) 88 | self.analyze_mock.assert_called_once_with() 89 | self.assertEqual(meta, self.meta) 90 | 91 | def test_ffprobe(self): 92 | try: 93 | self.ffprobe_patcher.stop() 94 | pi = ffprobe.ProbeInfo( 95 | streams=[{}], 96 | format={} 97 | ) 98 | # noinspection PyTypeChecker 99 | content = json.dumps(asdict(pi)) 100 | with mock.patch('video_transcoding.transcoding.extract.FFProbe', 101 | ) as m: 102 | m.return_value.run.return_value = (0, content, '') 103 | result = self.extractor.ffprobe('uri') 104 | m.assert_called_once_with( 105 | 'uri', 106 | show_format=True, 107 | show_streams=True, 108 | output_format='json', 109 | allowed_extensions='mkv', 110 | ) 111 | self.assertEqual(result, pi) 112 | finally: 113 | self.ffprobe_patcher.start() 114 | 115 | 116 | class VideoSegmentExtractorTestCase(MKVVideoSegmentTestsMixin, 117 | ExtractorBaseTestCase): 118 | analyzer = 'MKVSegmentAnalyzer' 119 | extractor_class = extract.VideoSegmentExtractor 120 | 121 | 122 | class VideoResultExtractorTestCase(MKVVideoSegmentTestsMixin, 123 | ExtractorBaseTestCase): 124 | analyzer = 'VideoResultAnalyzer' 125 | extractor_class = extract.VideoResultExtractor 126 | 127 | 128 | class SplitExtractorTestCase(ExtractorBaseTestCase): 129 | analyzer = 'MKVPlaylistAnalyzer' 130 | 131 | @staticmethod 132 | def extractor_class(): 133 | return extract.SplitExtractor(video_playlist='source-video.m3u8', 134 | audio_file='source-audio.mkv') 135 | 136 | def test_extract(self): 137 | self.video_meta = [s.meta for s in self.meta.streams if s.kind == VIDEO] 138 | self.audio_meta = [s.meta for s in self.meta.streams if s.kind == AUDIO] 139 | self.streams = None 140 | self.ffprobe_mock.side_effect = self.ffprobe 141 | self.analyze_mock.side_effect = self.analyze 142 | 143 | meta = self.extractor.get_meta_data('/dir/split.json') 144 | kw = dict(timeout=60.0, allowed_extensions='mkv') 145 | self.ffprobe_mock.assert_has_calls([ 146 | mock.call('/dir/source-video.m3u8', **kw), 147 | mock.call('/dir/source-audio.mkv', **kw), 148 | ]) 149 | self.analyze_mock.assert_has_calls([mock.call(), mock.call()]) 150 | self.meta.uri = '/dir/split.json' 151 | self.assertEqual(meta, self.meta) 152 | 153 | def test_ffprobe(self): 154 | try: 155 | self.ffprobe_patcher.stop() 156 | pi = ffprobe.ProbeInfo( 157 | streams=[{}], 158 | format={} 159 | ) 160 | # noinspection PyTypeChecker 161 | content = json.dumps(asdict(pi)) 162 | with mock.patch('video_transcoding.transcoding.extract.FFProbe', 163 | ) as m: 164 | m.return_value.run.return_value = (0, content, '') 165 | result = self.extractor.ffprobe('uri') 166 | m.assert_called_once_with( 167 | 'uri', 168 | show_format=True, 169 | show_streams=True, 170 | output_format='json', 171 | allowed_extensions='mkv', 172 | ) 173 | self.assertEqual(result, pi) 174 | finally: 175 | self.ffprobe_patcher.start() 176 | 177 | def ffprobe(self, uri: str, *_, **__): 178 | if 'video' in uri: 179 | self.streams = self.video_meta 180 | elif 'audio' in uri: 181 | self.streams = self.audio_meta 182 | else: # pragma: no cover 183 | raise ValueError('uri') 184 | return self.ffprobe_mock.return_value 185 | 186 | def analyze(self): 187 | return self.streams 188 | 189 | 190 | class HLSExtractorTestCase(ExtractorBaseTestCase): 191 | analyzer = 'FFProbeHLSAnalyzer' 192 | extractor_class = extract.HLSExtractor 193 | 194 | def test_extract(self): 195 | meta = self.extractor.get_meta_data('uri') 196 | 197 | self.analyzer_mock.assert_called_once_with(mock.sentinel.ffprobe) 198 | self.analyze_mock.assert_called_once_with() 199 | self.assertEqual(meta, self.meta) 200 | 201 | def test_ffprobe(self): 202 | try: 203 | self.ffprobe_patcher.stop() 204 | pi = ffprobe.ProbeInfo( 205 | streams=[{}], 206 | format={} 207 | ) 208 | # noinspection PyTypeChecker 209 | content = json.dumps(asdict(pi)) 210 | with mock.patch('video_transcoding.transcoding.extract.FFProbe', 211 | ) as m: 212 | m.return_value.run.return_value = (0, content, '') 213 | result = self.extractor.ffprobe('uri') 214 | m.assert_called_once_with( 215 | 'uri', 216 | show_format=True, 217 | show_streams=True, 218 | output_format='json', 219 | ) 220 | self.assertEqual(result, pi) 221 | finally: 222 | self.ffprobe_patcher.start() 223 | -------------------------------------------------------------------------------- /src/video_transcoding/tests/test_metadata.py: -------------------------------------------------------------------------------- 1 | from unittest import TestCase 2 | 3 | from fffw.encoding import Stream 4 | from fffw.graph import Scene, TS, VideoMeta, AudioMeta, VIDEO, AUDIO 5 | 6 | from video_transcoding.transcoding import metadata 7 | 8 | 9 | class MetadataTestCase(TestCase): 10 | def setUp(self): 11 | super().setUp() 12 | self.video_data = { 13 | 'bitrate': 100500, 14 | 'width': 1920, 15 | 'height': 1080, 16 | 'par': 1.0, 17 | 'dar': 1.7778, 18 | 'frame_rate': 30.0, 19 | 'frames': int(30 * 1.23), 20 | 'device': None, 21 | 'start': 1.23, 22 | 'duration': '1.23', 23 | 'scenes': [{ 24 | 'duration': '1.23', 25 | 'start': 1.23, 26 | 'stream': 's', 27 | 'position': 2460, 28 | }], 29 | 'streams': ['stream'], 30 | } 31 | self.audio_data = { 32 | 'bitrate': 100500, 33 | 'sampling_rate': 44100, 34 | 'samples': int(44100 * 1.23), 35 | 'channels': 2, 36 | 'start': 1.23, 37 | 'duration': '1.23', 38 | 'scenes': [{ 39 | 'duration': '1.23', 40 | 'start': 1.23, 41 | 'stream': 's', 42 | 'position': 2460, 43 | }], 44 | 'streams': ['stream'], 45 | } 46 | self.meta_data = { 47 | 'videos': [self.video_data], 48 | 'audios': [self.audio_data], 49 | 'uri': 'file:///tmp/source.mp4', 50 | } 51 | 52 | def test_scenes_from_native(self): 53 | data = { 54 | 'duration': '1.23', 55 | 'start': 1.23, 56 | 'stream': 's', 57 | 'position': 2460, 58 | } 59 | scene = metadata.scene_from_native(data) 60 | self.assertIsInstance(scene, Scene) 61 | self.assertIsInstance(scene.duration, TS) 62 | self.assertEqual(scene.duration, TS(1.23)) 63 | self.assertIsInstance(scene.start, TS) 64 | self.assertEqual(scene.start, TS(1.23)) 65 | self.assertIsInstance(scene.position, TS) 66 | self.assertEqual(scene.position, TS(2.46)) 67 | self.assertIsInstance(scene.stream, str) 68 | self.assertEqual(scene.stream, 's') 69 | 70 | def test_get_meta_kwargs(self): 71 | data = { 72 | 'sentinel': {'deep': 'copy'}, 73 | 'start': 1.23, 74 | 'duration': '1.23', 75 | 'scenes': [{ 76 | 'duration': '1.23', 77 | 'start': 1.23, 78 | 'stream': 's', 79 | 'position': 2460, 80 | }] 81 | } 82 | 83 | kwargs = metadata.get_meta_kwargs(data) 84 | 85 | data['sentinel']['deep'] = 'copied' 86 | self.assertEqual(kwargs['sentinel']['deep'], 'copy') 87 | self.assertIsInstance(kwargs['start'], TS) 88 | self.assertEqual(kwargs['start'], TS(1.23)) 89 | self.assertIsInstance(kwargs['duration'], TS) 90 | self.assertEqual(kwargs['duration'], TS(1.23)) 91 | self.assertIsInstance(kwargs['scenes'], list) 92 | self.assertEqual(len(kwargs['scenes']), len(data['scenes'])) 93 | self.assertIsInstance(kwargs['scenes'][0], Scene) 94 | 95 | def test_video_meta_from_native(self): 96 | m = metadata.video_meta_from_native(self.video_data) 97 | 98 | self.assertIsInstance(m, VideoMeta) 99 | expected = VideoMeta( 100 | streams=['stream'], 101 | scenes=[Scene( 102 | duration=TS(1.23), 103 | start=TS(1.23), 104 | stream='s', 105 | position=TS(2.46), 106 | )], 107 | device=None, 108 | start=TS(1.23), 109 | duration=TS(1.23), 110 | bitrate=100500, 111 | width=1920, 112 | height=1080, 113 | par=1.0, 114 | dar=1.7778, 115 | frame_rate=30.0, 116 | frames=int(30 * 1.23), 117 | ) 118 | self.assertEqual(m, expected) 119 | 120 | def test_audio_meta_from_native(self): 121 | m = metadata.audio_meta_from_native(self.audio_data) 122 | 123 | self.assertIsInstance(m, AudioMeta) 124 | expected = AudioMeta( 125 | streams=['stream'], 126 | scenes=[Scene( 127 | duration=TS(1.23), 128 | start=TS(1.23), 129 | stream='s', 130 | position=TS(2.46), 131 | )], 132 | start=TS(1.23), 133 | duration=TS(1.23), 134 | bitrate=100500, 135 | sampling_rate=44100, 136 | samples=int(44100 * 1.23), 137 | channels=2, 138 | ) 139 | self.assertEqual(m, expected) 140 | 141 | def test_metadata_from_native(self): 142 | m = metadata.Metadata.from_native(self.meta_data) 143 | self.assertIsInstance(m, metadata.Metadata) 144 | self.assertIsInstance(m.uri, str) 145 | self.assertEqual(m.uri, 'file:///tmp/source.mp4') 146 | self.assertIsInstance(m.videos, list) 147 | self.assertEqual(len(m.videos), 1) 148 | self.assertIsInstance(m.videos[0], VideoMeta) 149 | self.assertIsInstance(m.audios, list) 150 | self.assertEqual(len(m.audios), 1) 151 | self.assertIsInstance(m.audios[0], AudioMeta) 152 | 153 | def test_metadata_properties(self): 154 | m = metadata.Metadata.from_native(self.meta_data) 155 | self.assertIsInstance(m.video, VideoMeta) 156 | self.assertEqual(m.video, m.videos[0]) 157 | self.assertIsInstance(m.audio, AudioMeta) 158 | self.assertEqual(m.audio, m.audios[0]) 159 | self.assertIsInstance(m.streams, list) 160 | self.assertEqual(len(m.streams), 1 + 1) 161 | for s in m.streams: 162 | self.assertIsInstance(s, Stream) 163 | self.assertEqual(m.streams[0].kind, VIDEO) 164 | self.assertEqual(m.streams[0].meta, m.videos[0]) 165 | self.assertEqual(m.streams[1].kind, AUDIO) 166 | self.assertEqual(m.streams[1].meta, m.audios[0]) 167 | 168 | def test_metadata_repr_smoke(self): 169 | m = metadata.Metadata.from_native(self.meta_data) 170 | self.assertIsInstance(repr(m), str) 171 | -------------------------------------------------------------------------------- /src/video_transcoding/tests/test_models.py: -------------------------------------------------------------------------------- 1 | from unittest import mock 2 | from uuid import uuid4, UUID 3 | 4 | from celery.result import AsyncResult 5 | 6 | from video_transcoding import models 7 | from video_transcoding.tests.base import BaseTestCase 8 | 9 | 10 | class VideoModelTestCase(BaseTestCase): 11 | """ Video django model tests.""" 12 | def setUp(self): 13 | self.on_commit_patcher = mock.patch('django.db.transaction.on_commit', 14 | side_effect=self.on_commit) 15 | self.on_commit_mock = self.on_commit_patcher.start() 16 | self.apply_async_patcher = mock.patch( 17 | 'video_transcoding.tasks.transcode_video.apply_async', 18 | return_value=AsyncResult(str(uuid4()))) 19 | self.apply_async_mock = self.apply_async_patcher.start() 20 | 21 | def tearDown(self): 22 | self.on_commit_patcher.stop() 23 | self.apply_async_patcher.stop() 24 | 25 | @staticmethod 26 | def on_commit(func): 27 | func() 28 | 29 | def test_send_transcode_task(self): 30 | """ When new video is created, a transcode task is sent.""" 31 | v = models.Video.objects.create(source='http://ya.ru/1.mp4') 32 | self.on_commit_mock.assert_called() 33 | self.apply_async_mock.assert_called_once_with( 34 | args=(v.id,), 35 | countdown=10) 36 | v.refresh_from_db() 37 | self.assertEqual(v.status, models.Video.QUEUED) 38 | result = self.apply_async_mock.return_value 39 | self.assertEqual(v.task_id, UUID(result.task_id)) 40 | -------------------------------------------------------------------------------- /src/video_transcoding/tests/test_tasks.py: -------------------------------------------------------------------------------- 1 | from dataclasses import asdict 2 | from datetime import timedelta 3 | from unittest import mock 4 | from uuid import UUID, uuid4 5 | 6 | from billiard.exceptions import SoftTimeLimitExceeded 7 | from celery.exceptions import Retry 8 | 9 | from video_transcoding import models, tasks 10 | from video_transcoding.tests import base 11 | from video_transcoding.transcoding import profiles 12 | 13 | 14 | class TranscodeTaskVideoStateTestCase(base.BaseTestCase): 15 | """ Tests Video status handling in transcode task.""" 16 | 17 | def setUp(self): 18 | super().setUp() 19 | self.video = models.Video.objects.create( 20 | status=models.Video.QUEUED, 21 | task_id=uuid4(), 22 | source='ftp://ya.ru/1.mp4') 23 | cls = "video_transcoding.tasks.TranscodeVideo" 24 | self.handle_patcher = mock.patch( 25 | f'{cls}.process_video', 26 | return_value={"duration": 42.0}) 27 | self.handle_mock: mock.MagicMock = self.handle_patcher.start() 28 | self.retry_patcher = mock.patch(f'{cls}.retry', 29 | side_effect=Retry) 30 | self.retry_mock = self.retry_patcher.start() 31 | 32 | def tearDown(self): 33 | super().tearDown() 34 | self.handle_patcher.stop() 35 | self.retry_patcher.stop() 36 | 37 | def run_task(self): 38 | result = tasks.transcode_video.apply( 39 | task_id=str(self.video.task_id), 40 | args=(self.video.id,), 41 | throw=True) 42 | return result 43 | 44 | def test_lock_video(self): 45 | """ 46 | Video transcoding starts with status PROCESS and finished with DONE. 47 | """ 48 | self.video.error = "my error" 49 | self.video.save() 50 | 51 | result = self.run_task() 52 | 53 | video = self.handle_mock.call_args[0][0] 54 | self.assertEqual(video.status, models.Video.PROCESS) 55 | 56 | self.video.refresh_from_db() 57 | self.assertEqual(self.video.status, models.Video.DONE) 58 | self.assertIsNone(self.video.error) 59 | self.assertEqual(self.video.task_id, UUID(result.task_id)) 60 | self.assertIsNotNone(self.video.basename) 61 | 62 | def test_mark_error(self): 63 | """ 64 | Video transcoding failed with ERROR status and error message saved. 65 | """ 66 | error = RuntimeError("my error " * 100) 67 | self.handle_mock.side_effect = error 68 | 69 | self.run_task() 70 | 71 | self.video.refresh_from_db() 72 | self.assertEqual(self.video.status, models.Video.ERROR) 73 | self.assertEqual(self.video.error, repr(error)) 74 | 75 | def test_skip_incorrect_status(self): 76 | """ 77 | Unexpected video statuses lead to task retry. 78 | """ 79 | self.video.status = models.Video.ERROR 80 | self.video.save() 81 | 82 | with self.assertRaises(Retry): 83 | self.run_task() 84 | 85 | self.video.refresh_from_db() 86 | self.assertEqual(self.video.status, models.Video.ERROR) 87 | self.handle_mock.assert_not_called() 88 | 89 | def test_skip_locked(self): 90 | """ 91 | Locked video leads to task retry. 92 | """ 93 | # We can't simulate database lock, so just simulate this with 94 | # DoesNotExist in select_related(skip_locked=True) 95 | self.video.pk += 1 96 | 97 | with self.assertRaises(Retry): 98 | self.run_task() 99 | 100 | self.handle_mock.assert_not_called() 101 | 102 | def test_skip_unlock_incorrect_status(self): 103 | """ 104 | Video status is not changed in db if video was modified somewhere else. 105 | """ 106 | 107 | # noinspection PyUnusedLocal 108 | def change_status(video, *args, **kwargs): 109 | video.change_status(models.Video.QUEUED) 110 | 111 | self.handle_mock.side_effect = change_status 112 | 113 | with self.assertRaises(RuntimeError): 114 | self.run_task() 115 | 116 | self.video.refresh_from_db() 117 | self.assertEqual(self.video.status, models.Video.QUEUED) 118 | 119 | def test_skip_unlock_foreign_task_id(self): 120 | """ 121 | Video status is not changed in db if it was locked by another task. 122 | """ 123 | task_id = uuid4() 124 | 125 | # noinspection PyUnusedLocal 126 | def change_status(video, *args, **kwargs): 127 | video.task_id = task_id 128 | video.save() 129 | 130 | self.handle_mock.side_effect = change_status 131 | 132 | with self.assertRaises(RuntimeError): 133 | self.run_task() 134 | 135 | self.video.refresh_from_db() 136 | self.assertEqual(self.video.task_id, task_id) 137 | self.assertEqual(self.video.status, models.Video.PROCESS) 138 | 139 | def test_retry_task_on_worker_shutdown(self): 140 | """ 141 | For graceful restart Video status should be reverted to queued on task 142 | retry. 143 | """ 144 | exc = SoftTimeLimitExceeded() 145 | self.handle_mock.side_effect = exc 146 | 147 | with self.assertRaises(Retry): 148 | self.run_task() 149 | 150 | self.video.refresh_from_db() 151 | self.assertEqual(self.video.status, models.Video.QUEUED) 152 | self.assertEqual(self.video.error, repr(exc)) 153 | self.retry_mock.assert_called_once_with(countdown=10) 154 | 155 | def test_init_preset_default(self): 156 | preset = tasks.transcode_video.init_preset(None) 157 | self.assertEqual(preset, profiles.DEFAULT_PRESET) 158 | 159 | def test_init_preset_from_video(self): 160 | p = models.Preset.objects.create() 161 | vt = models.VideoTrack.objects.create( 162 | name='v', 163 | preset=p, 164 | params={ 165 | 'codec': 'libx264', 166 | 'constant_rate_factor': 23, 167 | 'preset': 'slow', 168 | 'max_rate': 1_500_000, 169 | 'buf_size': 3_000_000, 170 | 'profile': 'main', 171 | 'pix_fmt': 'yuv420p', 172 | 'width': 1920, 173 | 'height': 1080, 174 | 'frame_rate': 30.0, 175 | 'gop_size': 30, 176 | 'force_key_frames': 'formula' 177 | }) 178 | at = models.AudioTrack.objects.create( 179 | name='a', 180 | preset=p, 181 | params={ 182 | 'codec': 'libfdk_aac', 183 | 'bitrate': 128_000, 184 | 'channels': 2, 185 | 'sample_rate': 44100, 186 | } 187 | ) 188 | vp = models.VideoProfile.objects.create( 189 | preset=p, 190 | segment_duration=timedelta(seconds=1.0), 191 | condition={ 192 | 'min_width': 1, 193 | 'min_height': 2, 194 | 'min_bitrate': 3, 195 | 'min_frame_rate': 4.0, 196 | 'min_dar': 5.0, 197 | 'max_dar': 6.0, 198 | } 199 | ) 200 | ap = models.AudioProfile.objects.create( 201 | preset=p, 202 | condition={ 203 | 'min_sample_rate': 1, 204 | 'min_bitrate': 2, 205 | } 206 | ) 207 | vp.videoprofiletracks_set.create(track=vt) 208 | ap.audioprofiletracks_set.create(track=at) 209 | 210 | preset = tasks.transcode_video.init_preset(p) 211 | 212 | self.assertIsInstance(preset, profiles.Preset) 213 | self.assertEqual(len(preset.video_profiles), 1) 214 | vp = preset.video_profiles[0] 215 | self.assertEqual(vp, profiles.VideoProfile( 216 | condition=profiles.VideoCondition( 217 | min_width=1, 218 | min_height=2, 219 | min_bitrate=3, 220 | min_frame_rate=4.0, 221 | min_dar=5.0, 222 | max_dar=6.0, 223 | 224 | ), 225 | segment_duration=1.0, 226 | video=['v'] 227 | )) 228 | self.assertEqual(len(preset.video), 1) 229 | v = preset.video[0] 230 | self.assertEqual(v, profiles.VideoTrack( 231 | id='v', 232 | codec='libx264', 233 | constant_rate_factor=23, 234 | preset='slow', 235 | max_rate=1_500_000, 236 | buf_size=3_000_000, 237 | profile='main', 238 | pix_fmt='yuv420p', 239 | width=1920, 240 | height=1080, 241 | frame_rate=30.0, 242 | gop_size=30, 243 | force_key_frames='formula' 244 | )) 245 | 246 | self.assertEqual(len(preset.audio_profiles), 1) 247 | ap = preset.audio_profiles[0] 248 | self.assertEqual(ap, profiles.AudioProfile( 249 | condition=profiles.AudioCondition( 250 | min_sample_rate=1, 251 | min_bitrate=2, 252 | ), 253 | audio=['a'], 254 | )) 255 | self.assertEqual(len(preset.audio), 1) 256 | a = preset.audio[0] 257 | self.assertEqual(a, profiles.AudioTrack( 258 | id='a', 259 | codec='libfdk_aac', 260 | bitrate=128_000, 261 | channels=2, 262 | sample_rate=44100, 263 | )) 264 | 265 | 266 | class ProcessVideoTestCase(base.MetadataMixin, base.BaseTestCase): 267 | """ 268 | Tests video processing in terms of transcoding and uploading in 269 | transcode_video task. 270 | """ 271 | 272 | def setUp(self): 273 | super().setUp() 274 | self.video = models.Video.objects.create( 275 | status=models.Video.PROCESS, 276 | source='ftp://ya.ru/1.mp4') 277 | self.basename = uuid4().hex 278 | self.video.basename = UUID(self.basename) 279 | 280 | self.strategy_patcher = mock.patch( 281 | 'video_transcoding.strategy.ResumableStrategy') 282 | self.strategy_mock = self.strategy_patcher.start() 283 | self.meta = self.make_meta(30.0) 284 | # noinspection PyTypeChecker 285 | self.strategy_mock.return_value.return_value = self.meta 286 | 287 | def tearDown(self): 288 | super().tearDown() 289 | self.strategy_patcher.stop() 290 | 291 | def run_task(self): 292 | return tasks.transcode_video.process_video(self.video) 293 | 294 | def test_process_video(self): 295 | result = self.run_task() 296 | 297 | self.strategy_mock.assert_called_once_with( 298 | source_uri=self.video.source, 299 | basename=self.video.basename.hex, 300 | preset=tasks.transcode_video.init_preset(self.video.preset), 301 | ) 302 | self.strategy_mock.return_value.assert_called_once_with() 303 | 304 | # noinspection PyTypeChecker 305 | expected = asdict(self.meta) 306 | streams = expected['audios'] + expected['videos'] 307 | for s in streams: 308 | for f in ('scenes', 'streams', 'start', 'device'): 309 | s.pop(f, None) 310 | duration = min(s['duration'] for s in streams) 311 | expected['duration'] = duration 312 | self.assertEqual(result, expected) 313 | -------------------------------------------------------------------------------- /src/video_transcoding/tests/test_transcoder.py: -------------------------------------------------------------------------------- 1 | from copy import deepcopy 2 | from dataclasses import replace 3 | from unittest import mock 4 | 5 | from django.test import TestCase 6 | from fffw.encoding import Stream 7 | from fffw.graph import VIDEO, AUDIO 8 | from fffw.wrapper import ensure_binary 9 | 10 | from video_transcoding import defaults 11 | from video_transcoding.tests import base 12 | from video_transcoding.transcoding import ( 13 | transcoder, 14 | profiles, 15 | inputs, 16 | codecs, 17 | outputs, 18 | ) 19 | 20 | 21 | class ProcessorBaseTestCase(base.ProfileMixin, base.MetadataMixin, TestCase): 22 | def setUp(self): 23 | self.profile = self.default_profile() 24 | self.meta = self.make_meta(30.0, uri='src.ts') 25 | 26 | 27 | class TranscoderTestCase(ProcessorBaseTestCase): 28 | def setUp(self): 29 | super().setUp() 30 | self.transcoder = transcoder.Transcoder( 31 | 'src.ts', 32 | 'dst.ts', 33 | profile=self.profile, 34 | meta=self.meta, 35 | ) 36 | 37 | def test_call(self): 38 | with ( 39 | mock.patch.object(self.transcoder, 'process', 40 | return_value=mock.sentinel.rv) as m 41 | ): 42 | result = self.transcoder() 43 | self.assertEqual(result, mock.sentinel.rv) 44 | m.assert_called_once_with() 45 | 46 | def test_process(self): 47 | with ( 48 | mock.patch.object(self.transcoder, 'prepare_ffmpeg', 49 | return_value=mock.sentinel.ff) as prepare_ffmpeg, 50 | mock.patch.object(self.transcoder, 'run') as run, 51 | mock.patch.object(self.transcoder, 'get_result_metadata', 52 | return_value=mock.sentinel.rv) as get_result_metadata 53 | ): 54 | result = self.transcoder.process() 55 | 56 | prepare_ffmpeg.assert_called_once_with(self.meta) 57 | run.assert_called_once_with(mock.sentinel.ff) 58 | get_result_metadata.assert_called_once_with('dst.ts') 59 | self.assertEqual(result, mock.sentinel.rv) 60 | 61 | def test_run(self): 62 | ff = mock.MagicMock() 63 | ff.run.return_value = (0, 'output', 'error') 64 | 65 | self.transcoder.run(ff) 66 | 67 | ff.run.assert_called_once_with() 68 | 69 | ff.run.return_value = (1, 'output', 'error') 70 | 71 | with self.assertRaises(RuntimeError) as ctx: 72 | self.transcoder.run(ff) 73 | self.assertEqual(ctx.exception.args[0], 'error') 74 | 75 | ff.run.return_value = (2, 'output', '') 76 | with self.assertRaises(RuntimeError) as ctx: 77 | self.transcoder.run(ff) 78 | self.assertEqual(ctx.exception.args[0], 79 | 'invalid ffmpeg return code 2') 80 | 81 | def test_prepare_ffmpeg(self): 82 | with ( 83 | mock.patch.object( 84 | self.transcoder, 'prepare_input', 85 | return_value=mock.sentinel.source) as prepare_input, 86 | mock.patch.object( 87 | self.transcoder, 'prepare_video_codecs', 88 | return_value=mock.sentinel.video_codecs) as prepare_video_codecs, 89 | mock.patch.object( 90 | self.transcoder, 'prepare_output', 91 | return_value=mock.sentinel.dst) as prepare_output, 92 | mock.patch.object( 93 | self.transcoder, 'scale_and_encode', 94 | return_value=mock.Mock( 95 | ffmpeg=mock.sentinel.ffmpeg)) as scale_and_encode, 96 | ): 97 | ffmpeg = self.transcoder.prepare_ffmpeg(mock.sentinel.src) 98 | prepare_input.assert_called_once_with(mock.sentinel.src) 99 | prepare_video_codecs.assert_called_once_with() 100 | prepare_output.assert_called_once_with(mock.sentinel.video_codecs) 101 | scale_and_encode.assert_called_once_with( 102 | mock.sentinel.source, mock.sentinel.video_codecs, mock.sentinel.dst 103 | ) 104 | self.assertEqual(ffmpeg, mock.sentinel.ffmpeg) 105 | 106 | def test_scale_and_encode(self): 107 | self.profile.video.append(profiles.VideoTrack( 108 | frame_rate=30, 109 | width=1280, 110 | height=720, 111 | profile='main', 112 | pix_fmt='yuv420p', 113 | buf_size=1_500_000, 114 | gop_size=30, 115 | max_rate=750_000, 116 | id='v', 117 | force_key_frames='1.0', 118 | codec='libx264', 119 | preset='slow', 120 | constant_rate_factor=23, 121 | )) 122 | source = inputs.Input(streams=(Stream(VIDEO, meta=self.meta.video), 123 | Stream(AUDIO, meta=self.meta.audio))) 124 | 125 | video_codecs = [ 126 | codecs.VideoCodec('libx264', bitrate=1_500_000), 127 | codecs.VideoCodec('libx264', bitrate=750_000), 128 | ] 129 | 130 | dst = outputs.Output(codecs=video_codecs, output_file='out.m3u8') 131 | 132 | simd = self.transcoder.scale_and_encode(source, video_codecs, dst) 133 | 134 | fc = ';'.join([ 135 | '[0:v:0]split[v:split0][v:split1]', 136 | '[v:split0]scale=w=1920:h=1080[vout0]', 137 | '[v:split1]scale=w=1280:h=720[vout1]', 138 | ]) 139 | # ffmpeg 140 | expected = [ 141 | '-loglevel', 'repeat+level+info', 142 | '-y', 143 | '-filter_complex', fc, 144 | '-map', '[vout0]', 145 | '-c:v:0', 'libx264', 146 | '-b:v:0', 1500000, 147 | '-map', '[vout1]', 148 | '-c:v:1', 'libx264', 149 | '-b:v:1', 750000, 150 | '-an', 151 | 'out.m3u8' 152 | ] 153 | 154 | self.assertEqual(simd.ffmpeg.get_args(), ensure_binary(expected)) 155 | 156 | def test_prepare_input(self): 157 | src = self.transcoder.prepare_input(self.meta) 158 | self.assertIsInstance(src, inputs.Input) 159 | self.assertEqual(src.input_file, self.meta.uri) 160 | self.assertEqual(len(src.streams), len(self.meta.streams)) 161 | for x, y in zip(src.streams, self.meta.streams): 162 | self.assertIsInstance(x, Stream) 163 | self.assertEqual(x.kind, y.kind) 164 | self.assertEqual(x.meta, y.meta) 165 | 166 | def test_prepare_output(self): 167 | video_codecs = [ 168 | codecs.VideoCodec('libx264', bitrate=1_500_000), 169 | codecs.VideoCodec('libx264', bitrate=750_000), 170 | ] 171 | dst = self.transcoder.prepare_output(video_codecs) 172 | expected = outputs.FileOutput( 173 | output_file='dst.ts', 174 | method='PUT', 175 | codecs=video_codecs, 176 | format='mpegts', 177 | muxdelay='0', 178 | avoid_negative_ts='disabled', 179 | copyts=True, 180 | ) 181 | self.assertEqual(dst, expected) 182 | 183 | def test_prepare_video_codecs(self): 184 | video_codecs = self.transcoder.prepare_video_codecs() 185 | self.assertEqual(len(video_codecs), len(self.profile.video)) 186 | for c, v in zip(video_codecs, self.profile.video): 187 | expected = codecs.VideoCodec( 188 | codec=v.codec, 189 | force_key_frames=v.force_key_frames, 190 | constant_rate_factor=v.constant_rate_factor, 191 | preset=v.preset, 192 | max_rate=v.max_rate, 193 | buf_size=v.buf_size, 194 | profile=v.profile, 195 | pix_fmt=v.pix_fmt, 196 | gop=v.gop_size, 197 | rate=v.frame_rate, 198 | ) 199 | self.assertEqual(c, expected) 200 | 201 | def test_get_result_metadata(self): 202 | target = 'video_transcoding.transcoding.extract.VideoResultExtractor' 203 | with mock.patch(target, autospec=True) as m: 204 | m.return_value.get_meta_data.return_value = self.meta 205 | 206 | result = self.transcoder.get_result_metadata('uri') 207 | 208 | m.assert_called_once_with() 209 | m.return_value.get_meta_data.assert_called_once_with('uri') 210 | self.assertEqual(result, self.meta) 211 | 212 | 213 | class SplitterTestCase(ProcessorBaseTestCase): 214 | 215 | def setUp(self): 216 | super().setUp() 217 | self.splitter = transcoder.Splitter( 218 | 'src.mp4', 219 | '/dst/', 220 | profile=self.profile, 221 | meta=self.meta, 222 | source_video_playlist='source-video.m3u8', 223 | source_video_chunk='source-video-%05d.mkv', 224 | source_audio='source-audio.mkv', 225 | ) 226 | 227 | def test_get_result_metadata(self): 228 | meta = deepcopy(self.meta) 229 | for stream in meta.streams: 230 | # ensure that bitrate metadata is copied from source meta 231 | # noinspection PyTypeChecker 232 | stream._meta = replace(stream.meta, bitrate=stream.meta.bitrate + 1) 233 | target = 'video_transcoding.transcoding.extract.SplitExtractor' 234 | with mock.patch(target, autospec=True) as m: 235 | m.return_value.get_meta_data.return_value = meta 236 | 237 | result = self.splitter.get_result_metadata('uri') 238 | 239 | m.assert_called_once_with( 240 | video_playlist='source-video.m3u8', 241 | audio_file='source-audio.mkv', 242 | ) 243 | m.return_value.get_meta_data.assert_called_once_with('uri') 244 | self.assertEqual(result, self.meta) 245 | 246 | def test_prepare_ffmpeg(self): 247 | ff = self.splitter.prepare_ffmpeg(self.meta) 248 | 249 | # ffmpeg 250 | expected = [ 251 | '-loglevel', 'level+info', '-y', 252 | '-i', 'src.mp4', 253 | '-map', '0:v:0', 254 | '-c:v:0', 'copy', 255 | '-an', 256 | '-f', 'stream_segment', 257 | '-copyts', '-avoid_negative_ts', 'disabled', 258 | '-segment_format', 'mkv', 259 | '-segment_list', '/dst/source-video.m3u8', 260 | '-segment_list_type', 'm3u8', 261 | '-segment_time', defaults.VIDEO_CHUNK_DURATION, 262 | '/dst/source-video-%05d.mkv', 263 | '-map', '0:a:0', 264 | '-c:a:0', 'copy', 265 | '-vn', 266 | '-f', 'matroska', 267 | '-copyts', '-avoid_negative_ts', 'disabled', 268 | '/dst/source-audio.mkv', 269 | ] 270 | self.assertEqual(ff.get_args(), ensure_binary(expected)) 271 | 272 | 273 | class SegmentorTestCase(ProcessorBaseTestCase): 274 | 275 | def setUp(self): 276 | super().setUp() 277 | self.segmentor = transcoder.Segmentor( 278 | video_source='/results/source-video.m3u8', 279 | audio_source='/sources/source-audio.mkv', 280 | dst='/dst/', 281 | profile=self.profile, 282 | meta=self.meta, 283 | ) 284 | 285 | def test_get_result_metadata(self): 286 | target = 'video_transcoding.transcoding.extract.HLSExtractor' 287 | with mock.patch(target, autospec=True) as m: 288 | m.return_value.get_meta_data.return_value = self.meta 289 | 290 | result = self.segmentor.get_result_metadata('uri') 291 | 292 | m.assert_called_once_with() 293 | m.return_value.get_meta_data.assert_called_once_with('uri') 294 | self.assertEqual(result, self.meta) 295 | 296 | def test_prepare_ffmpeg(self): 297 | self.profile.video.append(profiles.VideoTrack( 298 | frame_rate=30, 299 | width=1280, 300 | height=720, 301 | profile='main', 302 | pix_fmt='yuv420p', 303 | buf_size=1_500_000, 304 | gop_size=30, 305 | max_rate=750_000, 306 | id='v', 307 | force_key_frames='1.0', 308 | codec='libx264', 309 | preset='slow', 310 | constant_rate_factor=23, 311 | )) 312 | self.meta.videos.append(deepcopy(self.meta.video)) 313 | 314 | ff = self.segmentor.prepare_ffmpeg(self.meta) 315 | vsm = ' '.join([ 316 | 'a:0,agroup:a0:bandwidth:128000', 317 | 'v:0,agroup:a0:bandwidth:1500000', 318 | 'v:1,agroup:a0:bandwidth:750000', 319 | ]) 320 | 321 | expected = [ 322 | '-loglevel', 'level+info', '-y', 323 | '-i', '/results/source-video.m3u8', 324 | '-i', '/sources/source-audio.mkv', 325 | '-map', '0:v:0', 326 | '-c:v:0', 'copy', 327 | '-b:v:0', 1500000, 328 | '-map', '0:v:1', 329 | '-c:v:1', 'copy', 330 | '-b:v:1', 750000, 331 | '-map', '1:a:0', 332 | '-c:a:0', 'libfdk_aac', 333 | '-b:a:0', 128000, 334 | '-ar:a:0', 48000, 335 | '-ac:a:0', 2, 336 | '-copyts', '-avoid_negative_ts', 'auto', 337 | '-hls_time', 1.0, 338 | '-hls_playlist_type', 'vod', 339 | '-var_stream_map', vsm, 340 | '-hls_segment_filename', '/dst/segment-%v-%05d.ts', 341 | '-muxdelay', 0, 342 | '-reset_timestamps', 1, 343 | '/dst/playlist-%v.m3u8' 344 | ] 345 | self.assertEqual(ff.get_args(), ensure_binary(expected)) 346 | -------------------------------------------------------------------------------- /src/video_transcoding/tests/test_transcoding.py: -------------------------------------------------------------------------------- 1 | from typing import TYPE_CHECKING, Dict, Any 2 | from unittest import mock, skip 3 | 4 | import pymediainfo 5 | from fffw.wrapper.helpers import ensure_text 6 | 7 | from video_transcoding.tests.base import BaseTestCase 8 | from video_transcoding.transcoding import transcoder 9 | from video_transcoding.transcoding.profiles import DEFAULT_PRESET 10 | 11 | if TYPE_CHECKING: 12 | MediaInfoMixinTarget = BaseTestCase 13 | else: 14 | MediaInfoMixinTarget = object 15 | 16 | 17 | class MediaInfoMixin(MediaInfoMixinTarget): 18 | """ Mixin to manipulate MediaInfo output.""" 19 | 20 | # Minimal mediainfo output template to mock MediaInfo.parse result 21 | media_info_xml = """ 22 | 23 | 24 | 1 25 | 1 26 | 27 | 28 | 29 | {video_duration:.3f} 30 | {video_bitrate} 31 | {width} 32 | {height} 33 | {par:.3f} 34 | {aspect:.3f} 35 | {video_frame_rate:.3f} 36 | {video_frames} 37 | 38 | 39 | 40 | {audio_duration:.3f} 41 | {audio_bitrate} 42 | {audio_sampling_rate} 43 | {audio_samples} 44 | 45 | 46 | 47 | """ 48 | 49 | # Default video file metadata 50 | metadata = { 51 | 'width': 1920, 52 | 'height': 1080, 53 | 'aspect': 1.778, 54 | 'par': 1.0, 55 | 56 | 'video_bitrate': 5000000, 57 | 58 | 'video_duration': 3600.22, 59 | 'video_frame_rate': 24.97, 60 | 'frames_count': round(3600.22 * 24.97), 61 | 62 | 'audio_bitrate': 192000, 63 | 64 | 'audio_duration': 3600.22, 65 | 'audio_sampling_rate': 48000, 66 | 'samples_count': round(3600.22 * 48000), 67 | } 68 | 69 | def setUp(self): 70 | super().setUp() 71 | self.media_info_patcher = mock.patch.object( 72 | pymediainfo.MediaInfo, 'parse', side_effect=self.get_media_info) 73 | self.media_info_mock = self.media_info_patcher.start() 74 | self.media_info: Dict[str, Dict[str, Any]] = {} 75 | 76 | def tearDown(self): 77 | self.media_info_patcher.stop() 78 | 79 | def get_media_info(self, filename: str) -> pymediainfo.MediaInfo: 80 | """ Prepares mediainfo result for file.""" 81 | metadata = self.media_info[filename].copy() 82 | rate = metadata['audio_sampling_rate'] 83 | audio_duration = metadata.pop('audio_duration') 84 | fps = metadata['video_frame_rate'] 85 | video_duration = metadata.pop('video_duration') 86 | xml = self.media_info_xml.format( 87 | filename=filename, 88 | audio_samples=metadata.get('samples_count', 89 | int(rate * audio_duration)), 90 | video_frames=metadata.get('frames_count', 91 | int(fps * video_duration)), 92 | audio_duration=audio_duration * 1000, # ms 93 | video_duration=video_duration * 1000, # ms 94 | **metadata) 95 | return pymediainfo.MediaInfo(xml) 96 | 97 | def prepare_metadata(self, **kwargs): 98 | """ 99 | Modifies metadata template with new values. 100 | """ 101 | media_info = self.metadata.copy() 102 | media_info.update(kwargs) 103 | return media_info 104 | 105 | 106 | # noinspection PyUnresolvedReferences,PyArgumentList 107 | @skip("refactor needed") 108 | class TranscodingTestCase(MediaInfoMixin, BaseTestCase): 109 | """ Video file transcoding tests.""" 110 | 111 | def setUp(self): 112 | self.source = 'http://ya.ru/source.mp4' 113 | self.dest = '/tmp/result.mp4' 114 | super().setUp() 115 | self.media_info = { 116 | self.source: self.prepare_metadata(), 117 | self.dest: self.prepare_metadata() 118 | } 119 | 120 | self.transcoder = transcoder.Transcoder(self.source, self.dest, 121 | DEFAULT_PRESET) 122 | 123 | self.runner_mock = mock.MagicMock( 124 | return_value=(0, '', '') 125 | ) 126 | 127 | self.runner_patcher = mock.patch( 128 | 'fffw.encoding.ffmpeg.FFMPEG.runner_class', 129 | return_value=self.runner_mock) 130 | self.ffmpeg_mock = self.runner_patcher.start() 131 | 132 | def tearDown(self): 133 | super().tearDown() 134 | self.runner_patcher.stop() 135 | 136 | def test_smoke(self): 137 | """ 138 | ffmpeg arguments test. 139 | """ 140 | self.transcoder.transcode() 141 | 142 | filter_complex = ';'.join([ 143 | '[0:v:0]split=4[v:split0][v:split1][v:split2][v:split3]', 144 | '[v:split0]scale=w=1920:h=1080[vout0]', 145 | '[v:split1]scale=w=1280:h=720[vout1]', 146 | '[v:split2]scale=w=854:h=480[vout2]', 147 | '[v:split3]scale=w=640:h=360[vout3]', 148 | ]) 149 | 150 | ffmpeg_args = [ 151 | 'ffmpeg', 152 | '-loglevel', 'repeat+level+info', 153 | '-y', 154 | '-i', self.source, 155 | 156 | '-filter_complex', filter_complex, 157 | 158 | '-map', '[vout0]', 159 | '-c:v:0', 'libx264', 160 | '-force_key_frames:0', 161 | 'expr:if(isnan(prev_forced_t),1,gte(t,prev_forced_t+4))', 162 | '-crf:0', '23', 163 | '-preset:0', 'slow', 164 | '-maxrate:0', '5000000', 165 | '-bufsize:0', '10000000', 166 | '-profile:v:0', 'high', 167 | '-g:0', '60', 168 | '-r:0', '30', 169 | '-pix_fmt:0', 'yuv420p', 170 | 171 | '-map', '[vout1]', 172 | '-c:v:1', 'libx264', 173 | '-force_key_frames:1', 174 | 'expr:if(isnan(prev_forced_t),1,gte(t,prev_forced_t+4))', 175 | '-crf:1', '23', 176 | '-preset:1', 'slow', 177 | '-maxrate:1', '3000000', 178 | '-bufsize:1', '6000000', 179 | '-profile:v:1', 'high', 180 | '-g:1', '60', 181 | '-r:1', '30', 182 | '-pix_fmt:1', 'yuv420p', 183 | 184 | '-map', '[vout2]', 185 | '-c:v:2', 'libx264', 186 | '-force_key_frames:2', 187 | 'expr:if(isnan(prev_forced_t),1,gte(t,prev_forced_t+4))', 188 | '-crf:2', '23', 189 | '-preset:2', 'slow', 190 | '-maxrate:2', '1500000', 191 | '-bufsize:2', '3000000', 192 | '-profile:v:2', 'main', 193 | '-g:2', '60', 194 | '-r:2', '30', 195 | '-pix_fmt:2', 'yuv420p', 196 | 197 | '-map', '[vout3]', 198 | '-c:v:3', 'libx264', 199 | '-force_key_frames:3', 200 | 'expr:if(isnan(prev_forced_t),1,gte(t,prev_forced_t+4))', 201 | '-crf:3', '23', 202 | '-preset:3', 'slow', 203 | '-maxrate:3', '800000', 204 | '-bufsize:3', '1600000', 205 | '-profile:v:3', 'main', 206 | '-g:3', '60', 207 | '-r:3', '30', 208 | '-pix_fmt:3', 'yuv420p', 209 | 210 | '-map', '0:a:0', 211 | '-c:a:0', 'aac', 212 | '-b:a:0', '192000', 213 | '-ar:0', '48000', 214 | '-ac:0', '2', 215 | 216 | '-f', 'mp4', self.dest, 217 | ] 218 | args, kwargs = self.ffmpeg_mock.call_args 219 | self.assertEqual(ensure_text(args), tuple(ffmpeg_args)) 220 | 221 | def test_select_profile(self): 222 | """ 223 | select another set of video and audio tracks to transcode. 224 | """ 225 | src = self.transcoder.get_media_info(self.source) 226 | p = self.transcoder.select_profile(src) 227 | self.assertEqual(len(p.video), 4) 228 | self.assertEqual(len(p.audio), 1) 229 | 230 | mi = self.media_info[self.source] 231 | vb = DEFAULT_PRESET.video_profiles[0].condition.min_bitrate 232 | mi['video_bitrate'] = vb - 1 233 | 234 | src = self.transcoder.get_media_info(self.source) 235 | p = self.transcoder.select_profile(src) 236 | self.assertEqual(len(p.video), 3) 237 | self.assertEqual(len(p.audio), 1) 238 | 239 | def test_handle_stderr_errors(self): 240 | self.runner_mock.return_value = ( 241 | 0, 'stdout', '[error] a warning captured', 242 | ) 243 | try: 244 | self.transcoder.transcode() 245 | except transcoder.TranscodeError: # pragma: no cover 246 | self.fail("False positive error") 247 | 248 | def test_handle_return_code_from_stderr(self): 249 | error = '[error] a warning captured' 250 | self.runner_mock.return_value = (1, 'stdout', error) 251 | 252 | with self.assertRaises(transcoder.TranscodeError) as ctx: 253 | self.transcoder.transcode() 254 | 255 | self.assertEqual(ctx.exception.message, error) 256 | 257 | def test_handle_return_code(self): 258 | self.runner_mock.return_value = (-9, '', '') 259 | 260 | with self.assertRaises(transcoder.TranscodeError) as ctx: 261 | self.transcoder.transcode() 262 | 263 | self.assertEqual(ctx.exception.message, "invalid ffmpeg return code -9") 264 | -------------------------------------------------------------------------------- /src/video_transcoding/tests/test_workspace.py: -------------------------------------------------------------------------------- 1 | from functools import partial 2 | from unittest import mock 3 | 4 | import requests 5 | from django.test import TestCase 6 | 7 | from video_transcoding import defaults 8 | from video_transcoding.transcoding import workspace 9 | 10 | 11 | class ResourceTestCase(TestCase): 12 | def setUp(self): 13 | super().setUp() 14 | self.file = workspace.File('first', 'second', 'file.txt') 15 | self.dir = workspace.Collection('first', 'second', 'dir') 16 | self.root = workspace.Collection() 17 | 18 | def test_basename(self): 19 | self.assertEqual(self.file.basename, 'file.txt') 20 | self.assertEqual(self.root.basename, '') 21 | 22 | def test_path(self): 23 | self.assertEqual(self.file.path, '/first/second/file.txt') 24 | self.assertEqual(self.root.path, '') 25 | 26 | def test_parent(self): 27 | self.assertIsInstance(self.file.parent, workspace.Collection) 28 | self.assertEqual(self.file.parent.path, '/first/second') 29 | self.assertIsNone(self.root.parent) 30 | 31 | def test_repr(self): 32 | self.assertIsInstance(repr(self.file), str) 33 | self.assertIsInstance(repr(self.dir), str) 34 | 35 | def test_trailing_slash(self): 36 | self.assertEqual(self.dir.trailing_slash, '/') 37 | self.assertEqual(self.file.trailing_slash, '') 38 | 39 | def test_child(self): 40 | c = self.dir.collection('new') 41 | self.assertIsInstance(c, workspace.Collection) 42 | self.assertEqual(c.path, '/first/second/dir/new') 43 | c = self.root.collection('new') 44 | self.assertEqual(c.path, '/new') 45 | f = self.dir.file('new.txt') 46 | self.assertIsInstance(f, workspace.File) 47 | self.assertEqual(f.path, '/first/second/dir/new.txt') 48 | 49 | 50 | class FileSystemWorkspaceTestCase(TestCase): 51 | def setUp(self): 52 | super().setUp() 53 | self.ws = workspace.FileSystemWorkspace('/tmp/dir') 54 | self.file = workspace.File('first', 'second', 'file.txt') 55 | self.dir = workspace.Collection('first', 'second', 'dir') 56 | 57 | def test_get_absolute_uri(self): 58 | uri = self.ws.get_absolute_uri(self.file).geturl() 59 | self.assertEqual(uri, 'file:///tmp/dir/first/second/file.txt') 60 | uri = self.ws.get_absolute_uri(self.dir).geturl() 61 | self.assertEqual(uri, 'file:///tmp/dir/first/second/dir/') 62 | 63 | @mock.patch('os.makedirs') 64 | def test_ensure_collection(self, m: mock.Mock): 65 | c = self.ws.ensure_collection('/another/collection') 66 | self.assertIsInstance(c, workspace.Collection) 67 | self.assertEqual(c.path, '/another/collection') 68 | m.assert_called_once_with('/tmp/dir/another/collection/', exist_ok=True) 69 | 70 | @mock.patch('os.makedirs') 71 | def test_create_collection(self, m: mock.Mock): 72 | c = workspace.Collection('another', 'collection') 73 | self.ws.create_collection(c) 74 | m.assert_called_once_with('/tmp/dir/another/collection/', exist_ok=True) 75 | 76 | @mock.patch('shutil.rmtree') 77 | def test_delete_collection(self, m: mock.Mock): 78 | c = workspace.Collection('another', 'collection') 79 | self.ws.delete_collection(c) 80 | m.assert_called_once_with('/tmp/dir/another/collection/') 81 | 82 | m.side_effect = FileNotFoundError() 83 | try: 84 | self.ws.delete_collection(c) 85 | except FileNotFoundError: # pragma: no cover 86 | self.fail("exception raised") 87 | 88 | @mock.patch('os.path.exists') 89 | def test_exists(self, m: mock.Mock): 90 | m.return_value = True 91 | self.assertTrue(self.ws.exists(self.file)) 92 | m.assert_called_once_with('/tmp/dir/first/second/file.txt') 93 | m.reset_mock() 94 | m.return_value = False 95 | self.assertFalse(self.ws.exists(self.dir)) 96 | m.assert_called_once_with('/tmp/dir/first/second/dir/') 97 | 98 | @mock.patch('builtins.open', 99 | new_callable=partial(mock.mock_open, read_data='read_data')) 100 | def test_read(self, m: mock.Mock): 101 | content = self.ws.read(self.file) 102 | self.assertEqual(content, 'read_data') 103 | m.assert_called_once_with('/tmp/dir/first/second/file.txt', 'r') 104 | 105 | @mock.patch('builtins.open', new_callable=mock.mock_open) 106 | def test_write(self, m: mock.Mock): 107 | self.ws.write(self.file, 'content') 108 | m.return_value.write.assert_called_once_with('content') 109 | 110 | 111 | class WebDAVWorkspaceTestCase(TestCase): 112 | def setUp(self): 113 | super().setUp() 114 | self.ws = workspace.WebDAVWorkspace('https://domain.com/path') 115 | self.file = workspace.File('first', 'second', 'file.txt') 116 | self.dir = workspace.Collection('first', 'second', 'dir') 117 | self.session_patcher = mock.patch('requests.Session.request') 118 | self.session_mock = self.session_patcher.start() 119 | self.response = requests.Response() 120 | self.response.status_code = requests.status_codes.codes.ok 121 | self.session_mock.return_value = self.response 122 | timeout = ( 123 | defaults.VIDEO_CONNECT_TIMEOUT, 124 | defaults.VIDEO_REQUEST_TIMEOUT, 125 | ) 126 | self.session_kwargs = {'timeout': timeout} 127 | self.status_patcher = mock.patch.object(self.response, 128 | 'raise_for_status') 129 | self.status_mock = self.status_patcher.start() 130 | 131 | def tearDown(self): 132 | super().tearDown() 133 | self.session_patcher.stop() 134 | self.status_patcher.stop() 135 | 136 | def test_get_absolute_uri(self): 137 | uri = self.ws.get_absolute_uri(self.file).geturl() 138 | self.assertEqual(uri, 'https://domain.com/path/first/second/file.txt') 139 | uri = self.ws.get_absolute_uri(self.dir).geturl() 140 | self.assertEqual(uri, 'https://domain.com/path/first/second/dir/') 141 | self.ws = workspace.WebDAVWorkspace('https://domain.com') 142 | uri = self.ws.get_absolute_uri(workspace.Collection()).geturl() 143 | self.assertEqual(uri, 'https://domain.com') 144 | uri = self.ws.get_absolute_uri(self.file).geturl() 145 | self.assertEqual(uri, 'https://domain.com/first/second/file.txt') 146 | uri = self.ws.get_absolute_uri(self.dir).geturl() 147 | self.assertEqual(uri, 'https://domain.com/first/second/dir/') 148 | 149 | def test_ensure_collection(self): 150 | c = self.ws.ensure_collection('/another/collection') 151 | self.assertIsInstance(c, workspace.Collection) 152 | self.assertEqual(c.path, '/another/collection') 153 | kw = self.session_kwargs 154 | self.session_mock.assert_has_calls([ 155 | mock.call('MKCOL', 'https://domain.com/path/', **kw), 156 | mock.call('MKCOL', 'https://domain.com/path/another/', **kw), 157 | mock.call('MKCOL', 'https://domain.com/path/another/collection/', 158 | **kw), 159 | ]) 160 | self.status_mock.assert_called() 161 | 162 | def test_ensure_collection_exists(self): 163 | self.response.status_code = requests.codes.method_not_allowed 164 | c = self.ws.ensure_collection('/another/collection') 165 | self.assertIsInstance(c, workspace.Collection) 166 | self.status_mock.assert_not_called() 167 | 168 | self.response.status_code = requests.codes.server_error 169 | self.status_mock.side_effect = requests.exceptions.HTTPError 170 | with self.assertRaises(requests.exceptions.HTTPError): 171 | self.ws.ensure_collection('/another/collection') 172 | self.status_mock.assert_called() 173 | 174 | def test_create_collection(self): 175 | c = workspace.Collection('another', 'collection') 176 | 177 | self.ws.create_collection(c) 178 | 179 | kw = self.session_kwargs 180 | self.session_mock.assert_has_calls([ 181 | mock.call('MKCOL', 'https://domain.com/path/', **kw), 182 | mock.call('MKCOL', 'https://domain.com/path/another/', **kw), 183 | mock.call('MKCOL', 'https://domain.com/path/another/collection/', 184 | **kw), 185 | ]) 186 | self.status_mock.assert_called() 187 | 188 | def test_create_collection_strip(self): 189 | c = workspace.Collection() 190 | 191 | self.ws.create_collection(c) 192 | 193 | kw = self.session_kwargs 194 | self.session_mock.assert_has_calls([ 195 | mock.call('MKCOL', 'https://domain.com/path/', **kw), 196 | ]) 197 | self.status_mock.assert_called() 198 | 199 | def test_create_collection_root(self): 200 | self.ws = workspace.WebDAVWorkspace('https://domain.com') 201 | c = workspace.Collection() 202 | 203 | self.ws.create_collection(c) 204 | kw = self.session_kwargs 205 | self.session_mock.assert_has_calls([ 206 | mock.call('MKCOL', 'https://domain.com/', **kw), 207 | ]) 208 | 209 | def test_delete_collection(self): 210 | c = workspace.Collection('another', 'collection') 211 | 212 | self.ws.delete_collection(c) 213 | 214 | self.session_mock.assert_has_calls([ 215 | mock.call('DELETE', 'https://domain.com/path/another/collection/', 216 | **self.session_kwargs) 217 | ]) 218 | self.status_mock.assert_called() 219 | 220 | self.response.status_code = requests.codes.not_found 221 | try: 222 | self.ws.delete_collection(c) 223 | except requests.exceptions.HTTPError: # pragma: no cover 224 | self.fail("exception raised") 225 | 226 | def test_exists(self): 227 | self.assertTrue(self.ws.exists(self.file)) 228 | self.session_mock.assert_has_calls([ 229 | mock.call('HEAD', 'https://domain.com/path/first/second/file.txt', 230 | **self.session_kwargs), 231 | ]) 232 | self.status_mock.assert_called() 233 | 234 | self.session_mock.reset_mock() 235 | self.response.status_code = requests.codes.not_found 236 | self.assertFalse(self.ws.exists(self.dir)) 237 | self.session_mock.assert_has_calls([ 238 | mock.call('HEAD', 'https://domain.com/path/first/second/dir/', 239 | **self.session_kwargs), 240 | ]) 241 | 242 | def test_read(self): 243 | self.response._content = b'read_data' 244 | content = self.ws.read(self.file) 245 | self.assertEqual(content, 'read_data') 246 | self.session_mock.assert_has_calls([ 247 | mock.call('GET', 'https://domain.com/path/first/second/file.txt', 248 | **self.session_kwargs) 249 | ]) 250 | self.status_mock.assert_called() 251 | 252 | def test_write(self): 253 | self.ws.write(self.file, 'content') 254 | self.session_mock.assert_has_calls([ 255 | mock.call('PUT', 'https://domain.com/path/first/second/file.txt', 256 | data='content') 257 | ]) 258 | self.status_mock.assert_called() 259 | 260 | 261 | class InitWorkspaceTestCase(TestCase): 262 | def test_init_file(self): 263 | ws = workspace.init('file:///tmp/root') 264 | self.assertIsInstance(ws, workspace.FileSystemWorkspace) 265 | uri = ws.get_absolute_uri(ws.root).geturl() 266 | self.assertEqual(uri, 'file:///tmp/root/') 267 | 268 | def test_init_dav(self): 269 | ws = workspace.init('dav://domain.com/root/') 270 | self.assertIsInstance(ws, workspace.WebDAVWorkspace) 271 | uri = ws.get_absolute_uri(ws.root).geturl() 272 | self.assertEqual(uri, 'http://domain.com/root/') 273 | 274 | def test_init_davs(self): 275 | ws = workspace.init('davs://domain.com/root/') 276 | self.assertIsInstance(ws, workspace.WebDAVWorkspace) 277 | uri = ws.get_absolute_uri(ws.root).geturl() 278 | self.assertEqual(uri, 'https://domain.com/root/') 279 | 280 | def test_init_value_error(self): 281 | with self.assertRaises(ValueError): 282 | workspace.init('not_a_scheme://domain.com/') 283 | -------------------------------------------------------------------------------- /src/video_transcoding/tests/test_wrappers.py: -------------------------------------------------------------------------------- 1 | from unittest import TestCase, mock 2 | 3 | from fffw.encoding import Stream 4 | from fffw.graph import VIDEO 5 | 6 | from video_transcoding.transcoding import inputs 7 | from video_transcoding.transcoding.ffprobe import FFProbe 8 | 9 | 10 | class FFProbeWrapperTestCase(TestCase): 11 | def test_handle_stdout(self): 12 | f = FFProbe() 13 | m = mock.MagicMock() 14 | with mock.patch.object(f, 'logger', 15 | new_callable=mock.PropertyMock(return_value=m)): 16 | s = mock.sentinel.stdout 17 | out = f.handle_stdout(s) 18 | self.assertIs(out, s) 19 | m.assert_not_called() 20 | 21 | def test_handle_stderr_error(self): 22 | f = FFProbe() 23 | m = mock.MagicMock() 24 | with mock.patch.object(f, 'logger', 25 | new_callable=mock.PropertyMock(return_value=m)): 26 | line = 'perfix [error] suffix' 27 | out = f.handle_stderr(line) 28 | self.assertEqual(out, '') 29 | m.error.assert_called_once_with(line) 30 | 31 | def test_handle_stderr_debug(self): 32 | f = FFProbe() 33 | m = mock.MagicMock() 34 | with mock.patch.object(f, 'logger', 35 | new_callable=mock.PropertyMock(return_value=m)): 36 | line = 'debug' 37 | out = f.handle_stderr(line) 38 | self.assertEqual(out, '') 39 | m.assert_not_called() 40 | 41 | 42 | class InputsTestCase(TestCase): 43 | def test_init_inputs(self): 44 | src = inputs.input_file('filename', Stream(kind=VIDEO), 45 | allowed_extensions='m3u8') 46 | self.assertIsInstance(src, inputs.Input) 47 | self.assertEqual(src.allowed_extensions, 'm3u8') 48 | -------------------------------------------------------------------------------- /src/video_transcoding/transcoding/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/just-work/django-video-transcoding/aa4a945a319bb4bd540c2acf4fcc3cbd6e3fad26/src/video_transcoding/transcoding/__init__.py -------------------------------------------------------------------------------- /src/video_transcoding/transcoding/analysis.py: -------------------------------------------------------------------------------- 1 | from typing import Dict, Any, List 2 | 3 | from fffw.analysis import ffprobe, mediainfo 4 | from fffw.graph import meta 5 | 6 | 7 | class SourceAnalyzer(mediainfo.Analyzer): 8 | """ 9 | Universal source media analyzer. 10 | """ 11 | 12 | 13 | class MKVPlaylistAnalyzer(ffprobe.Analyzer): 14 | """ 15 | Analyzer for HLS playlist with .mkv fragments. 16 | """ 17 | 18 | def get_duration(self, track: Dict[str, Any]) -> meta.TS: 19 | """ 20 | Augment track duration with a value from container. 21 | 22 | This is legit if media contains only a single stream. 23 | """ 24 | duration = super().get_duration(track) 25 | if not duration and len(self.info.streams) == 1: 26 | duration = self.maybe_parse_duration(self.info.format.get('duration')) 27 | return duration 28 | 29 | 30 | class MKVSegmentAnalyzer(MKVPlaylistAnalyzer): 31 | """ 32 | Analyzer for audio/video segments in .NUT container. 33 | """ 34 | 35 | def get_bitrate(self, track: Dict[str, Any]) -> int: 36 | bitrate = super().get_bitrate(track) 37 | if bitrate == 0 and len(self.info.streams) == 1: 38 | bitrate = int(self.info.format.get('bit_rate', 0)) 39 | return bitrate 40 | 41 | 42 | class VideoResultAnalyzer(ffprobe.Analyzer): 43 | """ 44 | Analyzer for multi-stream video segments in MPEGTS container. 45 | """ 46 | 47 | 48 | class FFProbeHLSAnalyzer(ffprobe.Analyzer): 49 | """ 50 | Analyzer for multi-variant HLS results. 51 | """ 52 | 53 | def analyze(self) -> List[meta.Meta]: 54 | streams: List[meta.Meta] = [] 55 | for stream in self.info.streams: 56 | if stream.get('tags', {}).get('comment'): 57 | # Skip HLS alternative groups 58 | continue 59 | if stream["codec_type"] == "video": 60 | streams.append(self.video_meta_data(**stream)) 61 | elif stream["codec_type"] == "audio": 62 | streams.append(self.audio_meta_data(**stream)) 63 | else: 64 | # Skip side data 65 | continue 66 | return streams 67 | 68 | def get_duration(self, track: Dict[str, Any]) -> meta.TS: 69 | duration = super().get_duration(track) 70 | if duration: 71 | return duration 72 | return self.maybe_parse_duration(self.info.format.get('duration')) 73 | 74 | def get_bitrate(self, track: Dict[str, Any]) -> int: 75 | bitrate = super().get_bitrate(track) 76 | if bitrate: 77 | return bitrate 78 | variant_bitrate = int(track.get('tags', {}).get('variant_bitrate', 0)) 79 | # Revert multiplying real bitrate on 1.1 80 | # https://github.com/FFmpeg/FFmpeg/blob/n7.0.1/libavformat/hlsenc.c#L1493 81 | return round(variant_bitrate / 1.1) 82 | -------------------------------------------------------------------------------- /src/video_transcoding/transcoding/codecs.py: -------------------------------------------------------------------------------- 1 | from dataclasses import dataclass 2 | from typing import Optional 3 | 4 | from fffw.encoding import codecs 5 | from fffw.wrapper import param 6 | 7 | 8 | @dataclass 9 | class Copy(codecs.Copy): 10 | bitrate: Optional[int] = param(name='b', stream_suffix=True) 11 | 12 | 13 | @dataclass 14 | class AudioCodec(codecs.AudioCodec): 15 | rate: float = param(name='ar', stream_suffix=True) 16 | channels: int = param(name='ac', stream_suffix=True) 17 | 18 | 19 | @dataclass 20 | class VideoCodec(codecs.VideoCodec): 21 | force_key_frames: str = param() 22 | constant_rate_factor: int = param(name='crf') 23 | preset: str = param() 24 | max_rate: int = param(name='maxrate') 25 | buf_size: int = param(name='bufsize') 26 | profile: str = param(stream_suffix=True) 27 | gop: int = param(name='g') 28 | rate: float = param(name='r') 29 | pix_fmt: str = param() 30 | -------------------------------------------------------------------------------- /src/video_transcoding/transcoding/extract.py: -------------------------------------------------------------------------------- 1 | import abc 2 | import json 3 | from typing import List, cast, Any 4 | 5 | from pymediainfo import MediaInfo 6 | 7 | from fffw.analysis import ffprobe 8 | from fffw.graph import meta 9 | from video_transcoding.transcoding import analysis 10 | from video_transcoding.transcoding.ffprobe import FFProbe 11 | from video_transcoding.transcoding.metadata import Metadata 12 | from video_transcoding.utils import LoggerMixin 13 | 14 | 15 | class Extractor(LoggerMixin, abc.ABC): 16 | @abc.abstractmethod 17 | def get_meta_data(self, uri: str) -> Metadata: # pragma: no cover 18 | raise NotImplementedError() 19 | 20 | def ffprobe(self, uri: str, timeout: float = 60.0, **kwargs: Any) -> ffprobe.ProbeInfo: 21 | self.logger.debug("Probing %s", uri) 22 | ff = FFProbe(uri, show_format=True, show_streams=True, output_format='json', **kwargs) 23 | ret, output, errors = ff.run(timeout=timeout) 24 | if ret != 0: # pragma: no cover 25 | raise RuntimeError(f"ffprobe returned {ret}") 26 | return ffprobe.ProbeInfo(**json.loads(output)) 27 | 28 | def mediainfo(self, uri: str) -> MediaInfo: 29 | self.logger.debug("Mediainfo %s", uri) 30 | return MediaInfo.parse(uri) 31 | 32 | 33 | class SourceExtractor(Extractor): 34 | 35 | def get_meta_data(self, uri: str) -> Metadata: 36 | info = self.mediainfo(uri) 37 | video_streams: List[meta.VideoMeta] = [] 38 | audio_streams: List[meta.AudioMeta] = [] 39 | for s in analysis.SourceAnalyzer(info).analyze(): 40 | if isinstance(s, meta.VideoMeta): 41 | video_streams.append(s) 42 | elif isinstance(s, meta.AudioMeta): 43 | audio_streams.append(s) 44 | else: # pragma: no cover 45 | raise RuntimeError("unexpected stream kind") 46 | return Metadata( 47 | uri=uri, 48 | videos=video_streams, 49 | audios=audio_streams, 50 | ) 51 | 52 | 53 | class MKVExtractor(Extractor, abc.ABC): 54 | """ 55 | Supports analyzing media from playlists with .mkv segments. 56 | """ 57 | def ffprobe(self, uri: str, timeout: float = 60.0, **kwargs: Any) -> ffprobe.ProbeInfo: 58 | kwargs.setdefault('allowed_extensions', 'mkv') 59 | return super().ffprobe(uri, timeout=timeout, **kwargs) 60 | 61 | 62 | class SplitExtractor(MKVExtractor): 63 | """ 64 | Extracts source metadata from video and audio HLS playlists. 65 | """ 66 | 67 | def __init__(self, video_playlist: str, audio_file: str) -> None: 68 | super().__init__() 69 | self.video_playlist = video_playlist 70 | self.audio_file = audio_file 71 | 72 | def get_meta_data(self, uri: str) -> Metadata: 73 | video_uri = uri.replace('/split.json', f'/{self.video_playlist}') 74 | video_streams = analysis.MKVPlaylistAnalyzer(self.ffprobe(video_uri)).analyze() 75 | audio_uri = uri.replace('/split.json', f'/{self.audio_file}') 76 | audio_streams = analysis.MKVPlaylistAnalyzer(self.ffprobe(audio_uri)).analyze() 77 | return Metadata( 78 | uri=uri, 79 | videos=cast(List[meta.VideoMeta], video_streams), 80 | audios=cast(List[meta.AudioMeta], audio_streams), 81 | ) 82 | 83 | 84 | class VideoSegmentExtractor(MKVExtractor): 85 | """ 86 | Extracts metadata from video segments 87 | """ 88 | 89 | def get_meta_data(self, uri: str) -> Metadata: 90 | streams = analysis.MKVSegmentAnalyzer(self.ffprobe(uri)).analyze() 91 | return Metadata( 92 | uri=uri, 93 | videos=cast(List[meta.VideoMeta], streams), 94 | audios=[], 95 | ) 96 | 97 | 98 | class VideoResultExtractor(MKVExtractor): 99 | """ 100 | Extracts metadata from video segment transcoding results. 101 | """ 102 | 103 | def get_meta_data(self, uri: str) -> Metadata: 104 | streams = analysis.VideoResultAnalyzer(self.ffprobe(uri)).analyze() 105 | # Missing bitrate is OK because it varies among segments. 106 | return Metadata( 107 | uri=uri, 108 | videos=cast(List[meta.VideoMeta], streams), 109 | audios=[], 110 | ) 111 | 112 | 113 | class HLSExtractor(Extractor): 114 | """ 115 | Extracts metadata from HLS results. 116 | """ 117 | 118 | def get_meta_data(self, uri: str) -> Metadata: 119 | info = self.ffprobe(uri) 120 | video_streams: List[meta.VideoMeta] = [] 121 | audio_streams: List[meta.AudioMeta] = [] 122 | for s in analysis.FFProbeHLSAnalyzer(info).analyze(): 123 | if isinstance(s, meta.VideoMeta): 124 | video_streams.append(s) 125 | elif isinstance(s, meta.AudioMeta): 126 | audio_streams.append(s) 127 | else: # pragma: no cover 128 | raise RuntimeError("invalid stream kind") 129 | return Metadata( 130 | uri=uri, 131 | videos=video_streams, 132 | audios=audio_streams, 133 | ) 134 | -------------------------------------------------------------------------------- /src/video_transcoding/transcoding/ffprobe.py: -------------------------------------------------------------------------------- 1 | from dataclasses import dataclass 2 | from typing import Optional 3 | 4 | from fffw.encoding import ffprobe 5 | 6 | 7 | @dataclass 8 | class FFProbe(ffprobe.FFProbe): 9 | """ 10 | Extends ffprobe wrapper with new arguments and output filtering. 11 | """ 12 | allowed_extensions: Optional[str] = None 13 | 14 | def handle_stderr(self, line: str) -> str: 15 | if '[error]' in line: 16 | self.logger.error(line) 17 | return '' 18 | 19 | def handle_stdout(self, line: str) -> str: 20 | return line 21 | -------------------------------------------------------------------------------- /src/video_transcoding/transcoding/inputs.py: -------------------------------------------------------------------------------- 1 | from dataclasses import dataclass 2 | from typing import Any 3 | 4 | from fffw.encoding import inputs, Stream 5 | from fffw.wrapper import param 6 | 7 | 8 | @dataclass 9 | class Input(inputs.Input): 10 | allowed_extensions: str = param() 11 | 12 | 13 | def input_file(filename: str, *streams: Stream, **kwargs: Any) -> Input: 14 | kwargs['input_file'] = filename 15 | if streams: 16 | # skip empty streams list to force Input.streams default_factory 17 | kwargs['streams'] = streams 18 | return Input(**kwargs) 19 | -------------------------------------------------------------------------------- /src/video_transcoding/transcoding/metadata.py: -------------------------------------------------------------------------------- 1 | from copy import deepcopy 2 | from dataclasses import dataclass, asdict 3 | from pprint import pformat 4 | from typing import List, TYPE_CHECKING 5 | 6 | from fffw.encoding import Stream 7 | from fffw.graph import meta 8 | 9 | if TYPE_CHECKING: # pragma: no cover 10 | from _typeshed import DataclassInstance 11 | else: 12 | DataclassInstance = object 13 | 14 | 15 | def scene_from_native(data: dict) -> meta.Scene: 16 | return meta.Scene( 17 | duration=meta.TS(data['duration']), 18 | start=meta.TS(data['start']), 19 | position=meta.TS(data['position']), 20 | stream=data['stream'] 21 | ) 22 | 23 | 24 | def get_meta_kwargs(data: dict) -> dict: 25 | kwargs = deepcopy(data) 26 | kwargs['start'] = meta.TS(data['start']) 27 | kwargs['duration'] = meta.TS(data['duration']) 28 | kwargs['scenes'] = [scene_from_native(s) for s in data['scenes']] 29 | return kwargs 30 | 31 | 32 | def video_meta_from_native(data: dict) -> meta.VideoMeta: 33 | return meta.VideoMeta(**get_meta_kwargs(data)) 34 | 35 | 36 | def audio_meta_from_native(data: dict) -> meta.AudioMeta: 37 | return meta.AudioMeta(**get_meta_kwargs(data)) 38 | 39 | 40 | @dataclass(repr=False) 41 | class Metadata(DataclassInstance): 42 | uri: str 43 | videos: List[meta.VideoMeta] 44 | audios: List[meta.AudioMeta] 45 | 46 | @classmethod 47 | def from_native(cls, data: dict) -> 'Metadata': 48 | return cls( 49 | videos=list(map(video_meta_from_native, data['videos'])), 50 | audios=list(map(audio_meta_from_native, data['audios'])), 51 | uri=data['uri'], 52 | ) 53 | 54 | @property 55 | def video(self) -> meta.VideoMeta: 56 | return self.videos[0] 57 | 58 | @property 59 | def audio(self) -> meta.AudioMeta: 60 | return self.audios[0] 61 | 62 | @property 63 | def streams(self) -> List[Stream]: 64 | streams = [] 65 | for vm in self.videos: 66 | streams.append(Stream(meta.VIDEO, vm)) 67 | for am in self.audios: 68 | streams.append(Stream(meta.AUDIO, am)) 69 | return streams 70 | 71 | def __repr__(self) -> str: 72 | return f'{self.__class__.__name__}\n{pformat(asdict(self))}' 73 | -------------------------------------------------------------------------------- /src/video_transcoding/transcoding/outputs.py: -------------------------------------------------------------------------------- 1 | from dataclasses import dataclass 2 | from typing import Optional 3 | 4 | from fffw.encoding import outputs 5 | from fffw.wrapper import param 6 | 7 | 8 | @dataclass 9 | class Output(outputs.Output): 10 | copyts: bool = param(default=False) 11 | avoid_negative_ts: str = param() 12 | 13 | 14 | @dataclass 15 | class HLSOutput(Output): 16 | hls_time: Optional[float] = param(default=2) 17 | hls_playlist_type: Optional[str] = None 18 | var_stream_map: Optional[str] = None 19 | hls_segment_filename: Optional[str] = None 20 | master_pl_name: Optional[str] = None 21 | muxdelay: Optional[str] = None 22 | reset_timestamps: Optional[int] = 0 23 | 24 | 25 | @dataclass 26 | class SegmentOutput(Output): 27 | """ 28 | Segment muxer 29 | """ 30 | segment_format: Optional[str] = None 31 | segment_list: Optional[str] = None 32 | segment_list_type: Optional[str] = None 33 | segment_time: Optional[float] = None 34 | 35 | 36 | @dataclass 37 | class FileOutput(Output): 38 | method: Optional[str] = param(default=None) 39 | muxdelay: Optional[str] = None 40 | -------------------------------------------------------------------------------- /src/video_transcoding/transcoding/profiles.py: -------------------------------------------------------------------------------- 1 | from dataclasses import dataclass 2 | from typing import List, Optional, Any, Dict 3 | 4 | from fffw.graph import VideoMeta, AudioMeta 5 | 6 | 7 | @dataclass 8 | class VideoTrack: 9 | """ 10 | Settings for a single video stream in resulting media file 11 | """ 12 | id: str 13 | codec: str 14 | constant_rate_factor: int 15 | preset: str 16 | max_rate: int 17 | buf_size: int 18 | profile: str 19 | pix_fmt: str 20 | width: int 21 | height: int 22 | frame_rate: float 23 | gop_size: int 24 | force_key_frames: str 25 | 26 | @classmethod 27 | def from_native(cls, data: Dict[str, Any]) -> "VideoTrack": 28 | return cls(**data) 29 | 30 | 31 | @dataclass 32 | class AudioTrack: 33 | """ 34 | Settings for a single audio stream in resulting media file 35 | """ 36 | id: str 37 | codec: str 38 | bitrate: int 39 | channels: int 40 | sample_rate: int 41 | 42 | @classmethod 43 | def from_native(cls, data: Dict[str, Any]) -> "AudioTrack": 44 | return cls(**data) 45 | 46 | 47 | @dataclass 48 | class VideoCondition: 49 | """ 50 | Condition for source video stream for video profile selection 51 | """ 52 | min_width: int = 0 53 | min_height: int = 0 54 | min_bitrate: int = 0 55 | min_frame_rate: float = 0.0 56 | min_dar: float = 0.0 57 | max_dar: float = 0.0 58 | 59 | def is_valid(self, meta: VideoMeta) -> bool: 60 | """ 61 | :param meta: source video stream metadata 62 | :return: True if source stream satisfies condition. 63 | """ 64 | return ( 65 | meta.width >= self.min_width and 66 | meta.height >= self.min_height and 67 | meta.bitrate >= self.min_bitrate and 68 | meta.frame_rate >= self.min_frame_rate and 69 | (not self.min_dar or meta.dar >= self.min_dar) and 70 | (not self.max_dar or meta.dar <= self.max_dar) 71 | ) 72 | 73 | 74 | @dataclass 75 | class AudioCondition: 76 | """ 77 | Condition for source audio stream for video profile selection 78 | """ 79 | 80 | min_sample_rate: int = 0 81 | min_bitrate: int = 0 82 | 83 | def is_valid(self, meta: AudioMeta) -> bool: 84 | """ 85 | :param meta: source video stream metadata 86 | :return: True if source stream satisfies condition. 87 | """ 88 | return ( 89 | meta.bitrate >= self.min_bitrate and 90 | meta.sampling_rate >= self.min_sample_rate 91 | ) 92 | 93 | 94 | @dataclass 95 | class VideoProfile: 96 | """ 97 | Video transcoding profile. 98 | """ 99 | condition: VideoCondition 100 | segment_duration: float 101 | video: List[str] # List of VideoTrack ids defined in a preset 102 | 103 | 104 | @dataclass 105 | class AudioProfile: 106 | """ 107 | Audio transcoding profile. 108 | """ 109 | condition: AudioCondition 110 | audio: List[str] # List of AudioTrack ids defined in a preset 111 | 112 | 113 | @dataclass 114 | class Container: 115 | """ 116 | Output file format 117 | """ 118 | segment_duration: Optional[float] = None 119 | 120 | @classmethod 121 | def from_native(cls, data: Dict[str, Any]) -> "Container": 122 | return cls(**data) 123 | 124 | 125 | @dataclass 126 | class Profile: 127 | """ 128 | Selected transcoding profile containing a number of audio and video streams. 129 | """ 130 | video: List[VideoTrack] 131 | audio: List[AudioTrack] 132 | container: Container 133 | 134 | @classmethod 135 | def from_native(cls, data: Dict[str, Any]) -> "Profile": 136 | return cls( 137 | video=list(map(VideoTrack.from_native, data['video'])), 138 | audio=list(map(AudioTrack.from_native, data['audio'])), 139 | container=Container.from_native(data['container']), 140 | ) 141 | 142 | 143 | @dataclass 144 | class Preset: 145 | """ 146 | A set of video and audio profiles to select from. 147 | """ 148 | video_profiles: List[VideoProfile] 149 | audio_profiles: List[AudioProfile] 150 | video: List[VideoTrack] 151 | audio: List[AudioTrack] 152 | 153 | def select_profile(self, 154 | video: VideoMeta, 155 | audio: AudioMeta) -> Profile: 156 | video_profile = None 157 | for vp in self.video_profiles: 158 | if vp.condition.is_valid(video): 159 | video_profile = vp 160 | break 161 | if video_profile is None: 162 | raise RuntimeError("No compatible video profiles") 163 | 164 | audio_profile = None 165 | for ap in self.audio_profiles: 166 | if ap.condition.is_valid(audio): 167 | audio_profile = ap 168 | break 169 | if audio_profile is None: 170 | raise RuntimeError("No compatible audio profiles") 171 | 172 | # noinspection PyTypeChecker 173 | return Profile( 174 | video=[v for v in self.video if v.id in video_profile.video], 175 | audio=[a for a in self.audio if a.id in audio_profile.audio], 176 | container=Container( 177 | segment_duration=video_profile.segment_duration), 178 | ) 179 | 180 | 181 | # Selecting HLS Segment duration 182 | # ============================== 183 | # 184 | # 48 kHz * 1024 samples per frame (AAC) and 30 fps (H264) have common 185 | # "pretty" duration 1.6 seconds - 48 H264-frames (one GOP) and 75 AAC-frames. 186 | # This duration satisfy integer equation: M * 1024/48000 = N * 1/30 187 | # * 48 kHz @ 25 fps - 4.8 seconds 188 | # * 44100 Hz @ 25 fps - 10.24 seconds 189 | # * 44100 Hz @ 30 fps - just don't use this (1536 frames or 51.2 seconds) 190 | 191 | # Default frame rate 192 | FRAME_RATE = 30 193 | # HLS Segment duration step, seconds 194 | SEGMENT_SIZE = 4.8 195 | # H.264 Group of pixels duration, seconds 196 | GOP_DURATION = 1.6 197 | # Force key frame every N seconds 198 | KEY_FRAMES = 'expr:if(isnan(prev_forced_t),1,gte(t,prev_forced_t+{sec}))' 199 | 200 | DEFAULT_PRESET = Preset( 201 | video_profiles=[ 202 | VideoProfile( 203 | condition=VideoCondition( 204 | min_width=1920, 205 | min_height=1080, 206 | min_bitrate=4_000_000, 207 | min_frame_rate=0.0, 208 | min_dar=0.0, 209 | max_dar=0.0, 210 | ), 211 | segment_duration=SEGMENT_SIZE, 212 | video=['1080p', '720p', '480p', '360p'] 213 | ), 214 | VideoProfile( 215 | condition=VideoCondition( 216 | min_width=1280, 217 | min_height=720, 218 | min_bitrate=2_500_000, 219 | min_frame_rate=0.0, 220 | min_dar=0.0, 221 | max_dar=0.0, 222 | ), 223 | segment_duration=SEGMENT_SIZE, 224 | video=['720p', '480p', '360p'] 225 | ), 226 | VideoProfile( 227 | condition=VideoCondition( 228 | min_width=854, 229 | min_height=480, 230 | min_bitrate=1_200_000, 231 | min_frame_rate=0.0, 232 | min_dar=0.0, 233 | max_dar=0.0, 234 | ), 235 | segment_duration=SEGMENT_SIZE, 236 | video=['480p', '360p'] 237 | ), 238 | VideoProfile( 239 | condition=VideoCondition( 240 | min_width=0, 241 | min_height=0, 242 | min_bitrate=0, 243 | min_frame_rate=0.0, 244 | min_dar=0.0, 245 | max_dar=0.0, 246 | ), 247 | segment_duration=SEGMENT_SIZE, 248 | video=['360p'] 249 | ), 250 | ], 251 | audio_profiles=[ 252 | AudioProfile( 253 | condition=AudioCondition( 254 | min_bitrate=0, 255 | min_sample_rate=0 256 | ), 257 | audio=['192k'] 258 | ), 259 | ], 260 | 261 | video=[ 262 | VideoTrack( 263 | id='1080p', 264 | codec='libx264', 265 | profile='high', 266 | preset='slow', 267 | constant_rate_factor=23, 268 | max_rate=5_000_000, 269 | buf_size=10_000_000, 270 | pix_fmt='yuv420p', 271 | width=1920, 272 | height=1080, 273 | force_key_frames=KEY_FRAMES.format(sec=SEGMENT_SIZE), 274 | gop_size=round(GOP_DURATION * FRAME_RATE), 275 | frame_rate=FRAME_RATE, 276 | ), 277 | VideoTrack( 278 | id='720p', 279 | codec='libx264', 280 | profile='high', 281 | preset='slow', 282 | constant_rate_factor=23, 283 | max_rate=3_000_000, 284 | buf_size=6_000_000, 285 | pix_fmt='yuv420p', 286 | width=1280, 287 | height=720, 288 | force_key_frames=KEY_FRAMES.format(sec=SEGMENT_SIZE), 289 | gop_size=round(GOP_DURATION * FRAME_RATE), 290 | frame_rate=FRAME_RATE, 291 | ), 292 | VideoTrack( 293 | id='480p', 294 | codec='libx264', 295 | profile='main', 296 | preset='slow', 297 | constant_rate_factor=23, 298 | max_rate=1_500_000, 299 | buf_size=3_000_000, 300 | pix_fmt='yuv420p', 301 | width=854, 302 | height=480, 303 | force_key_frames=KEY_FRAMES.format(sec=SEGMENT_SIZE), 304 | gop_size=round(GOP_DURATION * FRAME_RATE), 305 | frame_rate=FRAME_RATE, 306 | ), 307 | VideoTrack( 308 | id='360p', 309 | codec='libx264', 310 | profile='main', 311 | preset='slow', 312 | constant_rate_factor=23, 313 | max_rate=800_000, 314 | buf_size=1_600_000, 315 | pix_fmt='yuv420p', 316 | width=640, 317 | height=360, 318 | force_key_frames=KEY_FRAMES.format(sec=SEGMENT_SIZE), 319 | gop_size=round(GOP_DURATION * FRAME_RATE), 320 | frame_rate=FRAME_RATE, 321 | ), 322 | ], 323 | audio=[ 324 | AudioTrack( 325 | id='192k', 326 | codec='aac', 327 | bitrate=192000, 328 | channels=2, 329 | sample_rate=48000, 330 | ), 331 | ] 332 | ) 333 | -------------------------------------------------------------------------------- /src/video_transcoding/transcoding/workspace.py: -------------------------------------------------------------------------------- 1 | import abc 2 | import http 3 | import os 4 | import shutil 5 | from pathlib import Path 6 | from typing import Optional, Any 7 | from urllib.parse import urlparse, ParseResult 8 | 9 | import requests 10 | 11 | from video_transcoding import defaults 12 | from video_transcoding.utils import LoggerMixin 13 | 14 | 15 | class Resource(abc.ABC): 16 | """ 17 | Directory or file in a workspace. 18 | """ 19 | 20 | def __init__(self, *parts: str): 21 | self.parts = parts 22 | 23 | @property 24 | def basename(self) -> str: 25 | return self.parts[-1] if self.parts else '' 26 | 27 | @property 28 | @abc.abstractmethod 29 | def trailing_slash(self) -> str: # pragma: no cover 30 | raise NotImplementedError 31 | 32 | @property 33 | def path(self) -> str: 34 | return '/'.join(('', *self.parts)) 35 | 36 | @property 37 | def parent(self) -> Optional["Collection"]: 38 | if not self.parts: 39 | return None 40 | return Collection(*self.parts[:-1]) 41 | 42 | def __repr__(self) -> str: 43 | return self.path 44 | 45 | def __eq__(self, other: Any) -> bool: # pragma: no cover 46 | if not isinstance(other, Resource): 47 | return False 48 | return self.parts == other.parts 49 | 50 | 51 | class Collection(Resource): 52 | """ 53 | Directory in a workspace 54 | """ 55 | 56 | def __repr__(self) -> str: 57 | return '/'.join((self.path, '')) 58 | 59 | @property 60 | def trailing_slash(self) -> str: 61 | return '/' 62 | 63 | def collection(self, *parts: str) -> "Collection": 64 | return Collection(*self.parts, *parts) 65 | 66 | def file(self, *parts: str) -> "File": 67 | return File(*self.parts, *parts) 68 | 69 | 70 | class File(Resource): 71 | """ 72 | File in a workspace. 73 | """ 74 | 75 | @property 76 | def trailing_slash(self) -> str: 77 | return '' 78 | 79 | 80 | class Workspace(LoggerMixin, abc.ABC): 81 | 82 | @abc.abstractmethod 83 | def create_collection(self, c: Collection) -> None: # pragma: no cover 84 | raise NotImplementedError 85 | 86 | def get_absolute_uri(self, r: Resource) -> ParseResult: 87 | path = '/'.join((*Path(self.uri.path.lstrip('/')).parts, *r.parts)) 88 | if not path: 89 | return self.uri 90 | return self.uri._replace(path='/' + path + r.trailing_slash) 91 | 92 | @abc.abstractmethod 93 | def delete_collection(self, c: Collection) -> None: # pragma: no cover 94 | raise NotImplementedError 95 | 96 | @abc.abstractmethod 97 | def read(self, f: File) -> str: # pragma: no cover 98 | raise NotImplementedError 99 | 100 | @abc.abstractmethod 101 | def write(self, f: File, content: str) -> None: # pragma: no cover 102 | raise NotImplementedError 103 | 104 | @abc.abstractmethod 105 | def exists(self, r: Resource) -> bool: # pragma: no cover 106 | raise NotImplementedError 107 | 108 | def __init__(self, uri: ParseResult) -> None: 109 | super().__init__() 110 | self.uri = uri._replace(path=uri.path.rstrip('/')) 111 | self.root = Collection() 112 | 113 | def ensure_collection(self, path: str) -> Collection: 114 | """ 115 | Ensures that a directory with relative path exists. 116 | 117 | :returns: complete uri for a directory. 118 | """ 119 | c = self.root.collection(*Path(path.lstrip('/')).parts) 120 | self.create_collection(c) 121 | return c 122 | 123 | 124 | class FileSystemWorkspace(Workspace): 125 | 126 | def __init__(self, base: str) -> None: 127 | uri = urlparse(base, scheme='file') 128 | super().__init__(uri) 129 | 130 | def create_collection(self, c: Collection) -> None: 131 | uri = self.get_absolute_uri(c) 132 | self.logger.debug("mkdir %s", uri.path) 133 | os.makedirs(uri.path, exist_ok=True) 134 | 135 | def delete_collection(self, c: Collection) -> None: 136 | uri = self.get_absolute_uri(c) 137 | self.logger.debug("rmtree %s", uri.path) 138 | try: 139 | shutil.rmtree(uri.path) 140 | except FileNotFoundError: 141 | self.logger.warning("dir not found: %s", uri.path) 142 | 143 | def exists(self, r: Resource) -> bool: 144 | uri = self.get_absolute_uri(r) 145 | self.logger.debug("exists %s", uri.path) 146 | return os.path.exists(uri.path) 147 | 148 | def read(self, r: File) -> str: 149 | uri = self.get_absolute_uri(r) 150 | self.logger.debug("read %s", uri.path) 151 | with open(uri.path, 'r') as f: 152 | return f.read() 153 | 154 | def write(self, r: File, content: str) -> None: 155 | uri = self.get_absolute_uri(r) 156 | self.logger.debug("write %s", uri.path) 157 | with open(uri.path, 'w') as f: 158 | f.write(content) 159 | 160 | 161 | class WebDAVWorkspace(Workspace): 162 | def __init__(self, base: str) -> None: 163 | super().__init__(urlparse(base)) 164 | self.session = requests.Session() 165 | 166 | def create_collection(self, c: Collection) -> None: 167 | self._mkcol(self.root) 168 | tmp = self.root 169 | for p in c.parts: 170 | tmp = tmp.collection(p) 171 | self._mkcol(tmp) 172 | 173 | def delete_collection(self, c: Collection) -> None: 174 | uri = self.get_absolute_uri(c) 175 | self.logger.debug("delete %s", uri) 176 | timeout = (defaults.VIDEO_CONNECT_TIMEOUT, 177 | defaults.VIDEO_REQUEST_TIMEOUT,) 178 | resp = self.session.request("DELETE", uri.geturl(), timeout=timeout) 179 | if resp.status_code == http.HTTPStatus.NOT_FOUND: 180 | self.logger.warning("collection not found: %s", uri.geturl()) 181 | return 182 | resp.raise_for_status() 183 | 184 | def exists(self, r: Resource) -> bool: 185 | uri = self.get_absolute_uri(r) 186 | self.logger.debug("exists %s", uri.geturl()) 187 | timeout = (defaults.VIDEO_CONNECT_TIMEOUT, 188 | defaults.VIDEO_REQUEST_TIMEOUT,) 189 | resp = self.session.request("HEAD", uri.geturl(), timeout=timeout) 190 | if resp.status_code == http.HTTPStatus.NOT_FOUND: 191 | return False 192 | resp.raise_for_status() 193 | return True 194 | 195 | def read(self, r: File) -> str: 196 | uri = self.get_absolute_uri(r) 197 | self.logger.debug("get %s", uri.geturl()) 198 | timeout = (defaults.VIDEO_CONNECT_TIMEOUT, 199 | defaults.VIDEO_REQUEST_TIMEOUT,) 200 | resp = self.session.request("GET", uri.geturl(), timeout=timeout) 201 | resp.raise_for_status() 202 | return resp.text 203 | 204 | def write(self, r: File, content: str) -> None: 205 | uri = self.get_absolute_uri(r) 206 | self.logger.debug("put %s", uri.geturl()) 207 | resp = self.session.request("PUT", uri.geturl(), data=content) 208 | resp.raise_for_status() 209 | 210 | def _mkcol(self, c: Collection) -> None: 211 | uri = self.get_absolute_uri(c) 212 | if not uri.path.endswith('/'): 213 | uri = uri._replace(path=uri.path + '/') 214 | self.logger.debug("mkcol %s", uri.geturl()) 215 | timeout = (defaults.VIDEO_CONNECT_TIMEOUT, 216 | defaults.VIDEO_REQUEST_TIMEOUT,) 217 | resp = self.session.request("MKCOL", uri.geturl(), timeout=timeout) 218 | if resp.status_code != http.HTTPStatus.METHOD_NOT_ALLOWED: 219 | # MKCOL returns 405 if collection already exists and 220 | # 409 if existing resource is not a collection 221 | resp.raise_for_status() 222 | 223 | 224 | def init(base: str) -> Workspace: 225 | uri = urlparse(base) 226 | if uri.scheme == 'file': 227 | return FileSystemWorkspace(base) 228 | elif uri.scheme == 'dav': 229 | uri = uri._replace(scheme='http') 230 | return WebDAVWorkspace(uri.geturl()) 231 | elif uri.scheme == 'davs': 232 | uri = uri._replace(scheme='https') 233 | return WebDAVWorkspace(uri.geturl()) 234 | else: 235 | raise ValueError(base) 236 | -------------------------------------------------------------------------------- /src/video_transcoding/utils.py: -------------------------------------------------------------------------------- 1 | from logging import getLogger 2 | from typing import Any 3 | from django.utils.translation import gettext_lazy as _ 4 | 5 | 6 | class LoggerMixin: 7 | """ 8 | A mixin for logger injection. 9 | 10 | Should not be used with Django models, because Logger contains 11 | non-serializable threading.Lock object. 12 | """ 13 | 14 | def __init__(self, *args: Any, **kwargs: Any) -> None: 15 | super().__init__(*args, **kwargs) # type: ignore 16 | cls = self.__class__ 17 | self.logger = getLogger(f'{cls.__module__}.{cls.__name__}') 18 | 19 | 20 | # Adding missing translations for django-model-utils TimeStampedModel 21 | _('created') 22 | _('modified') 23 | -------------------------------------------------------------------------------- /tox.ini: -------------------------------------------------------------------------------- 1 | [tox] 2 | envlist = 3 | {py3.9,py3.10}-django3.2 4 | {py3.9,py3.10}-django4.0 5 | {py3.9,py3.10,py3.11}-django4.1 6 | {py3.9,py3.10,py3.11,py3.12}-django4.2 7 | {py3.10,py3.11,py3.12}-django5.0 8 | {py3.10,py3.11,py3.13}-django5.1 9 | 10 | [gh-actions] 11 | python = 12 | 3.9: py3.9 13 | 3.10: py3.10 14 | 3.11: py3.11 15 | 3.12: py3.12 16 | 3.13: py3.13 17 | 18 | [testenv] 19 | changedir = ./src 20 | basepython = 21 | py3.9: python3.9 22 | py3.10: python3.10 23 | py3.11: python3.11 24 | py3.12: python3.12 25 | py3.13: python3.13 26 | deps = 27 | -r requirements.txt 28 | django3.2: Django~=3.2.0 29 | django4.0: Django~=4.0.0 30 | django4.1: Django~=4.1.0 31 | django4.2: Django~=4.2.0 32 | django5.0: Django~=5.0.0 33 | django5.1: Django~=5.1.0 34 | commands = python manage.py test --------------------------------------------------------------------------------