├── .codecov.yml ├── clairmeta ├── utils │ ├── __init__.py │ ├── crypto.py │ ├── uuid.py │ ├── time.py │ ├── isdcf.py │ └── file.py ├── info.py ├── xsd │ ├── SMPTE-429-10-2008-Main-Stereo-Picture-CPL.xsd │ ├── PROTO-ASDCP-CC_CPL_20070926.xsd │ ├── 437-Y-2007-Main-Stereo-Picture-CPL.xsd │ ├── PROTO-ASDCP-PKL-20040311.xsd │ ├── SMPTE-429-8-2006-PKL.xsd │ ├── PROTO-ASDCP-AM-20040311.xsd │ ├── SMPTE-429-9-2007-AM.xsd │ ├── SMPTE-430-3-2006-ETM.xsd │ ├── catalog.xml │ ├── SMPTE-430-1-2006-KDM.xsd │ ├── xml.xsd │ ├── xmldsig-core-schema.dtd │ ├── xenc-schema.xsd │ ├── SMPTE-429-16-2014-CPL-Metadata.xsd │ └── SMPTE-429-7-2006-CPL.xsd ├── __init__.py ├── exception.py ├── dcp_check_vol.py ├── logger.py ├── sequence.py ├── dcp_check_execution.py ├── dcp_check_atmos.py ├── dcp_check_pkl.py ├── dcp_check_utils.py ├── profile.py ├── sequence_check.py ├── dcp_check_sound.py ├── cli.py ├── dcp_check_global.py ├── dcp_check.py ├── dcp_check_am.py ├── report.py └── settings.py ├── .git-blame-ignore-revs ├── CHANGELOG.rst ├── MANIFEST.IN ├── .github ├── workflows │ ├── ruff.yml │ ├── publish-to-pypi.yml │ └── test-package.yml └── scripts │ ├── linux │ └── apt │ │ └── install_test_env.sh │ ├── macos │ └── install_test_env.sh │ └── windows │ └── install_test_env.sh ├── tests ├── README.rst ├── test_xml_utils.py ├── test_profile.py ├── test_license.py ├── test_mxf_probe.py ├── __init__.py ├── test_sequence.py ├── test_cli.py ├── test_dcp_parse.py └── test_dcp_check.py ├── .gitignore ├── .codeclimate.yml ├── CONTRIBUTORS ├── LICENSE ├── pyproject.toml ├── scripts └── dcp_copy.py └── README.rst /.codecov.yml: -------------------------------------------------------------------------------- 1 | comment: false -------------------------------------------------------------------------------- /clairmeta/utils/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /clairmeta/info.py: -------------------------------------------------------------------------------- 1 | __license__ = "BSD" 2 | __author__ = "Ymagis" 3 | __version__ = "1.6.1" 4 | -------------------------------------------------------------------------------- /.git-blame-ignore-revs: -------------------------------------------------------------------------------- 1 | # Migrate code style to Black 2 | 3cdf4ab81d821048f91fda1e354b60b600ca2b95 -------------------------------------------------------------------------------- /CHANGELOG.rst: -------------------------------------------------------------------------------- 1 | Clairmeta - Changelog 2 | ===================== 3 | 4 | The releases changes are available on Github: https://github.com/Ymagis/ClairMeta/releases -------------------------------------------------------------------------------- /MANIFEST.IN: -------------------------------------------------------------------------------- 1 | include LICENSE CONTRIBUTORS MANIFEST.in *.rst 2 | graft clairmeta/xsd 3 | graft tests 4 | prune tests/resources 5 | global-exclude *.py[cod] 6 | global-exclude .DS_Store -------------------------------------------------------------------------------- /.github/workflows/ruff.yml: -------------------------------------------------------------------------------- 1 | name: Ruff 2 | 3 | on: [ push, pull_request ] 4 | 5 | jobs: 6 | ruff: 7 | runs-on: ubuntu-latest 8 | steps: 9 | - uses: actions/checkout@v4 10 | - uses: chartboost/ruff-action@v1 -------------------------------------------------------------------------------- /tests/README.rst: -------------------------------------------------------------------------------- 1 | Test materials can be found at https://github.com/Ymagis/ClairMeta_Data 2 | 3 | To download them at the expected location, please use : 4 | 5 | :: 6 | 7 | git clone https://github.com/Ymagis/ClairMeta_Data tests/resources -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *~ 2 | *.log 3 | *.pyc 4 | .*.swp 5 | .*.swo 6 | .DS_Store 7 | 8 | tests/resources 9 | build*/ 10 | dist 11 | clairmeta.egg-info 12 | __pycache__ 13 | .pybuild 14 | .coverage 15 | .mypy_cache/ 16 | .pytest_cache/ 17 | .vscode/ 18 | -------------------------------------------------------------------------------- /.codeclimate.yml: -------------------------------------------------------------------------------- 1 | engines: 2 | fixme: 3 | enabled: true 4 | pep8: 5 | enabled: true 6 | radon: 7 | enabled: true 8 | duplication: 9 | enabled: true 10 | config: 11 | languages: 12 | - python: 13 | 14 | ratings: 15 | paths: 16 | - clairmeta/** 17 | 18 | exclude_paths: 19 | - pkg/**/* 20 | - clairmeta/xsd/* 21 | - clairmeta/settings.py 22 | - tests/* 23 | -------------------------------------------------------------------------------- /CONTRIBUTORS: -------------------------------------------------------------------------------- 1 | # List of Contributors to this software - in alphabetic order 2 | # 3 | Antoine Cozzi 4 | Benjamin Gigon 5 | Brice Gros 6 | Carl Hetherington 7 | Carole Vasseur 8 | Cédric Lejeune 9 | Kevin Le Bihan 10 | Kieran O'Leary 11 | Laureen Gautier 12 | Laurent Bodson 13 | Leo Winter 14 | Mattias Mattsson 15 | Maxime Huynh 16 | Nelsy Zami 17 | Nicolas Mifsud 18 | Quentin Bisserie 19 | Rémi Achard 20 | Robin François 21 | Thomas Capricelli 22 | Tomasz Witkowski 23 | Willy Nuez 24 | # -- 25 | -------------------------------------------------------------------------------- /clairmeta/xsd/SMPTE-429-10-2008-Main-Stereo-Picture-CPL.xsd: -------------------------------------------------------------------------------- 1 | 2 | 6 | 7 | 8 | 9 | 10 | 11 | -------------------------------------------------------------------------------- /.github/scripts/linux/apt/install_test_env.sh: -------------------------------------------------------------------------------- 1 | set -ex 2 | 3 | sudo apt-get -y update 4 | 5 | ## 6 | # Install asdcplib 7 | 8 | sudo apt-get -y install libssl-dev libxerces-c-dev libexpat-dev 9 | 10 | BASE_DIR=`pwd` 11 | WORK_DIR=`mktemp -d` 12 | cd "$WORK_DIR" 13 | 14 | git clone https://github.com/cinecert/asdcplib.git && cd asdcplib 15 | git checkout rel_2_13_1 16 | mkdir build && cd build 17 | 18 | cmake .. 19 | sudo cmake --build . --target install --config Release -- -j$(nproc) 20 | 21 | cd "$BASE_DIR" 22 | rm -rf "$WORK_DIR" 23 | 24 | 25 | ## 26 | # ClairMeta dependencies 27 | 28 | sudo apt-get -y install mediainfo sox 29 | -------------------------------------------------------------------------------- /.github/scripts/macos/install_test_env.sh: -------------------------------------------------------------------------------- 1 | set -ex 2 | 3 | brew update 4 | 5 | ## 6 | # Install asdcplib 7 | 8 | brew install openssl xerces-c 9 | 10 | BASE_DIR=`pwd` 11 | WORK_DIR=`mktemp -d` 12 | cd "$WORK_DIR" 13 | 14 | git clone https://github.com/cinecert/asdcplib.git && cd asdcplib 15 | git checkout rel_2_13_1 16 | mkdir build && cd build 17 | 18 | cmake -DCMAKE_MACOSX_RPATH=ON -DCMAKE_POLICY_VERSION_MINIMUM=3.5 .. 19 | sudo cmake --build . --target install --config Release -- -j$(sysctl -n hw.ncpu) 20 | 21 | cd "$BASE_DIR" 22 | rm -rf "$WORK_DIR" 23 | 24 | 25 | ## 26 | # ClairMeta other dependencies 27 | brew install mediainfo sox 28 | -------------------------------------------------------------------------------- /clairmeta/__init__.py: -------------------------------------------------------------------------------- 1 | # Clairmeta - (C) YMAGIS S.A. 2 | # See LICENSE for more information 3 | 4 | from clairmeta.info import __license__, __author__, __version__ 5 | from clairmeta.dcp import DCP 6 | from clairmeta.sequence import Sequence 7 | from clairmeta.logger import get_log 8 | from clairmeta.utils.probe import check_command, PROBE_DEPS 9 | 10 | 11 | __all__ = ["DCP", "Sequence"] 12 | __license__ = __license__ 13 | __author__ = __author__ 14 | __version__ = __version__ 15 | 16 | 17 | # External dependencies check 18 | for d in PROBE_DEPS: 19 | if not check_command(d): 20 | get_log().warning("Missing dependency : {}".format(d)) 21 | -------------------------------------------------------------------------------- /clairmeta/xsd/PROTO-ASDCP-CC_CPL_20070926.xsd: -------------------------------------------------------------------------------- 1 | 2 | 7 | 9 | 10 | 11 | -------------------------------------------------------------------------------- /clairmeta/xsd/437-Y-2007-Main-Stereo-Picture-CPL.xsd: -------------------------------------------------------------------------------- 1 | 2 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | -------------------------------------------------------------------------------- /clairmeta/exception.py: -------------------------------------------------------------------------------- 1 | # Clairmeta - (C) YMAGIS S.A. 2 | # See LICENSE for more information 3 | 4 | 5 | class ClairMetaException(Exception): 6 | """Base class for all exception raised by this library.""" 7 | 8 | pass 9 | 10 | 11 | class CommandException(ClairMetaException): 12 | """Raised when external command fails.""" 13 | 14 | pass 15 | 16 | 17 | class ProbeException(ClairMetaException): 18 | """Raised when probing a DCP fails.""" 19 | 20 | def __init__(self, msg): 21 | super(ProbeException, self).__init__(str(msg)) 22 | 23 | 24 | class CheckException(ClairMetaException): 25 | """Non recoverable errors while checking a DCP. 26 | 27 | This is not to be used for regular check errors, where we instead use 28 | ``error()`` and ``fatal_error()`` methods. 29 | 30 | """ 31 | 32 | pass 33 | -------------------------------------------------------------------------------- /.github/workflows/publish-to-pypi.yml: -------------------------------------------------------------------------------- 1 | name: Publish package to PyPI 2 | 3 | on: 4 | push: 5 | tags: 6 | - '*' 7 | 8 | jobs: 9 | build-n-publish: 10 | runs-on: ubuntu-latest 11 | environment: 12 | name: pypi 13 | url: https://pypi.org/project/clairmeta/ 14 | permissions: 15 | id-token: write 16 | steps: 17 | - uses: actions/checkout@v4 18 | - name: Set up Python 3.9 19 | uses: actions/setup-python@v5 20 | with: 21 | python-version: "3.9" 22 | - name: Install pypa/build 23 | run: >- 24 | python -m 25 | pip install 26 | build 27 | --user 28 | - name: Build a binary wheel and a source tarball 29 | run: >- 30 | python -m 31 | build 32 | --sdist 33 | --wheel 34 | --outdir dist/ 35 | . 36 | - name: Publish distribution to PyPI 37 | uses: pypa/gh-action-pypi-publish@release/v1 38 | -------------------------------------------------------------------------------- /tests/test_xml_utils.py: -------------------------------------------------------------------------------- 1 | # Clairmeta - (C) YMAGIS S.A. 2 | # See LICENSE for more information 3 | 4 | import unittest 5 | import os 6 | 7 | from clairmeta.utils.xml import parse_xml 8 | from clairmeta.utils.sys import remove_key_dict 9 | 10 | 11 | class ParseTest(unittest.TestCase): 12 | def get_file_path(self, name): 13 | file_path = os.path.join(os.path.dirname(__file__), "resources", "XML", name) 14 | 15 | self.assertTrue(os.path.exists(file_path)) 16 | return file_path 17 | 18 | def test_attributes(self): 19 | xml_with_attrib = parse_xml(self.get_file_path("CPL_SMPTE.xml")) 20 | xml_without_attrib = parse_xml( 21 | self.get_file_path("CPL_SMPTE.xml"), xml_attribs=False 22 | ) 23 | self.assertNotEqual(xml_with_attrib, xml_without_attrib) 24 | 25 | xml_with_attrib = remove_key_dict(xml_with_attrib, ["@"]) 26 | self.assertEqual(xml_with_attrib, xml_without_attrib) 27 | 28 | 29 | if __name__ == "__main__": 30 | unittest.main() 31 | -------------------------------------------------------------------------------- /tests/test_profile.py: -------------------------------------------------------------------------------- 1 | # Clairmeta - (C) YMAGIS S.A. 2 | # See LICENSE for more information 3 | 4 | import unittest 5 | import os 6 | 7 | from clairmeta.utils.file import temporary_file 8 | from clairmeta.profile import load_profile, save_profile, get_default_profile 9 | 10 | 11 | class ProfileTest(unittest.TestCase): 12 | def get_file_path(self, name): 13 | file_path = os.path.join(os.path.dirname(__file__), "resources", name) 14 | 15 | return file_path 16 | 17 | def test_load_profile(self): 18 | p = load_profile(self.get_file_path("myprofile.json")) 19 | self.assertEqual(p["log_level"], "INFO") 20 | self.assertEqual(p["bypass"], ["check_assets_pkl_hash"]) 21 | self.assertEqual(p["criticality"]["default"], "ERROR") 22 | 23 | def test_save_profile(self): 24 | with temporary_file(suffix=".json") as f: 25 | p_gold = get_default_profile() 26 | save_profile(p_gold, f) 27 | p = load_profile(f) 28 | self.assertEqual(p, p_gold) 29 | 30 | 31 | if __name__ == "__main__": 32 | unittest.main() 33 | -------------------------------------------------------------------------------- /tests/test_license.py: -------------------------------------------------------------------------------- 1 | # Clairmeta - (C) YMAGIS S.A. 2 | # See LICENSE for more information 3 | 4 | import unittest 5 | import os 6 | 7 | 8 | template_lines = [ 9 | "# Clairmeta - (C) YMAGIS S.A.\n", 10 | "# See LICENSE for more information\n", 11 | ] 12 | 13 | source_folder = os.path.dirname(os.path.dirname(__file__)) 14 | source_folder = os.path.join(source_folder, "sources") 15 | 16 | 17 | class LicenseTest(unittest.TestCase): 18 | def file_contain_license(self, path): 19 | with open(path, "r") as fhandle: 20 | lines = fhandle.readlines() 21 | lines = [ln for ln in lines if ln != "" and not ln.startswith("#!")] 22 | return all([a == b for a, b in zip(template_lines, lines)]) 23 | 24 | def test_sources_have_license(self): 25 | for dirpath, dirnames, filenames in os.walk(source_folder): 26 | for f in filenames: 27 | if f.endswith(".py"): 28 | fpath = os.path.join(dirpath, f) 29 | self.assertTrue( 30 | self.file_contain_license(fpath), 31 | msg="Missing license for file {}".format(f), 32 | ) 33 | 34 | 35 | if __name__ == "__main__": 36 | unittest.main() 37 | -------------------------------------------------------------------------------- /clairmeta/utils/crypto.py: -------------------------------------------------------------------------------- 1 | # Clairmeta - (C) YMAGIS S.A. 2 | # See LICENSE for more information 3 | 4 | import os 5 | import base64 6 | from cryptography.hazmat.backends import default_backend 7 | from cryptography.hazmat.primitives import serialization, hashes 8 | from cryptography.hazmat.primitives.asymmetric import padding 9 | 10 | 11 | def decrypt_b64(cipher, key): 12 | """Decrypt encoded cipher with specified private key. 13 | 14 | Args: 15 | cipher (str): Base64 encoded message. 16 | key (str): Absolute path to PEM private key. 17 | 18 | Returns: 19 | Decoded message. 20 | 21 | Raises: 22 | ValueError: If ``key`` is not a valid file. 23 | 24 | """ 25 | if not os.path.isfile(key): 26 | raise ValueError("{} file not found".format(key)) 27 | 28 | with open(key, "rb") as f: 29 | key = serialization.load_pem_private_key( 30 | f.read(), password=None, backend=default_backend() 31 | ) 32 | 33 | return key.decrypt( 34 | base64.b64decode(cipher), 35 | padding.OAEP( 36 | mgf=padding.MGF1(algorithm=hashes.SHA1()), 37 | algorithm=hashes.SHA1(), 38 | label=None, 39 | ), 40 | ) 41 | -------------------------------------------------------------------------------- /.github/scripts/windows/install_test_env.sh: -------------------------------------------------------------------------------- 1 | set -ex 2 | 3 | ## 4 | # Install asdcplib 5 | 6 | export CMAKE_PREFIX_PATH="$VCPKG_INSTALLATION_ROOT/installed/x64-windows" 7 | vcpkg install openssl:x64-windows 8 | vcpkg install xerces-c:x64-windows 9 | 10 | BASE_DIR=`pwd` 11 | WORK_DIR=`mktemp -d` 12 | cd "$WORK_DIR" 13 | 14 | git clone https://github.com/cinecert/asdcplib.git && cd asdcplib 15 | git checkout rel_2_13_1 16 | mkdir build && cd build 17 | 18 | cmake \ 19 | -DCMAKE_INSTALL_PREFIX=$VCPKG_INSTALLATION_ROOT/installed/x64-windows \ 20 | -DOpenSSLLib_PATH=$VCPKG_INSTALLATION_ROOT/installed/x64-windows/lib/libcrypto.lib \ 21 | -DOpenSSLLib_include_DIR=$VCPKG_INSTALLATION_ROOT/installed/x64-windows/include \ 22 | -DXercescppLib_PATH=$VCPKG_INSTALLATION_ROOT/installed/x64-windows/lib/xerces-c_3.lib \ 23 | -DXercescppLib_Debug_PATH=$VCPKG_INSTALLATION_ROOT/installed/x64-windows/lib/xerces-c_3.lib \ 24 | -DXercescppLib_include_DIR=$VCPKG_INSTALLATION_ROOT/installed/x64-windows/include \ 25 | .. 26 | cmake --build . --target install --config Release 27 | 28 | cd "$BASE_DIR" 29 | rm -rf "$WORK_DIR" 30 | 31 | 32 | ## 33 | # ClairMeta other dependencies 34 | choco install mediainfo-cli 35 | 36 | ls -lR $VCPKG_INSTALLATION_ROOT/installed/x64-windows 37 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright © 2017 YMAGIS S.A. 2 | 3 | All rights reserved. 4 | Redistribution and use in source and binary forms, with or without 5 | modification, are permitted provided that the following conditions 6 | are met: 7 | 8 | * Redistributions of source code must retain the above copyright 9 | notice, this list of conditions and the following disclaimer. 10 | * Redistributions in binary form must reproduce the above copyright 11 | notice, this list of conditions and the following disclaimer in the 12 | documentation and/or other materials provided with the distribution. 13 | * Neither the name of YMAGIS S.A., nor the names of its contributors 14 | may be used to endorse or promote products derived from this software 15 | without specific prior written permission. 16 | 17 | 18 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 19 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, 20 | THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 21 | PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL YMAGIS S.A. AND CONTRIBUTORS 22 | BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 23 | CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 24 | SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 25 | INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 26 | CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 27 | ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF 28 | THE POSSIBILITY OF SUCH DAMAGE. 29 | -------------------------------------------------------------------------------- /pyproject.toml: -------------------------------------------------------------------------------- 1 | [project] 2 | name = "clairmeta" 3 | version = "1.6.1" 4 | description = "ClairMeta is a python package for Digital Cinema Package (DCP) probing and checking." 5 | authors = [ 6 | { name = "Rémi Achard", email = "remiachard@gmail.com" }, 7 | ] 8 | license = { text = "BSD-3-Clause" } 9 | readme = "README.rst" 10 | requires-python = ">=3.8" 11 | classifiers = [ 12 | "Development Status :: 5 - Production/Stable", 13 | "Intended Audience :: Developers", 14 | "Topic :: Utilities", 15 | "Topic :: Multimedia :: Video", 16 | "Topic :: Software Development :: Libraries :: Python Modules", 17 | "License :: OSI Approved :: BSD License", 18 | "Programming Language :: Python :: 3", 19 | "Programming Language :: Python :: 3.8", 20 | "Programming Language :: Python :: 3.9", 21 | "Programming Language :: Python :: 3.10", 22 | "Programming Language :: Python :: 3.11", 23 | "Programming Language :: Python :: 3.12", 24 | "Programming Language :: Python :: 3.13", 25 | ] 26 | dependencies = [ 27 | "cryptography>=44.0.0", 28 | "dicttoxml>=1.7.16", 29 | "freetype-py>=2.5.1", 30 | "lxml>=5.3.0", 31 | "pycountry>=24.6.1", 32 | "python-dateutil>=2.9.0.post0", 33 | "xmltodict>=0.14.2", 34 | ] 35 | keywords = ["digital", "cinema", "dcp", "dcdm", "dsm", "check", "probe", "smpte", "interop"] 36 | 37 | [project.urls] 38 | Repository = "https://github.com/Ymagis/ClairMeta" 39 | 40 | [dependency-groups] 41 | dev = [ 42 | "black>=24.8.0", 43 | "coverage>=7.6.1", 44 | "pytest>=8.3.4", 45 | "ruff>=0.9.5", 46 | ] 47 | -------------------------------------------------------------------------------- /clairmeta/utils/uuid.py: -------------------------------------------------------------------------------- 1 | # Clairmeta - (C) YMAGIS S.A. 2 | # See LICENSE for more information 3 | 4 | import re 5 | 6 | # ruff: noqa: E501 7 | # fmt: off 8 | 9 | RE = '(^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$)' 10 | FILE_RE = '([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12})' 11 | RFC4122_RE = '(^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-([1-5])[0-9a-fA-F]{3}\ 12 | -[8-9a-bA-B][0-9a-fA-F]{3}-[0-9a-fA-F]{12}$)' 13 | 14 | # fmt: on 15 | 16 | 17 | def check_uuid(uuid, regex=RE): 18 | """Check UUID validity against one of the available regex. 19 | 20 | Args: 21 | uuid (str): UUID string. 22 | regex: Pattern to validate ``uuid``. 23 | 24 | Returns: 25 | True if successful, False otherwise. 26 | 27 | >>> check_uuid('123e4567-e89b-12d3-a456-426655440000') 28 | True 29 | >>> check_uuid('123E4567-E89B-12D3-A456-426655440000') 30 | True 31 | >>> check_uuid('23e4567-e89b-12d3-a456-426655440000') 32 | False 33 | 34 | """ 35 | return re.match(regex, uuid) is not None 36 | 37 | 38 | def extract_uuid(in_str, regex=FILE_RE): 39 | """Extract UUID from a string. 40 | 41 | Args: 42 | in_str (str): Input string. 43 | regex: Pattern to extract the UUID. 44 | 45 | Returns: 46 | UUID extracted if successful. 47 | 48 | >>> extract_uuid('jp2k_123e4567-e89b-12d3-a456-426655440000_ecl') 49 | '123e4567-e89b-12d3-a456-426655440000' 50 | >>> extract_uuid('abcdefg') is None 51 | True 52 | 53 | """ 54 | match = re.search(regex, in_str) 55 | if match: 56 | return match.group(0) 57 | -------------------------------------------------------------------------------- /clairmeta/dcp_check_vol.py: -------------------------------------------------------------------------------- 1 | # Clairmeta - (C) YMAGIS S.A. 2 | # See LICENSE for more information 3 | 4 | from clairmeta.dcp_check import CheckerBase 5 | from clairmeta.dcp_check_utils import check_xml 6 | 7 | 8 | class Checker(CheckerBase): 9 | def __init__(self, dcp): 10 | super(Checker, self).__init__(dcp) 11 | 12 | def run_checks(self): 13 | for source in self.dcp._list_vol: 14 | checks = self.find_check("vol") 15 | [ 16 | self.run_check(check, source, stack=[source["FileName"]]) 17 | for check in checks 18 | ] 19 | 20 | return self.checks 21 | 22 | def check_vol_xml(self, vol): 23 | """VolIndex XML syntax and structure check. 24 | 25 | References: 26 | SMPTE ST 429-9:2014 27 | mpeg_ii_am_spec.doc (v3.4) 28 | https://interop-docs.cinepedia.com/Document_Release_2.0/mpeg_ii_am_spec.pdf 29 | """ 30 | if self.dcp.schema == "Interop": 31 | return 32 | 33 | check_xml( 34 | self, 35 | vol["FilePath"], 36 | vol["Info"]["VolumeIndex"]["__xmlns__"], 37 | vol["Info"]["VolumeIndex"]["Schema"], 38 | self.dcp.schema, 39 | ) 40 | 41 | def check_vol_name(self, vol): 42 | """VolIndex file name respect DCP standard. 43 | 44 | References: N/A 45 | """ 46 | schema = vol["Info"]["VolumeIndex"]["Schema"] 47 | mandatory_name = {"Interop": "VOLINDEX", "SMPTE": "VOLINDEX.xml"} 48 | 49 | if mandatory_name[schema] != vol["FileName"]: 50 | self.error( 51 | "{} VolIndex must be named {}, got {} instead".format( 52 | schema, mandatory_name[schema], vol["FileName"] 53 | ) 54 | ) 55 | -------------------------------------------------------------------------------- /tests/test_mxf_probe.py: -------------------------------------------------------------------------------- 1 | # Clairmeta - (C) YMAGIS S.A. 2 | # See LICENSE for more information 3 | 4 | import unittest 5 | import os 6 | 7 | from clairmeta.utils.probe import probe_mxf 8 | from clairmeta.exception import CommandException 9 | 10 | 11 | class TestAssetProbe(unittest.TestCase): 12 | def get_path(self, name): 13 | file_path = os.path.join(os.path.dirname(__file__), "resources", "MXF", name) 14 | 15 | return file_path 16 | 17 | def test_video_iop(self): 18 | metadata = probe_mxf(self.get_path("picture_2D_iop.mxf"), stereoscopic=False) 19 | self.assertTrue(metadata["EditRate"] == 24) 20 | self.assertTrue(metadata["LabelSetType"] == "MXFInterop") 21 | 22 | def test_video_smpte(self): 23 | metadata = probe_mxf(self.get_path("picture_2D_smpte.mxf"), stereoscopic=False) 24 | self.assertTrue(metadata["EditRate"] == 24) 25 | self.assertTrue(metadata["LabelSetType"] == "SMPTE") 26 | 27 | def test_video_bitrate(self): 28 | metadata = probe_mxf( 29 | self.get_path("picture_over_250_mb.mxf"), stereoscopic=False 30 | ) 31 | self.assertTrue(metadata["LabelSetType"] == "SMPTE") 32 | self.assertTrue(metadata["AverageBitRate"] > 250) 33 | self.assertTrue(metadata["MaxBitRate"] > 350) 34 | 35 | def test_audio_iop(self): 36 | metadata = probe_mxf(self.get_path("audio_iop.mxf")) 37 | self.assertTrue(metadata["AudioSamplingRate"] == 48000) 38 | 39 | def test_audio_smpte(self): 40 | metadata = probe_mxf(self.get_path("audio_smpte.mxf")) 41 | self.assertTrue(metadata["AudioSamplingRate"] == 48000) 42 | 43 | def test_atmos(self): 44 | metadata = probe_mxf(self.get_path("atmos.mxf")) 45 | self.assertTrue(metadata["AtmosVersion"] == 1) 46 | 47 | def test_subtitle_smpte(self): 48 | metadata = probe_mxf(self.get_path("subtitle_smpte.mxf")) 49 | self.assertTrue(metadata["LabelSetType"] == "SMPTE") 50 | self.assertEqual( 51 | metadata["NamespaceName"], 52 | r"http://www.smpte-ra.org/schemas/428-7/2007/DCST", 53 | ) 54 | 55 | def test_fake(self): 56 | with self.assertRaises(CommandException): 57 | probe_mxf("null") 58 | 59 | 60 | if __name__ == "__main__": 61 | unittest.main() 62 | -------------------------------------------------------------------------------- /scripts/dcp_copy.py: -------------------------------------------------------------------------------- 1 | """ 2 | Implement DCP copy and subsequent check. 3 | 4 | Require python 3.8 for shutil.copytree() dirs_exist_ok kwargs. 5 | Require python 3.2 for concurrent.futures.ThreadPoolExecutor. 6 | """ 7 | 8 | import argparse 9 | import concurrent.futures 10 | import time 11 | import shutil 12 | import sys 13 | 14 | import clairmeta 15 | from clairmeta import DCP 16 | from clairmeta.logger import get_log 17 | from clairmeta.utils.file import ConsoleProgress, folder_size 18 | 19 | 20 | def cli_copy(args): 21 | dcp = DCP(args.source) 22 | dcp_size = dcp.size 23 | 24 | try: 25 | log = get_log() 26 | log.info("Copy {} to {}".format(args.source, args.dest)) 27 | 28 | start = time.time() 29 | progress = ConsoleProgress() 30 | progress._total_size = dcp_size 31 | 32 | with concurrent.futures.ThreadPoolExecutor(max_workers=1) as executor: 33 | future = executor.submit( 34 | shutil.copytree, args.source, args.dest, dirs_exist_ok=args.overwrite 35 | ) 36 | 37 | while future.running(): 38 | copy_size = folder_size(args.dest) 39 | elapsed = time.time() - start 40 | progress(args.source, copy_size, dcp_size, elapsed) 41 | time.sleep(1.0) 42 | 43 | future.result() 44 | 45 | progress(args.source, dcp_size, dcp_size, elapsed) 46 | log.info("Total time : {:.2f} sec".format(time.time() - start)) 47 | 48 | DCP(args.dest) 49 | status, _ = dcp.check(hash_callback=ConsoleProgress()) 50 | 51 | return status 52 | 53 | except Exception as e: 54 | print(str(e)) 55 | return False 56 | 57 | 58 | def get_parser(): 59 | parser = argparse.ArgumentParser( 60 | description="Clairmeta Copy Sample Utility {}".format(clairmeta.__version__) 61 | ) 62 | 63 | parser.add_argument("source", help="absolute source package path") 64 | parser.add_argument("dest", help="absolute destination copy path") 65 | parser.add_argument("-progress", action="store_true", help="progress bar") 66 | parser.add_argument("-overwrite", action="store_true", help="overwrite dst") 67 | parser.set_defaults(func=cli_copy) 68 | 69 | return parser 70 | 71 | 72 | if __name__ == "__main__": 73 | parser = get_parser() 74 | args = parser.parse_args() 75 | if len(sys.argv) == 1: 76 | parser.print_help() 77 | else: 78 | status = args.func(args) 79 | sys.exit(0 if status else 1) 80 | -------------------------------------------------------------------------------- /clairmeta/logger.py: -------------------------------------------------------------------------------- 1 | # Clairmeta - (C) YMAGIS S.A. 2 | # See LICENSE for more information 3 | 4 | import os 5 | import logging 6 | from logging.handlers import RotatingFileHandler 7 | 8 | from clairmeta.settings import LOG_SETTINGS 9 | 10 | 11 | def init_log(): 12 | """Initialize logging utilities. 13 | 14 | Returns: 15 | logging.Logger object with appropriate handler initialized. 16 | 17 | """ 18 | log = logging.getLogger("Clairmeta") 19 | 20 | formatter = logging.Formatter( 21 | "%(asctime)s - %(name)s - %(levelname)s - %(message)s" 22 | ) 23 | 24 | stream_handler = logging.NullHandler() 25 | stream_handler.setFormatter(formatter) 26 | log.addHandler(stream_handler) 27 | 28 | if LOG_SETTINGS["enable_console"] == "ON": 29 | init_console(log, formatter) 30 | if LOG_SETTINGS["enable_file"] == "ON" and LOG_SETTINGS["file_name"]: 31 | init_file(log, formatter) 32 | 33 | return log 34 | 35 | 36 | def init_console(log, formatter): 37 | """Initialize console stream handler.""" 38 | stream_handler = logging.StreamHandler() 39 | stream_handler.setFormatter(formatter) 40 | log.addHandler(stream_handler) 41 | 42 | 43 | def init_file(log, formatter): 44 | """Initialize file handler.""" 45 | try: 46 | log_dir = os.path.expanduser(LOG_SETTINGS["file_name"]) 47 | log_file = log_dir 48 | log_dir = os.path.dirname(log_dir) 49 | if not os.path.exists(log_dir): 50 | os.mkdir(log_dir) 51 | 52 | file_handler = RotatingFileHandler( 53 | log_file, 54 | maxBytes=LOG_SETTINGS["file_size"], 55 | backupCount=LOG_SETTINGS["file_count"], 56 | ) 57 | file_handler.setFormatter(formatter) 58 | log.addHandler(file_handler) 59 | except Exception as e: 60 | log.error("Could not intialize log file : {}".format(str(e))) 61 | 62 | 63 | def enable_log(): 64 | logging.disable(logging.NOTSET) 65 | 66 | 67 | def disable_log(): 68 | logging.disable(logging.CRITICAL) 69 | 70 | 71 | def get_log(): 72 | """Returns package logging.Logger global object.""" 73 | return cm_log 74 | 75 | 76 | def set_level(level): 77 | """Set logging threshold level. 78 | 79 | Args: 80 | level (str): Minimum level for a log event to be recorded. List 81 | include : CRITICAL, ERROR, WARNING, INFO, DEBUG, NOTSET. 82 | 83 | """ 84 | cm_log.setLevel(level) 85 | [h.setLevel(level) for h in cm_log.handlers] 86 | 87 | 88 | cm_log = init_log() 89 | set_level(LOG_SETTINGS["level"]) 90 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- 1 | # ruff: noqa: E501 2 | 3 | DCP_MAP = { 4 | 1: "ECL01-SINGLE-CPL_TST_S_EN-XX_UK-U_71_2K_DI_20171218_ECL_IOP_OV", 5 | 2: "ECL02-SINGLE-CPL_TST_S_EN-XX_UK-U_71_2K_DI_20171218_ECL_IOP_VF", 6 | 7: "ECL07-SINGLE-CPL_TST-3D-48_S_EN-XX_UK-U_71-ATMOS_2K_ECL_20180301_ECL_SMPTE-3D_OV", 7 | 8: "ECL08-SINGLE-CPL_TST_S_EN-EN_UK-U_51_2K_DI_20180125_ECL_SMPTE_VF", 8 | 9: "ECL09-SINGLE-CPL_TST_S_EN-XX_UK-U_51-ATMOS_2K_DI_20171220_ECL_SMPTE_OV", 9 | 10: "ECL10-SINGLE-CPL_TST_S_EN-XX_UK-U_51-ATMOS_2K_DI_20171220_ECL_SMPTE_VF", 10 | 11: "ECL11-SINGLE-CPL_TST_S_EN-XX_UK-U_51_2K_DI_20171220_ECL_SMPTE_OV", 11 | 22: "ECL22-SINGLE-CPL_TST-48_S_EN-XX_UK-U_51_2K_DI_20180125_ECL_SMPTE_OV", 12 | 23: "ECL23-SINGLE-CPL_TST-60_S_EN-XX_UK-U_51_2K_DI_20180125_ECL_SMPTE_OV", 13 | 25: "ECL25-SINGLE-CPL_TST-48-600_S_EN-XX_UK-U_51_2K_DI_20180301_ECL_SMPTE_OV", 14 | 26: "ECL26-SINGLE-CPL_TST_F-177_EN-XX_UK-U_51_2K_DI_20180103_ECL_SMPTE_OV", 15 | 27: "ECL27-SINGLE-CPL_TST_F-177-UHD_EN-XX_UK-U_51_4K_DI_20180103_ECL_SMPTE_OV", 16 | 28: "ECL28-SINGLE-CPL_TST_F_EN-XX_UK-U_51_4K_DI_20180103_ECL_SMPTE_OV", 17 | 29: "ECL29-SINGLE-CPL-CRYPT_TST_S_EN-EN_UK-U_51_2K_DI_20180103_ECL_IOP_OV", 18 | 30: "ECL30-SINGLE-CPL-CRYPT_TST_S_EN-EN_UK-U_51_2K_DI_20180125_ECL_SMPTE_OV", 19 | 31: "ECL31-SINGLE-CPL-NCCRYPT_TST_S_EN-XX_UK-U_51_2K_DI_20180103_ECL_IOP_OV", 20 | 32: "ECL32-SINGLE-CPL-NCCRYPT_TST_S_EN-XX_UK-U_51_2K_DI_20180103_ECL_SMPTE_OV", 21 | 33: "ECL33-SINGLE-CPL_TST_S_EN-EN_UK-U_51_2K_DI_20180103_ECL_IOP_OV", 22 | 38: "ECL38-SINGLE-CPL_TST_S-300_EN-XX_UK-U_51_2K_DI_20180103_ECL_SMPTE_OV", 23 | 39: "ECL39-SINGLE-CPL-LVLS_TST_C_EN-XX_UK-U_51_4K_DI_20180103_ECL_SMPTE_OV", 24 | 40: "ECL40-SINGLE-CPL-MPG_TST_S_EN-XX_UK-U_51_2K_DI_20180103_ECL_SMPTE_OV", 25 | 41: "ECL41-SINGLE-CPL_TST-60-400_F_XX-XX_UK-U_51_2K_DI_20180301_ECL_SMPTE_OV", 26 | 42: "ECL42-SINGLE-CPL_TST-3D-48-600_F_XX-XX_UK-U_51_2K_ECL_20180301_ECL_SMPTE-3D_OV", 27 | 43: "ECL43-SINGLE-CPL_TST-50_F_XX-XX_UK-U_51_2K_ECL_20180301_ECL_SMPTE_OV", 28 | 44: "ECL44-SINGLE-CPL_TST-3D-48_F_XX-XX_UK-U_51_2K_ECL_20180301_ECL_SMPTE-3D_OV", 29 | 45: "ECL45-SINGLE-CPL_TST-3D-50_F_XX-XX_UK-U_51_2K_ECL_20180301_ECL_SMPTE-3D_OV", 30 | 46: "ECL46-SINGLE-CPL_TST-3D-60_F_XX-XX_UK-U_51_2K_ECL_20180301_ECL_SMPTE-3D_OV", 31 | 47: "ECL47-MULTI-PKL_TST_S_EN-XX_UK-U_51-71_2K_DI_20180301_ECL_SMPTE_OV", 32 | } 33 | 34 | KDM_MAP = { 35 | 29: "KDM_ECL29_FR-Y-LABO_Leaf_c2b01941-a633-4ee1-9101-4645cca97be1.xml", 36 | 30: "KDM_ECL30_180125_FR-Y-LABO_Leaf_57e32d37-050c-4d43-a15d-7943e673cc25.xml", 37 | } 38 | 39 | KEY = "leaf.key" 40 | -------------------------------------------------------------------------------- /clairmeta/sequence.py: -------------------------------------------------------------------------------- 1 | # Clairmeta - (C) YMAGIS S.A. 2 | # See LICENSE for more information 3 | 4 | import os 5 | 6 | from clairmeta.sequence_check import check_sequence 7 | from clairmeta.utils.sys import key_by_path_dict 8 | from clairmeta.utils.probe import probe_folder 9 | 10 | 11 | class Sequence(object): 12 | """Image file sequence abstraction.""" 13 | 14 | def __init__(self, path): 15 | """Sequence constructor. 16 | 17 | Args: 18 | path (str): Absolute path to directory. 19 | 20 | Raises: 21 | ValueError: ``path`` directory not found. 22 | 23 | """ 24 | if not os.path.isdir(path): 25 | raise ValueError("{} is not a valid folder".format(path)) 26 | 27 | self.path = path 28 | self.probe_folder = probe_folder(path) 29 | 30 | def parse(self): 31 | """Extract metadata.""" 32 | return self.probe_folder 33 | 34 | def check(self, setting): 35 | """Check validity. 36 | 37 | Raises: 38 | ValueError: Validity check failure. 39 | 40 | """ 41 | check_sequence( 42 | self.path, 43 | setting["allowed_extensions"], 44 | setting["file_white_list"], 45 | setting["directory_white_list"], 46 | ) 47 | 48 | for folder, seqs in self.probe_folder.items(): 49 | for seq, keys in seqs.items(): 50 | ext = keys.get("Extension") 51 | check_keys = setting["allowed_extensions"].get("." + ext) 52 | probe_keys = keys.get("Probe") 53 | 54 | if not probe_keys or not check_keys: 55 | continue 56 | 57 | self._check_keys(check_keys, probe_keys, folder) 58 | 59 | return True 60 | 61 | def _check_keys(self, check_keys, probe_keys, folder): 62 | """Compare expected and detected file probe informations. 63 | 64 | Raises: 65 | ValueError: Mismatch. 66 | 67 | """ 68 | for key, expect_val in check_keys.items(): 69 | val = key_by_path_dict(probe_keys, key) 70 | 71 | if isinstance(expect_val, list): 72 | if val not in expect_val: 73 | raise ValueError( 74 | "{} - Invalid {}, got {} but expected {}".format( 75 | folder, key, val, expect_val 76 | ) 77 | ) 78 | else: 79 | if val != expect_val: 80 | raise ValueError( 81 | "{} - Invalid {}, got {} but expected {}".format( 82 | folder, key, val, expect_val 83 | ) 84 | ) 85 | -------------------------------------------------------------------------------- /clairmeta/xsd/PROTO-ASDCP-PKL-20040311.xsd: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 12 | 13 | 17 | 19 | 21 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | -------------------------------------------------------------------------------- /clairmeta/xsd/SMPTE-429-8-2006-PKL.xsd: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | -------------------------------------------------------------------------------- /clairmeta/xsd/PROTO-ASDCP-AM-20040311.xsd: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 11 | 12 | 16 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | -------------------------------------------------------------------------------- /clairmeta/xsd/SMPTE-429-9-2007-AM.xsd: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | -------------------------------------------------------------------------------- /clairmeta/utils/time.py: -------------------------------------------------------------------------------- 1 | # Clairmeta - (C) YMAGIS S.A. 2 | # See LICENSE for more information 3 | 4 | 5 | def compare_ratio(a, b, precision=0.05): 6 | """Compare decimal numbers up to a given precision.""" 7 | return abs(a - b) <= precision 8 | 9 | 10 | def format_ratio(in_str, separator="/"): 11 | """Convert a string representing a rational value to a decimal value. 12 | 13 | Args: 14 | in_str (str): Input string. 15 | separator (str): Separator character used to extract numerator and 16 | denominator, if not found in ``in_str`` whitespace is used. 17 | 18 | Returns: 19 | An integer or float value with 2 digits precision or ``in_str`` if 20 | formating has failed. 21 | 22 | >>> format_ratio('48000/1') 23 | 48000 24 | >>> format_ratio('24000 1000') 25 | 24 26 | >>> format_ratio('24000 1001') 27 | 23.98 28 | >>> format_ratio('1,77') 29 | '1,77' 30 | >>> format_ratio(1.77) 31 | 1.77 32 | 33 | """ 34 | if not isinstance(in_str, str): 35 | return in_str 36 | 37 | try: 38 | sep = separator if separator in in_str else " " 39 | ratio = in_str.split(sep) 40 | 41 | if len(ratio) == 2: 42 | ratio = round(float(ratio[0]) / float(ratio[1]), 2) 43 | else: 44 | ratio = float(ratio[0]) 45 | 46 | if ratio.is_integer(): 47 | ratio = int(ratio) 48 | 49 | return ratio 50 | except ValueError: 51 | return in_str 52 | 53 | 54 | def frame_to_tc(edit_count, edit_rate): 55 | """Convert sample count to timecode. 56 | 57 | Args: 58 | edit_count(int): number of samples. 59 | edit_rate (int): number of sample per second. 60 | 61 | Returns: 62 | Timecode string (format HH:MM:SS:FF). 63 | 64 | >>> frame_to_tc(48, 24) 65 | '00:00:02:00' 66 | 67 | """ 68 | if edit_rate != 0 and edit_count != 0: 69 | s, f = divmod(edit_count, edit_rate) 70 | m, s = divmod(s, 60) 71 | h, m = divmod(m, 60) 72 | return "%02d:%02d:%02d:%02d" % (h, m, s, f) 73 | else: 74 | return "00:00:00:00" 75 | 76 | 77 | def tc_to_frame(tc, edit_rate): 78 | """Convert timecode to sample count. 79 | 80 | Args: 81 | tc (str): Timecode string (format HH:MM:SS:FF). 82 | edit_rate (int): number of samples per second. 83 | 84 | Returns: 85 | Total samples count. 86 | 87 | >>> tc_to_frame('00:00:02:00', 24) 88 | 48 89 | 90 | """ 91 | hours, minutes, seconds, frames = map(int, tc.split(":")) 92 | framePerHour = edit_rate * 60 * 60 93 | framePerMinute = edit_rate * 60 94 | framePerSecond = edit_rate 95 | 96 | return ( 97 | hours * framePerHour 98 | + minutes * framePerMinute 99 | + seconds * framePerSecond 100 | + frames 101 | ) 102 | -------------------------------------------------------------------------------- /clairmeta/xsd/SMPTE-430-3-2006-ETM.xsd: -------------------------------------------------------------------------------- 1 | 2 | 3 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | -------------------------------------------------------------------------------- /tests/test_sequence.py: -------------------------------------------------------------------------------- 1 | # Clairmeta - (C) YMAGIS S.A. 2 | # See LICENSE for more information 3 | 4 | import unittest 5 | import os 6 | 7 | from clairmeta import Sequence 8 | from clairmeta.logger import disable_log 9 | from clairmeta.settings import SEQUENCE_SETTINGS 10 | 11 | 12 | class SequenceTestBase(unittest.TestCase): 13 | def __init__(self, *args, **kwargs): 14 | super(SequenceTestBase, self).__init__(*args, **kwargs) 15 | disable_log() 16 | 17 | def get_path(self, name): 18 | file_path = os.path.join(os.path.dirname(__file__), "resources", "SEQ", name) 19 | 20 | return file_path 21 | 22 | def check_dsm(self, path): 23 | return Sequence(self.get_path(path)).check(SEQUENCE_SETTINGS["DSM"]) 24 | 25 | def check_dcdm(self, path): 26 | return Sequence(self.get_path(path)).check(SEQUENCE_SETTINGS["DCDM"]) 27 | 28 | 29 | class ParseTest(SequenceTestBase): 30 | def test_parse_dsm(self): 31 | res = Sequence(self.get_path("DSM_PKG/MINI_DSM1")).parse() 32 | self.assertTrue(isinstance(res, dict)) 33 | self.assertEqual(len(res.keys()), 1) 34 | 35 | def test_parse_dsm_package(self): 36 | res = Sequence(self.get_path("DSM_PKG")).parse() 37 | self.assertTrue(isinstance(res, dict)) 38 | self.assertEqual(len(res.keys()), 3) 39 | 40 | def test_parse_dcdm(self): 41 | res = Sequence(self.get_path("DCDM")).parse() 42 | self.assertTrue(isinstance(res, dict)) 43 | self.assertEqual(len(res.keys()), 1) 44 | 45 | 46 | class CheckTest(SequenceTestBase): 47 | def test_check_raise_not_folder(self): 48 | with self.assertRaises(ValueError): 49 | self.check_dsm("null") 50 | 51 | def test_check_raise_empty_folder(self): 52 | with self.assertRaises(ValueError): 53 | self.check_dsm("EMPTY") 54 | 55 | def test_check_raise_foreign(self): 56 | with self.assertRaises(ValueError): 57 | self.check_dsm("DSM_FOREIGN_FILE") 58 | 59 | def test_check_raise_j2k(self): 60 | with self.assertRaises(ValueError): 61 | self.check_dsm("REEL_J2K") 62 | 63 | def test_check_raise_length(self): 64 | with self.assertRaises(ValueError): 65 | self.check_dsm("DSM_BAD_FILE_NAME_LENGTH") 66 | 67 | def test_check_raise_jump(self): 68 | with self.assertRaises(ValueError): 69 | self.check_dsm("DSM_BAD_JUMP") 70 | 71 | def test_check_raise_desc(self): 72 | with self.assertRaises(ValueError): 73 | self.check_dsm("DSM_BAD_DESC") 74 | 75 | def test_check_dsm_ok(self): 76 | self.assertTrue(self.check_dsm("DSM_PKG/MINI_DSM1")) 77 | 78 | def test_check_dsm_empty_name(self): 79 | self.assertTrue(self.check_dsm("DSM_EMPTY_NAME")) 80 | 81 | def test_check_dsm_no_padding(self): 82 | self.assertTrue(self.check_dsm("DSM_NO_PADDING")) 83 | 84 | def test_check_package_ok(self): 85 | self.assertTrue(self.check_dsm("DSM_PKG")) 86 | 87 | def test_check_dcdm_ok(self): 88 | self.assertTrue(self.check_dcdm("DCDM")) 89 | 90 | 91 | if __name__ == "__main__": 92 | unittest.main() 93 | -------------------------------------------------------------------------------- /clairmeta/xsd/catalog.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | -------------------------------------------------------------------------------- /.github/workflows/test-package.yml: -------------------------------------------------------------------------------- 1 | name: Build and test package 2 | 3 | on: [push, pull_request] 4 | 5 | jobs: 6 | test-linux: 7 | runs-on: ubuntu-latest 8 | strategy: 9 | matrix: 10 | python-version: ["3.8", "3.9", "3.10", "3.11", "3.12", "3.13"] 11 | steps: 12 | - name: Clone repository 13 | uses: actions/checkout@v4 14 | - name: Clone tests data repository 15 | uses: actions/checkout@v4 16 | with: 17 | repository: Ymagis/ClairMeta_Data 18 | path: tests/resources 19 | - name: Install uv 20 | uses: astral-sh/setup-uv@v5 21 | with: 22 | python-version: ${{ matrix.python-version }} 23 | - name: Set up Python ${{ matrix.python-version }} 24 | uses: actions/setup-python@v5 25 | with: 26 | python-version: ${{ matrix.python-version }} 27 | - name: Set up environment 28 | run: .github/scripts/linux/apt/install_test_env.sh 29 | - name: Install python dependencies 30 | run: | 31 | uv sync --all-extras --dev 32 | uv tree 33 | - name: Test 34 | run: | 35 | uv run python -m compileall clairmeta 36 | uv run pytest --doctest-modules 37 | 38 | test-macos: 39 | runs-on: macos-latest 40 | strategy: 41 | matrix: 42 | python-version: ["3.8", "3.13"] 43 | steps: 44 | - name: Clone repository 45 | uses: actions/checkout@v4 46 | - name: Clone tests data repository 47 | uses: actions/checkout@v4 48 | with: 49 | repository: Ymagis/ClairMeta_Data 50 | path: tests/resources 51 | - name: Install uv 52 | uses: astral-sh/setup-uv@v5 53 | with: 54 | python-version: ${{ matrix.python-version }} 55 | - name: Set up Python ${{ matrix.python-version }} 56 | uses: actions/setup-python@v5 57 | with: 58 | python-version: ${{ matrix.python-version }} 59 | - name: Set up environment 60 | run: .github/scripts/macos/install_test_env.sh 61 | - name: Install python dependencies 62 | run: | 63 | uv sync --all-extras --dev 64 | uv tree 65 | - name: Test 66 | run: | 67 | uv run python -m compileall clairmeta 68 | uv run pytest --doctest-modules 69 | 70 | test-windows: 71 | runs-on: windows-latest 72 | strategy: 73 | matrix: 74 | python-version: ["3.8", "3.13"] 75 | steps: 76 | - name: Clone repository 77 | uses: actions/checkout@v4 78 | - name: Clone tests data repository 79 | uses: actions/checkout@v4 80 | with: 81 | repository: Ymagis/ClairMeta_Data 82 | path: tests/resources 83 | - name: Install uv 84 | uses: astral-sh/setup-uv@v5 85 | with: 86 | python-version: ${{ matrix.python-version }} 87 | - name: Set up Python ${{ matrix.python-version }} 88 | uses: actions/setup-python@v5 89 | with: 90 | python-version: ${{ matrix.python-version }} 91 | - name: Set up environment 92 | run: .github/scripts/windows/install_test_env.sh 93 | shell: bash 94 | - name: Install python dependencies 95 | run: | 96 | uv sync --all-extras --dev 97 | uv tree 98 | shell: bash 99 | - name: Test 100 | run: | 101 | export PATH="/c/vcpkg/installed/x64-windows/bin:$PATH" 102 | uv run python -m compileall clairmeta 103 | uv run pytest --doctest-modules 104 | shell: bash 105 | -------------------------------------------------------------------------------- /clairmeta/xsd/SMPTE-430-1-2006-KDM.xsd: -------------------------------------------------------------------------------- 1 | 2 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | 74 | 75 | 76 | 77 | 78 | 79 | 80 | 81 | 82 | -------------------------------------------------------------------------------- /clairmeta/dcp_check_execution.py: -------------------------------------------------------------------------------- 1 | # Clairmeta - (C) YMAGIS S.A. 2 | # See LICENSE for more information 3 | 4 | ERROR_SILENT = 0 5 | ERROR_INFO = 1 6 | ERROR_WARNING = 2 7 | ERROR_ERROR = 3 8 | 9 | ERROR_FROM_STR = { 10 | "SILENT": ERROR_SILENT, 11 | "INFO": ERROR_INFO, 12 | "WARNING": ERROR_WARNING, 13 | "ERROR": ERROR_ERROR, 14 | } 15 | 16 | STR_FROM_ERROR = {v: k for k, v in ERROR_FROM_STR.items()} 17 | 18 | 19 | def ErrorLevelFromString(error_str): 20 | return ERROR_FROM_STR[error_str] 21 | 22 | 23 | def ErrorLevelToString(error_level): 24 | return STR_FROM_ERROR[error_level] 25 | 26 | 27 | class CheckError(object): 28 | """Error reporting from whithin checks accumulate a list of errors.""" 29 | 30 | def __init__(self, msg, name="", doc=""): 31 | self.name = name 32 | self.parent_name = "" 33 | self.doc = doc 34 | self.parent_doc = doc 35 | self.message = msg 36 | self.criticality = "ERROR" 37 | 38 | def full_name(self): 39 | if self.name: 40 | return "{}_{}".format(self.parent_name, self.name) 41 | else: 42 | return self.parent_name 43 | 44 | def short_desc(self): 45 | """Returns first line of the documentation.""" 46 | lines = list(filter(None, self.doc.split("\n"))) 47 | return lines[0].strip() if lines else "" 48 | 49 | def to_dict(self): 50 | """Returns a dictionary representation.""" 51 | return { 52 | "name": self.name, 53 | "pretty_name": self.short_desc(), 54 | "doc": self.doc, 55 | "message": self.message, 56 | "criticality": self.criticality, 57 | } 58 | 59 | 60 | class CheckExecution(object): 61 | """Check execution with status and related metadatas.""" 62 | 63 | def __init__(self, func): 64 | """Constructor for CheckExecution. 65 | 66 | Args: 67 | func (function): Check function. 68 | 69 | """ 70 | self.name = func.__name__ 71 | self.doc = func.__doc__ 72 | self.bypass = False 73 | self.seconds_elapsed = 0 74 | self.asset_stack = [] 75 | self.errors = [] 76 | 77 | def short_desc(self): 78 | """Returns first line of the docstring or function name.""" 79 | docstring_lines = self.doc.split("\n") if self.doc else "" 80 | return docstring_lines[0].strip() if docstring_lines else self.name 81 | 82 | def is_valid(self, criticality="ERROR"): 83 | """Returns whether check raised any errors is above ``criticality``. 84 | 85 | Args: 86 | criticality (str, optional): Maximum error level to be 87 | considered invalid. 88 | 89 | Returns: 90 | Boolean 91 | 92 | """ 93 | error_level = ErrorLevelFromString(criticality) 94 | return not any( 95 | [ErrorLevelFromString(e.criticality) >= error_level for e in self.errors] 96 | ) 97 | 98 | def has_errors(self, criticality=None): 99 | """Returns whether check raised any errors of ``criticality``. 100 | 101 | Args: 102 | criticality (str, optional): Error level to consider, if empty 103 | will look for all errors. 104 | 105 | Returns: 106 | Boolean 107 | 108 | """ 109 | if not criticality: 110 | return self.errors != [] 111 | else: 112 | error_level = ErrorLevelFromString(criticality) 113 | return any([e for e in self.errors if e.criticality == error_level]) 114 | 115 | def to_dict(self): 116 | """Returns a dictionary representation.""" 117 | return { 118 | "name": self.name, 119 | "pretty_name": self.short_desc(), 120 | "doc": self.doc, 121 | "bypass": self.bypass, 122 | "seconds_elapsed": self.seconds_elapsed, 123 | "asset_stack": self.asset_stack, 124 | "errors": [e.to_dict() for e in self.errors], 125 | } 126 | -------------------------------------------------------------------------------- /clairmeta/dcp_check_atmos.py: -------------------------------------------------------------------------------- 1 | # Clairmeta - (C) YMAGIS S.A. 2 | # See LICENSE for more information 3 | 4 | from clairmeta.dcp_check import CheckerBase 5 | from clairmeta.dcp_utils import list_cpl_assets 6 | from clairmeta.settings import DCP_SETTINGS 7 | 8 | 9 | class Checker(CheckerBase): 10 | def __init__(self, dcp): 11 | super(Checker, self).__init__(dcp) 12 | 13 | def run_checks(self): 14 | for source in self.dcp._list_cpl: 15 | asset_checks = self.find_check("atmos_cpl") 16 | [ 17 | self.run_check( 18 | check, 19 | source, 20 | asset, 21 | stack=[source["FileName"], asset[1].get("Path") or asset[1]["Id"]], 22 | ) 23 | for asset in list_cpl_assets( 24 | source, filters="AuxData", required_keys=["Probe"] 25 | ) 26 | if asset[1]["Schema"] == "Atmos" 27 | for check in asset_checks 28 | ] 29 | 30 | return self.checks 31 | 32 | def check_atmos_cpl_essence_encoding(self, playlist, asset): 33 | """Atmos data essence coding universal label. 34 | 35 | References: 36 | SMPTE ST 429-18:2019 11 37 | """ 38 | _, asset = asset 39 | ul = DCP_SETTINGS["atmos"]["smpte_ul"] 40 | cpl_ul = asset.get("DataType", "").replace("urn:smpte:ul:", "").strip() 41 | mxf_ul = asset["Probe"].get("DataEssenceCoding", "") 42 | 43 | if not cpl_ul: 44 | self.fatal_error("Missing Atmos DataType tag (CPL/AuxData)", "missing_cpl") 45 | elif not mxf_ul: 46 | self.fatal_error("Missing Atmos Essence Coding UL (MXF)", "missing_mxf") 47 | 48 | cpl_ul, mxf_ul = cpl_ul.lower(), mxf_ul.lower() 49 | if cpl_ul != mxf_ul: 50 | self.error( 51 | "Incoherent Atmos Data Essence Coding, CPL {} / MXF {}".format( 52 | cpl_ul, mxf_ul 53 | ), 54 | "incoherent", 55 | ) 56 | elif mxf_ul != ul: 57 | self.error( 58 | "Unknown Atmos Data Essence Coding, expecting {} but got {}".format( 59 | ul, mxf_ul 60 | ), 61 | "unknown", 62 | ) 63 | 64 | def check_atmos_cpl_channels(self, playlist, asset): 65 | """Atmos maximum channels count. 66 | 67 | This field will be optional (429-18). 68 | 69 | References: 70 | Dolby S14/26858/27819 71 | https://web.archive.org/web/20190407130138/https://www.dolby.com/us/en/technologies/dolby-atmos/dolby-atmos-next-generation-audio-for-cinema-white-paper.pdf 72 | SMPTE ST 429-18:2019 12 Table 4 73 | """ 74 | _, asset = asset 75 | max_atmos = DCP_SETTINGS["atmos"]["max_channel_count"] 76 | max_cc = asset["Probe"].get("MaxChannelCount") 77 | 78 | if not max_cc: 79 | self.error("Missing MaxChannelCount field", "missing") 80 | elif max_cc > max_atmos: 81 | self.error( 82 | "Invalid Atmos MaxChannelCount, got {} but maximum is {}".format( 83 | max_cc, max_atmos 84 | ), 85 | "invalid", 86 | ) 87 | 88 | def check_atmos_cpl_objects(self, playlist, asset): 89 | """Atmos maximum objects count. 90 | 91 | This field will be optional (429-18). 92 | 93 | References: 94 | Dolby S14/26858/27819 95 | https://web.archive.org/web/20190407130138/https://www.dolby.com/us/en/technologies/dolby-atmos/dolby-atmos-next-generation-audio-for-cinema-white-paper.pdf 96 | SMPTE ST 429-18:2019 12 Table 4 97 | """ 98 | _, asset = asset 99 | max_atmos = DCP_SETTINGS["atmos"]["max_object_count"] 100 | max_obj = asset["Probe"].get("MaxObjectCount") 101 | 102 | if not max_obj: 103 | self.error("Missing MaxObjectCount field", "missing") 104 | elif max_obj > max_atmos: 105 | self.error( 106 | "Invalid Atmos MaxObjectCount, got {} but maximum is {}".format( 107 | max_obj, max_atmos 108 | ), 109 | "invalid", 110 | ) 111 | -------------------------------------------------------------------------------- /clairmeta/dcp_check_pkl.py: -------------------------------------------------------------------------------- 1 | # Clairmeta - (C) YMAGIS S.A. 2 | # See LICENSE for more information 3 | 4 | import os 5 | 6 | from clairmeta.utils.file import shaone_b64 7 | from clairmeta.dcp_check import CheckerBase 8 | from clairmeta.dcp_check_utils import check_xml, check_issuedate 9 | from clairmeta.dcp_utils import list_pkl_assets 10 | 11 | 12 | class Checker(CheckerBase): 13 | def __init__(self, dcp): 14 | super(Checker, self).__init__(dcp) 15 | 16 | def run_checks(self): 17 | # Accumulate hash by UUID, useful for multi PKL package 18 | self.hash_map = {} 19 | 20 | for source in self.dcp._list_pkl: 21 | asset_stack = [source["FileName"]] 22 | 23 | checks = self.find_check("pkl") 24 | [self.run_check(check, source, stack=asset_stack) for check in checks] 25 | 26 | asset_checks = self.find_check("assets_pkl") 27 | [ 28 | self.run_check( 29 | check, 30 | source, 31 | asset, 32 | stack=asset_stack + [asset[2].get("Path", asset[2]["Id"])], 33 | ) 34 | for asset in list_pkl_assets(source) 35 | for check in asset_checks 36 | ] 37 | 38 | return self.checks 39 | 40 | def check_pkl_xml(self, pkl): 41 | """PKL XML syntax and structure check.""" 42 | pkl_node = pkl["Info"]["PackingList"] 43 | check_xml( 44 | self, 45 | pkl["FilePath"], 46 | pkl_node["__xmlns__"], 47 | pkl_node["Schema"], 48 | self.dcp.schema, 49 | ) 50 | 51 | def check_pkl_empty_text_fields(self, am): 52 | """PKL empty text fields check. 53 | 54 | References: N/A 55 | """ 56 | fields = ["Creator", "Issuer", "AnnotationText"] 57 | madatory_fields = ["Creator"] 58 | empty_fields = [] 59 | missing_fields = [] 60 | 61 | for f in fields: 62 | am_f = am["Info"]["PackingList"].get(f) 63 | if am_f == "": 64 | empty_fields.append(f) 65 | elif am_f is None and f in madatory_fields: 66 | missing_fields.append(f) 67 | 68 | if empty_fields: 69 | self.error("Empty {} field(s)".format(", ".join(empty_fields))) 70 | if missing_fields: 71 | self.error( 72 | "Missing {} field(s)".format(", ".join(missing_fields)), "missing" 73 | ) 74 | 75 | def check_pkl_issuedate(self, pkl): 76 | """PKL Issue Date validation. 77 | 78 | References: N/A 79 | """ 80 | check_issuedate(self, pkl["Info"]["PackingList"]["IssueDate"]) 81 | 82 | def check_assets_pkl_referenced_by_assetamp(self, pkl, asset): 83 | """PKL assets shall be present in AssetMap. 84 | 85 | References: N/A 86 | """ 87 | uuid, _, _ = asset 88 | # Note : dcp._list_asset is directly extracted from Assetmap 89 | if uuid not in self.dcp._list_asset.keys(): 90 | self.error("Not present in Assetmap") 91 | 92 | def check_assets_pkl_size(self, pkl, asset): 93 | """PKL assets size check. 94 | 95 | References: 96 | SMPTE ST 429-8:2007 6.4 97 | """ 98 | _, path, asset = asset 99 | if not path or not os.path.exists(path): 100 | return 101 | 102 | asset_size = asset["Size"] 103 | actual_size = os.path.getsize(path) 104 | 105 | if actual_size != asset_size: 106 | self.error( 107 | "Invalid size, expected {} but got {}".format(asset_size, actual_size) 108 | ) 109 | 110 | def check_assets_pkl_hash(self, pkl, asset): 111 | """PKL assets hash check. 112 | 113 | References: 114 | SMPTE ST 429-8:2007 6.3 115 | """ 116 | _, path, asset = asset 117 | if not path or not os.path.exists(path): 118 | return 119 | 120 | asset_hash = asset["Hash"] 121 | asset_id = asset["Id"] 122 | 123 | if asset_id not in self.hash_map: 124 | self.hash_map[asset_id] = shaone_b64(path, self.hash_callback) 125 | 126 | if self.hash_map[asset_id] != asset_hash: 127 | self.error( 128 | "Corrupt file, expected hash {} but got {}".format( 129 | asset_hash, self.hash_map[asset_id] 130 | ) 131 | ) 132 | -------------------------------------------------------------------------------- /clairmeta/dcp_check_utils.py: -------------------------------------------------------------------------------- 1 | # Clairmeta - (C) YMAGIS S.A. 2 | # See LICENSE for more information 3 | 4 | import re 5 | from datetime import datetime 6 | from dateutil import parser 7 | 8 | from clairmeta.logger import get_log 9 | from clairmeta.utils.uuid import check_uuid 10 | from clairmeta.utils.xml import validate_xml 11 | from clairmeta.settings import DCP_SETTINGS 12 | 13 | 14 | def get_schema(name): 15 | for k, v in DCP_SETTINGS["xmlns"].items(): 16 | if name == k or name == v: 17 | return v 18 | 19 | 20 | def check_xml_constraints(checker, xml_path): 21 | """Check D-Cinema XML Contraints 22 | 23 | References: 24 | TI Subtitle Operational Recommendation for DLP Cinema Projectors (Draft A) 25 | https://web.archive.org/web/20140924153620/http://dlp.com/downloads/pdf_dlp_cinema_subtitle_operational_recommendation_rev_a.pdf 26 | SMPTE ST 429-17:2017 27 | W3C Extensible Markup Language v (1.0) 28 | """ 29 | # ruff: noqa: E501 30 | # fmt: off 31 | 32 | # Follow the XML spec precicely for the definition of XMLDecl, except for: 33 | # VersionNum := '1.0' 34 | # EncName := 'UTF-8' 35 | # EncodingDecl not optional 36 | # SDDecl must have 'no' 37 | RE_XML_S = r'([\x20\x09\x0D\x0A])' 38 | RE_XML_Eq = '(' + RE_XML_S + '?=' + RE_XML_S + '?)' 39 | RE_XML_SDDecl = '(' + RE_XML_S + 'standalone' + RE_XML_Eq + r'(\'no\'|"no"))' 40 | RE_XML_EncName = r'(UTF\-8)' 41 | RE_XML_EncodingDecl = '(' + RE_XML_S + 'encoding' + RE_XML_Eq + '("' + RE_XML_EncName + r'"|\'' + RE_XML_EncName + r'\'))' 42 | RE_XML_VersionNum = r'(1\.0)' 43 | RE_XML_VersionInfo = '(' + RE_XML_S + 'version' + RE_XML_Eq + r'(\'' + RE_XML_VersionNum + r'\'|"' + RE_XML_VersionNum + '"))' 44 | RE_XML_XMLDecl = r'<\?xml' + RE_XML_VersionInfo + RE_XML_EncodingDecl + RE_XML_SDDecl + '?' + RE_XML_S + '?' + r'\?>' 45 | 46 | # fmt: on 47 | 48 | try: 49 | with open(xml_path, encoding="utf-8-sig") as file: 50 | xml_file = file.read() 51 | newlines = file.newlines 52 | except IOError as e: 53 | get_log().error("Error opening XML file {} : {}".format(xml_path, str(e))) 54 | return 55 | 56 | if re.match("\ufeff", xml_file): 57 | checker.error("BOM not allowed in XML file", "constraints_bom") 58 | 59 | if not ( 60 | re.match(RE_XML_XMLDecl, xml_file) 61 | or re.match("\ufeff" + RE_XML_XMLDecl, xml_file) 62 | ): 63 | checker.error("Invalid XML Declaration", "constraints_declaration") 64 | 65 | # Some files might not have newlines at all (single line) 66 | if newlines not in ["\n", "\r\n", None]: 67 | checker.error( 68 | "XML file has invalid ending: {}".format(repr(file.newlines)), 69 | "constraints_line_ending", 70 | ) 71 | 72 | 73 | def check_xml(checker, xml_path, xml_ns, schema_type, schema_dcp): 74 | # XML constraints 75 | check_xml_constraints(checker, xml_path) 76 | 77 | # Correct namespace 78 | schema_id = get_schema(xml_ns) 79 | if not schema_id: 80 | checker.error("Namespace unknown : {}".format(xml_ns), "namespace") 81 | 82 | # Coherence with package schema 83 | if schema_type != schema_dcp: 84 | message = "Schema is not valid got {} but was expecting {}".format( 85 | schema_type, schema_dcp 86 | ) 87 | checker.error(message, "schema_coherence") 88 | 89 | # XSD schema validation 90 | try: 91 | validate_xml(xml_path, schema_id) 92 | except LookupError: 93 | get_log().info("Schema validation skipped : {}".format(xml_path)) 94 | except Exception as e: 95 | message = "Schema validation error : {}\n" "Using schema : {}".format( 96 | str(e), schema_id 97 | ) 98 | checker.error(message, "schema_validation") 99 | 100 | 101 | def check_issuedate(checker, date): 102 | # As a reminder, date should be already correctly formatted as checked 103 | # by XSD validation. 104 | parse_date = parser.parse(date).astimezone(tz=None) 105 | now_date = datetime.now().astimezone(tz=None) 106 | 107 | if parse_date > now_date: 108 | checker.error("IssueDate is post dated : {}".format(parse_date)) 109 | 110 | 111 | def compare_uuid(checker, uuid_to_check, uuid_reference): 112 | name, uuid = uuid_to_check 113 | name_ref, uuid_ref = uuid_reference 114 | 115 | if not check_uuid(uuid): 116 | checker.error("Invalid {} uuid found : {}".format(name, uuid)) 117 | if uuid.lower() != uuid_ref.lower(): 118 | checker.error( 119 | "Uuid {} ({}) not equal to {} ({})".format(name, uuid, name_ref, uuid_ref) 120 | ) 121 | -------------------------------------------------------------------------------- /clairmeta/profile.py: -------------------------------------------------------------------------------- 1 | # Clairmeta - (C) YMAGIS S.A. 2 | # See LICENSE for more information 3 | 4 | import json 5 | import os 6 | import copy 7 | 8 | from clairmeta.exception import ClairMetaException 9 | 10 | 11 | DCP_CHECK_PROFILE = { 12 | # Checker criticality 13 | # Base level is default and can be overrided per check using its name 14 | # Incomplete name allowed, the best match will be selected automatically 15 | # Wildcard character allowed for regex based matching 16 | # 4 levels : ERROR, WARNING, INFO and SILENT. 17 | "criticality": { 18 | "default": "ERROR", 19 | "certif_der_decoding": "WARNING", 20 | "check_dcnc_": "INFO", 21 | "check_dcp_foreign_files": "WARNING", 22 | "check_assets_am_volindex_one": "WARNING", 23 | "check_*_empty_text_fields": "WARNING", 24 | "check_*_empty_text_fields_missing": "ERROR", 25 | "check_*_xml_constraints_line_ending": "WARNING", 26 | "check_cpl_contenttitle_annotationtext_match": "WARNING", 27 | "check_cpl_contenttitle_pklannotationtext_match": "WARNING", 28 | "check_assets_cpl_missing_from_vf": "WARNING", 29 | "check_assets_cpl_labels_schema": "WARNING", 30 | "check_assets_cpl_filename_uuid": "WARNING", 31 | "check_certif_multi_role": "WARNING", 32 | "check_certif_date_expired": "INFO", 33 | "check_certif_date_overflow": "WARNING", 34 | "check_picture_cpl_avg_bitrate": "WARNING", 35 | "check_picture_cpl_resolution": "WARNING", 36 | "check_subtitle_cpl_reel_number": "WARNING", 37 | "check_subtitle_cpl_empty": "WARNING", 38 | "check_subtitle_cpl_uuid_case": "WARNING", 39 | "check_subtitle_cpl_duplicated_uuid": "WARNING", 40 | "check_subtitle_cpl_first_tt_event": "WARNING", 41 | "check_picture_cpl_archival_framerate": "WARNING", 42 | "check_picture_cpl_hfr_framerate": "WARNING", 43 | "check_sound_cpl_format": "WARNING", 44 | "check_sound_cpl_channel_assignments": "WARNING", 45 | "check_atmos_cpl_channels": "WARNING", 46 | "check_atmos_cpl_objects": "WARNING", 47 | }, 48 | # Checker options 49 | # Bypass is a list of check names (function names) 50 | "bypass": [], 51 | # Allowed foreign files, paths are relative to the DCP root 52 | "allowed_foreign_files": [], 53 | } 54 | 55 | 56 | def get_default_profile(): 57 | """Returns the default DCP checking profile""" 58 | return copy.deepcopy(DCP_CHECK_PROFILE) 59 | 60 | 61 | def load_profile(file_path): 62 | """Load a check profile config file. 63 | 64 | ``file_path`` must be a valid json configuration file, this function 65 | include a basic check for correctness (required keys and type). 66 | 67 | Args: 68 | file_path (str): Config file (json) absolute path. 69 | 70 | Returns: 71 | Dictionary containing check profile settings. 72 | 73 | Raise: 74 | ClairMetaException: ``file_path`` is not a valid file. 75 | ClairMetaException: ``file_path`` is not a json file (.json). 76 | ClairMetaException: ``file_path`` json parsing error. 77 | ClairMetaException: ``file_path`` miss some required keys or values 78 | type are wrong. 79 | 80 | """ 81 | if not os.path.isfile(file_path): 82 | raise ClairMetaException("Load Profile : {} file not found".format(file_path)) 83 | 84 | allowed_ext = [".json"] 85 | file_ext = os.path.splitext(file_path)[-1] 86 | if file_ext not in allowed_ext: 87 | raise ClairMetaException( 88 | "Load Profile : {} must be a valid json file".format(file_path) 89 | ) 90 | 91 | profile_format = {"criticality": dict, "bypass": list} 92 | 93 | try: 94 | with open(file_path) as f: 95 | profile = json.load(f) 96 | except Exception as e: 97 | raise ClairMetaException( 98 | "Load Profile {} : loading error - {}".format(file_path, str(e)) 99 | ) 100 | 101 | for k, v in profile_format.items(): 102 | if k not in profile: 103 | raise ClairMetaException( 104 | "Load Profile {} : missing key {}".format(file_path, k) 105 | ) 106 | if not isinstance(profile[k], v): 107 | raise ClairMetaException( 108 | "Load Profile {} : key {} should be a {}".format(file_path, k, v) 109 | ) 110 | 111 | return profile 112 | 113 | 114 | def save_profile(profile, file_path): 115 | """Save a check profile to json config file. 116 | 117 | Args: 118 | profile (dict): Check profile to save. 119 | file_path (str): Config file (json) absolute path. 120 | 121 | """ 122 | with open(file_path, "w") as f: 123 | json.dump(profile, f) 124 | -------------------------------------------------------------------------------- /clairmeta/sequence_check.py: -------------------------------------------------------------------------------- 1 | # Clairmeta - (C) YMAGIS S.A. 2 | # See LICENSE for more information 3 | 4 | import os 5 | 6 | from clairmeta.settings import SEQUENCE_SETTINGS 7 | from clairmeta.utils.sys import number_is_close 8 | from clairmeta.utils.file import parse_name 9 | 10 | 11 | def check_sequence(path, allowed_extensions, ignore_files=None, ignore_dirs=None): 12 | """Check image file sequence coherence recursively. 13 | 14 | Args: 15 | path (str): Base directory path. 16 | allowed_extensions (dict): Dictionary mapping extensions. 17 | ignore_files (list): List of files name to ignore. 18 | ignore_dirs (list): List of directory name to ignore. 19 | 20 | Raises: 21 | ValueError: If ``path`` is not a valid directory. 22 | ValueError: If ``path`` is an empty directory. 23 | ValueError: If ``allowed_extensions`` is not a dictionary. 24 | 25 | """ 26 | if not os.path.isdir(path): 27 | raise ValueError("Folder not found : {}".format(path)) 28 | if not os.listdir(path): 29 | raise ValueError("Empty folder") 30 | if not isinstance(allowed_extensions, dict): 31 | raise ValueError("Wrong arguments, allowed_extensions must be a dict") 32 | 33 | for dirpath, dirnames, filenames in os.walk(path, topdown=True): 34 | # Filter out explicitly ignored files 35 | if ignore_files: 36 | filenames = [f for f in filenames if f not in ignore_files] 37 | if ignore_dirs: 38 | # Why dirnames[:] ? Quote from the documentation : 39 | # When topdown is True, the caller can modify the dirnames list 40 | # in-place (perhaps using del or slice assignment). 41 | dirnames[:] = [d for d in dirnames if d not in ignore_dirs] 42 | 43 | # No files in folder, nothing to check.. 44 | if not filenames: 45 | continue 46 | 47 | # First file in folder is the reference 48 | check_sequence_folder(dirpath, filenames, allowed_extensions) 49 | 50 | 51 | def check_sequence_folder(dirpath, filenames, allowed_extensions): 52 | """Check image file sequence coherence. 53 | 54 | This function checks : 55 | - Image extension and Mime type is authorized 56 | - No jump (missing frame) are found in the whole sequence 57 | - All images must have the same file name (excluding index) 58 | - All images must have the same extension and Mime type 59 | - All images must have the same size (we work on uncompressed files 60 | only) 61 | 62 | Args: 63 | dirpath (str): Directory path. 64 | filenames (list): List of files to check in ``dirpath``. 65 | allowed_extensions (dict): Dictionary mapping extensions. 66 | 67 | Raises: 68 | ValueError: If image file sequence check failed. 69 | 70 | """ 71 | settings = SEQUENCE_SETTINGS["ALL"] 72 | size_rtol = settings["size_diff_tol"] / 1e2 73 | 74 | # First file in folder is the reference 75 | fileref = filenames[0] 76 | fullpath_ref = os.path.join(dirpath, fileref) 77 | filename, idx = parse_name(fileref) 78 | filesize = os.path.getsize(fullpath_ref) 79 | extension = os.path.splitext(fileref)[-1] 80 | sequence_idx = [idx] 81 | 82 | # Check that this reference is conform 83 | if extension not in allowed_extensions: 84 | raise ValueError("extension {} not authorized".format(extension)) 85 | 86 | # Then check that all subsequent files are identical 87 | for f in filenames[1:]: 88 | fullpath = os.path.join(dirpath, f) 89 | current_ext = os.path.splitext(f)[-1] 90 | current_filename, current_idx = parse_name(f) 91 | current_filesize = os.path.getsize(fullpath) 92 | sequence_idx.append(current_idx) 93 | 94 | if current_filename != filename: 95 | raise ValueError( 96 | "Filename difference, {} but expected {}".format( 97 | current_filename, filename 98 | ) 99 | ) 100 | if current_ext != extension: 101 | raise ValueError( 102 | "File extension difference, {} but expected {}".format( 103 | current_filename, extension 104 | ) 105 | ) 106 | if not number_is_close(current_filesize, filesize, rtol=size_rtol): 107 | raise ValueError( 108 | "{} : file size difference got {} but expected {}" 109 | " - tolerance of {}%".format( 110 | current_filename, 111 | current_filesize, 112 | filesize, 113 | settings["size_diff_tol"], 114 | ) 115 | ) 116 | 117 | # Check for jump in sequence (ie. missing frame(s)) 118 | sequence_idx.sort() 119 | for idx, fno in enumerate(sequence_idx, sequence_idx[0]): 120 | if idx != fno: 121 | raise ValueError("File sequence jump found, file {} not found".format(idx)) 122 | -------------------------------------------------------------------------------- /clairmeta/xsd/xml.xsd: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | See http://www.w3.org/XML/1998/namespace.html and 8 | http://www.w3.org/TR/REC-xml for information about this namespace. 9 | 10 | This schema document describes the XML namespace, in a form 11 | suitable for import by other schema documents. 12 | 13 | Note that local names in this namespace are intended to be defined 14 | only by the World Wide Web Consortium or its subgroups. The 15 | following names are currently defined in this namespace and should 16 | not be used with conflicting semantics by any Working Group, 17 | specification, or document instance: 18 | 19 | base (as an attribute name): denotes an attribute whose value 20 | provides a URI to be used as the base for interpreting any 21 | relative URIs in the scope of the element on which it 22 | appears; its value is inherited. This name is reserved 23 | by virtue of its definition in the XML Base specification. 24 | 25 | lang (as an attribute name): denotes an attribute whose value 26 | is a language code for the natural language of the content of 27 | any element; its value is inherited. This name is reserved 28 | by virtue of its definition in the XML specification. 29 | 30 | space (as an attribute name): denotes an attribute whose 31 | value is a keyword indicating what whitespace processing 32 | discipline is intended for the content of the element; its 33 | value is inherited. This name is reserved by virtue of its 34 | definition in the XML specification. 35 | 36 | Father (in any context at all): denotes Jon Bosak, the chair of 37 | the original XML Working Group. This name is reserved by 38 | the following decision of the W3C XML Plenary and 39 | XML Coordination groups: 40 | 41 | In appreciation for his vision, leadership and dedication 42 | the W3C XML Plenary on this 10th day of February, 2000 43 | reserves for Jon Bosak in perpetuity the XML name 44 | xml:Father 45 | 46 | 47 | 48 | 49 | This schema defines attributes and an attribute group 50 | suitable for use by 51 | schemas wishing to allow xml:base, xml:lang or xml:space attributes 52 | on elements they define. 53 | 54 | To enable this, such a schema must import this schema 55 | for the XML namespace, e.g. as follows: 56 | <schema . . .> 57 | . . . 58 | <import namespace="http://www.w3.org/XML/1998/namespace" 59 | schemaLocation="http://www.w3.org/2001/03/xml.xsd"/> 60 | 61 | Subsequently, qualified reference to any of the attributes 62 | or the group defined below will have the desired effect, e.g. 63 | 64 | <type . . .> 65 | . . . 66 | <attributeGroup ref="xml:specialAttrs"/> 67 | 68 | will define a type which will schema-validate an instance 69 | element with any of those attributes 70 | 71 | 72 | 73 | In keeping with the XML Schema WG's standard versioning 74 | policy, this schema document will persist at 75 | http://www.w3.org/2001/03/xml.xsd. 76 | At the date of issue it can also be found at 77 | http://www.w3.org/2001/xml.xsd. 78 | The schema document at that URI may however change in the future, 79 | in order to remain compatible with the latest version of XML Schema 80 | itself. In other words, if the XML Schema namespace changes, the version 81 | of this document at 82 | http://www.w3.org/2001/xml.xsd will change 83 | accordingly; the version at 84 | http://www.w3.org/2001/03/xml.xsd will not change. 85 | 86 | 87 | 88 | 89 | 90 | In due course, we should install the relevant ISO 2- and 3-letter 91 | codes as the enumerated possible values . . . 92 | 93 | 94 | 95 | 96 | 97 | 98 | 99 | 100 | 101 | 102 | 103 | 104 | 105 | 106 | See http://www.w3.org/TR/xmlbase/ for 107 | information about this attribute. 108 | 109 | 110 | 111 | 112 | 113 | 114 | 115 | 116 | 117 | 118 | -------------------------------------------------------------------------------- /clairmeta/xsd/xmldsig-core-schema.dtd: -------------------------------------------------------------------------------- 1 | 19 | 20 | 21 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 55 | 56 | 57 | 59 | 60 | 62 | 65 | 66 | 67 | 69 | 70 | 71 | 73 | 74 | 75 | 79 | 80 | 81 | 82 | 83 | 84 | 86 | 87 | 88 | 89 | 90 | 92 | 93 | 94 | 95 | 97 | 99 | 100 | 101 | 102 | 103 | 104 | 105 | 106 | 107 | 110 | 111 | 112 | 113 | 115 | 116 | 117 | 118 | 119 | 120 | 121 | 122 | 123 | 124 | 125 | 126 | 127 | 128 | 129 | 130 | 131 | 132 | 133 | 134 | 135 | 136 | 137 | 141 | 142 | 143 | 145 | 146 | 147 | 149 | 150 | 151 | 154 | 155 | 156 | 157 | 158 | 159 | 160 | 161 | 162 | 163 | 164 | 165 | 166 | 167 | 168 | 169 | 170 | 171 | 172 | -------------------------------------------------------------------------------- /clairmeta/dcp_check_sound.py: -------------------------------------------------------------------------------- 1 | # Clairmeta - (C) YMAGIS S.A. 2 | # See LICENSE for more information 3 | 4 | from clairmeta.dcp_check import CheckerBase 5 | from clairmeta.dcp_utils import list_cpl_assets 6 | from clairmeta.settings import DCP_SETTINGS 7 | 8 | 9 | class Checker(CheckerBase): 10 | def __init__(self, dcp): 11 | super(Checker, self).__init__(dcp) 12 | 13 | def run_checks(self): 14 | for source in self.dcp._list_cpl: 15 | asset_checks = self.find_check("sound_cpl") 16 | [ 17 | self.run_check( 18 | check, 19 | source, 20 | asset, 21 | stack=[source["FileName"], asset[1].get("Path") or asset[1]["Id"]], 22 | ) 23 | for asset in list_cpl_assets( 24 | source, filters="Sound", required_keys=["Probe"] 25 | ) 26 | for check in asset_checks 27 | ] 28 | 29 | return self.checks 30 | 31 | def check_sound_cpl_channels(self, playlist, asset): 32 | """Sound max channels count. 33 | 34 | References: 35 | SMPTE ST 428-2:2006 3.3 36 | """ 37 | channels = DCP_SETTINGS["sound"]["max_channel_count"] 38 | _, asset = asset 39 | cc = asset["Probe"]["ChannelCount"] 40 | 41 | if cc > channels: 42 | self.error( 43 | "Invalid Sound ChannelCount, should be less than {} but got {}" 44 | "".format(channels, cc) 45 | ) 46 | 47 | def check_sound_cpl_channels_odd(self, playlist, asset): 48 | """Sound channels count must be an even number. 49 | 50 | Extract fom ISDCF recommandation : Note 1. Not all channels need to 51 | be present in a given DCP. For instance, only the first 8 channels 52 | should be used when delivering 5.1 + HI/VI content. In all cases, 53 | an even number of channels shall be used. 54 | 55 | References: 56 | ISDCF Doc 04 57 | https://isdcf.com/papers/ISDCF-Doc4-Audio-channel-recommendations.pdf 58 | """ 59 | _, asset = asset 60 | cc = asset["Probe"]["ChannelCount"] 61 | 62 | if cc % 2 != 0: 63 | self.error( 64 | "Invalid Sound ChannelCount, should be an even number, got {}" 65 | "".format(cc) 66 | ) 67 | 68 | def check_sound_cpl_channel_assignments(self, playlist, asset): 69 | """Sound channel configuration shall be Wild Track (4). 70 | 71 | References: 72 | ISDCF Doc 04 73 | SMPTE RDD 52:2020 10.3.1 74 | SMPTE ST 429-2:2013 A.1.2 75 | """ 76 | configurations = DCP_SETTINGS["sound"]["configuration_channels"] 77 | _, asset = asset 78 | cf = asset["Probe"]["ChannelFormat"] 79 | 80 | if cf in configurations and cf != 4: 81 | self.error( 82 | 'Detected channel assignments "{}", but expected "{}"'.format( 83 | configurations[cf][0], configurations[4][0] 84 | ) 85 | ) 86 | 87 | def check_sound_cpl_format(self, playlist, asset): 88 | """Sound channels count coherence with format. 89 | 90 | References: 91 | SMPTE ST 429-2:2013 A.1.2 92 | """ 93 | configurations = DCP_SETTINGS["sound"]["configuration_channels"] 94 | _, asset = asset 95 | cf = asset["Probe"]["ChannelFormat"] 96 | cc = asset["Probe"]["ChannelCount"] 97 | 98 | if cf in configurations: 99 | label, min_cc, max_cc = configurations[cf] 100 | if label and cc < min_cc or cc > max_cc: 101 | self.error( 102 | "Invalid Sound ChannelCount, {} require between {} and {} " 103 | "channels, got {}".format(label, min_cc, max_cc, cc) 104 | ) 105 | 106 | def check_sound_cpl_sampling(self, playlist, asset): 107 | """Sound sampling rate check. 108 | 109 | References: 110 | SMPTE ST 428-2:2006 3.2 111 | """ 112 | rates = DCP_SETTINGS["sound"]["sampling_rate"] 113 | _, asset = asset 114 | sr = asset["Probe"]["AudioSamplingRate"] 115 | 116 | if sr not in rates: 117 | self.error( 118 | "Invalid Sound SamplingRate, expected {} but got {}".format(rates, sr) 119 | ) 120 | 121 | def check_sound_cpl_quantization(self, playlist, asset): 122 | """Sound quantization check. 123 | 124 | References: 125 | SMPTE ST 428-2:2006 3.1 126 | """ 127 | bitdepth = DCP_SETTINGS["sound"]["quantization"] 128 | _, asset = asset 129 | depth = asset["Probe"]["QuantizationBits"] 130 | 131 | if depth != bitdepth: 132 | self.error( 133 | "Invalid Sound Quantization, expected {} but got {}".format( 134 | bitdepth, depth 135 | ) 136 | ) 137 | 138 | def check_sound_cpl_blockalign(self, playlist, asset): 139 | """Sound block alignement check. 140 | 141 | References: N/A 142 | """ 143 | align = DCP_SETTINGS["sound"]["quantization"] / 8 144 | _, asset = asset 145 | al = asset["Probe"]["BlockAlign"] 146 | cc = asset["Probe"]["ChannelCount"] 147 | 148 | if al != cc * align: 149 | self.error( 150 | "Invalid Sound BlockAlign, expected {} but got {} (it should " 151 | "be ChannelCount x 3)".format(cc * align, al) 152 | ) 153 | -------------------------------------------------------------------------------- /clairmeta/xsd/xenc-schema.xsd: -------------------------------------------------------------------------------- 1 | 2 | 8 | 9 | 10 | 11 | ]> 12 | 13 | 18 | 19 | 21 | 22 | 23 | 24 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | 74 | 75 | 76 | 77 | 78 | 79 | 80 | 81 | 82 | 83 | 84 | 85 | 86 | 87 | 88 | 89 | 90 | 92 | 93 | 94 | 95 | 96 | 97 | 98 | 99 | 100 | 101 | 102 | 103 | 104 | 105 | 106 | 107 | 108 | 109 | 110 | 111 | 112 | 113 | 114 | 115 | 116 | 117 | 118 | 119 | 120 | 121 | 122 | 123 | 124 | 125 | 126 | 127 | 128 | 129 | 130 | 131 | 132 | 133 | 134 | 135 | 136 | 137 | 138 | 139 | 140 | 141 | 142 | 143 | 144 | 145 | 146 | 147 | -------------------------------------------------------------------------------- /clairmeta/cli.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # Clairmeta - (C) YMAGIS S.A. 3 | # See LICENSE for more information 4 | 5 | from __future__ import print_function 6 | 7 | import os 8 | import argparse 9 | import sys 10 | import json 11 | import dicttoxml 12 | import pprint 13 | 14 | from clairmeta import DCP, Sequence 15 | from clairmeta.logger import disable_log 16 | from clairmeta.info import __version__ 17 | from clairmeta.profile import load_profile, DCP_CHECK_PROFILE 18 | from clairmeta.settings import SEQUENCE_SETTINGS 19 | from clairmeta.utils.xml import prettyprint_xml 20 | from clairmeta.utils.file import ConsoleProgress 21 | 22 | 23 | package_type_map = { 24 | "dcp": DCP, 25 | "dcdm": Sequence, 26 | "dsm": Sequence, 27 | "scan": Sequence, 28 | } 29 | 30 | package_check_settings = { 31 | "dcdm": SEQUENCE_SETTINGS["DCDM"], 32 | "dsm": SEQUENCE_SETTINGS["DSM"], 33 | "scan": SEQUENCE_SETTINGS["SCAN"], 34 | } 35 | 36 | 37 | def cli_check(args): 38 | try: 39 | if args.type == "dcp": 40 | check_profile = DCP_CHECK_PROFILE 41 | callback = None 42 | 43 | if args.profile: 44 | path = os.path.abspath(args.profile) 45 | check_profile = load_profile(path) 46 | if args.log: 47 | check_profile["log_level"] = args.log 48 | if args.progress: 49 | callback = ConsoleProgress() 50 | if args.format != "text": 51 | disable_log() 52 | 53 | status, report = DCP(args.path, kdm=args.kdm, pkey=args.key).check( 54 | profile=check_profile, ov_path=args.ov, hash_callback=callback 55 | ) 56 | 57 | if args.format == "dict": 58 | msg = pprint.pformat(report.to_dict()) 59 | elif args.format == "json": 60 | msg = json.dumps( 61 | report.to_dict(), sort_keys=True, indent=2, separators=(",", ": ") 62 | ) 63 | elif args.format == "xml": 64 | xml_str = dicttoxml.dicttoxml( 65 | report.to_dict(), 66 | custom_root="ClairmetaCheck", 67 | ids=False, 68 | attr_type=False, 69 | ) 70 | msg = prettyprint_xml(xml_str) 71 | 72 | if args.format != "text": 73 | return True, msg 74 | 75 | else: 76 | obj_type = package_type_map[args.type] 77 | setting = package_check_settings[args.type] 78 | status = obj_type(args.path).check(setting) 79 | 80 | except Exception as e: 81 | status = False 82 | print("Error : {}".format(e)) 83 | 84 | msg = "{} - {} - Check {}".format( 85 | args.type.upper(), args.path, "succeeded" if status else "failed" 86 | ) 87 | return status, msg 88 | 89 | 90 | def cli_probe(args): 91 | try: 92 | disable_log() 93 | kwargs = {} 94 | 95 | if args.type == "dcp": 96 | kwargs["kdm"] = args.kdm 97 | kwargs["pkey"] = args.key 98 | 99 | obj_type = package_type_map[args.type] 100 | res = obj_type(args.path, **kwargs).parse() 101 | 102 | if args.format == "dict": 103 | msg = pprint.pformat(res) 104 | elif args.format == "json": 105 | msg = json.dumps(res, sort_keys=True, indent=2, separators=(",", ": ")) 106 | elif args.format == "xml": 107 | xml_str = dicttoxml.dicttoxml( 108 | res, custom_root="ClairmetaProbe", ids=False, attr_type=False 109 | ) 110 | msg = prettyprint_xml(xml_str) 111 | 112 | return True, msg 113 | except Exception as e: 114 | return False, "Error : {}".format(e) 115 | 116 | 117 | def get_parser(): 118 | global_parser = argparse.ArgumentParser( 119 | description="Clairmeta Command Line Interface {}".format(__version__) 120 | ) 121 | subparsers = global_parser.add_subparsers() 122 | 123 | # DCP 124 | parser = subparsers.add_parser("check", help="Package validation") 125 | parser.add_argument("path", help="absolute package path") 126 | parser.add_argument("-log", default=None, help="logging level [dcp]") 127 | parser.add_argument("-profile", default=None, help="json profile [dcp]") 128 | parser.add_argument("-kdm", default=None, help="kdm with encrypted keys [dcp]") 129 | parser.add_argument("-key", default=None, help="recipient private key [dcp]") 130 | parser.add_argument( 131 | "-format", 132 | default="text", 133 | choices=["text", "dict", "xml", "json"], 134 | help="output format [dcp]", 135 | ) 136 | parser.add_argument( 137 | "-progress", action="store_true", help="hash progress bar [dcp]" 138 | ) 139 | parser.add_argument("-ov", default=None, help="ov package path [dcp]") 140 | parser.add_argument( 141 | "-type", choices=package_type_map.keys(), required=True, help="package type" 142 | ) 143 | parser.set_defaults(func=cli_check) 144 | 145 | parser = subparsers.add_parser("probe", help="Package metadata extraction") 146 | parser.add_argument("path", help="absolute package path") 147 | parser.add_argument("-kdm", default=None, help="kdm with encrypted keys") 148 | parser.add_argument("-key", default=None, help="recipient private key") 149 | parser.add_argument( 150 | "-format", default="dict", choices=["dict", "xml", "json"], help="output format" 151 | ) 152 | parser.add_argument( 153 | "-type", choices=package_type_map.keys(), required=True, help="package type" 154 | ) 155 | parser.set_defaults(func=cli_probe) 156 | 157 | return global_parser 158 | 159 | 160 | if __name__ == "__main__": 161 | parser = get_parser() 162 | args = parser.parse_args() 163 | if len(sys.argv) == 1: 164 | parser.print_help() 165 | else: 166 | status, msg = args.func(args) 167 | print(msg) 168 | sys.exit(0 if status else 1) 169 | -------------------------------------------------------------------------------- /clairmeta/dcp_check_global.py: -------------------------------------------------------------------------------- 1 | # Clairmeta - (C) YMAGIS S.A. 2 | # See LICENSE for more information 3 | 4 | import os 5 | import re 6 | 7 | from clairmeta.dcp_utils import list_cpl_assets, cpl_probe_asset 8 | from clairmeta.dcp_check import CheckerBase 9 | 10 | 11 | class Checker(CheckerBase): 12 | def __init__(self, dcp): 13 | super(Checker, self).__init__(dcp) 14 | 15 | def run_checks(self): 16 | """Execute all checks.""" 17 | dcp_checks = self.find_check("dcp") 18 | [self.run_check(c, stack=[self.dcp.path]) for c in dcp_checks] 19 | self.setup_dcp_link_ov() 20 | 21 | return self.checks 22 | 23 | def check_dcp_empty_dir(self): 24 | """Empty directory detection. 25 | 26 | References: N/A 27 | """ 28 | list_empty_dir = [] 29 | for dirpath, dirnames, filenames in os.walk(self.dcp.path): 30 | for d in dirnames: 31 | fullpath = os.path.join(dirpath, d) 32 | if not os.listdir(fullpath): 33 | list_empty_dir.append(os.path.relpath(fullpath, self.dcp.path)) 34 | 35 | if list_empty_dir: 36 | self.error("Empty directories detected : {}".format(list_empty_dir)) 37 | 38 | def check_dcp_hidden_files(self): 39 | """Hidden files detection. 40 | 41 | References: N/A 42 | """ 43 | hidden_files = [ 44 | os.path.relpath(f, self.dcp.path) 45 | for f in self.dcp._list_files 46 | if os.path.basename(f).startswith(".") 47 | ] 48 | if hidden_files: 49 | self.error("Hidden files detected : {}".format(hidden_files)) 50 | 51 | def check_dcp_foreign_files(self): 52 | """Foreign files detection (not listed in AssetMap). 53 | 54 | References: N/A 55 | """ 56 | list_asset_path = [ 57 | os.path.join(self.dcp.path, a) for a in self.dcp._list_asset.values() 58 | ] 59 | list_asset_path += self.dcp._list_vol_path 60 | list_asset_path += self.dcp._list_am_path 61 | 62 | self.dcp.foreign_files = [ 63 | os.path.relpath(a, self.dcp.path) 64 | for a in self.dcp._list_files 65 | if a not in list_asset_path 66 | and not any([re.search(i, a) for i in self.allowed_foreign_files]) 67 | ] 68 | if self.dcp.foreign_files: 69 | self.error("\n".join(self.dcp.foreign_files)) 70 | 71 | def check_dcp_multiple_am_or_vol(self): 72 | """Only one AssetMap and VolIndex shall be present. 73 | 74 | References: N/A 75 | """ 76 | restricted_lists = { 77 | "VolIndex": self.dcp._list_vol, 78 | "Assetmap": self.dcp._list_am, 79 | } 80 | 81 | for k, v in restricted_lists.items(): 82 | if len(v) == 0: 83 | self.error("Missing {} file".format(k)) 84 | if len(v) > 1: 85 | self.error("Multiple {} files found".format(k)) 86 | 87 | def setup_dcp_link_ov(self): 88 | """Setup the link VF to OV check and run for each assets.""" 89 | if not self.ov_path: 90 | return 91 | 92 | self.run_check(self.check_link_ov_coherence, stack=[self.dcp.path]) 93 | for cpl in self.dcp._list_cpl: 94 | for essence, asset in list_cpl_assets(cpl): 95 | self.run_check( 96 | self.check_link_ov_asset, asset, essence, stack=[self.dcp.path] 97 | ) 98 | 99 | def check_dcp_signed(self): 100 | """Encrypted DCP must be digitally signed (XMLs include Signer and Signature). 101 | 102 | References: 103 | DCI DCSS (v1.3) 5.4.3.7 104 | DCI DCSS (v1.3) 5.5.2.3 105 | """ 106 | for cpl in self.dcp._list_cpl: 107 | cpl_name = cpl["FileName"] 108 | cpl_node = cpl["Info"]["CompositionPlaylist"] 109 | if not cpl_node["Encrypted"]: 110 | continue 111 | 112 | xmls = [ 113 | (pkl["FileName"], pkl["Info"]["PackingList"]) 114 | for pkl in self.dcp._list_pkl 115 | if pkl["Info"]["PackingList"]["Id"] == cpl_node.get("PKLId") 116 | ] 117 | xmls.append((cpl_name, cpl_node)) 118 | 119 | for name, xml in xmls: 120 | for field in ["Signer", "Signature"]: 121 | if field not in xml.keys(): 122 | self.error("Missing {} element in {}".format(field, name)) 123 | 124 | def check_link_ov_coherence(self): 125 | """Relink OV/VF sanity checks. 126 | 127 | References: N/A 128 | """ 129 | if self.ov_path and self.dcp.package_type != "VF": 130 | self.error("Package checked must be a VF") 131 | 132 | from clairmeta.dcp import DCP 133 | 134 | self.ov_dcp = DCP(self.ov_path) 135 | self.ov_dcp.parse() 136 | if self.ov_dcp.package_type != "OV": 137 | self.error("Package referenced must be a OV") 138 | 139 | def check_link_ov_asset(self, asset, essence): 140 | """VF package shall reference assets present in OV. 141 | 142 | References: N/A 143 | """ 144 | if not self.ov_dcp: 145 | return 146 | 147 | ov_dcp_dict = self.ov_dcp.parse() 148 | 149 | if not asset.get("Path"): 150 | uuid = asset["Id"] 151 | path_ov = ov_dcp_dict["asset_list"].get(uuid) 152 | 153 | if not path_ov: 154 | self.error("Asset missing ({}) from OV : {}".format(essence, uuid)) 155 | 156 | asset_path = os.path.join(self.ov_dcp.path, path_ov) 157 | if not os.path.exists(asset_path): 158 | self.error( 159 | "Asset missing ({}) from OV (MXF not found) : {}" 160 | "".format(essence, path_ov) 161 | ) 162 | 163 | # Probe asset for later checks 164 | asset["AbsolutePath"] = asset_path 165 | cpl_probe_asset(asset, essence, asset_path) 166 | -------------------------------------------------------------------------------- /tests/test_cli.py: -------------------------------------------------------------------------------- 1 | # Clairmeta - (C) YMAGIS S.A. 2 | # See LICENSE for more information 3 | 4 | import collections 5 | import platform 6 | import unittest 7 | import os 8 | import json 9 | from collections import OrderedDict 10 | from xml.etree import ElementTree as ET 11 | 12 | from tests import DCP_MAP 13 | from clairmeta.logger import disable_log 14 | from clairmeta.cli import get_parser 15 | 16 | 17 | class CliTest(unittest.TestCase): 18 | def __init__(self, *args, **kwargs): 19 | super(CliTest, self).__init__(*args, **kwargs) 20 | disable_log() 21 | 22 | def get_file_path(self, name): 23 | file_path = os.path.join(os.path.dirname(__file__), "resources", name) 24 | 25 | return file_path 26 | 27 | def get_dcp_path(self, dcp_id): 28 | if dcp_id in DCP_MAP: 29 | dcp_folder = os.path.join( 30 | os.path.dirname(__file__), "resources", "DCP", "ECL-SET" 31 | ) 32 | dcp_name = DCP_MAP[dcp_id] 33 | 34 | folder_path = os.path.join(dcp_folder, dcp_name) 35 | self.assertTrue(os.path.exists(folder_path)) 36 | return os.path.relpath(folder_path) 37 | 38 | def get_dsm_path(self, name): 39 | file_path = os.path.join(os.path.dirname(__file__), "resources", "SEQ", name) 40 | 41 | return file_path 42 | 43 | def launch_command(self, args): 44 | parser = get_parser() 45 | args = parser.parse_args(args) 46 | return args.func(args) 47 | 48 | def test_dcp_probe_formating_dict(self): 49 | status, msg = self.launch_command( 50 | ["probe", self.get_dcp_path(1), "-type", "dcp", "-format", "dict"] 51 | ) 52 | self.assertTrue(isinstance(eval(msg), collections.abc.Mapping)) 53 | 54 | def test_dcp_probe_formating_xml(self): 55 | status, msg = self.launch_command( 56 | ["probe", self.get_dcp_path(1), "-type", "dcp", "-format", "xml"] 57 | ) 58 | ET.XML(msg) 59 | 60 | def test_dcp_probe_formating_json(self): 61 | # Reference file contains Unix specific values (path formating) 62 | if platform.system() == "Windows": 63 | return 64 | 65 | status, msg = self.launch_command( 66 | ["probe", self.get_dcp_path(1), "-type", "dcp", "-format", "json"] 67 | ) 68 | 69 | json_test = json.loads(msg, object_pairs_hook=OrderedDict) 70 | with open(self.get_file_path("ECL01.json")) as f: 71 | json_gold = json.load(f, object_pairs_hook=OrderedDict) 72 | 73 | # Prefer comparing strings for better diagnostic messages 74 | self.assertEqual( 75 | json.dumps(json_test, indent=4, sort_keys=True), 76 | json.dumps(json_gold, indent=4, sort_keys=True), 77 | ) 78 | 79 | def test_dcp_check_formating_dict(self): 80 | status, msg = self.launch_command( 81 | ["check", self.get_dcp_path(1), "-type", "dcp", "-format", "dict"] 82 | ) 83 | self.assertTrue(isinstance(eval(msg), collections.abc.Mapping)) 84 | 85 | def test_dcp_check_formating_xml(self): 86 | status, msg = self.launch_command( 87 | ["check", self.get_dcp_path(1), "-type", "dcp", "-format", "xml"] 88 | ) 89 | ET.XML(msg) 90 | 91 | def test_dcp_check_formating_json(self): 92 | status, msg = self.launch_command( 93 | ["check", self.get_dcp_path(1), "-type", "dcp", "-format", "json"] 94 | ) 95 | json.loads(msg, object_pairs_hook=OrderedDict) 96 | 97 | def test_dcp_check_good(self): 98 | status, msg = self.launch_command( 99 | ["check", self.get_dcp_path(1), "-type", "dcp", "-log", "CRITICAL"] 100 | ) 101 | self.assertTrue(status) 102 | 103 | def test_dcp_check_good_progress(self): 104 | status, msg = self.launch_command( 105 | [ 106 | "check", 107 | self.get_dcp_path(1), 108 | "-type", 109 | "dcp", 110 | "-log", 111 | "CRITICAL", 112 | "-progress", 113 | ] 114 | ) 115 | self.assertTrue(status) 116 | 117 | def test_dcp_check_good_relink(self): 118 | status, msg = self.launch_command( 119 | [ 120 | "check", 121 | self.get_dcp_path(2), 122 | "-type", 123 | "dcp", 124 | "-log", 125 | "CRITICAL", 126 | "-ov", 127 | self.get_dcp_path(1), 128 | ] 129 | ) 130 | self.assertTrue(status) 131 | 132 | def test_dcp_check_wrong_relink(self): 133 | status, msg = self.launch_command( 134 | [ 135 | "check", 136 | self.get_dcp_path(1), 137 | "-type", 138 | "dcp", 139 | "-log", 140 | "CRITICAL", 141 | "-ov", 142 | self.get_dcp_path(2), 143 | ] 144 | ) 145 | self.assertFalse(status) 146 | 147 | def test_dcp_check_bad(self): 148 | status, msg = self.launch_command( 149 | ["check", self.get_dcp_path(25), "-type", "dcp", "-log", "CRITICAL"] 150 | ) 151 | self.assertFalse(status) 152 | 153 | def test_dsm_probe(self): 154 | status, msg = self.launch_command( 155 | ["probe", "-type", "dsm", self.get_dsm_path("DSM_PKG/MINI_DSM1")] 156 | ) 157 | self.assertTrue(status) 158 | 159 | def test_dsm_check_good(self): 160 | status, msg = self.launch_command( 161 | ["check", "-type", "dsm", self.get_dsm_path("DSM_PKG/MINI_DSM1")] 162 | ) 163 | self.assertTrue(status) 164 | 165 | def test_dsm_check_bad(self): 166 | status, msg = self.launch_command( 167 | ["check", "-type", "dsm", self.get_dsm_path("DSM_BAD_FILE_NAME_LENGTH")] 168 | ) 169 | self.assertFalse(status) 170 | 171 | 172 | if __name__ == "__main__": 173 | unittest.main() 174 | -------------------------------------------------------------------------------- /clairmeta/dcp_check.py: -------------------------------------------------------------------------------- 1 | # Clairmeta - (C) YMAGIS S.A. 2 | # See LICENSE for more information 3 | 4 | import re 5 | import time 6 | import importlib 7 | import inspect 8 | import traceback 9 | 10 | from clairmeta.settings import DCP_CHECK_SETTINGS 11 | from clairmeta.logger import get_log 12 | from clairmeta.dcp_check_execution import CheckError, CheckExecution 13 | from clairmeta.utils.file import ConsoleProgress 14 | from clairmeta.exception import CheckException 15 | 16 | 17 | class CheckerBase(object): 18 | """Digital Cinema Package checker. 19 | 20 | Base class for check module, provide check discover and run utilities. 21 | All check module shall derive from this class. 22 | 23 | """ 24 | 25 | ERROR_NAME_RE = re.compile(r"^\w+$") 26 | 27 | def __init__( 28 | self, 29 | dcp, 30 | ov_path=None, 31 | hash_callback=None, 32 | bypass_list=None, 33 | allowed_foreign_files=None, 34 | ): 35 | """CheckerBase constructor. 36 | 37 | Args: 38 | dcp (clairmeta.DCP): DCP object. 39 | ov_path (str, optional): Absolute path of OriginalVersion DCP. 40 | hash_callback (function, optional): Callback function to report 41 | file hash progression. 42 | bypass_list (list, optional): List of checks to bypass. 43 | allowed_foreign_files (list, optional): List of files allowed 44 | in the DCP folder (don't trigger foreign files check). 45 | 46 | """ 47 | self.dcp = dcp 48 | self.log = get_log() 49 | self.checks = [] 50 | self.errors = [] 51 | self.report = None 52 | self.bypass_list = bypass_list or [] 53 | self.allowed_foreign_files = allowed_foreign_files or [] 54 | self.check_modules = {} 55 | self.ov_path = ov_path 56 | self.ov_dcp = None 57 | 58 | self.hash_callback = hash_callback 59 | if not self.hash_callback: 60 | pass 61 | elif isinstance(self.hash_callback, ConsoleProgress): 62 | self.hash_callback._total_size = self.dcp.size 63 | elif inspect.isclass(self.hash_callback): 64 | raise CheckException( 65 | "Invalid callback, please provide a function" 66 | " or instance of ConsoleProgress (or derivate)." 67 | ) 68 | 69 | def load_modules(self): 70 | prefix = DCP_CHECK_SETTINGS["module_prefix"] 71 | for k, v in DCP_CHECK_SETTINGS["modules"].items(): 72 | try: 73 | module_path = "clairmeta." + prefix + k 74 | module = importlib.import_module(module_path) 75 | checker = module.Checker(self.dcp) 76 | checker.ov_path = self.ov_path 77 | checker.allowed_foreign_files = self.allowed_foreign_files 78 | checker.bypass_list = self.bypass_list 79 | checker.hash_callback = self.hash_callback 80 | self.check_modules[v] = checker 81 | except (ImportError, Exception) as e: 82 | self.log.critical("Import error {} : {}".format(module_path, str(e))) 83 | 84 | def check(self): 85 | """Execute the complete check process. 86 | 87 | Returns: 88 | List of checks executed. 89 | 90 | """ 91 | self.load_modules() 92 | return self.run_checks() 93 | 94 | def find_check(self, prefix): 95 | """Discover checks functions (using introspection). 96 | 97 | Args: 98 | prefix (str): Prefix of the checks to find (excluding leading 99 | 'check_'). 100 | 101 | Returns: 102 | List of check functions. 103 | 104 | """ 105 | checks = [] 106 | 107 | member_list = inspect.getmembers(self, predicate=inspect.ismethod) 108 | for k, v in member_list: 109 | check_prefix = k.startswith("check_" + prefix) 110 | check_bypass = any([k.startswith(c) for c in self.bypass_list]) 111 | 112 | if check_prefix and not check_bypass: 113 | checks.append(v) 114 | elif check_bypass: 115 | check_exec = CheckExecution(v) 116 | check_exec.bypass = True 117 | self.checks.append(check_exec) 118 | 119 | return checks 120 | 121 | def run_checks(self): 122 | """Execute all checks.""" 123 | self.log.info("Checking DCP : {}".format(self.dcp.path)) 124 | 125 | for _, checker in self.check_modules.items(): 126 | self.checks += checker.run_checks() 127 | return self.checks 128 | 129 | def run_check(self, check, *args, **kwargs): 130 | """Execute a check. 131 | 132 | Args: 133 | check (function): Check function. 134 | *args: Variable list of check function arguments. 135 | **kwargs: Variable list of keywords arguments. 136 | error_prefix (str): error message prefix 137 | 138 | Returns: 139 | Check function return value 140 | 141 | """ 142 | self._check_setup() 143 | 144 | check_exec = CheckExecution(check) 145 | 146 | try: 147 | start = time.time() 148 | check_res = None 149 | check_res = check(*args) 150 | except CheckException: 151 | pass 152 | except Exception: 153 | error = CheckError("{}".format(traceback.format_exc())) 154 | error.name = "internal_error" 155 | error.parent_name = check_exec.name 156 | error.doc = "ClairMeta internal error" 157 | check_exec.errors.append(error) 158 | self.log.error(error.message) 159 | finally: 160 | for error in self.errors: 161 | error.parent_name = check_exec.name 162 | error.parent_doc = check_exec.doc 163 | check_exec.errors.append(error) 164 | 165 | check_exec.asset_stack = kwargs.get("stack", [self.dcp.path]) 166 | check_exec.seconds_elapsed = time.time() - start 167 | 168 | self.checks.append(check_exec) 169 | 170 | return check_res 171 | 172 | def _check_setup(self): 173 | """Internal setup executed before each check is run.""" 174 | self.errors = [] 175 | 176 | def error(self, message, name="", doc=""): 177 | """Append an error to the current check execution. 178 | 179 | Args: 180 | message (str): Error message. 181 | name (str): Error name that will be appended to the check name 182 | to uniquely identify this error. Only alphanumeric 183 | characters allowed. 184 | doc (str): Error description. 185 | 186 | """ 187 | if name and not re.match(self.ERROR_NAME_RE, name): 188 | raise Exception("Error name invalid : {}".format(name)) 189 | 190 | self.errors.append(CheckError(message, name.lower(), doc)) 191 | 192 | def fatal_error(self, message, name="", doc=""): 193 | """Append an error and halt the current check execution.""" 194 | self.error(message, name, doc) 195 | raise CheckException() 196 | -------------------------------------------------------------------------------- /README.rst: -------------------------------------------------------------------------------- 1 | |Build Status| |PyPI version| |Code coverage| 2 | 3 | ClairMeta 4 | ========= 5 | 6 | ClairMeta is a python package for Digital Cinema Package (DCP) probing 7 | and checking. 8 | 9 | Features 10 | -------- 11 | 12 | - DCP Probe: 13 | - Metadata extraction of the whole DCP, including all XML fields and MXF 14 | assets inspection. 15 | - DCP Checker: 16 | - SMPTE / Interop standard convention 17 | - Integrity (MIME type, size, hash) of all assets 18 | - Foreign file identification 19 | - XSD Schema validation for XML files (VOLINDEX, ASSETMAP, CPL, PKL) 20 | - Digital signature validation (CPL, PKL) 21 | - Intra / Inter Reels integrity and coherence 22 | - Metadata match between CPL assets and MXF headers 23 | - Re-link VF / OV 24 | - Picture tests : FrameRate, BitRate 25 | - Sound tests : Channels, Sampling 26 | - Subtitle : Deep inspection of Interop and SMPTE subtitles 27 | - DSM / DCDM Checker: 28 | - Basic image file sequence validation with some specific rules. 29 | 30 | Installation 31 | ------------ 32 | 33 | Requirements: 34 | 35 | - Python: 3.8 or later 36 | - Platform: Windows (with limitations), macOS, Linux 37 | - External (non-python) dependencies: 38 | - asdcplib 39 | - mediainfo (opt) 40 | - sox (opt) 41 | 42 | Install from PyPI package (this does not install external dependencies): 43 | 44 | .. code-block:: bash 45 | 46 | pip install clairmeta 47 | 48 | If you need help installing the external dependencies, you can have a look at 49 | our continuous integration system, specifically the **.github** folder. 50 | 51 | 52 | Usage 53 | ----- 54 | 55 | General 56 | ~~~~~~~ 57 | 58 | As a command line tool: 59 | 60 | .. code-block:: python 61 | 62 | # Probing 63 | python3 -m clairmeta.cli probe -type dcp path/to/dcp 64 | python3 -m clairmeta.cli probe -type dcp path/to/dcp -format json > dcp.json 65 | python3 -m clairmeta.cli probe -type dcp path/to/dcp -format xml > dcp.xml 66 | 67 | # Checking 68 | python3 -m clairmeta.cli check -type dcp path/to/dcp 69 | python3 -m clairmeta.cli check -type dcp path/to/dcp -format json > check.json 70 | python3 -m clairmeta.cli check -type dcp path/to/dcp -format xml > check.xml 71 | python3 -m clairmeta.cli check -type dcp path/to/dcp -kdm /path/to/kdm -key /path/to/privatekey 72 | python3 -m clairmeta.cli check -type dcp path/to/dcp -progress 73 | python3 -m clairmeta.cli check -type dcp path/to/dcp_vf -ov path/to/dcp_ov 74 | 75 | As a python library: 76 | 77 | .. code-block:: python 78 | 79 | from clairmeta import DCP 80 | 81 | dcp = DCP("path/to/dcp") 82 | dcp.parse() 83 | status, report = dcp.check() 84 | 85 | .. code-block:: python 86 | 87 | # Check DCP VF against OV 88 | status, report = dcp.check(ov_path="/path/to/dcp_ov") 89 | 90 | .. code-block:: python 91 | 92 | # DCP check with console progression report 93 | from clairmeta.utils.file import ConsoleProgress 94 | 95 | status, report = dcp.check(hash_callback=ConsoleProgress()) 96 | # Alternatives 97 | # - function matching utils.file.ConsoleProgress.__call__ signature 98 | # - derived class from utils.file.ConsoleProgress 99 | 100 | 101 | Profiles 102 | ~~~~~~~~ 103 | 104 | Check profile allow custom configuration of the DCP check process such 105 | as bypassing some unwanted tests or error level specification. To 106 | implement a check profile, simply write a JSON file derived from this 107 | template (actual content listed below is for demonstration purposes only): 108 | 109 | - *criticality* key allow custom criteria level specification, check 110 | name can be incomplete to quickly ignore a bunch of tests, *default* is 111 | used if no other match where found. 112 | - *bypass* key allow specific test bypass, incomplete names are not allowed. 113 | - *allowed_foreign_files* key specify files that are allowed in the DCP 114 | folder and should not trigger the foreign file check. 115 | 116 | .. code-block:: python 117 | 118 | { 119 | "criticality": { 120 | "default": "ERROR", 121 | "check_dcnc_": "WARNING", 122 | "check_cpl_reel_duration_picture_subtitles": "WARNING", 123 | "check_picture_cpl_avg_bitrate": "WARNING", 124 | "check_picture_cpl_resolution": "WARNING" 125 | }, 126 | "bypass": ["check_assets_pkl_hash"], 127 | "allowed_foreign_files": ["md5.md5"] 128 | } 129 | 130 | Custom profile check: 131 | 132 | .. code-block:: python 133 | 134 | python3 -m clairmeta.cli check -type dcp path/to/dcp -profile path/to/profile.json 135 | 136 | .. code-block:: python 137 | 138 | from clairmeta import DCP 139 | from clairmeta.profile import load_profile 140 | 141 | dcp = DCP("path/to/dcp") 142 | profile = load_profile("/path/to/profile.json") 143 | status, report = dcp.check(profile=profile) 144 | 145 | Logging 146 | ~~~~~~~ 147 | 148 | Logging is customizable, see the *settings.py* file or below. By default 149 | ClairMeta logs to stdout and a rotated log file. 150 | 151 | .. code-block:: python 152 | 153 | 'level': 'INFO' # Minimum log level 154 | 'enable_console': True # Enable / Disable stdout logging 155 | 'enable_file': True # Enable / Disable file logging 156 | 'file_name': '/log/path/clairmeta.log' # Log file absolute path 157 | 'file_size': 1e6 # Individual log file maximum size 158 | 'file_count': 10 # Number of files to rotate on 159 | 160 | Contributing 161 | ------------ 162 | 163 | - To setup your environment follow these steps: 164 | 165 | .. code-block:: bash 166 | 167 | git clone https://github.com/Ymagis/ClairMeta.git 168 | cd clairmeta 169 | git clone https://github.com/Ymagis/ClairMeta_Data tests/resources 170 | 171 | uv sync --all-extras --dev 172 | 173 | # Code... 174 | 175 | uv run ruff check 176 | uv run black . 177 | uv run pytest --doctest-modules 178 | 179 | - Open a Pull Request 180 | - Open an Issue 181 | 182 | Changes 183 | ------- 184 | 185 | The release changes are available on Github: 186 | https://github.com/Ymagis/ClairMeta/releases 187 | 188 | References 189 | ---------- 190 | 191 | The following sources / software were used: 192 | 193 | - asdcp-lib: http://www.cinecert.com/asdcplib/ 194 | - sox: http://sox.sourceforge.net/ 195 | - mediainfo: https://mediaarea.net/ 196 | - SMPTE Digital Cinema standards: https://www.smpte.org/ 197 | - Interop Digital Cinema specifications: https://cinepedia.com/interop/ 198 | - Digital Cinema Initiative specifications: http://www.dcimovies.com/specification/index.html 199 | - ISDCF Naming Convention: http://isdcf.com/dcnc/ 200 | - Texas Instrument Digital Cinema Subtitles specifications 201 | 202 | About 203 | ----- 204 | 205 | http://www.ymagis.com/ 206 | 207 | .. |Build Status| image:: https://github.com/Ymagis/ClairMeta/actions/workflows/test-package.yml/badge.svg 208 | :target: https://github.com/Ymagis/ClairMeta/actions/workflows/test-package.yml 209 | .. |PyPI version| image:: https://badge.fury.io/py/clairmeta.svg 210 | :target: https://badge.fury.io/py/clairmeta 211 | .. |Code coverage| image:: https://codecov.io/gh/Ymagis/ClairMeta/branch/develop/graph/badge.svg 212 | :target: https://codecov.io/gh/Ymagis/ClairMeta -------------------------------------------------------------------------------- /clairmeta/xsd/SMPTE-429-16-2014-CPL-Metadata.xsd: -------------------------------------------------------------------------------- 1 | 2 | 9 | 10 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | 74 | 75 | 76 | 77 | 78 | 79 | 80 | 81 | 82 | 83 | 84 | 85 | 86 | 87 | 88 | 89 | 90 | 91 | 92 | 93 | 94 | 95 | 96 | 97 | 98 | 99 | 100 | 101 | 102 | 103 | 104 | 105 | 106 | 107 | 108 | 109 | 110 | 111 | 112 | 113 | 114 | 115 | 116 | 117 | 118 | 119 | 120 | 121 | 122 | 123 | 124 | 125 | 126 | 127 | 128 | 129 | 130 | 131 | 132 | 133 | 134 | 135 | 136 | 137 | 138 | 139 | 140 | 141 | 142 | 143 | 144 | 145 | 146 | 147 | -------------------------------------------------------------------------------- /tests/test_dcp_parse.py: -------------------------------------------------------------------------------- 1 | # Clairmeta - (C) YMAGIS S.A. 2 | # See LICENSE for more information 3 | 4 | import unittest 5 | import os 6 | 7 | from tests import DCP_MAP 8 | from clairmeta.logger import disable_log 9 | from clairmeta.dcp import DCP 10 | 11 | 12 | class ParserTestBase(unittest.TestCase): 13 | def __init__(self, *args, **kwargs): 14 | super(ParserTestBase, self).__init__(*args, **kwargs) 15 | disable_log() 16 | 17 | def get_dcp_path(self, dcp_id): 18 | if dcp_id in DCP_MAP: 19 | dcp_folder = os.path.join( 20 | os.path.dirname(__file__), "resources", "DCP", "ECL-SET" 21 | ) 22 | dcp_name = DCP_MAP[dcp_id] 23 | 24 | folder_path = os.path.join(dcp_folder, dcp_name) 25 | self.assertTrue(os.path.exists(folder_path)) 26 | return folder_path 27 | 28 | def parse(self, dcp_id): 29 | self.dcp = DCP(self.get_dcp_path(dcp_id)) 30 | return self.dcp.parse() 31 | 32 | 33 | class DCPParseTest(ParserTestBase): 34 | vf_missing = "check_assets_cpl_missing_from_vf" 35 | 36 | def __init__(self, *args, **kwargs): 37 | super(DCPParseTest, self).__init__(*args, **kwargs) 38 | 39 | def test_dcp_01(self): 40 | res = self.parse(1) 41 | self.assertEqual(len(res["asset_list"]), 14) 42 | self.assertEqual(len(res["volindex_list"]), 1) 43 | self.assertEqual(len(res["assetmap_list"]), 1) 44 | self.assertEqual(len(res["cpl_list"]), 1) 45 | self.assertEqual(len(res["pkl_list"]), 1) 46 | self.assertTrue(res["package_type"], "OV") 47 | self.assertTrue(res["count_file"], "16") 48 | self.assertTrue(res["schema"], "Interop") 49 | self.assertTrue(res["type"], "DCP") 50 | 51 | cpl = res["cpl_list"][0]["Info"]["CompositionPlaylist"] 52 | self.assertFalse(cpl["AuxData"]) 53 | self.assertFalse(cpl["HighFrameRate"]) 54 | self.assertFalse(cpl["Stereoscopic"]) 55 | self.assertFalse(cpl["Subtitle"]) 56 | 57 | def test_dcp_02(self): 58 | res = self.parse(2) 59 | self.assertTrue(res["schema"], "Interop") 60 | self.assertTrue(res["package_type"], "VF") 61 | 62 | def test_dcp_07(self): 63 | res = self.parse(7) 64 | self.assertTrue(res["schema"], "SMPTE") 65 | 66 | cpl = res["cpl_list"][0]["Info"]["CompositionPlaylist"] 67 | self.assertTrue(cpl["Stereoscopic"]) 68 | 69 | def test_dcp_08(self): 70 | res = self.parse(8) 71 | self.assertTrue(res["schema"], "SMPTE") 72 | 73 | cpl = res["cpl_list"][0]["Info"]["CompositionPlaylist"] 74 | self.assertTrue(cpl["Subtitle"]) 75 | 76 | def test_dcp_09(self): 77 | res = self.parse(9) 78 | self.assertTrue(res["schema"], "SMPTE") 79 | self.assertTrue(res["package_type"], "OV") 80 | 81 | cpl = res["cpl_list"][0]["Info"]["CompositionPlaylist"] 82 | self.assertTrue(cpl["AuxData"]) 83 | 84 | def test_dcp_10(self): 85 | res = self.parse(10) 86 | self.assertTrue(res["schema"], "SMPTE") 87 | self.assertTrue(res["package_type"], "VF") 88 | 89 | def test_dcp_11(self): 90 | res = self.parse(11) 91 | self.assertTrue(res["schema"], "Interop") 92 | self.assertTrue(res["package_type"], "VF") 93 | 94 | def test_dcp_22(self): 95 | res = self.parse(22) 96 | self.assertTrue(res["schema"], "SMPTE") 97 | self.assertTrue(res["package_type"], "OV") 98 | 99 | cpl = res["cpl_list"][0]["Info"]["CompositionPlaylist"] 100 | self.assertEqual(cpl["FrameRate"], 48) 101 | 102 | def test_dcp_23(self): 103 | res = self.parse(23) 104 | self.assertTrue(res["schema"], "SMPTE") 105 | self.assertTrue(res["package_type"], "OV") 106 | 107 | cpl = res["cpl_list"][0]["Info"]["CompositionPlaylist"] 108 | self.assertEqual(cpl["FrameRate"], 60) 109 | 110 | def test_dcp_25(self): 111 | res = self.parse(25) 112 | self.assertTrue(res["schema"], "SMPTE") 113 | self.assertTrue(res["package_type"], "OV") 114 | 115 | cpl = res["cpl_list"][0]["Info"]["CompositionPlaylist"] 116 | self.assertEqual(cpl["FrameRate"], 48) 117 | 118 | def test_dcp_26(self): 119 | res = self.parse(26) 120 | self.assertTrue(res["schema"], "SMPTE") 121 | self.assertTrue(res["package_type"], "OV") 122 | 123 | cpl = res["cpl_list"][0]["Info"]["CompositionPlaylist"] 124 | self.assertEqual(cpl["Resolution"], "1920x1080") 125 | 126 | def test_dcp_27(self): 127 | res = self.parse(27) 128 | self.assertTrue(res["schema"], "SMPTE") 129 | self.assertTrue(res["package_type"], "OV") 130 | 131 | cpl = res["cpl_list"][0]["Info"]["CompositionPlaylist"] 132 | self.assertEqual(cpl["Resolution"], "3840x2160") 133 | 134 | def test_dcp_28(self): 135 | res = self.parse(28) 136 | self.assertTrue(res["schema"], "SMPTE") 137 | self.assertTrue(res["package_type"], "OV") 138 | 139 | cpl = res["cpl_list"][0]["Info"]["CompositionPlaylist"] 140 | self.assertEqual(cpl["Resolution"], "4096x2160") 141 | 142 | def test_dcp_29(self): 143 | res = self.parse(29) 144 | self.assertTrue(res["schema"], "Interop") 145 | self.assertTrue(res["package_type"], "OV") 146 | 147 | cpl = res["cpl_list"][0]["Info"]["CompositionPlaylist"] 148 | self.assertTrue(cpl["Encrypted"]) 149 | 150 | def test_dcp_30(self): 151 | res = self.parse(30) 152 | self.assertTrue(res["schema"], "SMPTE") 153 | self.assertTrue(res["package_type"], "OV") 154 | 155 | cpl = res["cpl_list"][0]["Info"]["CompositionPlaylist"] 156 | self.assertTrue(cpl["Encrypted"]) 157 | 158 | def test_dcp_31(self): 159 | res = self.parse(31) 160 | self.assertTrue(res["schema"], "Interop") 161 | self.assertTrue(res["package_type"], "OV") 162 | 163 | cpl = res["cpl_list"][0]["Info"]["CompositionPlaylist"] 164 | self.assertEqual(cpl["Encrypted"], "Mixed") 165 | 166 | def test_dcp_32(self): 167 | res = self.parse(32) 168 | self.assertTrue(res["schema"], "SMPTE") 169 | self.assertTrue(res["package_type"], "OV") 170 | 171 | cpl = res["cpl_list"][0]["Info"]["CompositionPlaylist"] 172 | self.assertEqual(cpl["Encrypted"], "Mixed") 173 | 174 | def test_dcp_33(self): 175 | res = self.parse(33) 176 | self.assertTrue(res["schema"], "Interop") 177 | self.assertTrue(res["package_type"], "OV") 178 | 179 | cpl = res["cpl_list"][0]["Info"]["CompositionPlaylist"] 180 | self.assertTrue(cpl["Subtitle"]) 181 | 182 | def test_dcp_38(self): 183 | res = self.parse(38) 184 | self.assertTrue(res["schema"], "SMPTE") 185 | self.assertTrue(res["package_type"], "OV") 186 | 187 | def test_dcp_39(self): 188 | res = self.parse(39) 189 | self.assertTrue(res["schema"], "SMPTE") 190 | self.assertTrue(res["package_type"], "OV") 191 | 192 | cpl = res["cpl_list"][0]["Info"]["CompositionPlaylist"] 193 | self.assertEqual(cpl["DecompositionLevels"], "Mixed") 194 | 195 | def test_dcp_40(self): 196 | res = self.parse(40) 197 | self.assertTrue(res["schema"], "SMPTE") 198 | self.assertTrue(res["package_type"], "OV") 199 | 200 | def test_dcp_41_46(self): 201 | for dcp_id in [41, 42, 43, 44, 45, 46]: 202 | res = self.parse(dcp_id) 203 | self.assertTrue(res["schema"], "SMPTE") 204 | self.assertTrue(res["package_type"], "OV") 205 | 206 | cpl = res["cpl_list"][0]["Info"]["CompositionPlaylist"] 207 | self.assertTrue(cpl["HighFrameRate"]) 208 | -------------------------------------------------------------------------------- /clairmeta/utils/isdcf.py: -------------------------------------------------------------------------------- 1 | # Clairmeta - (C) YMAGIS S.A. 2 | # See LICENSE for more information 3 | 4 | import re 5 | from collections import OrderedDict 6 | from itertools import islice 7 | 8 | from clairmeta.settings import DCP_SETTINGS 9 | 10 | # fmt: off 11 | 12 | RULES_ORDER = [ 13 | 'FilmTitle', 14 | 'ContentType', 15 | 'ProjectorAspectRatio', 16 | 'Language', 17 | 'TerritoryRating', 18 | 'AudioType', 19 | 'Resolution', 20 | 'Studio', 21 | 'Date', 22 | 'Facility', 23 | 'Standard', 24 | 'PackageType' 25 | ] 26 | 27 | RULES = { 28 | '9.6': { 29 | 'FilmTitle': r'(^[a-zA-Z0-9-]{1,14}$)', 30 | 'ContentType': 31 | r'(^' 32 | r'(?PFTR|EPS|TLR|TSR|PRO|TST|RTG-F|RTG-T|SHR|ADV|XSN|PSA|POL)' 33 | r'(-(?P\d))?' 34 | r'(-(?PTemp))?' 35 | r'(-(?PPre))?' 36 | r'(-(?PRedBand))?' 37 | r'(-(?P[a-zA-Z0-9]))?' 38 | r'(-(?P(2D|3D)))?' 39 | r'(-(?P\d+fl))?' 40 | r'(-(?P\d+))?' 41 | r'(-(?PDVis))?' 42 | r'(-(?PEC))?' 43 | r'$)', 44 | 'ProjectorAspectRatio': 45 | r'(^' 46 | r'(?PF|S|C)' 47 | r'(-(?P\d{1,3}))?' 48 | r'$)', 49 | 'Language': 50 | r'(^' 51 | r'(?P[A-Z]{2,3})' 52 | r'-(?P[A-Za-z]{2,3})' 53 | r'(-(?P[A-Za-z]{2,3}))?' 54 | r'(-(?P(CCAP|OCAP)))?' 55 | r'$)', 56 | 'TerritoryRating': 57 | r'(^' 58 | r'(?P([A-Z]{2,3}))' 59 | r'(-(?P[A-Z0-9\+]{1,3}))?' 60 | r'$)', 61 | 'AudioType': 62 | r'(^' 63 | r'(?P(10|20|51|71|MOS))' 64 | r'(-(?PHI))?' 65 | r'(-(?PVI))?' 66 | r'(-(?PSL))?' 67 | r'(-(?P(ATMOS|Atmos|AURO|DTS-X)))?' 68 | r'(-(?P(DBOX|Dbox)))?' 69 | r'$)', 70 | 'Resolution': r'(^2K|4K$)', 71 | 'Studio': r'(^[A-Z0-9]{2,4}$)', 72 | 'Date': r'(^\d{8}$)', 73 | 'Facility': r'(^[A-Z0-9]{2,3}$)', 74 | 'Standard': 75 | r'(^' 76 | r'(?P(IOP|SMPTE))' 77 | r'(-(?P3D))?' 78 | r'$)', 79 | 'PackageType': 80 | r'(^' 81 | r'(?P(OV|VF))' 82 | r'(-(?P\d))?' 83 | r'$)', 84 | } 85 | } 86 | 87 | 88 | DEFAULT = '' 89 | DEFAULTS = { 90 | 'Temporary': False, 91 | 'PreRelease': False, 92 | 'RedBand': False, 93 | 'DolbyVision': False, 94 | 'EclairColor': False, 95 | 'Caption': False, 96 | 'HearingImpaired': False, 97 | 'VisionImpaired': False, 98 | 'SignLanguage': False, 99 | 'ImmersiveSound': False, 100 | 'MotionSimulator': False, 101 | } 102 | 103 | # fmt: on 104 | 105 | 106 | def parse_isdcf_string(isdcf_str): 107 | """Regex based check of ISDCF Naming convention. 108 | 109 | Digital Cinema Naming Convention as defined by ISDCF 110 | ISDCF : Inter Society Digital Cinema Forum 111 | DCNC : Digital Cinema Naming Convention 112 | See : http://isdcf.com/dcnc/index.html 113 | 114 | Args: 115 | isdcf_str (str): ContentTitle to check. 116 | 117 | Returns: 118 | Tuple consisting of a dictionary of all extracted fiels and a list 119 | of errors. 120 | 121 | """ 122 | fields_dict = {} 123 | error_list = [] 124 | 125 | if not isinstance(isdcf_str, str): 126 | error_list.append("ContentTitle invalid type") 127 | return fields_dict, error_list 128 | 129 | # Sort the fields to respect DCNC order 130 | # Note : in python3 we can declare an OrderedDict({...}) and the field 131 | # order is preserved so this is not needed, but not in python 2.7 132 | dcnc_version = DCP_SETTINGS["naming_convention"] 133 | rules = OrderedDict( 134 | sorted(RULES[dcnc_version].items(), key=lambda f: RULES_ORDER.index(f[0])) 135 | ) 136 | 137 | fields_dict = init_dict_isdcf(rules) 138 | fields_list = isdcf_str.split("_") 139 | 140 | if len(fields_list) != 12: 141 | error_list.append( 142 | "ContentTitle should have 12 parts to be fully compliant with" 143 | " ISDCF naming convention version {}, {} part(s) found".format( 144 | dcnc_version, len(fields_list) 145 | ) 146 | ) 147 | 148 | # Parsing title with some robustness to missing / additionals fields 149 | # Find a match in nearby fields only 150 | max_field_shift = 3 151 | 152 | fields_matched = [] 153 | 154 | for idx_field, field in enumerate(fields_list): 155 | matched = False 156 | 157 | for idx_rule, (name, regex) in enumerate(rules.items()): 158 | pattern = re.compile(regex) 159 | match = re.match(pattern, field) 160 | 161 | if idx_field == 0 and not match: 162 | error_list.append( 163 | "ContentTitle Film Name does not respect naming convention" 164 | " rules : {}".format(field) 165 | ) 166 | elif match and idx_rule < max_field_shift: 167 | fields_dict[name].update(match.groupdict(DEFAULT)) 168 | else: 169 | continue 170 | 171 | fields_dict[name]["Value"] = field 172 | fields_matched.append(name) 173 | sliced = islice(rules.items(), idx_rule + 1, None) 174 | rules = OrderedDict(sliced) 175 | matched = True 176 | break 177 | 178 | if not matched: 179 | error_list.append( 180 | "ContentTitle Part {} not matching any naming convention field".format( 181 | field 182 | ) 183 | ) 184 | 185 | for name, _ in RULES[dcnc_version].items(): 186 | if name not in fields_matched: 187 | error_list.append("Field {} not found in ContentTitle".format(name)) 188 | 189 | fields_dict = post_parse_isdcf(fields_dict) 190 | return fields_dict, error_list 191 | 192 | 193 | def init_dict_isdcf(rules): 194 | """Initialize naming convention metadata dictionary. 195 | 196 | Args: 197 | rules (dict): Dictionary of the rules. 198 | """ 199 | res = {} 200 | 201 | for name, regex in rules.items(): 202 | pattern = re.compile(regex) 203 | 204 | res[name] = {} 205 | res[name]["Value"] = "" 206 | res[name].update({k: DEFAULT for k in pattern.groupindex.keys()}) 207 | 208 | return res 209 | 210 | 211 | def post_parse_isdcf(fields): 212 | """Use additional deduction rules to augment dictionary. 213 | 214 | Args: 215 | fields (dict): Dictionary of parsed ISDCF fields. 216 | 217 | """ 218 | # Custom default values 219 | for field, groups in fields.items(): 220 | for key, value in groups.items(): 221 | if value == DEFAULT and key in DEFAULTS: 222 | fields[field][key] = DEFAULTS[key] 223 | 224 | # Adjust schema format 225 | schema = fields["Standard"]["Schema"] 226 | schema_map = {"SMPTE": "SMPTE", "IOP": "Interop"} 227 | if schema and schema in schema_map: 228 | fields["Standard"]["Schema"] = schema_map[schema] 229 | 230 | # See Appendix 1. Subtitles 231 | st_lang = fields["Language"].get("SubtitleLanguage") 232 | has_subtitle = st_lang != "" and st_lang != "XX" 233 | has_burn_st = fields["Language"].get("SubtitleLanguage", "").islower() 234 | fields["Language"]["BurnedSubtitle"] = has_burn_st 235 | fields["Language"]["Subtitle"] = has_subtitle 236 | 237 | return fields 238 | -------------------------------------------------------------------------------- /clairmeta/utils/file.py: -------------------------------------------------------------------------------- 1 | # Clairmeta - (C) YMAGIS S.A. 2 | # See LICENSE for more information 3 | 4 | from __future__ import division 5 | from __future__ import absolute_import 6 | import os 7 | import sys 8 | import contextlib 9 | import shutil 10 | import tempfile 11 | import base64 12 | import hashlib 13 | import time 14 | import re 15 | 16 | 17 | def folder_size(folder): 18 | """Compute total size of a folder. 19 | 20 | Args: 21 | folder (str): Folder path. 22 | 23 | Returns: 24 | Total folder size in bytes. 25 | 26 | """ 27 | size = 0 28 | 29 | for dirpath, dirnames, filenames in os.walk(folder): 30 | for f in filenames: 31 | filename = os.path.join(dirpath, f) 32 | size += os.path.getsize(filename) 33 | 34 | return size 35 | 36 | 37 | def human_size(nbytes): 38 | """Convert size in bytes to a human readable representation. 39 | 40 | Args: 41 | nbytes (int): Size in bytes. 42 | 43 | Returns: 44 | Human friendly string representation of ``nbytes``, unit is power 45 | of 1024. 46 | 47 | >>> human_size(65425721) 48 | '62.39 MiB' 49 | >>> human_size(0) 50 | '0.00 B' 51 | 52 | """ 53 | for unit in ["", "ki", "Mi", "Gi", "Ti", "Pi", "Ei", "Zi"]: 54 | if abs(nbytes) < 1024.0: 55 | return "{:.2f} {}B".format(nbytes, unit) 56 | nbytes /= 1024.0 57 | return "{:.2f} {}B".format(nbytes, "Yi") 58 | 59 | 60 | @contextlib.contextmanager 61 | def temporary_file(prefix="tmp", suffix=""): 62 | """Context managed temporary file. 63 | 64 | Yields: 65 | str: Absolute path of the temporary file. 66 | 67 | """ 68 | try: 69 | _, filepath = tempfile.mkstemp(prefix=prefix, suffix=suffix) 70 | yield filepath 71 | finally: 72 | try: 73 | os.remove(filepath) 74 | except PermissionError: 75 | pass 76 | 77 | 78 | @contextlib.contextmanager 79 | def temporary_dir(): 80 | """Context managed temporary directory. 81 | 82 | Yields: 83 | str: Absolute path of the temporary directory. 84 | 85 | """ 86 | try: 87 | dirpath = tempfile.mkdtemp() 88 | yield dirpath 89 | finally: 90 | shutil.rmtree(dirpath) 91 | 92 | 93 | class ConsoleProgress(object): 94 | def __init__(self): 95 | """ConsoleProgress constructor.""" 96 | self._total_size = None 97 | 98 | self.total_processed = 0 99 | self.total_elapsed = 0 100 | 101 | def __call__(self, file_path, file_processed, file_size, file_elapsed): 102 | """Callback for shaone_b64. 103 | 104 | Args: 105 | file_path (str): File absolute path. 106 | file_processed (int): Bytes processed for the current file 107 | file_size (int): Size of the current file 108 | file_elapsed (float): Seconds elapsed for the current file 109 | 110 | """ 111 | col_width = 15 112 | complete_col_width = 60 113 | # Avoid division by zero if time resolution is too small 114 | file_elapsed = max(sys.float_info.epsilon, file_elapsed) 115 | 116 | if file_processed != file_size: 117 | elapsed = self.total_elapsed + file_elapsed 118 | processed = self.total_processed + file_processed 119 | 120 | file_progress = min(1, (file_processed / file_size)) 121 | file_progress_size = int(file_progress * col_width) 122 | file_bar_size = col_width - file_progress_size 123 | 124 | total_progress = min(1, (processed / self._total_size)) 125 | total_progress_size = int(total_progress * col_width) 126 | total_bar_size = col_width - total_progress_size 127 | 128 | if processed > 0: 129 | eta_sec = (self._total_size - processed) / (processed / elapsed) 130 | else: 131 | eta_sec = 0 132 | 133 | eta_str = time.strftime("%H:%M:%S", time.gmtime(eta_sec)) 134 | 135 | sys.stdout.write( 136 | "ETA {} [{}] {:.2f}% - File [{}] {:.2f}% - {}\r".format( 137 | eta_str, 138 | "{}{}".format("=" * total_progress_size, " " * total_bar_size), 139 | total_progress * 100.0, 140 | "{}{}".format("=" * file_progress_size, " " * file_bar_size), 141 | file_progress * 100.0, 142 | os.path.basename(file_path), 143 | ) 144 | ) 145 | sys.stdout.flush() 146 | else: 147 | file_size = os.path.getsize(file_path) 148 | 149 | speed_report = "{} in {:.2f} sec (at {:.2f} MBytes/s)".format( 150 | human_size(file_size), file_elapsed, (file_size / 1e6) / file_elapsed 151 | ) 152 | 153 | sys.stdout.write( 154 | "[ {}] 100.00% - {}\r".format( 155 | speed_report.ljust(complete_col_width - 2), 156 | os.path.basename(file_path), 157 | ) 158 | ) 159 | sys.stdout.write("\n") 160 | 161 | self.total_processed += file_size 162 | self.total_elapsed += file_elapsed 163 | 164 | 165 | def shaone_b64(file_path, callback=None): 166 | """Compute file hash using sha1 algorithm. 167 | 168 | Args: 169 | file_path (str): File absolute path. 170 | callback (func, optional): Callback function, see 171 | ``console_progress_bar`` for an example implementation. 172 | 173 | Returns: 174 | String representation of ``file`` sha1 (encoded in base 64). 175 | 176 | Raises: 177 | ValueError: If ``file_path`` is not a valid file. 178 | 179 | """ 180 | if not os.path.isfile(file_path): 181 | raise ValueError("{} file not found".format(file_path)) 182 | 183 | BUF_SIZE = 65536 184 | file_size = os.path.getsize(file_path) 185 | run_size = 0 186 | sha1 = hashlib.sha1() 187 | start = time.time() 188 | last_cb_time = start 189 | 190 | with open(file_path, "rb") as f: 191 | while True: 192 | data = f.read(BUF_SIZE) 193 | if not data: 194 | break 195 | 196 | run_size += len(data) 197 | sha1.update(data) 198 | 199 | time_cb = time.time() 200 | call_cb = time_cb - last_cb_time > 0.2 201 | complete = run_size == file_size 202 | if callback and (call_cb or complete): 203 | last_cb_time = time_cb 204 | callback(file_path, run_size, file_size, time_cb - start) 205 | 206 | # Encode base64 and remove carriage return 207 | sha1b64 = base64.b64encode(sha1.digest()) 208 | return sha1b64.decode("utf-8") 209 | 210 | 211 | IMAGENO_REGEX = re.compile(r"[\._]?(?P\d+)(?=[\._])") 212 | 213 | 214 | def parse_name(filename, regex=IMAGENO_REGEX): 215 | """Extract image name and index from filename. 216 | 217 | Args: 218 | filename (str): Image file name. 219 | regex (RegexObject): Extraction rule. 220 | 221 | Returns: 222 | Tuple (name, index) extracted from filename. 223 | 224 | Raises: 225 | ValueError: If image index not found in ``filename``. 226 | 227 | >>> parse_name('myfile.0001.tiff') 228 | ('myfile', 1) 229 | >>> parse_name('myfile_0001.tiff') 230 | ('myfile', 1) 231 | >>> parse_name('myfile.123.0001.tiff') 232 | ('myfile.123', 1) 233 | >>> parse_name('00123060.tiff') 234 | ('', 123060) 235 | >>> parse_name('123060.tiff') 236 | ('', 123060) 237 | >>> parse_name('myfile.tiff') 238 | Traceback (most recent call last): 239 | ... 240 | ValueError: myfile.tiff : image index not found 241 | >>> parse_name('myfile.abcdef.tiff') 242 | Traceback (most recent call last): 243 | ... 244 | ValueError: myfile.abcdef.tiff : image index not found 245 | 246 | """ 247 | m = list(regex.finditer(filename)) 248 | if m == []: 249 | raise ValueError("{} : image index not found".format(filename)) 250 | 251 | lastm = m[-1] 252 | name = filename[: lastm.start()] 253 | index = lastm.groupdict()["Index"] 254 | return name, int(index) 255 | -------------------------------------------------------------------------------- /clairmeta/dcp_check_am.py: -------------------------------------------------------------------------------- 1 | # Clairmeta - (C) YMAGIS S.A. 2 | # See LICENSE for more information 3 | 4 | import os 5 | import re 6 | 7 | from clairmeta.utils.uuid import check_uuid 8 | from clairmeta.dcp_check import CheckerBase 9 | from clairmeta.dcp_check_utils import check_xml 10 | from clairmeta.dcp_utils import list_am_assets 11 | 12 | 13 | class Checker(CheckerBase): 14 | def __init__(self, dcp): 15 | super(Checker, self).__init__(dcp) 16 | 17 | def run_checks(self): 18 | for source in self.dcp._list_am: 19 | asset_stack = [source["FileName"]] 20 | 21 | checks = self.find_check("am") 22 | [self.run_check(check, source, stack=asset_stack) for check in checks] 23 | 24 | asset_checks = self.find_check("assets_am") 25 | [ 26 | self.run_check( 27 | check, 28 | source, 29 | asset, 30 | stack=asset_stack + [asset[2]["ChunkList"]["Chunk"]["Path"]], 31 | ) 32 | for asset in list_am_assets(source) 33 | for check in asset_checks 34 | ] 35 | 36 | return self.checks 37 | 38 | def check_am_xml(self, am): 39 | """AssetMap XML syntax and structure check. 40 | 41 | References: N/A 42 | """ 43 | check_xml( 44 | self, 45 | am["FilePath"], 46 | am["Info"]["AssetMap"]["__xmlns__"], 47 | am["Info"]["AssetMap"]["Schema"], 48 | self.dcp.schema, 49 | ) 50 | 51 | def check_am_volume_count(self, am): 52 | """The VolumeCount element shall be 1 53 | 54 | References: 55 | SMPTE ST 429-9:2014 5.4 56 | 57 | """ 58 | volume_count = am["Info"]["AssetMap"]["VolumeCount"] 59 | if volume_count != 1: 60 | self.error("Invalid VolumeCount value: {}".format(volume_count)) 61 | 62 | def check_am_name(self, am): 63 | """AssetMap file name respect DCP standard. 64 | 65 | References: 66 | mpeg_ii_am_spec.doc (v3.4) 6.2 67 | https://interop-docs.cinepedia.com/Document_Release_2.0/mpeg_ii_am_spec.pdf 68 | SMPTE ST 429-9:2014 A.4 69 | 70 | """ 71 | schema = am["Info"]["AssetMap"]["Schema"] 72 | mandatory_name = {"Interop": "ASSETMAP", "SMPTE": "ASSETMAP.xml"} 73 | 74 | if mandatory_name[schema] != am["FileName"]: 75 | self.error( 76 | "{} Assetmap must be named {}, got {} instead".format( 77 | schema, mandatory_name[schema], am["FileName"] 78 | ) 79 | ) 80 | 81 | def check_am_empty_text_fields(self, am): 82 | """AssetMap empty text fields check. 83 | 84 | This check for empty 'Creator', 'Issuer' or 'AnnotationText' text 85 | fields. While not invalid per specification, it appears other 86 | checking tools might trigger error / warning here so we provide 87 | this to align with other check reports. 88 | 89 | References: 90 | mpeg_ii_am_spec.doc (v3.4) 4.1.2, 4.1.5, 4.1.6 91 | SMPTE ST 429-9:2014 5.2, 5.3, 5.6 92 | 93 | """ 94 | fields = ["Creator", "Issuer", "AnnotationText"] 95 | madatory_fields = ["Creator"] 96 | empty_fields = [] 97 | missing_fields = [] 98 | 99 | for f in fields: 100 | am_f = am["Info"]["AssetMap"].get(f) 101 | if am_f == "": 102 | empty_fields.append(f) 103 | elif am_f is None and f in madatory_fields: 104 | missing_fields.append(f) 105 | 106 | if empty_fields: 107 | self.error("Empty {} field(s)".format(", ".join(empty_fields))) 108 | if missing_fields: 109 | self.error( 110 | "Missing {} field(s)".format(", ".join(missing_fields)), "missing" 111 | ) 112 | 113 | def check_assets_am_uuid(self, am, asset): 114 | """AssetMap UUIDs validation. 115 | 116 | References: 117 | mpeg_ii_am_spec.doc (v3.4) 4.1.1 118 | https://interop-docs.cinepedia.com/Document_Release_2.0/mpeg_ii_am_spec.pdf 119 | SMPTE ST 429-9:2014 5.1 120 | 121 | ST 429-9 references the final version of RFC 4122 (July 2005) 122 | whereas mpeg_ii_am_spec.doc references Draft 03 (January 2004). 123 | 124 | Diff here: 125 | https://tools.ietf.org/rfcdiff?url1=draft-mealling-uuid-urn-03.txt&url2=rfc4122.txt 126 | 127 | """ 128 | uuid, _, _ = asset 129 | if not check_uuid(uuid): 130 | self.error("Invalid uuid found : {}".format(uuid)) 131 | 132 | def check_assets_am_volindex_one(self, am, asset): 133 | """AssetMap Asset VolumeIndex element shall be 1 or absent. 134 | 135 | References: 136 | SMPTE ST 429-9:2014 7.2 137 | """ 138 | _, _, asset = asset 139 | asset_vol = asset["ChunkList"]["Chunk"].get("VolumeIndex") 140 | if asset_vol and asset_vol != 1: 141 | self.error( 142 | "VolIndex is now deprecated and shall always be 1, got {}".format( 143 | asset_vol 144 | ) 145 | ) 146 | 147 | def check_assets_am_path(self, am, asset): 148 | """AssetMap assets path validation. 149 | 150 | References: 151 | mpeg_ii_am_spec.doc (v3.4) 4.3.1, 5.3, 6.4 152 | https://interop-docs.cinepedia.com/Document_Release_2.0/mpeg_ii_am_spec.pdf 153 | SMPTE ST 429-9:2014 7.1, A.2 154 | 155 | """ 156 | _, path, _ = asset 157 | 158 | path_segments = list(filter(None, path.split("/"))) 159 | path_segments_count = len(path_segments) 160 | if path_segments_count > 10: 161 | self.error(">10 path segments: {}".format(path_segments_count)) 162 | 163 | max_path_seg = max(map(len, path_segments)) 164 | if max_path_seg > 100: 165 | self.error("Path segment >100 characters: {}".format(max_path_seg)) 166 | 167 | if len(path) > 100: 168 | self.error("Path >100 characters: {}".format(len(path))) 169 | 170 | path_invalid_chars = re.findall(r"[^a-zA-Z0-9._/-]", path) 171 | if path_invalid_chars: 172 | unique_char_str = ", ".join(sorted(set(path_invalid_chars))) 173 | self.error("Invalid characters in path: {}".format(unique_char_str)) 174 | 175 | if path[0] == "/": 176 | self.error("Path is not relative") 177 | 178 | rel_path = os.path.relpath(os.path.join(self.dcp.path, path), self.dcp.path) 179 | if rel_path.startswith("../"): 180 | self.error("Path points outside of DCP root") 181 | 182 | if not os.path.isfile(os.path.join(self.dcp.path, path)): 183 | self.error("Missing asset file: {}".format(os.path.basename(path))) 184 | 185 | def check_assets_am_offset(self, am, asset): 186 | """AssetMap Chunk Offset check 187 | 188 | References: 189 | SMPTE ST 429-9:2014 7.3 190 | """ 191 | _, _, asset = asset 192 | chunk = asset["ChunkList"]["Chunk"] 193 | 194 | if "Offset" not in chunk: 195 | return 196 | 197 | offset = chunk["Offset"] 198 | if offset != 0: 199 | self.error("Invalid offset value {}".format(offset)) 200 | 201 | def check_assets_am_size(self, am, asset): 202 | """AssetMap assets size check. 203 | 204 | References: 205 | mpeg_ii_am_spec.doc (v3.4) 4.3.4 206 | https://interop-docs.cinepedia.com/Document_Release_2.0/mpeg_ii_am_spec.pdf 207 | SMPTE ST 429-9:2014 7.4 208 | 209 | """ 210 | _, path, asset = asset 211 | path = os.path.join(self.dcp.path, path) 212 | chunk = asset["ChunkList"]["Chunk"] 213 | 214 | if "Length" not in chunk: 215 | return 216 | if os.path.isfile(path): 217 | actual_size = os.path.getsize(path) 218 | length = chunk["Length"] 219 | 220 | if length != actual_size: 221 | self.error( 222 | "Invalid size value, expected {} but got " 223 | "{}".format(length, actual_size) 224 | ) 225 | -------------------------------------------------------------------------------- /clairmeta/report.py: -------------------------------------------------------------------------------- 1 | # Clairmeta - (C) YMAGIS S.A. 2 | # See LICENSE for more information 3 | 4 | import re 5 | from collections import defaultdict 6 | from datetime import datetime 7 | 8 | from clairmeta.utils.file import human_size 9 | 10 | 11 | class CheckReport(object): 12 | """Check report listing all checks executions.""" 13 | 14 | ORDERED_STATUS = [ 15 | "ERROR", 16 | "WARNING", 17 | "INFO", 18 | "SILENT", 19 | "BYPASS", 20 | ] 21 | 22 | PRETTY_STATUS = { 23 | "ERROR": "Error(s)", 24 | "WARNING": "Warning(s)", 25 | "INFO": "Info(s)", 26 | "SILENT": "Supressed(s)", 27 | "BYPASS": "Bypass(s)", 28 | } 29 | 30 | def __init__(self, dcp, profile): 31 | """Constructor for CheckReport. 32 | 33 | Args: 34 | dcp (clairmeta.DCP): DCP. 35 | profile (dict): Checker profile. 36 | 37 | """ 38 | self.dcp = dcp 39 | self.checks = dcp.checks 40 | self.profile = profile 41 | self.date = datetime.now().strftime("%d/%m/%Y %H:%M:%S") 42 | self.duration = sum([c.seconds_elapsed for c in self.checks]) 43 | 44 | self._detect_check_criticality() 45 | 46 | def checks_count(self): 47 | """Return the number of different checks executed.""" 48 | check_unique = set([c.name for c in self.checks if not c.bypass]) 49 | return len(check_unique) 50 | 51 | def checks_failed(self): 52 | """Returns a list of all failed checks.""" 53 | return [c for c in self.checks if c.has_errors()] 54 | 55 | def checks_succeeded(self): 56 | """Returns a list of all succeeded checks.""" 57 | return [c for c in self.checks if not c.has_errors() and not c.bypass] 58 | 59 | def checks_bypassed(self): 60 | """Returns a set of all bypassed unique checks.""" 61 | return [c for c in self.checks if c.bypass] 62 | 63 | def checks_by_criticality(self, criticality): 64 | """Returns a list of failed checks with ``criticality``.""" 65 | return [ 66 | check 67 | for check in self.checks 68 | for error in check.errors 69 | if error.criticality == criticality 70 | ] 71 | 72 | def errors_by_criticality(self, criticality): 73 | """Returns a list of failed checks with ``criticality``.""" 74 | return [ 75 | error 76 | for check in self.checks 77 | for error in check.errors 78 | if error.criticality == criticality 79 | ] 80 | 81 | def is_valid(self): 82 | """Returns validity of checked DCP.""" 83 | return all([c.is_valid() for c in self.checks]) 84 | 85 | def pretty_str(self): 86 | """Format the report in a human friendly way.""" 87 | report = "" 88 | report += "Status : {}\n".format("Success" if self.is_valid() else "Fail") 89 | report += "Path : {}\n".format(self.dcp.path) 90 | report += "Size : {}\n".format(human_size(self.dcp.size)) 91 | report += "Total check : {}\n".format(self.checks_count()) 92 | report += "Total time : {:.2f} sec\n".format(self.duration) 93 | report += "\n" 94 | 95 | def nested_dict(): 96 | return defaultdict(nested_dict) 97 | 98 | status_map = nested_dict() 99 | 100 | # Accumulate all failed check and stack them by asset 101 | for check in self.checks_failed(): 102 | lines = [". {}".format(check.short_desc())] 103 | 104 | for error in check.errors: 105 | asset = status_map[str(error.criticality)] 106 | 107 | for filename in check.asset_stack: 108 | asset = asset[filename] 109 | 110 | desc = error.doc 111 | desc = ". {}\n".format(desc) if desc else "" 112 | lines.append("{}{}".format(desc, error.message)) 113 | 114 | asset["msg"] = asset.get("msg", []) + ["\n".join(lines)] 115 | 116 | # Ignore silenced checks 117 | status_map.pop("SILENT", None) 118 | 119 | for status in self.ORDERED_STATUS: 120 | out_stack = [] 121 | for k, v in status_map[status].items(): 122 | out_stack += [self._dump_stack("", k, v, indent_level=0)] 123 | if out_stack: 124 | report += "{}\n{}\n".format( 125 | self.PRETTY_STATUS[status] + ":", "\n".join(out_stack) 126 | ) 127 | 128 | bypassed = "\n".join( 129 | set([" . " + c.short_desc() for c in self.checks_bypassed()]) 130 | ) 131 | if bypassed: 132 | report += "{}\n{}\n".format(self.PRETTY_STATUS["BYPASS"] + ":", bypassed) 133 | 134 | return report 135 | 136 | def _detect_check_criticality(self): 137 | """Assign criticality for each errors.""" 138 | levels = self.profile["criticality"] 139 | default = levels.get("default", "ERROR") 140 | # Translate Perl like syntax to Python 141 | levels = {k.replace("*", ".*"): v for k, v in levels.items()} 142 | 143 | for check in self.checks: 144 | for error in check.errors: 145 | score_profile = {0: default} 146 | for c_name, c_level in levels.items(): 147 | # Assumes python is internally caching regex compilation 148 | if re.search(c_name, error.full_name()): 149 | score_profile[len(c_name)] = c_level 150 | 151 | error.criticality = score_profile[max(score_profile.keys())] 152 | 153 | def _dump_stack(self, out_str, key, values, indent_level): 154 | """Recursively iterate through the error message stack. 155 | 156 | Args: 157 | out_str (str): Accumulate messages to ``out_str`` 158 | key (str): Filename of the current asset. 159 | values (dict): Message stack to dump. 160 | indent_level (int): Current indentation level. 161 | 162 | Returns: 163 | Output error message string 164 | 165 | """ 166 | indent_offset = 2 167 | indent_step = 2 168 | indent_char = " " 169 | ind = indent_offset + indent_level 170 | 171 | filename = key 172 | desc = self._title_from_filename(filename) 173 | messages = values.pop("msg", []) 174 | 175 | out_str = "" if indent_level == 0 else "\n" 176 | out_str += indent_char * ind + "+ " 177 | out_str += filename 178 | out_str += " " + desc if desc else "" 179 | 180 | ind += indent_step 181 | for m in messages: 182 | out_str += "\n" 183 | out_str += indent_char * ind 184 | # Correct indentation for multi-lines messages 185 | out_str += ("\n" + indent_char * (ind + 2)).join(m.split("\n")) 186 | ind -= indent_step 187 | 188 | for k, v in values.items(): 189 | out_str += self._dump_stack(out_str, k, v, indent_level + indent_step) 190 | 191 | return out_str 192 | 193 | def _title_from_filename(self, filename): 194 | """Returns a human friendly title for the given file.""" 195 | for cpl in self.dcp._list_cpl: 196 | if cpl["FileName"] == filename: 197 | desc = "({})".format( 198 | cpl["Info"]["CompositionPlaylist"].get("ContentTitleText", "") 199 | ) 200 | return desc 201 | 202 | for pkl in self.dcp._list_pkl: 203 | if pkl["FileName"] == filename: 204 | desc = "({})".format( 205 | pkl["Info"]["PackingList"].get("AnnotationText", "") 206 | ) 207 | return desc 208 | 209 | return "" 210 | 211 | def to_dict(self): 212 | """Returns a dictionary representation.""" 213 | return { 214 | "dcp_path": self.dcp.path, 215 | "dcp_size": self.dcp.size, 216 | "valid": self.is_valid(), 217 | "profile": self.profile, 218 | "date": self.date, 219 | "duration_seconds": self.duration, 220 | "message": self.pretty_str(), 221 | "unique_checks_count": self.checks_count(), 222 | "checks": [c.to_dict() for c in self.checks], 223 | } 224 | -------------------------------------------------------------------------------- /clairmeta/xsd/SMPTE-429-7-2006-CPL.xsd: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | 72 | 73 | 74 | 75 | 76 | 77 | 78 | 79 | 80 | 81 | 82 | 83 | 84 | 85 | 86 | 87 | 88 | 89 | 90 | 91 | 92 | 93 | 94 | 95 | 96 | 97 | 98 | 99 | 100 | 101 | 102 | 103 | 104 | 105 | 106 | 107 | 108 | 109 | 110 | 111 | 112 | 113 | 114 | 115 | 116 | 117 | 118 | 119 | 120 | 121 | 122 | 123 | 124 | 125 | 126 | 127 | 128 | 129 | 130 | 131 | 132 | 133 | 134 | 135 | 136 | 137 | 138 | 139 | 140 | 141 | 142 | 143 | 144 | 145 | 146 | 147 | 148 | 149 | 150 | 151 | 152 | 153 | 154 | 155 | 156 | 157 | 158 | 159 | 160 | 161 | 162 | 163 | 164 | 165 | 166 | 167 | 168 | 169 | 170 | 171 | 172 | 173 | 174 | 175 | 176 | 177 | 178 | 179 | 180 | 181 | 182 | 183 | 184 | 185 | 186 | 187 | 188 | 189 | 190 | 191 | 192 | 193 | 194 | 195 | 196 | 197 | 198 | 199 | 200 | 201 | 202 | 203 | 204 | 205 | 206 | 207 | 208 | 209 | -------------------------------------------------------------------------------- /tests/test_dcp_check.py: -------------------------------------------------------------------------------- 1 | # Clairmeta - (C) YMAGIS S.A. 2 | # See LICENSE for more information 3 | 4 | import unittest 5 | import os 6 | import platform 7 | from datetime import datetime 8 | 9 | from tests import DCP_MAP, KDM_MAP, KEY 10 | from clairmeta.logger import disable_log 11 | from clairmeta.profile import get_default_profile 12 | from clairmeta.dcp import DCP 13 | 14 | # ruff: noqa: E501 15 | 16 | 17 | class CheckerTestBase(unittest.TestCase): 18 | def __init__(self, *args, **kwargs): 19 | super(CheckerTestBase, self).__init__(*args, **kwargs) 20 | disable_log() 21 | self.profile = get_default_profile() 22 | self.profile["bypass"] = ["check_assets_pkl_hash"] 23 | 24 | def get_dcp_folder(self): 25 | return os.path.join(os.path.dirname(__file__), "resources", "DCP", "ECL-SET") 26 | 27 | def get_dcp_path(self, dcp_id): 28 | if dcp_id in DCP_MAP: 29 | dcp_name = DCP_MAP[dcp_id] 30 | folder_path = os.path.join(self.get_dcp_folder(), dcp_name) 31 | self.assertTrue(os.path.exists(folder_path)) 32 | return folder_path 33 | 34 | def get_kdm_path(self, dcp_id): 35 | if dcp_id in KDM_MAP: 36 | kdm_name = KDM_MAP[dcp_id] 37 | file_path = os.path.join(self.get_dcp_folder(), kdm_name) 38 | self.assertTrue(os.path.exists(file_path)) 39 | return file_path 40 | 41 | def check(self, dcp_id, ov_id=None, kdm=None, pkey=None): 42 | self.dcp = DCP(self.get_dcp_path(dcp_id)) 43 | self.status, self.report = self.dcp.check( 44 | profile=self.profile, 45 | ov_path=self.get_dcp_path(ov_id), 46 | ) 47 | return self.status 48 | 49 | def has_succeeded(self): 50 | return self.status 51 | 52 | def has_failed(self, check_name): 53 | failed = self.report.checks_failed() 54 | return check_name in [c.name for c in failed] 55 | 56 | 57 | class DCPCheckTest(CheckerTestBase): 58 | vf_missing = "check_assets_cpl_missing_from_vf" 59 | 60 | def __init__(self, *args, **kwargs): 61 | super(DCPCheckTest, self).__init__(*args, **kwargs) 62 | 63 | def test_iop_ov(self): 64 | self.assertTrue(self.check(1)) 65 | self.assertTrue(self.check(33)) 66 | 67 | def test_iop_vf(self): 68 | self.assertTrue(self.check(2)) 69 | self.assertTrue(self.has_failed(DCPCheckTest.vf_missing)) 70 | self.assertTrue(self.check(2, ov_id=1)) 71 | self.assertFalse(self.has_failed(DCPCheckTest.vf_missing)) 72 | 73 | def test_smpte_ov(self): 74 | self.assertTrue(self.check(7)) 75 | self.assertTrue(self.check(9)) 76 | self.assertTrue(self.check(11)) 77 | self.assertTrue(self.check(28)) 78 | self.assertTrue(self.check(38)) 79 | 80 | @unittest.skipIf( 81 | platform.system() == "Windows", 82 | "asdcp-unwrap on Windows doesn't properly unwrap resources files (including fonts) from MXF making check fails. Help wanted.", 83 | ) 84 | def test_smpte_vf(self): 85 | self.assertTrue(self.check(8)) 86 | self.assertTrue(self.has_failed(DCPCheckTest.vf_missing)) 87 | self.assertTrue(self.check(8, ov_id=11)) 88 | self.assertFalse(self.has_failed(DCPCheckTest.vf_missing)) 89 | 90 | self.assertTrue(self.check(10)) 91 | self.assertTrue(self.has_failed(DCPCheckTest.vf_missing)) 92 | self.assertTrue(self.check(10, ov_id=9)) 93 | self.assertFalse(self.has_failed(DCPCheckTest.vf_missing)) 94 | 95 | def test_smpte_ov_hfr(self): 96 | self.assertTrue(self.check(22)) 97 | self.assertTrue(self.check(23)) 98 | 99 | def test_over_bitrate(self): 100 | self.check(25) 101 | self.assertFalse(self.has_succeeded()) 102 | self.assertTrue(self.has_failed("check_picture_cpl_max_bitrate")) 103 | self.assertTrue(self.has_failed("check_picture_cpl_avg_bitrate")) 104 | 105 | self.check(42) 106 | self.assertFalse(self.has_succeeded()) 107 | self.assertTrue(self.has_failed("check_picture_cpl_max_bitrate")) 108 | self.assertTrue(self.has_failed("check_picture_cpl_avg_bitrate")) 109 | 110 | def test_nondci_resolution(self): 111 | self.assertTrue(self.check(26)) 112 | self.assertTrue(self.has_failed("check_picture_cpl_resolution")) 113 | 114 | self.assertTrue(self.check(27)) 115 | self.assertTrue(self.has_failed("check_picture_cpl_resolution")) 116 | 117 | def test_encrypted(self): 118 | self.assertTrue(self.check(29)) 119 | self.assertTrue(self.check(30)) 120 | 121 | def test_encrypted_kdm(self): 122 | self.assertTrue(self.check(29, kdm=self.get_kdm_path(29), pkey=KEY)) 123 | self.assertTrue(self.check(30, kdm=self.get_kdm_path(30), pkey=KEY)) 124 | 125 | def test_noncoherent_encryption(self): 126 | self.assertFalse(self.check(31)) 127 | self.assertTrue(self.has_failed("check_cpl_reel_coherence")) 128 | self.assertFalse(self.check(32)) 129 | self.assertTrue(self.has_failed("check_cpl_reel_coherence")) 130 | 131 | def test_iop_subtitle_png(self): 132 | self.assertTrue(self.check(33)) 133 | self.assertFalse(self.has_failed("check_subtitle_cpl_image")) 134 | 135 | def test_noncoherent_jp2k(self): 136 | self.assertFalse(self.check(39)) 137 | self.assertTrue(self.has_failed("check_picture_cpl_encoding")) 138 | self.assertTrue(self.has_failed("check_cpl_reel_coherence")) 139 | 140 | def test_mpeg(self): 141 | self.assertFalse(self.check(40)) 142 | 143 | def test_hfr(self): 144 | self.assertTrue(self.check(41)) 145 | self.assertTrue(self.check(43)) 146 | self.assertTrue(self.check(44)) 147 | self.assertTrue(self.check(45)) 148 | self.assertTrue(self.check(46)) 149 | 150 | def test_multi_pkl(self): 151 | self.assertTrue(self.check(47)) 152 | 153 | 154 | class DCPCheckReportTest(CheckerTestBase): 155 | def __init__(self, *args, **kwargs): 156 | super(DCPCheckReportTest, self).__init__(*args, **kwargs) 157 | self.check(25) 158 | 159 | def test_report_metadata(self): 160 | self.assertTrue(isinstance(self.report.profile, dict)) 161 | self.assertTrue(datetime.strptime(self.report.date, "%d/%m/%Y %H:%M:%S")) 162 | self.assertGreaterEqual(self.report.duration, 0) 163 | 164 | def test_report_checks(self): 165 | self.assertGreaterEqual(len(self.report.checks), self.report.checks_count()) 166 | 167 | failed = self.report.checks_failed() 168 | success = self.report.checks_succeeded() 169 | bypass = self.report.checks_bypassed() 170 | 171 | all_names = [] 172 | for checks in [failed, success, bypass]: 173 | all_names += [c.name for c in checks] 174 | self.assertEqual( 175 | sorted(all_names), sorted([c.name for c in self.report.checks]) 176 | ) 177 | self.assertEqual( 178 | len(failed) + len(success) + len(bypass), len(self.report.checks) 179 | ) 180 | 181 | self.report.errors_by_criticality("ERROR") 182 | self.assertEqual(9, len(self.report.checks_failed())) 183 | self.assertEqual(1, len(self.report.errors_by_criticality("ERROR"))) 184 | self.assertEqual(1, len(self.report.errors_by_criticality("WARNING"))) 185 | self.assertEqual(7, len(self.report.errors_by_criticality("INFO"))) 186 | 187 | check = self.report.checks_by_criticality("ERROR")[0] 188 | self.assertEqual(check.name, "check_picture_cpl_max_bitrate") 189 | self.assertFalse(check.is_valid()) 190 | self.assertFalse(check.bypass) 191 | self.assertGreaterEqual(check.seconds_elapsed, 0) 192 | self.assertEqual( 193 | check.asset_stack, 194 | [ 195 | "CPL_ECL25SingleCPL_TST-48-600_S_EN-XX_UK-U_51_2K_DI_20180301_ECL_SMPTE_OV.xml", 196 | "ECL25SingleCPL_TST-48-600_S_EN-XX_UK-U_51_2K_DI_20180301_ECL_SMPTE_OV_01.mxf", 197 | ], 198 | ) 199 | 200 | error = check.errors[0] 201 | self.assertEqual(error.full_name(), "check_picture_cpl_max_bitrate") 202 | self.assertEqual( 203 | error.message, "Exceed DCI maximum bitrate (250.05 Mb/s) : 358.25 Mb/s" 204 | ) 205 | self.assertTrue(error.criticality == "ERROR") 206 | 207 | def test_report_output(self): 208 | self.assertEqual(False, self.report.is_valid()) 209 | 210 | report = self.report.pretty_str() 211 | self.assertTrue(report) 212 | self.assertTrue("Picture maximum bitrate DCI compliance." in report) 213 | 214 | self.assertTrue(self.report.to_dict()) 215 | 216 | 217 | if __name__ == "__main__": 218 | unittest.main() 219 | -------------------------------------------------------------------------------- /clairmeta/settings.py: -------------------------------------------------------------------------------- 1 | # Clairmeta - (C) YMAGIS S.A. 2 | # See LICENSE for more information 3 | 4 | import os 5 | 6 | 7 | LOG_SETTINGS = { 8 | "level": os.getenv("CLAIRMETA_LOG_LEVEL", "INFO"), 9 | "enable_console": os.getenv("CLAIRMETA_LOG_CONSOLE", "ON"), 10 | "enable_file": os.getenv("CLAIRMETA_LOG_FILE", "OFF"), 11 | "file_name": os.getenv("CLAIRMETA_LOG_FILE_NAME", ""), 12 | "file_size": os.getenv("CLAIRMETA_LOG_FILE_SIZE", 1e6), 13 | "file_count": os.getenv("CLAIRMETA_LOG_FILE_COUNT", 10), 14 | } 15 | 16 | DCP_SETTINGS = { 17 | # ISDCF Naming Convention enforced 18 | "naming_convention": "9.6", 19 | # Recognized XML namespaces 20 | "xmlns": { 21 | "xml": "http://www.w3.org/XML/1998/namespace", 22 | "xmldsig": "http://www.w3.org/2000/09/xmldsig#", 23 | "cpl_metadata_href": "http://isdcf.com/schemas/draft/2011/cpl-metadata", 24 | "interop_pkl": "http://www.digicine.com/PROTO-ASDCP-PKL-20040311#", 25 | "interop_cpl": "http://www.digicine.com/PROTO-ASDCP-CPL-20040511#", 26 | "interop_am": "http://www.digicine.com/PROTO-ASDCP-AM-20040311#", 27 | "interop_vl": "http://www.digicine.com/PROTO-ASDCP-VL-20040311#", 28 | "interop_stereo": "http://www.digicine.com/schemas/437-Y/2007/Main-Stereo-Picture-CPL", 29 | "interop_subtitle": "interop_subtitle", 30 | "interop_cc_cpl": "http://www.digicine.com/PROTO-ASDCP-CC-CPL-20070926#", 31 | "smpte_pkl_2006": "http://www.smpte-ra.org/schemas/429-8/2006/PKL", 32 | "smpte_pkl_2007": "http://www.smpte-ra.org/schemas/429-8/2007/PKL", 33 | "smpte_cpl": "http://www.smpte-ra.org/schemas/429-7/2006/CPL", 34 | "smpte_cpl_metadata": "http://www.smpte-ra.org/schemas/429-16/2014/CPL-Metadata", 35 | "smpte_am_2006": "http://www.smpte-ra.org/schemas/429-9/2006/AM", 36 | "smpte_am_2007": "http://www.smpte-ra.org/schemas/429-9/2007/AM", 37 | "smpte_stereo_2007": "http://www.smpte-ra.org/schemas/429-10/2007/Main-Stereo-Picture-CPL", 38 | "smpte_stereo_2008": "http://www.smpte-ra.org/schemas/429-10/2008/Main-Stereo-Picture-CPL", 39 | "smpte_subtitles_2007": "http://www.smpte-ra.org/schemas/428-7/2007/DCST", 40 | "smpte_subtitles_2010": "http://www.smpte-ra.org/schemas/428-7/2010/DCST", 41 | "smpte_subtitles_2014": "http://www.smpte-ra.org/schemas/428-7/2014/DCST", 42 | "smpte_tt": "http://www.smpte-ra.org/schemas/429-12/2008/TT", 43 | "smpte_etm": "http://www.smpte-ra.org/schemas/430-3/2006/ETM", 44 | "smpte_kdm": "http://www.smpte-ra.org/schemas/430-1/2006/KDM", 45 | "atmos": "http://www.dolby.com/schemas/2012/AD", 46 | }, 47 | # Recognized XML identifiers 48 | "xmluri": { 49 | "interop_sig": "http://www.w3.org/2000/09/xmldsig#rsa-sha1", 50 | "smpte_sig": "http://www.w3.org/2001/04/xmldsig-more#rsa-sha256", 51 | "enveloped_sig": "http://www.w3.org/2000/09/xmldsig#enveloped-signature", 52 | "c14n": "http://www.w3.org/TR/2001/REC-xml-c14n-20010315", 53 | "sha1": "http://www.w3.org/2000/09/xmldsig#sha1", 54 | "dolby_edr": "http://www.dolby.com/schemas/2014/EDR-Metadata", 55 | "eidr": "http://eidr.org/EIDR/2016", 56 | }, 57 | "picture": { 58 | # Standard resolutions 59 | "resolutions": { 60 | "2K": ["1998x1080", "2048x858", "2048x1080"], 61 | "4K": ["3996x2160", "4096x1716", "4096x2160"], 62 | "HD": ["1920x1080"], 63 | "UHD": ["3840x2160"], 64 | }, 65 | # Valid pixel array sizes according to SMPTE RDD 52:2020 66 | "pixel_array_sizes": { 67 | "2K": ["1998x1080", "2048x858"], 68 | "4K": ["3996x2160", "4096x1716"], 69 | }, 70 | # Standard editrate 71 | "editrates": { 72 | "2K": {"2D": [24, 25, 30, 48, 50, 60], "3D": [24, 25, 30, 48, 50, 60]}, 73 | "4K": {"2D": [24, 25, 30], "3D": []}, 74 | }, 75 | # Archival editrate 76 | "editrates_archival": [16, 200.0 / 11, 20, 240.0 / 11], 77 | # HFR capable quipements (projection servers) 78 | "editrates_min_series2": { 79 | "2D": 96, 80 | "3D": 48, 81 | }, 82 | # Standard aspect ratio 83 | "aspect_ratio": { 84 | "F": {"ratio": 1.85, "resolutions": ["1998x1080", "3996x2160"]}, 85 | "S": {"ratio": 2.39, "resolutions": ["2048x858", "4096x1716"]}, 86 | "C": {"ratio": 1.90, "resolutions": ["2048x1080", "4096x2160"]}, 87 | }, 88 | # For metadata tagging, decoupled from bitrate thresholds 89 | "min_hfr_editrate": 48, 90 | # As stated in http://www.dcimovies.com/Recommended_Practice/ 91 | # These are in Mb/s 92 | # Note : asdcplib use a 400Mb/s threshold for HFR, why ? 93 | "max_dci_bitrate": 250, 94 | "max_hfr_bitrate": 500, 95 | "max_dvi_bitrate": 400, 96 | "min_editrate_hfr_bitrate": { 97 | "2K": {"2D": 60, "3D": 48}, 98 | "4K": {"2D": 48, "3D": 0}, 99 | }, 100 | # We allow a small offset above DCI specification : 101 | # asdcplib use a method of computation that can only give an 102 | # approximation (worst case scenario) of the actual max bitrate. 103 | # asdcplib basically find the biggest frame in the whole track and 104 | # multiply it by the editrate. 105 | # Note : DCI specification seems to limit individual j2c frame size, 106 | # the method used by asdcplib should be valid is this regard, it seems 107 | # that the observed bitrate between 250 and 250.05 are due to the 108 | # encryption overhead in the KLV packaging. 109 | "bitrate_tolerance": 0.05, 110 | # This is a percentage below max_bitrate 111 | "average_bitrate_margin": 2.0, 112 | # As stated in SMPTE 429-2 113 | "dwt_levels_2k": 5, 114 | "dwt_levels_4k": 6, 115 | }, 116 | "sound": { 117 | "sampling_rate": [48000, 96000], 118 | "max_channel_count": 16, 119 | "quantization": 24, 120 | # This maps SMPTE 429-2 AudioDescriptor.ChannelFormat to a label and 121 | # a min / max number of allowed channels. 122 | # See. Section A.1.2 'Channel Configuration Tables' 123 | "configuration_channels": { 124 | 1: ("5.1 with optional HI/VI", 6, 8), 125 | 2: ("6.1 (5.1 + center surround) with optional HI/VI", 7, 10), 126 | 3: ("7.1 (SDDS) with optional HI/VI", 8, 10), 127 | 4: ("Wild Track Format", 1, 16), 128 | 5: ("7.1 DS with optional HI/VI", 8, 10), 129 | }, 130 | "format_channels": { 131 | "10": 1, 132 | "20": 2, 133 | "51": 6, 134 | "61": 7, 135 | "71": 8, 136 | "11.1": 12, 137 | }, 138 | }, 139 | "atmos": { 140 | "max_channel_count": 64, 141 | "max_object_count": 118, 142 | "smpte_ul": "060e2b34.04010105.0e090604.00000000", 143 | }, 144 | "subtitle": { 145 | # In bytes 146 | "font_max_size": 655360, 147 | }, 148 | } 149 | 150 | DCP_CHECK_SETTINGS = { 151 | # List of check modules for DCP check, these modules will be imported 152 | # dynamically during the check process. 153 | "module_prefix": "dcp_check_", 154 | "modules": { 155 | "global": "Global checks", 156 | "vol": "VolIndex checks", 157 | "am": "AssetMap checks", 158 | "pkl": "PackingList checks", 159 | "cpl": "CompositionPlayList checks", 160 | "sign": "Digital signature checks", 161 | "isdcf_dcnc": "Naming Convention checks", 162 | "picture": "Picture essence checks", 163 | "sound": "Sound essence checks", 164 | "subtitle": "Subtitle essence checks", 165 | "atmos": "Atmos essence checks", 166 | }, 167 | } 168 | 169 | IMP_SETTINGS = { 170 | "xmlns": { 171 | "xmldsig": "http://www.w3.org/2000/09/xmldsig#", 172 | "imp_am": "http://www.smpte-ra.org/schemas/429-9/2007/AM", 173 | "imp_pkl": "http://www.smpte-ra.org/schemas/429-8/2007/PKL", 174 | "imp_opl": "http://www.smpte-ra.org/schemas/2067-100/", 175 | "imp_cpl": "http://www.smpte-ra.org/schemas/2067-3/", 176 | } 177 | } 178 | 179 | SEQUENCE_SETTINGS = { 180 | "ALL": { 181 | # In percentage 182 | "size_diff_tol": 2.5, 183 | }, 184 | "SCAN": { 185 | "allowed_extensions": { 186 | ".dpx": { 187 | "Format": "DPX", 188 | }, 189 | ".cri": {}, 190 | }, 191 | "directory_white_list": [".thumbnails"], 192 | "file_white_list": [".DS_Store"], 193 | }, 194 | "DSM": { 195 | "allowed_extensions": { 196 | ".dpx": { 197 | "Format": "DPX", 198 | }, 199 | ".tiff": {"Format": "TIFF"}, 200 | ".tif": {"Format": "TIFF"}, 201 | ".exr": {"Format": "EXR"}, 202 | ".cin": {}, 203 | }, 204 | "directory_white_list": [".thumbnails"], 205 | "file_white_list": [".DS_Store"], 206 | }, 207 | "DCDM": { 208 | "allowed_extensions": { 209 | ".tiff": { 210 | "Format": "TIFF", 211 | "ProbeImage.ColorSpace": ["RGB", "XYZ"], 212 | "ProbeImage.BitDepth": "16", 213 | }, 214 | ".tif": { 215 | "Format": "TIFF", 216 | "ProbeImage.ColorSpace": ["RGB", "XYZ"], 217 | "ProbeImage.BitDepth": "16", 218 | }, 219 | }, 220 | "directory_white_list": [".thumbnails"], 221 | "file_white_list": [".DS_Store"], 222 | }, 223 | } 224 | --------------------------------------------------------------------------------