├── .gitignore ├── CHANGES ├── Cluster.md ├── Jenkinsfile ├── LICENSE ├── Pipfile ├── Pipfile.lock ├── README.md ├── build.sh ├── config-samples ├── cluster-sample.yml ├── new.yml ├── simple.yml ├── transcode-sample2.yml └── transcode.yml ├── docs ├── Makefile ├── conf.py ├── configuration │ ├── cluster.rst │ ├── concurrency.rst │ ├── configuration.rst │ ├── installation.rst │ └── quickstart.rst ├── index.rst ├── make.bat ├── simple.yml └── usage │ ├── includes.rst │ ├── mixins.rst │ ├── running-clustered.rst │ └── running-local.rst ├── main.py ├── mixintests.py ├── pytranscoder ├── __init__.py ├── __main__.py ├── agent.py ├── cluster.py ├── config.py ├── ffmpeg.py ├── media.py ├── processor.py ├── profile.py ├── rule.py ├── template.py ├── transcode.py └── utils.py ├── requirements.txt ├── run-tests.sh ├── setup.py ├── tests ├── ffmpeg.out ├── ffmpeg2.out ├── ffmpeg3.out ├── ffmpeg4.out └── mixinstest.yml ├── transcodertests.py └── upload.sh /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | __pycache__ 3 | .vscode 4 | build 5 | dist 6 | docs/_build 7 | pytranscoder_ffmpeg.egg-info 8 | -------------------------------------------------------------------------------- /CHANGES: -------------------------------------------------------------------------------- 1 | 2 | ### Version History 3 | 4 | Version 2.2.7: 5 | * Fixed streaming host class where spaces in filenames weren't being recognized by Windows 6 | 7 | Version 2.2.6: 8 | * Added new templates which can be used instead of the more complicated profiles 9 | 10 | Version 2.2.5: 11 | * Much overdue doc updates 12 | * Addition of agent mode as alternative to using ssh. 13 | * Fixed issue cleaning up temp files on Windows host in streaming mode. 14 | 15 | Version 2.2.4: 16 | * Fixed issue where sometimes the wrong video stream is detected when there are multiple 17 | 18 | Version 2.2.3: 19 | * Minor issue with host name status during clustered transcoding 20 | 21 | Version 2.2.1: 22 | * Bug fix when no mixins given. 23 | 24 | Version 2.2: 25 | * Removed unportable is_mount detection. Now always default to any designated temp drive/path for transcoding. 26 | * Added mixins as a better alternative to includes (see docs). 27 | 28 | Version 2.1.0: 29 | * Added experimental support for HandBrakeCLI. 30 | * Requires Python 3.7 or higher 31 | * Removed Plex integration. 32 | * Removed Sonarr integration 33 | * Fixed typo in sample.yml and updated Cuda examples. 34 | 35 | Version 2.0.14: 36 | * Fixed ffprobe output parsing problem - thanks Grant. 37 | 38 | Version 2.0.13 39 | * Added support for ffprobe media analysis as a fallback if normal ffmpeg info dump parsing fails. 40 | * Fixed a regex issue where certain patterns in a "path" criteria didn't match 41 | * Tweaks to the sample transcode.yml 42 | 43 | Version 2.0.12 44 | * Fixed bad unlink() reference in threshold cleanup. 45 | 46 | Version 2.0.11 47 | * Fixed bug preventing completed work items from being removed from the queue file. 48 | 49 | Version 2.0.10 50 | * Fixed but on Windows trying to detect mounted filesystem. Not available under Windows. 51 | 52 | Version 2.0.9 53 | * Added 'fls_path' option to transcode.yml, which allows designating a "fast local storage" path where 54 | output is written during transcode rather than thrashing your network share/NAS with many small reads 55 | and writes. Final output is moved to the share when complete. Only applicable if the input file is on 56 | a detected network share. 57 | 58 | * Configuration files now require lists to use the multi-line YAML format for consistency (see updated example included) 59 | 60 | Version 2.0.8 61 | * Fixed automap=No recognition problem using non-clustered transcoding. 62 | 63 | Version 2.0.7 64 | * Fixed minor string formatting problem on abort of encode. 65 | 66 | Version 2.0.6 67 | * Suppress summary on --dry-run 68 | 69 | Version 2.0.5 70 | * Fixed another unhandled exception collecting statistics when using non-clustered transcoding. 71 | 72 | Version 2.0.4 73 | * Fixed unhandled exception collecting statistics when using non-clustered transcoding. 74 | 75 | Version 2.0.3 76 | * Prevent error message when remote host health testing times out. 77 | * Fixed erroneous "queue file not found" message. 78 | 79 | Version 2.0.2 80 | * Added summary output of full job with elapsed time 81 | * Fixed output bug which failed to report proper filename on termination of encode job. 82 | 83 | Version 2.0.1 84 | * Added "automap" setting to Global and Profile sections to expressly enable or disable mapping (default is ENABLED). 85 | * Added include_languages to "audio" and "subtitle" profile sections. Use Include or Exclude, not both. If you 86 | use both then Includes will take precedence. 87 | * Enabled audio and subtitle filtering in cluster mode too. 88 | * Fixed bug in inverted vcodec check (ex. vcodec !h264) 89 | * Fixed bug where cluster mode wasn't removing completed jobs from the queue file. 90 | * Been running reliably and well for months now - time for a version bump to 2.0! 91 | * Added a ctrl-c (sigint) handler to prevent threads from removing original media during termination. 92 | This seems to be a python 3.7 thing. 93 | 94 | Version 1.5.12 95 | * Merging of input_options and output_options supported at the option level. For example, if the included 96 | profiles has "-c:a copy" and the including profile has "-c:a libmp3lame", the including profile option will 97 | take precedence. 98 | 99 | Version 1.5.11 100 | * Default behavior is to now auto-map all audio and subtitle streams during transcode. 101 | May be disabled at the global or profile level with automap: no 102 | * More unit test fixes and improvements. 103 | * Updated sample transcode.yml for better feature reference 104 | 105 | Version 1.5.10 106 | * Fixed broken unit tests from previous release 107 | * Fixed rule predicates matching on runtime after conversion to seconds-level detail. 108 | 109 | Version 1.5.9 110 | * Added new optional "include" directive in profiles to help reduce profile repetition. 111 | * Fixed error message when input file not found. 112 | * Fixed divide-by-zero error when video runtime < 1 minute. Calculations how include seconds rather than just whole minutes 113 | 114 | Version 1.5.8 115 | * Updated yaml loader to support latest pyyaml library. 116 | 117 | Version 1.5.7 118 | * Threshold check termination resulted in a misleading error message. 119 | * Windows wildcard expansion fixed. 120 | * Changed rule criteria for range matching to be inclusive. So 30-60 now mean between 30 and 60 inclusively. 121 | 122 | Version 1.5.5 123 | * Threshold check termination resulted in a misleading error message. 124 | * Doc link to video series part 2. 125 | * Cluster documentation revised and corrected. 126 | 127 | Version 1.5.4 128 | * Fixed bug that suppressed certain ffmpeg error condition messages. 129 | * -k flag ignored under certain situations. 130 | * Doc link to video series part 1. 131 | * Fixed some code inconsistencies regarding global flags 132 | 133 | Version 1.5.3 134 | * Removed requirement for queue configuration if not using queues. 135 | * Documentation and Pipfile tweaks. 136 | 137 | Version 1.5.2 138 | * Fixed media info parsing problem with file generated from Final Cut Pro X 139 | * Fixed regex problem matching pathname in a rule 140 | * Fixed divide by zero error on status output when not using hardware acceleration 141 | * Tightened up the output 142 | * Fixed color output 143 | * Better test coverage with mocking. 144 | 145 | Version 1.4.7 146 | * Multi-queue support in cluster mode. Run multiple concurrent jobs on remote machines now. 147 | * import error in utils.py 148 | * -t option removed from ssh as it was screwing up the local terminal output 149 | 150 | Version 1.4.6 151 | * Configuration file change: "rules" section of each rule renamed to "criteria" for clarity. 152 | 153 | Version 1.4.5 154 | * Calculate and show compression % in progress output 155 | * Allow cluster mode profile override 156 | * Added threshold_check to enable threshold checking to start at designated percentage done. 157 | * Cluster mode -k option fix. 158 | 159 | Version 1.4.4 160 | * Optional colorized terminal output. 161 | * Encode runs are logged for inspection if something goes wrong. 162 | * Cluster mode not honoring default queue file. 163 | * Cluster mode --host override fixed. 164 | 165 | Version 1.4.1 166 | * Regex problem monitoring ffmpeg output. 167 | * Refactored ffmpeg control into a class 168 | 169 | Version 1.3 170 | * Added support for local host in the cluster so that the same host managing work also does work. 171 | * Monitoring of ffmpeg jobs and progress reporting. 172 | * Support for multiple allowed profiles for a host. 173 | * New --host commandline option to force cluster encodes on a designated host only 174 | * Large code refactoring. 175 | 176 | Version 1.2.2 177 | * Path to ssh is now configurable. 178 | 179 | Version 1.2.1 180 | * Multiple queue definitions, allowing for more fine threading control. 181 | * New cluster mode to allow for agent-less, multi-machine encode offloading. 182 | * Cluster host definitions support Linux, MacOS, and Windows. 183 | * Support for running on Windows 10. 184 | * Rule criteria now allows for range testing with numeric types. 185 | * Continue to improve unit testing suite. 186 | * Documentation revisions. 187 | 188 | Version 1.1.0 189 | * Fixed bug that would not allow empty input options in profile definition. 190 | 191 | Version 1.0.1 192 | * Added support for use as a Sonarr connection custom script. 193 | * Fixed broken unit test. 194 | * Install sample transcode.yml file in /usr/share/pytranscoder folder. 195 | * Revised some details in README.md 196 | * Started this CHANGES file. 197 | 198 | Version 1.0.0 199 | * Initial Release to pypi.org 200 | -------------------------------------------------------------------------------- /Cluster.md: -------------------------------------------------------------------------------- 1 | ## pytranscoder - cluster mode 2 | 3 | Python wrapper for ffmpeg for batch, concurrent, or clustered transcoding 4 | 5 | This script is intended to help automate encoding for people who do a lot of it. 6 | 7 | This documentation is specific to the clustering functionality of the tool. It is separate from the main 8 | docs, README.md, because it is a more advanced feature. 9 | 10 | ### System Setup 11 | We will refer to the machine you use pytranscode on as your *cluster manager* and other machines as *hosts*. 12 | 13 | Setting up remote hosts is as follows, if not already setup for ssh access and ffmpeg: 14 | #### Linux 15 | Linux is natively supported as long as the following conditions are true: 16 | * Each host machine in the cluster is running an ssh server. 17 | * Each host has ffmpeg installed. 18 | * If using hardware encoding, your machine (and ffmpeg) have been setup and tested to make sure it is working. Setup of hardware encoding is beyond the scope of this doc. 19 | * The *cluster manager* machine must be able to ssh to each host without a password prompt (see `man ssh-copy-id`). 20 | #### MacOS 21 | MacOS, being based on BSD, is also natively supported. See Linux section. Check your MacOS version of ffmpeg for what hardware acceleration support is available, if any. At the time of this writing there was nothing available of appreciable quality, only VAAPI and the quality was dismal. 22 | #### Windows 10 23 | **Windows is only supported as a cluster manager if installed and run under WSL** 24 | 25 | There are 2 ways to enable SSH access for Windows. Each method is further complicated depending on which ffmpeg you use. These instructions assume a certain level of proficiency with Windows and optionally WSL. 26 | 27 | **Windows SSH**: 28 | > This method will only allow streaming cluster support due to Windows OpenSSH not being able to access network shares. 29 | 30 | > * Enable OpenSSH server 31 | * Enable your OpenSSH server via the Windows Services control panel. 32 | * Start server 33 | 34 | > In the home folder of the user account create a directory called **.ssh**. Then from your *cluster manager* copy your **$HOME/.ssh/id_rsa.pub** to **c:/Users/*username*/.ssh/authorized_keys** on Windows. 35 | > 36 | >Finally, if you have a supported nVidia card download the nVidia CUDA drivers and install if you plan on using CUDA encoding. 37 | It's a large download. Choose Custom install and deselect all the documentation and other things you don't need if you want to 38 | minimize space usage. 39 | 40 | When pytranscoder starts it will verify that it can ssh to each host using the provided configuration before continuing. 41 | 42 | 43 | #### Configuration 44 | 45 | If you skipped right to this documentation you need to read the [README](https://github.com/mlsmithjr/transcoder/blob/master/README.md) first. 46 | 47 | Each of your hosts can be designated as *mounted*, *streaming*, or *agent*. 48 | A *mounted* host is one that has network access to the same media you are transcoding via a NFS or Samba/CIFS mount. This is the ideal configuration since files need not be copied to and from a host - they are already shared. 49 | A *streaming* host is one that does not have network access to the media. This most likely will be due to not having the ability to setup a network share on that host. Each file to be encoded is copied to that host using **scp** (ssh), encoded, then the output copied back to your *cluster manager*. This is very inefficient, but works. 50 | A *agent* host has pytranscoder installed and performs the encoding operations for the *cluster manager* (no ssh enablement required). 51 | 52 | There is a section in the transcode.yml file, under the global *config* section, called *clusters*. A cluster is a 53 | group of one or more host machines you will use for encoding. You may define multiple clusters if you have a large 54 | network of machines at your disposal. **Note**: all hosts in the cluster do not need to be available at runtime - they 55 | will simply be ignored and other hosts in the cluster used. 56 | 57 | Add an **ssh** item to your global configuration (location of your *cluster manager* ssh client), then define the cluster: 58 | 59 | Sample 60 | ```yaml 61 | config: 62 | 63 | ssh: '/usr/bin/ssh' # used only in cluster mode 64 | 65 | ###################### 66 | # cluster definitions 67 | ###################### 68 | clusters: 69 | household: # name for this cluster 70 | 71 | ################################# 72 | # cluster manager, which will 73 | # also participate in the cluster 74 | ################################# 75 | mediacenter: 76 | type: local # Indicates this is where pytranscoder is running and can be used in the cluster as well. 77 | ffmpeg: '/usr/bin/ffmpeg' 78 | status: 'enabled' 79 | 80 | ################################# 81 | # My old MacPro booted into Ubuntu 82 | ################################# 83 | macpro: # name of this host (does not need to be the same as network hostname) 84 | type: mounted # machine with source media and host share a filesystem (nfs, samba, etc) 85 | os: macos # choices are linux, macos, win10 86 | ip: 192.168.2.65 87 | user: sshuser # user account used to ssh to this host 88 | ffmpeg: '/usr/bin/ffmpeg' 89 | path-substitutions: # optional, map source pathnames to equivalent on host 90 | - /volume1/media/ /media/ 91 | profiles: # profiles allowed on this host 92 | - hevc 93 | - h264 94 | status: 'enabled' # set to disabled to temporarily stop using 95 | 96 | ################################# 97 | # My son's gaming machine 98 | ################################# 99 | gamer: # machine configured with Windows OpenSSH server 100 | type: streaming # host not using shared filesystem 101 | os: win10 # choices are linux, macos, win10 102 | ip: 192.168.2.64 # address of host 103 | user: matt # ssh login user 104 | working_dir: 'c:\temp' # working folder on remote host, required for streaming type 105 | ffmpeg: 'c:/ffmpeg/bin/ffmpeg' 106 | profiles: # profiles allowed on this host 107 | - hevc_cuda 108 | - hevc_qsv 109 | queues: 110 | qsv: 1 111 | cuda: 2 112 | status: 'enabled' # set to disabled to temporarily stop using 113 | 114 | ################################# 115 | # Spare family machine 116 | ################################# 117 | family: 118 | type: agent 119 | os: win10 120 | ip: 192.168.2.66 121 | user: chris 122 | ffmpeg: c:/ffmpeg/bin/ffmpeg.exe # using Windows ffmpeg.exe build 123 | profiles: # profiles allowed on this host 124 | - hevc_cuda 125 | - hevc_cuda_10bit 126 | queues: 127 | qsv: 1 128 | cuda: 1 129 | status: enabled 130 | ``` 131 | 132 | | setting | purpose | 133 | | ----------- | ----------- | 134 | | type | Host type, either *mounted*, *streaming*, or *agent*. There can be one host in all clusters with type *local*. A *mounted* type indicates the input media files are accessible via a shared filesystem mounted on the host. A *streaming* type indicates no sharing, and each media file being encoded is copied to that host, encoded, then copied back. A *local* type is used to also include the *cluster manager* machine (system running pytranscoder) in the cluster so it won't sit idle, and is optional. There are fewer required configuration attributes for this type. | 135 | | os | One of linux,macos, or win10. **[1]** | 136 | | ip | Address or host name of the host. **[1]** | 137 | | user | User to log into this host as via ssh. The user must be pre-authenticated to the host so that a password is not required. See https://www.ssh.com/ssh/copy-id. **[1]** | 138 | | ffmpeg | Path on the host to ffmpeg. | 139 | | working_dir | Indicates the temporary directory to use for encoding. **[2]** | 140 | | profiles | The allowed profiles to use for all encodes on this host. If not provided, assumes all. A video input matching a profile that is not assigned to a particular host will be run on a host that will, if any. This is how, for example, you restrict CPU-based encodings to hosts with no hardware acceleration - or vice versa. In other words, you control how each host is used by which profiles it supports. | 141 | | path-substitutions | Optional. Applicable only to *mounted* type hosts. Use when the server media files and host mount paths are different. | 142 | | queues | Optional. You can define per-host queues to enable concurrent jobs on each host. If not given, encoding jobs will run 1 at a time. See README.md for further discussion of queues. | 143 | | status | *enabled* or *disabled*. Disabled hosts will be skipped. Default is *enabled*.| 144 | 145 | **[1]** Required for *mounted* and *streaming* types. 146 | **[2]** Required for *streaming* type. 147 | 148 | 149 | #### Sample Walkthrough 150 | 151 | You have a media server called **mediaserver**. It has an NFS-exported path to the root of your media storage. This folder is 152 | on a RAID mounted at /volume1/media. You want to enable all the machines in your household to be used for encoding. You want to 153 | transcode all media to HEVC because it's just better, but it's very time-consuming so you decide to use other machines that 154 | are under-utilized for such tasks. You create a special user account on all hosts just for encoding and setup password-less ssh login to each host using *ssh-copy-id*. 155 | You plan to kick off clustered encoding from **mediaserver**, using 2 other machines to do the work. 156 | 157 | You have another machine, your main workstation, which is called **workstation**. This machine mounts the **mediaserver** export as /mnt/media for easy sharing. It has no CUDA-enabled graphics card but does have an 8th generation Intel i5 supporting QSV. 158 | 159 | Your last machine is a shared machine your kids sometimes use, called **shared**. It has a great nVidia graphics card, but does not mount the media filesystem exported from **mediaserver**. 160 | 161 | Here is the configuration for the scenario above: 162 | 163 | ```yaml 164 | clusters: 165 | household: # name for this cluster 166 | workstation: 167 | type: mounted 168 | ip: 192.168.2.63 169 | user: encodeuser 170 | ffmpeg: '/usr/bin/ffmpeg' 171 | path-substitutions: # optional, map source pathnames to equivalent on host 172 | - /volume1/media/ /mnt/media/ 173 | profiles: 174 | - qsv 175 | 176 | shared: 177 | type: streaming 178 | ip: 192.168.2.64 179 | user: encodeuser 180 | working_dir: '/tmp' 181 | ffmpeg: '/usr/bin/ffmpeg' 182 | profiles: 183 | - hevc_cuda 184 | 185 | ``` 186 | 187 | Not really much of a cluster, but just for illustration purposes. 188 | Now, assuming you have a bunch of media files on **mediaserver** you want to transcode: 189 | 190 | ```bash 191 | ls /volume1/media 192 | 193 | file1.mp4 file2.mp4 file3.mp4 194 | ``` 195 | Let's do a dry run to see what will happen: 196 | 197 | ```bash 198 | pytranscoder --dry-run -c household /volume1/media/*.mp4 199 | 200 | ---------------------------------------- 201 | Filename : file1.mp4 202 | Host: workstation (mounted) 203 | Profile : qsv 204 | ffmpeg : -y -hwaccel vaapi -hwaccel_device /dev/dri/renderD128 -hwaccel_output_format vaapi -i /volume1/media/file1.mp4 -vf scale_vaapi=format=p010 -c:v hevc_vaapi -crf 18 -c:a copy -c:s copy -f matroska -max_muxing_queue_size 1024 /volume1/media/file1.mkv.tmp 205 | 206 | ---------------------------------------- 207 | Filename : file2.mp4 208 | Host: shared (streaming) 209 | Profile : hevc_cuda 210 | ffmpeg : -y -hwaccel vaapi -hwaccel_device /dev/dri/renderD128 -hwaccel_output_format vaapi -i /volume1/media/file2.mp4 -vf scale_vaapi=format=p010 -c:v hevc_vaapi -crf 18 -c:a copy -c:s copy -f matroska -max_muxing_queue_size 1024 /volume1/media/file2.mkv.tmp 211 | 212 | ... 213 | ``` 214 | Running a --dry-run will check that the cluster machines are up, that **ssh** login works, and then perform the profile matchines, if applicable. Execution will then stop without doing any work. 215 | 216 | To run for real: 217 | 218 | ```bash 219 | pytranscoder -c household /volume1/media/* 220 | ``` 221 | 222 | This will pick up each file in /volume1/media and queue them for encoding. Two threads are started - one for **workstation** and 223 | the other for **shared**. Each thread examines the queue, pulling the next video to be transcoded until all files are 224 | processed. 225 | 226 | For **workstation**, a file is pulled from the queue, /volume1/media/file1.mp4 for instance. Since there is a *path-substitution* 227 | configured, change the path to /mnt/media/file1.mp4. Finally, ssh to **workstation** as _encodeuser_ and run ffmpeg to encode /mnt/media/file1.mp4 using QSV. 228 | The temporary encoded file will be placed in the same folder as the source. 229 | 230 | For **shared**, a file is pulled from the queue, /volume1/media/file2.mp4 for instance, and copied to /tmp on that host. 231 | Then ssh to **shared** as _encodeuser_and run ffmpeg to encode /tmp/file2.mp4 using hevc_cuda. When finished, copy the encoded 232 | file from **shared** back to **mediaserver** and remove temporary files from **shared** /tmp. 233 | 234 | The last file, /volume1/media/file3.mp4, will be handled by the first host to finish the previous encodes. Once all have 235 | been encoded the process will exit. 236 | 237 | No encoding was performed on **mediaserver** - it was only used as the manager for the hosts in the cluster. You can easily add it to the cluster though and have 3 machines working. 238 | 239 | Any defined host that isn't up and available when pytranscoder 240 | is run will be ignored and transcoding will continue on other available hosts. 241 | 242 | #### Testing your Setup 243 | 244 | You should always do a dry-run test before committing to a configuration change. It will help you see that your defined rules are matching as expected and that hosts can be connected to via ssh. 245 | 246 | ```bash 247 | pytranscoder --dry-run -c mycluster /volume1/media/any_video_file 248 | ``` 249 | 250 | #### Running 251 | 252 | Usage: 253 | ```bash 254 | pytranscoder -c files ... -c files ... 255 | ``` 256 | 257 | You can start 1 or multiple clusters, comprised of 1 or more hosts each. You can assign different files to different 258 | clusters, if your needs are that complex. 259 | 260 | However, most people will simple run as: 261 | 262 | ```bash 263 | pytranscoder -c mycluster /volume1/media/*.mp4 264 | ``` 265 | 266 | To troubleshoot problems, use verbose mode 267 | ```bash 268 | pytranscoder -v -c mycluster /volume1/media/*.mp4 269 | 270 | ``` 271 | 272 | To force encode(s) to a specific host named 'wopr' 273 | ```bash 274 | pytranscoder -c mycluster -h wopr /volume1/media/*.mp4 275 | ``` 276 | To force all jobs to use a specific profile: 277 | ```bash 278 | pytranscoder -c mycluster -p best_profile /volume1/media/*.mp4 279 | ``` 280 | 281 | Or you can do combinations: 282 | ```bash 283 | pytranscoder -v -c mycluster -p best_profile -h wopr /volume1/media/*.mp4 284 | ``` 285 | -------------------------------------------------------------------------------- /Jenkinsfile: -------------------------------------------------------------------------------- 1 | pipeline { 2 | agent any 3 | stages { 4 | stage('build') { 5 | steps { 6 | python setup.py sdist bdist_wheel 7 | } 8 | } 9 | } 10 | } 11 | 12 | -------------------------------------------------------------------------------- /Pipfile: -------------------------------------------------------------------------------- 1 | [[source]] 2 | name = "pypi" 3 | url = "https://pypi.org/simple" 4 | verify_ssl = true 5 | 6 | [dev-packages] 7 | 8 | [packages] 9 | PlexAPI = "*" 10 | crayons = "*" 11 | pyyaml = "*" 12 | mock = "*" 13 | 14 | [requires] 15 | python_version = "3.10" 16 | -------------------------------------------------------------------------------- /Pipfile.lock: -------------------------------------------------------------------------------- 1 | { 2 | "_meta": { 3 | "hash": { 4 | "sha256": "2b6ad49ebb90ed4372e2ea249af62bd9ac0c2fbaafc91e4ece3fd00f62f3b97b" 5 | }, 6 | "pipfile-spec": 6, 7 | "requires": { 8 | "python_version": "3.10" 9 | }, 10 | "sources": [ 11 | { 12 | "name": "pypi", 13 | "url": "https://pypi.org/simple", 14 | "verify_ssl": true 15 | } 16 | ] 17 | }, 18 | "default": { 19 | "certifi": { 20 | "hashes": [ 21 | "sha256:35824b4c3a97115964b408844d64aa14db1cc518f6562e8d7261699d1350a9e3", 22 | "sha256:4ad3232f5e926d6718ec31cfc1fcadfde020920e278684144551c91769c7bc18" 23 | ], 24 | "index": "pypi", 25 | "markers": "python_version >= '3.6'", 26 | "version": "==2022.12.7" 27 | }, 28 | "charset-normalizer": { 29 | "hashes": [ 30 | "sha256:5a3d016c7c547f69d6f81fb0db9449ce888b418b5b9952cc5e6e66843e9dd845", 31 | "sha256:83e9a75d1911279afd89352c68b45348559d1fc0506b054b346651b5e7fee29f" 32 | ], 33 | "markers": "python_full_version >= '3.6.0'", 34 | "version": "==2.1.1" 35 | }, 36 | "colorama": { 37 | "hashes": [ 38 | "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44", 39 | "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6" 40 | ], 41 | "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5, 3.6'", 42 | "version": "==0.4.6" 43 | }, 44 | "crayons": { 45 | "hashes": [ 46 | "sha256:bd33b7547800f2cfbd26b38431f9e64b487a7de74a947b0fafc89b45a601813f", 47 | "sha256:e73ad105c78935d71fe454dd4b85c5c437ba199294e7ffd3341842bc683654b1" 48 | ], 49 | "index": "pypi", 50 | "version": "==0.4.0" 51 | }, 52 | "idna": { 53 | "hashes": [ 54 | "sha256:814f528e8dead7d329833b91c5faa87d60bf71824cd12a7530b5526063d02cb4", 55 | "sha256:90b77e79eaa3eba6de819a0c442c0b4ceefc341a7a2ab77d7562bf49f425c5c2" 56 | ], 57 | "markers": "python_version >= '3.5'", 58 | "version": "==3.4" 59 | }, 60 | "mock": { 61 | "hashes": [ 62 | "sha256:c41cfb1e99ba5d341fbcc5308836e7d7c9786d302f995b2c271ce2144dece9eb", 63 | "sha256:e3ea505c03babf7977fd21674a69ad328053d414f05e6433c30d8fa14a534a6b" 64 | ], 65 | "index": "pypi", 66 | "version": "==5.0.1" 67 | }, 68 | "plexapi": { 69 | "hashes": [ 70 | "sha256:2a67b5739ec966e10dec957fea8abe38e9a4ff9d6b58dd0ec6a55ae758cead8e", 71 | "sha256:4a7cd6729061419abd600de9c436bdf9565976a873d0a74487606c4126b98439" 72 | ], 73 | "index": "pypi", 74 | "version": "==4.13.2" 75 | }, 76 | "pyyaml": { 77 | "hashes": [ 78 | "sha256:01b45c0191e6d66c470b6cf1b9531a771a83c1c4208272ead47a3ae4f2f603bf", 79 | "sha256:0283c35a6a9fbf047493e3a0ce8d79ef5030852c51e9d911a27badfde0605293", 80 | "sha256:055d937d65826939cb044fc8c9b08889e8c743fdc6a32b33e2390f66013e449b", 81 | "sha256:07751360502caac1c067a8132d150cf3d61339af5691fe9e87803040dbc5db57", 82 | "sha256:0b4624f379dab24d3725ffde76559cff63d9ec94e1736b556dacdfebe5ab6d4b", 83 | "sha256:0ce82d761c532fe4ec3f87fc45688bdd3a4c1dc5e0b4a19814b9009a29baefd4", 84 | "sha256:1e4747bc279b4f613a09eb64bba2ba602d8a6664c6ce6396a4d0cd413a50ce07", 85 | "sha256:213c60cd50106436cc818accf5baa1aba61c0189ff610f64f4a3e8c6726218ba", 86 | "sha256:231710d57adfd809ef5d34183b8ed1eeae3f76459c18fb4a0b373ad56bedcdd9", 87 | "sha256:277a0ef2981ca40581a47093e9e2d13b3f1fbbeffae064c1d21bfceba2030287", 88 | "sha256:2cd5df3de48857ed0544b34e2d40e9fac445930039f3cfe4bcc592a1f836d513", 89 | "sha256:40527857252b61eacd1d9af500c3337ba8deb8fc298940291486c465c8b46ec0", 90 | "sha256:432557aa2c09802be39460360ddffd48156e30721f5e8d917f01d31694216782", 91 | "sha256:473f9edb243cb1935ab5a084eb238d842fb8f404ed2193a915d1784b5a6b5fc0", 92 | "sha256:48c346915c114f5fdb3ead70312bd042a953a8ce5c7106d5bfb1a5254e47da92", 93 | "sha256:50602afada6d6cbfad699b0c7bb50d5ccffa7e46a3d738092afddc1f9758427f", 94 | "sha256:68fb519c14306fec9720a2a5b45bc9f0c8d1b9c72adf45c37baedfcd949c35a2", 95 | "sha256:77f396e6ef4c73fdc33a9157446466f1cff553d979bd00ecb64385760c6babdc", 96 | "sha256:81957921f441d50af23654aa6c5e5eaf9b06aba7f0a19c18a538dc7ef291c5a1", 97 | "sha256:819b3830a1543db06c4d4b865e70ded25be52a2e0631ccd2f6a47a2822f2fd7c", 98 | "sha256:897b80890765f037df3403d22bab41627ca8811ae55e9a722fd0392850ec4d86", 99 | "sha256:98c4d36e99714e55cfbaaee6dd5badbc9a1ec339ebfc3b1f52e293aee6bb71a4", 100 | "sha256:9df7ed3b3d2e0ecfe09e14741b857df43adb5a3ddadc919a2d94fbdf78fea53c", 101 | "sha256:9fa600030013c4de8165339db93d182b9431076eb98eb40ee068700c9c813e34", 102 | "sha256:a80a78046a72361de73f8f395f1f1e49f956c6be882eed58505a15f3e430962b", 103 | "sha256:afa17f5bc4d1b10afd4466fd3a44dc0e245382deca5b3c353d8b757f9e3ecb8d", 104 | "sha256:b3d267842bf12586ba6c734f89d1f5b871df0273157918b0ccefa29deb05c21c", 105 | "sha256:b5b9eccad747aabaaffbc6064800670f0c297e52c12754eb1d976c57e4f74dcb", 106 | "sha256:bfaef573a63ba8923503d27530362590ff4f576c626d86a9fed95822a8255fd7", 107 | "sha256:c5687b8d43cf58545ade1fe3e055f70eac7a5a1a0bf42824308d868289a95737", 108 | "sha256:cba8c411ef271aa037d7357a2bc8f9ee8b58b9965831d9e51baf703280dc73d3", 109 | "sha256:d15a181d1ecd0d4270dc32edb46f7cb7733c7c508857278d3d378d14d606db2d", 110 | "sha256:d4b0ba9512519522b118090257be113b9468d804b19d63c71dbcf4a48fa32358", 111 | "sha256:d4db7c7aef085872ef65a8fd7d6d09a14ae91f691dec3e87ee5ee0539d516f53", 112 | "sha256:d4eccecf9adf6fbcc6861a38015c2a64f38b9d94838ac1810a9023a0609e1b78", 113 | "sha256:d67d839ede4ed1b28a4e8909735fc992a923cdb84e618544973d7dfc71540803", 114 | "sha256:daf496c58a8c52083df09b80c860005194014c3698698d1a57cbcfa182142a3a", 115 | "sha256:dbad0e9d368bb989f4515da330b88a057617d16b6a8245084f1b05400f24609f", 116 | "sha256:e61ceaab6f49fb8bdfaa0f92c4b57bcfbea54c09277b1b4f7ac376bfb7a7c174", 117 | "sha256:f84fbc98b019fef2ee9a1cb3ce93e3187a6df0b2538a651bfb890254ba9f90b5" 118 | ], 119 | "index": "pypi", 120 | "version": "==6.0" 121 | }, 122 | "requests": { 123 | "hashes": [ 124 | "sha256:7c5599b102feddaa661c826c56ab4fee28bfd17f5abca1ebbe3e7f19d7c97983", 125 | "sha256:8fefa2a1a1365bf5520aac41836fbee479da67864514bdb821f31ce07ce65349" 126 | ], 127 | "markers": "python_version >= '3.7' and python_version < '4'", 128 | "version": "==2.28.1" 129 | }, 130 | "urllib3": { 131 | "hashes": [ 132 | "sha256:47cc05d99aaa09c9e72ed5809b60e7ba354e64b59c9c173ac3018642d8bb41fc", 133 | "sha256:c083dd0dce68dbfbe1129d5271cb90f9447dea7d52097c6e0126120c521ddea8" 134 | ], 135 | "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'", 136 | "version": "==1.26.13" 137 | } 138 | }, 139 | "develop": {} 140 | } 141 | -------------------------------------------------------------------------------- /build.sh: -------------------------------------------------------------------------------- 1 | python3 setup.py sdist bdist_wheel 2 | 3 | -------------------------------------------------------------------------------- /config-samples/cluster-sample.yml: -------------------------------------------------------------------------------- 1 | #### 2 | # This clustering sample defines some hosts and only 1 profile, for simplicity 3 | #### 4 | 5 | 6 | ## 7 | # global configuration 8 | ## 9 | config: 10 | ffmpeg: '/home/mark/bin/ffmpeg' 11 | queues: 12 | qsv: 1 13 | cuda: 1 14 | colorize: yes 15 | fls_path: '/tmp' 16 | 17 | # 18 | # Sample cluster definition with a few hosts in a home network. Each of these hosts are used to transcode files for a job until all files processed 19 | # 20 | clusters: 21 | home: 22 | homeserver: 23 | os: linux 24 | type: local 25 | ffmpeg: '/home/mark/bin/ffmpeg' 26 | queues: 27 | qsv: 1 28 | profiles: 29 | - hevc_qsv 30 | status: enabled 31 | working_dir: '/tmp' 32 | 33 | workpc: 34 | os: linux 35 | type: mounted 36 | working_dir: /tmp 37 | ip: 192.168.2.70 38 | user: mark 39 | ffmpeg: '/usr/bin/ffmpeg' 40 | queues: 41 | cuda: 1 42 | qsv: 1 43 | profiles: 44 | - hevc_qsv 45 | 46 | # you must specify how a paths on the fileserver side maps to the client(host) side if using a "mounted" host type 47 | 48 | path-substitutions: 49 | - '/mnt/merger/media/ /mnt/homeserver/media/' 50 | - '/mnt/downloads/ /mnt/homeserver/downloads/' 51 | status: disabled 52 | 53 | winpc: 54 | os: win10 55 | type: mounted 56 | ip: 192.168.2.61 57 | user: mark 58 | ffmpeg: 'c:\ffmpeg\bin\ffmpeg.exe' 59 | profiles: 60 | - hevc_qsv 61 | - hevc_qsv_medium 62 | queues: 63 | qsv: 1 64 | working_dir: 'c:\temp' 65 | 66 | # you must specify how a paths on the fileserver side maps to the client(host) side if using a "mounted" host type 67 | 68 | path-substitutions: 69 | - '/mnt/merger/media/video/Television/ m:video\Television\' 70 | - '/mnt/merger/media/video/ m:video\' 71 | - '/mnt/downloads/ z:\' 72 | status: disabled 73 | 74 | workstation: 75 | os: linux 76 | type: mounted 77 | working_dir: /tmp 78 | ip: 192.168.2.63 79 | user: mark 80 | ffmpeg: '/home/mark/bin/ffmpeg' 81 | queues: 82 | cuda: 1 83 | qsv: 1 84 | profiles: 85 | - hevc_cuda 86 | - hevc_cuda_medium 87 | - hevc_qsv 88 | - hevc_qsv_medium 89 | - hevc_cuda_10bit 90 | 91 | # you must specify how a paths on the fileserver side maps to the client(host) side if using a "mounted" host type 92 | 93 | path-substitutions: 94 | 95 | - '/mnt/merger/media/ /mnt/server/media/' 96 | - '/mnt/downloads/ /mnt/server/downloads/' 97 | status: enabled 98 | 99 | 100 | 101 | ## 102 | # profile definitions. You can model all your transcoding combinations here. 103 | ## 104 | profiles: 105 | 106 | # 107 | # define a common, base profile that others inherit from. Optional, but handy to save time 108 | # 109 | base: 110 | # 111 | # ffmpeg output-related options 112 | # 113 | output_options: 114 | - "-c:s copy" # copy all subtitles as-is 115 | - "-f matroska" # mkv format 116 | - "-max_muxing_queue_size 1024" 117 | output_options_audio: 118 | - "-c:a copy" # copy all audio as-is 119 | extension: '.mkv' 120 | threshold: 10 # minimum of 18% compression required 121 | threshold_check: 20 # start checking threshold at 20% complete 122 | 123 | # 124 | # audio handling 125 | # 126 | audio: 127 | # 128 | # Included languages take precedent. By default all languages are retained. 129 | # Any specifically included languages here automatically discard all others. 130 | # 131 | include_languages: 132 | - "eng" 133 | default_language: eng 134 | # 135 | # subtitle handling 136 | # 137 | subtitle: 138 | # see audio comment above 139 | include_languages: 140 | - "eng" 141 | default_language: eng 142 | 143 | ### 144 | # Intel QSV common options. Like "base" above, these are common to all QSV operations 145 | ### 146 | qsv: 147 | input_options: 148 | # enable hardware decoding 149 | output_options: 150 | - "-c:v hevc_qsv" 151 | - "-preset medium" 152 | - "-qp 21" 153 | - "-c:s copy" 154 | output_options_video: 155 | - "-b:v 7000K" 156 | output_options_audio: 157 | - "-c:a copy" 158 | 159 | ### 160 | # Intel QSV (HEVC) 161 | ### 162 | 163 | # full quality 164 | hevc_qsv: 165 | include: "base qsv" # Note we are including base and qsv defined above so we don't have to retype everything 166 | output_options: 167 | queue: 'qsv' 168 | 169 | # medium quality 170 | hevc_qsv_medium: 171 | include: "base qsv" 172 | output_options_video: 173 | - "-b:v 4000K" 174 | output_options_audio: 175 | - "-c:a ac3" 176 | - "-b:a 768k" 177 | queue: 'qsv' 178 | 179 | # 180 | # Automatching happens when a profile isn't provided on the command line. These rules are evalulated to find the 181 | # most appropriate profile for each video to be transcoded. 182 | # 183 | # Rules are evaluated in order. First matching rule wins so order wisely. 184 | # Rules with a profile of "SKIP" mean to skip processing of the matched video 185 | # 186 | rules: 187 | 188 | 'skip video if already encoded in hevc/h265': 189 | profile: SKIP 190 | criteria: 191 | vcodec: 'hevc' 192 | 193 | 'small enough already': 194 | profile: SKIP 195 | criteria: 196 | filesize_mb: '<2300' 197 | res_height: '721-1081' 198 | runtime: '30-65' 199 | 200 | 'small enough already-2': 201 | profile: SKIP 202 | criteria: 203 | filesize_mb: '<1000' 204 | res_height: '721-1081' 205 | runtime: '20-29' 206 | 207 | 'default HD': 208 | profile: hevc_qsv 209 | criteria: 210 | res_height: '721-1081' 211 | vcodec: '!hevc' 212 | runtime: '>35' 213 | filesize_mb: '>2300' 214 | 215 | -------------------------------------------------------------------------------- /config-samples/new.yml: -------------------------------------------------------------------------------- 1 | 2 | config: 3 | ffmpeg: '/usr/bin/ffmpeg' 4 | colorize: yes 5 | queues: 6 | qsv: 1 7 | cuda: 1 8 | fls_path: '/tmp' # fast local storage for work files, ideally an SSD (optional) 9 | 10 | clusters: 11 | home: 12 | workstation: 13 | type: mounted 14 | os: linus 15 | ffmpeg: '/usr/bin/ffmpeg' 16 | working_dir: /tmp 17 | queues: 18 | qsv: 1 19 | cuda: 1 20 | templates: 21 | - qsv 22 | - qsv_medium 23 | - qsv_anime 24 | status: 'enabled' 25 | 26 | homeserver: 27 | os: linux 28 | type: mounted 29 | ffmpeg: '/usr/bin/ffmpeg' 30 | working_dir: /tmp 31 | ip: 192.168.2.61 32 | user: mark 33 | queues: 34 | cuda: 1 35 | templates: 36 | - qsv 37 | - qsv_medium 38 | - qsv_anime 39 | status: 'enabled' 40 | 41 | 42 | # {input-options} {video-codec} {audio-codec} {subtitles} 43 | 44 | templates: 45 | qsv: 46 | cli: 47 | video-codec: "-c:v hevc_qsv -preset medium -qp 21 -b:v 7000K" 48 | audio-codec: "-c:a copy" 49 | subtitles: "-c:s copy" 50 | threshold: 15 51 | threshold_check: 30 52 | queue: "qsv" 53 | extension: '.mkv' 54 | 55 | qsv_medium: 56 | cli: 57 | video-codec: "-c:v hevc_qsv -preset medium -qp 21 -b:v 4000K" 58 | audio-codec: "-c:a ac3 -b:a 768k" 59 | subtitles: "-c:s copy" 60 | threshold: 15 61 | threshold_check: 30 62 | queue: "qsv" 63 | extension: '.mkv' 64 | 65 | qsv_anime: 66 | cli: 67 | video-codec: "-c:v hevc_qsv -preset medium -qp 21 -b:v 3000K" 68 | audio-codec: "-c:a ac3 -b:a 768k" 69 | subtitles: "-c:s copy" 70 | audio-lang: "eng jpn" 71 | subtitle-lang: eng 72 | threshold: 15 73 | threshold_check: 30 74 | queue: "qsv" 75 | extension: '.mkv' 76 | 77 | -------------------------------------------------------------------------------- /config-samples/simple.yml: -------------------------------------------------------------------------------- 1 | ## 2 | # NOTE: Very simple starter config for pytranscode 3 | # 4 | # Run using: pytranscoder -y simple.yml ... 5 | ## 6 | 7 | ## 8 | # global configuration 9 | ## 10 | config: 11 | ffmpeg: '/usr/bin/ffmpeg' 12 | colorize: yes 13 | 14 | ## 15 | # profile definitions. You can model all your transcoding combinations here. 16 | ## 17 | profiles: 18 | 19 | # 20 | # Sample cpu-based transcode, uses built-in default sequential queue 21 | # 22 | h264: 23 | input_options: 24 | output_options: 25 | - "-threads 4" 26 | - "-c:v x264 " 27 | - "-crf 20 " 28 | - "-c:a copy" 29 | - "-c:s copy " 30 | - "-f matroska" 31 | extension: '.mkv' 32 | threshold: 20 33 | 34 | # 35 | # Sample nVidia transcode setup, uses 'cuda' queue defined above 36 | # 37 | hevc_cuda: 38 | input_options: 39 | - "-hwaccel cuvid" 40 | output_options: 41 | - "-cq:v 19" # crf option passed to CUDA engine 42 | - "-rc vbr_hq" # variable bit-rate, high quality 43 | - "-rc-lookahead 32" 44 | - "-bufsize 8M" 45 | - "-b:v 8M" 46 | - "-profile:v main" 47 | - "-maxrate:v 8M" 48 | - "-c:v hevc_nvenc" 49 | - "-preset slow" 50 | - "-pix_fmt yuv420p" 51 | extension: '.mkv' 52 | threshold: 20 # 20% minimum size reduction %, otherwise source is preserved as-is 53 | 54 | # 55 | # Rules are evaluated in order. First matching rule wins so order wisely. 56 | # Rules with a profile of "SKIP" mean to skip processing of the matched video 57 | # 58 | rules: 59 | 'skip video if already encoded in hevc/h265': 60 | profile: SKIP 61 | criteria: 62 | vcodec: 'hevc' 63 | 64 | 'skip video if resolution < 700': 65 | profile: SKIP 66 | criteria: 67 | res_height: '<700' 68 | 69 | 'content just too big': 70 | profile: hevc_cuda 71 | criteria: 72 | runtime: '>90' # more than 90 minutes 73 | filesize_mb: '>4000' # ..and larger than 4 gigabytes 74 | 75 | 'small enough already': # skip if <2.5g size and higher than 720p and between 30 and 64 minutes long. 76 | profile: SKIP # transcoding these will probably cause a noticeable quality loss so skip. 77 | criteria: 78 | filesize_mb: '<2500' # less than 2.5 gigabytes 79 | res_height: '720-1081' # 1080p, allowing for random oddball resolutions still in the HD range 80 | runtime: '30-65' # between 30 and 65 minutes long 81 | 82 | 'default': # this will be the DEFAULT (no criteria implies a match) 83 | profile: hevc_cuda 84 | criteria: 85 | vcodec: '!hevc' 86 | 87 | -------------------------------------------------------------------------------- /config-samples/transcode-sample2.yml: -------------------------------------------------------------------------------- 1 | ## 2 | # global configuration 3 | ## 4 | config: 5 | ffmpeg: '/home/mark/bin/ffmpeg' 6 | queues: 7 | qsv: 1 8 | cuda: 1 9 | colorize: yes 10 | fls_path: '/tmp' 11 | 12 | 13 | ## 14 | # profile definitions. You can model all your transcoding combinations here. 15 | ## 16 | profiles: 17 | 18 | ############################################ 19 | # nVidia CUDA-specific common options 20 | ############################################ 21 | 22 | base: 23 | # 24 | # ffmpeg output-related options 25 | # 26 | output_options: 27 | - "-c:s copy" # copy all subtitles as-is 28 | - "-f matroska" # mkv format 29 | - "-max_muxing_queue_size 1024" 30 | 31 | output_options_audio: 32 | - "-c:a copy" # copy all audio as-is 33 | 34 | extension: '.mkv' 35 | threshold: 10 # minimum of 18% compression required 36 | threshold_check: 20 # start checking threshold at 20% complete 37 | 38 | # 39 | # audio handling 40 | # 41 | audio: 42 | # 43 | # Included languages take precedent. By default all languages are retained. 44 | # Any specifically included languages here automatically discard all others. 45 | # 46 | 47 | include_languages: 48 | - "eng" 49 | default_language: eng 50 | 51 | # 52 | # subtitle handling 53 | # 54 | subtitle: 55 | # see audio comment above 56 | include_languages: 57 | - "eng" 58 | default_language: eng 59 | 60 | ############################################ 61 | # nVidia CUDA-specific common options 62 | # (for include-use only) 63 | ############################################ 64 | cuda: 65 | input_options: 66 | # - "-hwaccel cuvid" # use hardware decoding 67 | # - "-c:v h264_cuvid" # only works if source is h264 !! 68 | output_options_video: 69 | - "-cq:v 21" # crf option passed to CUDA engine 70 | - "-rc vbr" # variable bit-rate, high quality 71 | - "-rc-lookahead 20" 72 | - "-bufsize 5M" 73 | - "-b:v 7M" 74 | - "-profile:v main" 75 | - "-maxrate:v 7M" 76 | - "-c:v hevc_nvenc" 77 | - "-preset slow" 78 | - "-pix_fmt yuv420p" 79 | 80 | cuda_medium: 81 | output_options_video: 82 | - "-cq:v 23" # crf option passed to CUDA engine 83 | - "-rc vbrq" # variable bit-rate, high quality 84 | - "-rc-lookahead 20" 85 | - "-bufsize 3M" 86 | - "-b:v 5M" 87 | - "-profile:v main" 88 | - "-maxrate:v 5M" 89 | - "-c:v hevc_nvenc" 90 | - "-preset medium" 91 | - "-pix_fmt yuv420p" 92 | 93 | ############################################ 94 | # nVidia CUDA-specific common options 95 | # without hardware decoding 96 | # (for include-use only) 97 | ############################################ 98 | cuda_nohwd: 99 | output_options: 100 | - "-cq:v 23" # crf option passed to CUDA engine 101 | - "-rc vbr" # variable bit-rate, high quality 102 | - "-b:v 3M" 103 | - "-maxrate:v 5M" 104 | - "-c:v hevc_nvenc" 105 | - "-profile:v main" 106 | 107 | 108 | ############################################## 109 | # nVidia CUDA-specific for 10bit video support 110 | # (Pascal-based cards or newer) 111 | ############################################## 112 | hevc_cuda_10bit: 113 | include: "base cuda" 114 | output_options: 115 | - "-pix_fmt yuv420p10le" 116 | - "-profile:v main10" # redefine the profile:v value from "base" options 117 | 118 | ############################################### 119 | # HEVC CUDA high quality, for just about 120 | # everything. 121 | ############################################### 122 | hevc_cuda: 123 | include: "base cuda" # include "base" and "cuda" definitions 124 | output_options_video: 125 | output_options_audio: 126 | queue: 'cuda' 127 | 128 | hevc_cuda_medium: 129 | include: "base cuda" 130 | output_options_video: 131 | - "-b:v 4000K" 132 | output_options_audio: 133 | - "-c:a libfdk_aac" 134 | - "-b:a 768k" 135 | queue: 'qsv' 136 | 137 | # 138 | # MIXINS - snippets of configuration you can specific on the commandline to override what is defined in the selected profile 139 | # 140 | sdr: 141 | output_options_video: 142 | - "-vf zscale=t=linear:npl=100,format=gbrpf32le,zscale=p=bt709,tonemap=tonemap=hable:desat=0,zscale=t=bt709:m=bt709:r=tv,format=yuv420p" 143 | 144 | aac_hq: 145 | output_options_audio: 146 | - "-c:a libfdk_aac" 147 | - "-b:a 384k" 148 | 149 | aac_movie: 150 | output_options_audio: 151 | - "-c:a libfdk_aac" 152 | - "-b:a 768k" 153 | 154 | denoise: 155 | output_options_video: 156 | - "-vf nlmeans=p=5:r=11" 157 | 158 | 159 | ############################################### 160 | # Intel QSV common options 161 | ############################################### 162 | qsv: 163 | input_options: 164 | output_options: 165 | - "-c:v hevc_qsv" 166 | - "-preset medium" 167 | - "-qp 21" 168 | - "-c:s copy" 169 | output_options_video: 170 | - "-b:v 7000K" 171 | output_options_audio: 172 | - "-c:a copy" 173 | 174 | ############################################### 175 | # Intel QSV (HEVC) 176 | ############################################### 177 | # standard quality 178 | hevc_qsv: 179 | include: "base qsv" # includes/inherts from base and qsv profiles 180 | output_options: 181 | queue: 'qsv' 182 | 183 | # medium quality 184 | hevc_qsv_medium: 185 | include: "base qsv" 186 | output_options_video: 187 | - "-b:v 4000K" 188 | output_options_audio: 189 | - "-c:a ac3" 190 | - "-b:a 768k" 191 | queue: 'qsv' 192 | 193 | # standard definition video 194 | hevc_qsv_stddef: 195 | include: "base qsv" 196 | output_options_video: 197 | - "-b:v 2000K" 198 | output_options_audio: 199 | - "-c:a ac3" 200 | - "-b:a 512k" 201 | queue: 'qsv' 202 | 203 | 204 | # anime - keep Japanese subtitles 205 | hevc_qsv_anime: 206 | include: "base qsv" 207 | output_options_video: 208 | - "-b:v 3000K" 209 | output_options_audio: 210 | - "-c:a libfdk_aac" 211 | - "-b:a 768k" 212 | audio: 213 | include_languages: 214 | - "jpn" 215 | - "eng" 216 | subtitle: 217 | include_languages: 218 | - "eng" 219 | queue: 'qsv' 220 | 221 | ############################################### 222 | # Intel QSV (x264) 223 | ############################################### 224 | x264_qsv: 225 | include: "base qsv" 226 | output_options: 227 | - "-c:v h264_vaapi" 228 | queue: 'qsv' 229 | threshold: 10 230 | 231 | ############################################### 232 | # Finally, come CPU-only profiles. Not sure 233 | # why you would use if you have hardware 234 | # support, but including for documentation. 235 | ############################################### 236 | # 237 | x264_mp4: 238 | include: "base" 239 | output_options: 240 | - "-threads 4" 241 | - "-c:v h264" 242 | - "-crf 18" 243 | - "-c:a copy" 244 | - "-c:s copy" 245 | - "-f mp4" 246 | threshold: 15 247 | threshold_check: 30 248 | 249 | # just copy to a new mkv container and process subtitles based on "base" profile definition 250 | copy_mkv: 251 | include: "base" 252 | output_options: 253 | - "-c:v copy" 254 | - "-c:a copy" 255 | - "-c:s copy" 256 | - "-f mkv" 257 | extension: '.mkv' 258 | 259 | ################################################# 260 | # The following are scenario-specific profiles. 261 | # They are examples of how to treat various 262 | # media differently from the generic ones above. 263 | ################################################# 264 | 265 | hevc_30fps: # when movie source is just too big, cut down fps 266 | include: "base cuda" 267 | output_options: 268 | - "-r 30" 269 | queue: 'cuda' 270 | 271 | 272 | x264_mp4: 273 | output_options: 274 | - "-threads 4" 275 | - "-c:v h264" 276 | - "-crf 18" 277 | - "-c:a copy" 278 | - "-c:s copy" 279 | - "-f mp4" 280 | extension: '.mp4' 281 | threshold: 15 282 | 283 | # 284 | # Automatching happens when a profile isn't provided on the command line. These rules are evalulated to find the 285 | # most appropriate profile for each video to be transcoded. 286 | # 287 | # rule predicates: 288 | # 289 | # vcodec Video codec of the source ('ffmpeg -codecs' to see full list), may preceed with ! for not-equal test 290 | # res_height Source video resolution height, operators < and > allowed 291 | # res_width Source video resolution width, operators < and > allowed 292 | # filesize_mb Size of the source file (in megabytes), operators allowed 293 | # runtime Source runtime in minutes, operators allowed 294 | # fps Framerate of the source 295 | # path Full path of the source file. Value can be a regular expression (ie. '.*/Television/.*'). 296 | # 297 | # Rules are evaluated in order. First matching rule wins so order wisely. 298 | # Rules with a profile of "SKIP" mean to skip processing of the matched video 299 | # 300 | rules: 301 | 302 | 'skip video if already encoded in hevc/h265': 303 | profile: SKIP 304 | criteria: 305 | vcodec: 'hevc' 306 | 307 | 'small enough already': 308 | profile: SKIP 309 | criteria: 310 | filesize_mb: '<2300' 311 | res_height: '721-1081' 312 | runtime: '30-65' 313 | 314 | 'small enough already-2': 315 | profile: SKIP 316 | criteria: 317 | filesize_mb: '<1000' 318 | res_height: '721-1081' 319 | runtime: '20-29' 320 | 321 | 'high frame rate': 322 | profile: hevc_30fps 323 | criteria: 324 | fps: '>30' 325 | filesize_mb: '>500' 326 | 327 | 'skip video if resolution < 700': 328 | profile: SKIP 329 | criteria: 330 | res_height: '<700' 331 | 332 | 'content just too big and framey': 333 | profile: hevc_hd_25fps 334 | criteria: 335 | runtime: '<180' # less than 3 hours 336 | filesize_mb: '>6000' # ..and larger than 6 gigabytes 337 | fps: '>25' 338 | 339 | 'special HD': 340 | profile: SKIP 341 | criteria: 342 | path: 'Mandalorian|Walking|Expanse|Vikings|Westworld|Outlander' 343 | filesize_mb: '<3200' 344 | 345 | # default HD profile for large files 346 | 'default HD': 347 | profile: hevc_qsv 348 | criteria: 349 | res_height: '721-1081' 350 | vcodec: '!hevc' 351 | runtime: '>35' 352 | filesize_mb: '>2300' 353 | 354 | # default HD profile for files < 35 minutes runtime 355 | 'default HD 30': 356 | profile: hevc_qsv 357 | criteria: 358 | res_height: '721-1081' 359 | vcodec: '!hevc' 360 | runtime: '<35' 361 | filesize_mb: '>1000' 362 | 363 | # default profile for low HD resolution 364 | 'default MQ': 365 | profile: hevc_qsv_sd 366 | criteria: 367 | res_height: '720' 368 | vcodec: '!hevc' 369 | 370 | # default profile for standard definition files 371 | 'default SD': 372 | profile: hevc_cuda_sd 373 | criteria: 374 | res_height: '<720' 375 | vcodec: '!hevc' 376 | 377 | -------------------------------------------------------------------------------- /config-samples/transcode.yml: -------------------------------------------------------------------------------- 1 | ## 2 | # NOTE: This file is a sample configuration starting point. Copy it to customize. 3 | ## 4 | 5 | ########################################## 6 | # global configuration section 7 | ########################################## 8 | config: 9 | default_queue_file: '/volume1/config/sonarr/transcode_queue.txt' 10 | ffmpeg: '/usr/local/bin/ffmpeg' 11 | docker: 12 | image: linuxserver/ffmpeg 13 | # devices: 14 | # - /dev/dir:/dev/dri 15 | 16 | queues: 17 | qsv: 1 18 | cuda: 1 19 | colorize: yes # use colors for text output 20 | fls_path: '/tmp' # fast local storage for work files, ideally an SSD (optional) 21 | 22 | # 23 | # Cluster machine definitions (optional). 24 | # You can omit this entire section if you don't use clustering 25 | # 26 | clusters: 27 | home: 28 | workstation: 29 | type: local 30 | ffmpeg: '/usr/bin/ffmpeg' 31 | queues: 32 | qsv: 1 33 | cuda: 1 34 | status: 'enabled' 35 | chrispc: 36 | os: win10 37 | type: mounted 38 | ip: 192.168.2.66 39 | user: mark 40 | ffmpeg: '/mnt/c/ffmpeg/bin/ffmpeg.exe' 41 | profiles: 42 | - hevc_cuda 43 | queues: 44 | qsv: 1 45 | cuda: 1 46 | working_dir: 'c:\temp' 47 | path-substitutions: 48 | - '/tv/ m:Video\Television\' 49 | - '/downloads/ n:\' 50 | status: enabled 51 | homeserver: 52 | os: linux 53 | type: mounted 54 | working_dir: /tmp 55 | ip: 192.168.2.61 56 | user: mark 57 | ffmpeg: '/home/mark/ffmpeg_sources/ffmpeg/ffmpeg' 58 | queues: 59 | cuda: 1 60 | profiles: 61 | - hevc_cuda 62 | - hevc_cuda_10bit 63 | status: 'enabled' 64 | macpro: 65 | os: linux 66 | type: mounted 67 | ip: 192.168.2.64 68 | user: mark 69 | ffmpeg: '/usr/bin/ffmpeg' 70 | profiles: 71 | - hevc 72 | status: 'enabled' 73 | 74 | ####################################################### 75 | # Profile definitions section 76 | # You can model all your transcoding combinations here. 77 | ####################################################### 78 | 79 | profiles: 80 | 81 | # 82 | # Common options for all encodes, included in other 83 | # profiles as defaults (but can be overridden) 84 | # (for include-use only) 85 | # 86 | base: 87 | # 88 | # ffmpeg output-related options 89 | # 90 | output_options: 91 | - "-f matroska" # mkv format 92 | - "-max_muxing_queue_size 1024" 93 | 94 | output_options_subtitle: 95 | - "-c:s copy" # copy all subtitles as-is 96 | output_options_audio: 97 | - "-c:a copy" # copy all audio as-is 98 | extension: '.mkv' 99 | threshold: 18 # minimum of 18% compression required 100 | threshold_check: 20 # start checking threshold at 20% complete 101 | 102 | # 103 | # audio drop/keep handling 104 | # 105 | audio: 106 | # 107 | # Included languages take precedent. By default all languages are retained. 108 | # Any specifically included languages here automatically discard all others. 109 | # 110 | include_languages: 111 | - "eng" 112 | default_language: eng 113 | # 114 | # subtitle drop/keep handling 115 | # 116 | subtitle: 117 | # see audio comment above 118 | include_languages: 119 | - "eng" 120 | default_language: eng 121 | 122 | ############################################ 123 | # nVidia CUDA-specific common options 124 | # (for include-use only) 125 | ############################################ 126 | cuda: 127 | input_options: 128 | # optionally you can enable hardware decoding here 129 | output_options_video: 130 | - "-cq:v 19" # crf option passed to CUDA engine 131 | - "-rc vbr_hq" # variable bit-rate, high quality 132 | - "-rc-lookahead 20" 133 | - "-bufsize 5M" 134 | - "-b:v 7M" 135 | - "-profile:v main" 136 | - "-maxrate:v 7M" 137 | - "-c:v hevc_nvenc" 138 | - "-preset slow" 139 | - "-pix_fmt yuv420p" 140 | 141 | hevc_cuda: 142 | include: "base cuda" 143 | output_options_video: 144 | - "-b:v 4000K" 145 | output_options_audio: 146 | - "-c:a ac3" 147 | - "-b:a 768k" 148 | queue: 'cuda' 149 | 150 | hevc_cuda_anime: 151 | include: "base cuda" 152 | output_options_video: 153 | - "-b:v 3000K" 154 | output_options_audio: 155 | - "-c:a ac3" 156 | - "-b:a 768k" 157 | audio: 158 | include_languages: 159 | - "jpn" 160 | - "eng" 161 | subtitle: 162 | include_languages: 163 | - "eng" 164 | queue: 'cuda' 165 | 166 | ############################################## 167 | # nVidia CUDA-specific for 10bit video support 168 | # (Pascal-based cards or newer) 169 | ############################################## 170 | hevc_cuda_10bit: 171 | include: "base cuda" # include "base" and "cuda" definitions 172 | output_options: 173 | - "-pix_fmt yuv420p10le" 174 | - "-profile:v main10" # redefine the profile:v value from "base" options 175 | 176 | ############################################### 177 | # Intel QSV common options 178 | ############################################### 179 | qsv: 180 | input_options: 181 | # enable hardware decoding 182 | - "-init_hw_device vaapi=intel:/dev/dri/renderD128" 183 | - "-hwaccel vaapi" 184 | - "-hwaccel_output_format vaapi" 185 | - "-hwaccel_device intel" 186 | output_options_video: 187 | - "-vf format=vaapi,hwupload" 188 | - "-preset slow" 189 | - "-rc_mode 1" 190 | - "-qp 20" 191 | - "-b:v 7M" 192 | output_options_subtitle: 193 | - "-c:s copy" 194 | output_options_audio: 195 | - "-c:a copy" 196 | 197 | ############################################### 198 | # Intel QSV (HEVC) 199 | ############################################### 200 | hevc_qsv: 201 | include: "base qsv" 202 | output_options_video: 203 | - "-c:v hevc_vaapi" 204 | - "-max_muxing_queue_size 1024" 205 | queue: 'qsv' 206 | 207 | ############################################### 208 | # Intel QSV (x264) 209 | ############################################### 210 | x264_qsv: 211 | include: "base qsv" 212 | output_options_video: 213 | - "-c:v h264_vaapi" 214 | queue: 'qsv' 215 | threshold: 20 216 | 217 | ############################################### 218 | # Finally, come CPU-only profiles. Not sure 219 | # why you would use if you have hardware 220 | # support, but including for documentation. 221 | ############################################### 222 | # 223 | x264_mp4: 224 | include: "base" 225 | output_options: 226 | - "-f mp4" 227 | output_options_video: 228 | - "-threads 4" 229 | - "-c:v h264" 230 | - "-crf 18" 231 | output_options_audio: 232 | - "-c:a copy" 233 | output_options_subtitle: 234 | - "-c:s copy" 235 | threshold: 15 236 | threshold_check: 30 237 | 238 | copy_mp4: # mostly used to fix badly formatted containers 239 | output_options_video: 240 | - "-c:v copy" 241 | output_options_audio: 242 | - "-c:a copy" 243 | output_options_subtitle: 244 | - "-c:s copy" 245 | - "-f mp4" 246 | extension: '.mp4' 247 | 248 | # 249 | # example audio-only partial profiles, to be used as mixins. 250 | # 251 | mp3_hq: 252 | output_options_audio: 253 | - "-c:a libmp3lame" 254 | - "-q:a 330" 255 | 256 | aac_hq: 257 | output_options_audio: 258 | - "-c:a libfdk_aac" 259 | - "-b:a 384k" 260 | 261 | ################################################# 262 | # The following are scenario-specific profiles. 263 | # They are examples of how to treat various 264 | # media differently from the generic ones above. 265 | ################################################# 266 | 267 | # 268 | # When you get those oddball encodes with a stupid high 269 | # frame rate (Alan Partridge comes to mind) 270 | # 271 | hevc_cuda_30fps: 272 | include: "base cuda" 273 | output_options: 274 | - "-r 30" 275 | queue: 'cuda' 276 | 277 | # 278 | # Automatching happens when a profile isn't provided on the command line. These rules are evalulated to find the 279 | # most appropriate profile for each video to be transcoded. 280 | # 281 | # rule predicates: 282 | # 283 | # vcodec Video codec of the source ('ffmpeg -codecs' to see full list), may preceed with ! for not-equal test 284 | # res_height Source video resolution height, operators < and > allowed 285 | # res_width Source video resolution width, operators < and > allowed 286 | # filesize_mb Size of the source file (in megabytes), operators allowed 287 | # runtime Source runtime in minutes, operators allowed 288 | # fps Framerate of the source 289 | # path Full path of the source file. Value can be a regular expression (ie. '.*/Television/.*'). 290 | # 291 | # Rules are evaluated in order. First matching rule wins so order wisely. 292 | # Rules with a profile of "SKIP" mean to skip processing of the matched video 293 | # 294 | rules: 295 | 296 | 'skip video if already encoded in hevc/h265': 297 | profile: SKIP 298 | criteria: 299 | vcodec: 'hevc' 300 | 301 | 'small enough already': 302 | profile: SKIP 303 | criteria: 304 | filesize_mb: '<2000' 305 | res_height: '721-1081' 306 | runtime: '30-65' 307 | 308 | 'special HD higher quality, keep original': 309 | profile: SKIP 310 | criteria: 311 | filesize_mb: '<3500' 312 | runtime: '45-65' 313 | path: 'Westworld|Walking|Vikings|Expanse' 314 | 315 | 'high frame rate': 316 | profile: hevc_cuda_30fps 317 | criteria: 318 | fps: '>30' 319 | filesize_mb: '>500' 320 | 321 | 'skip video if resolution < 700': 322 | profile: SKIP 323 | criteria: 324 | res_height: '<700' 325 | 326 | 'anime': 327 | profile: hevc_cuda_anime 328 | criteria: 329 | path: '.*/anime/.*' 330 | 331 | 'default': 332 | profile: hevc_cuda 333 | criteria: 334 | vcodec: '!hevc' 335 | 336 | -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Minimal makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line. 5 | SPHINXOPTS = 6 | SPHINXBUILD = sphinx-build 7 | SOURCEDIR = . 8 | BUILDDIR = _build 9 | 10 | # Put it first so that "make" without argument is like "make help". 11 | help: 12 | @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 13 | 14 | .PHONY: help Makefile 15 | 16 | # Catch-all target: route all unknown targets to Sphinx using the new 17 | # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). 18 | %: Makefile 19 | @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) -------------------------------------------------------------------------------- /docs/conf.py: -------------------------------------------------------------------------------- 1 | # Configuration file for the Sphinx documentation builder. 2 | # 3 | # This file only contains a selection of the most common options. For a full 4 | # list see the documentation: 5 | # http://www.sphinx-doc.org/en/master/config 6 | 7 | # -- Path setup -------------------------------------------------------------- 8 | 9 | # If extensions (or modules to document with autodoc) are in another directory, 10 | # add these directories to sys.path here. If the directory is relative to the 11 | # documentation root, use os.path.abspath to make it absolute, like shown here. 12 | # 13 | # import os 14 | # import sys 15 | # sys.path.insert(0, os.path.abspath('.')) 16 | 17 | 18 | # -- Project information ----------------------------------------------------- 19 | 20 | project = 'pytranscoder' 21 | copyright = '2019-2023, Marshall L Smith Jr' 22 | author = 'Marshall L Smith Jr' 23 | 24 | # The full version, including alpha/beta/rc tags 25 | release = '2.2.5' 26 | 27 | 28 | # -- General configuration --------------------------------------------------- 29 | 30 | # Add any Sphinx extension module names here, as strings. They can be 31 | # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom 32 | # ones. 33 | extensions = [ 34 | ] 35 | 36 | # Add any paths that contain templates here, relative to this directory. 37 | templates_path = ['_templates'] 38 | 39 | # List of patterns, relative to source directory, that match files and 40 | # directories to ignore when looking for source files. 41 | # This pattern also affects html_static_path and html_extra_path. 42 | exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] 43 | 44 | 45 | # -- Options for HTML output ------------------------------------------------- 46 | 47 | # The theme to use for HTML and HTML Help pages. See the documentation for 48 | # a list of builtin themes. 49 | # 50 | html_theme = 'alabaster' 51 | #html_theme = 'sphinx_rtd_theme' 52 | 53 | # Add any paths that contain custom static files (such as style sheets) here, 54 | # relative to this directory. They are copied after the builtin static files, 55 | # so a file named "default.css" will overwrite the builtin "default.css". 56 | html_static_path = ['_static'] 57 | master_doc = "index" 58 | 59 | -------------------------------------------------------------------------------- /docs/configuration/cluster.rst: -------------------------------------------------------------------------------- 1 | ===================== 2 | Cluster Configuration 3 | ===================== 4 | 5 | Watching multiple machines doing your bidding is a beautiful thing. Watching each of them doing concurrent bidding is even more beautiful. 6 | With a well-thought-out configuration this is easy to do. 7 | 8 | We will refer to the machine you use pytranscode on as your *cluster manager* and other machines as *hosts*. 9 | Note that the *cluster manager* machine can also be designated as a host. 10 | 11 | There are 2 methods to run transcodes on hosts - agent-less using *ssh* or via pytranscoder *agent mode*. 12 | 13 | ==== 14 | SSH 15 | ==== 16 | 17 | ---- 18 | Linux 19 | ---- 20 | 21 | Setting up remote hosts is as follows, if not already setup for ssh access and ffmpeg: 22 | Linux is natively supported as long as the following conditions are true: 23 | * Each host machine in the cluster is running an *ssh* server. 24 | * Each host has *ffmpeg* installed. 25 | * If using hardware encoding, your machine (and *ffmpeg*) have been setup and tested to make sure it is working. Setup of hardware encoding is 26 | beyond the scope of this document. 27 | * The *cluster manager* machine must be able to *ssh* to each host without a password prompt (see `man ssh-copy-id`). 28 | 29 | ----------- 30 | MacOS 31 | ----------- 32 | 33 | MacOS, being based on BSD, is also natively supported. See Linux section. 34 | Check your MacOS version of *ffmpeg* for what hardware acceleration support is available, if any. 35 | 36 | ---------------- 37 | Windows 10/11 38 | ---------------- 39 | Enable SSH access. 40 | 41 | This method will only allow *streaming mode* cluster support due to Windows OpenSSH not being able to access network shares. 42 | 43 | * Open your Windows Services manager and scroll down to OpenSSH Server. Enable it, preferably setting to auto-start. 44 | * In the home folder of the user account create a directory called **.ssh**. Then from your *cluster manager* copy your $HOME/.ssh/id_rsa.pub to c:/Users/*username*/.ssh/authorized_keys on Windows. 45 | 46 | Finally, if you have a supported nVidia card download the nVidia CUDA drivers and install if you plan on using CUDA encoding. 47 | It's a large download. Choose Custom install and deselect all the documentation and other things you don't need if you want to 48 | minimize space usage. 49 | 50 | ==== 51 | Pytranscoder Agent 52 | ==== 53 | 54 | As of 2.2.5 pytranscoder can run in *agent* mode. This allows you to install pytranscoder on any host and use it instead of *ssh*. 55 | The *cluster manager* will "talk" to the hosts via this agent directly and function just like ssh, but without the need to enable ssh or share keys. 56 | For the security-conscious (and everyone should be), the agent code is called agent.py for you to review. Pytranscoder talks to agents via a custom protocol. It accepts the file to transcode from the *cluster manager*, transcodes it, 57 | then sends it back. That is all. It does not access any system resources other than the temp folder you specify. 58 | 59 | For all platforms, use the installation instructions to install pytranscoder on all machines that will act as hosts. 60 | When ready to start a transcode session, start them with pytranscoder --agent. They will talk to each other on port 9567 so this port needs to be open in your firewall. 61 | 62 | ------------------ 63 | Cluster Definition 64 | ------------------ 65 | 66 | First, a word about queues. The *queues:* definition in the Global section only applies when running pytranscoder on 67 | a single host (not clustered using -c). These queues are not used when running in cluster mode. This is because you can define queues for each host doing work. 68 | So whatever queues you have defined there, just ignore for the purposes of cluster setup. 69 | 70 | Setting up your clusters is as it sounds - you must define some information about each host participating in the cluster, even 71 | including the one your are running pytranscoder from, if applicable. 72 | 73 | *Sample, 4-host configuration*: 74 | 75 | .. code-block:: yaml 76 | 77 | config: 78 | ... 79 | ssh: '/usr/bin/ssh' # used only in cluster mode 80 | ... 81 | ###################### 82 | # cluster definitions 83 | ###################### 84 | clusters: 85 | household: # name for this cluster 86 | 87 | ################################# 88 | # cluster manager, which will 89 | # also participate in the cluster 90 | ################################# 91 | mediacenter: 92 | type: local # Indicates this is where pytranscoder is running and can be used in the cluster as well. 93 | ffmpeg: '/usr/bin/ffmpeg' 94 | status: 'enabled' 95 | 96 | ################################## 97 | # My old MacPro 98 | ################################## 99 | macpro: # name of this host (does not need to be the same as network hostname) 100 | type: mounted # machine with source media and host share a filesystem (nfs, samba, etc) 101 | os: macos # choices are linux, macos, win10 102 | ip: 192.168.2.65 103 | user: sshuser # user account used to ssh to this host 104 | ffmpeg: '/usr/local/bin/ffmpeg' 105 | path-substitutions: # optional, map source pathnames to equivalent on host 106 | - "/volume1/media/ /mnt/media/" 107 | - "/downloads/ /mnt/downloads/" 108 | profiles: # profiles allowed on this host 109 | - hevc 110 | - h264 111 | status: 'enabled' # set to disabled to temporarily stop using 112 | 113 | ################################# 114 | # gaming machine (Windows OpenSSH) 115 | ################################# 116 | gamer: 117 | type: streaming # host not using a mounted filesystem (yuck) 118 | os: win10 # choices are linux, macos, win10 119 | ip: 192.168.2.64 # address of host 120 | user: matt # ssh login user 121 | working_dir: 'c:\temp' # working folder on remote host, required for streaming type 122 | ffmpeg: 'c:/ffmpeg/bin/ffmpeg' 123 | profiles: # profiles allowed on this host 124 | - hevc_cuda 125 | - hevc_qsv 126 | queues: 127 | qsv: 1 # allow only 1 encode on the CPU at a time 128 | cuda: 2 # allow 2 concurrent encodes on the nVidia card 129 | status: 'enabled' # set to disabled to temporarily stop using 130 | 131 | #################################################### 132 | # Spare family machine - Windows w/o SSH 133 | #################################################### 134 | family: # machine configured to use WSL ssh server 135 | type: agent 136 | os: win10 137 | ip: 192.168.2.66 138 | user: chris 139 | ffmpeg: 'c:/ffmpeg/bin/ffmpeg' 140 | profiles: # profiles allowed on this host 141 | - hevc_cuda 142 | - hevc_cuda_10bit 143 | - handbrake_qsv_hevc 144 | - handbrake_qsv_h264 145 | queues: 146 | qsv: 1 147 | cuda: 2 148 | status: enabled 149 | 150 | This sample is based on a setup where a Linux machine is used as a media server, and all media is stored on that machine. The 151 | relevant root paths on that machine are */downloads* and */volume1/media*. These folders are also shared via Samba (SMB) and NFS 152 | and accessible to all other machines on the network. 153 | 154 | The first machine, **mediacenter**, is of type *local* which means it's the same machine we're running pytranscoder on. This is just 155 | a simplified way of adding the machine without requiring ssh into itself. Status is either *enabled* or *disabled*. If disabled it will not participate in the cluster. 156 | 157 | .. note:: 158 | pytranscoder will check that each machine in the cluster is up and accessible when you start a job. If a host is down it will 159 | be ignored and processing will continue with the others. 160 | 161 | Skipping down to **macpro**, the type is *mounted*. The *local* and *mounted* types are most preferred as they are faster. What this means 162 | is the host has mounted shared folders from the server and can access media directly. In the Windows world this is a mapped drive, in Linux 163 | and MacOS it's an NFS mount. In the case of Linux or MacOS, if your mountpoints are not named the same as on the server you must use 164 | the *path-substitutions* configuration. 165 | 166 | For example, there is a video file on the server in */downloads/mymedia.mp4*. The */downloads* folder is exported via NFS and mounted on 167 | **macpro** machine under */mnt/downloads*. Once the *ffmpeg* job starts on **macpro** it will be passed */downloads/mymedia.mp4* as the input 168 | filename. Well, that path does not exist on **macpro**, but *mymedia.mp4* IS accessible as */mnt/downloads/mymedia.mp4*. So we setup 169 | the *path-substitutions* patterns to account for this. Now, the input pathname will be changed from 170 | */downloads/*... to */mnt/downloads/*... 171 | 172 | Likewise, a file under */volume1/media/tv/series/season1/show.s01e01.mp4* is accessible on **macpro** as 173 | */mnt/media/tv/series/season1/show.s01e01.mp4*. 174 | 175 | Whew, hope that was clear enough. 176 | 177 | Continuing on down the **macpro** configuration, and others, you'll see *profiles:*. This indicates a list of profiles suitable for this 178 | host. Note in this example that *h264* and *hevc* are given. These are basic profiles that perform CPU-based encoding without assistance 179 | since this host is incapable of any hardware encoding. If I put *hevc_cuda* as a supported profile the job would fail since this host 180 | has no nVidia GPU. So this host will only be called on to encode video matching those profiles. 181 | 182 | Skipping down to the **gamer** host we see a type of *streaming*. The streaming type is not encouraged but there in case you cannot or will not 183 | map a server drive to the host. 184 | Notice there are no *path-substitutions*. This is because for *streaming* they are not used. 185 | Hosts of the *streaming* type will be sent the media file via scp (secured copy) to the *working_dir* folder, *ffmpeg* will encode the file into the same 186 | the same folder, and the result will be copied back to the server. Finally, the 2 artifacts in *working_dir* are removed. 187 | 188 | Notice the differences between the **gamer** and **family** machines. They are both Windows 10 but are configured very differently. This 189 | is discussed in detail in Windows Installation. But the driving difference is that **gamer** only has Microsoft's own OpenSSH server 190 | installed, along with Windows *ffmpeg*, but the **family** host uses WSL. Both type get the job done, but with caveats. For Windows OpenSSH, 191 | the remote shell can access the c: drive normally (see **gamer** ffmpeg path). For WSL, the path is convoluted (see **family** ffmpeg path). 192 | 193 | -------------------------------------------------------------------------------- /docs/configuration/concurrency.rst: -------------------------------------------------------------------------------- 1 | =========== 2 | Concurrency 3 | =========== 4 | 5 | Concurrency, or multitasking, is simply the act of doing multiple tasks at once. Computers do this very well. 6 | However, some activities do not lend themselves well to concurrency - encoding and decoding video is one of them. 7 | On a typical machine this is a very CPU-intensive activity, especially encoding, and leaves little room or other processes to run well. 8 | So CPU and video card vendors have stepped up to provide dedicated hardware for this purpose. Modern Intel CPUs have hardware collectively 9 | referred to as QSV - QuickSync Video - for doing this. AMD's version is AMF/VCE. Also, modern media players like VLC will use these 10 | extensions to make HD video playback smooth and less CPU-intensive. 11 | 12 | For people who encode a lot, using QSV or AMF can speed up your job up to 4x. But if you want a faster solution both nVidia and AMD produce 13 | graphics cards (GPU) cable of encoding at over 10x - that's basically 6 minutes to encode a 1 hour HD video. Furthermore, you can run multiple 14 | encodes at the same time since most processing is handled by the GPU and not the CPU. For example, an nVidia 970 can handle 2 concurrent 15 | jobs hardware decoding and encoding of H264 or HEVC(H265) video. So this means we can encode 2 of our theoretical 1 hour videos in 6 minutes. 16 | 17 | pytranscoder was originally created to manage a pair of jobs running on a local host in this manner, handing out jobs to the next available 18 | slot as other jobs finished. It has since grown into a configurable workflow manager with multi-host clustering support. 19 | 20 | You can successfully achieve concurrency with either (or both) of these approaches: 21 | 22 | ----------------- 23 | Non-Clustered 24 | ----------------- 25 | 26 | A cluster is just a group of machines working together. If you have access to multiple machines skip down to the next section. 27 | But if you just want to encode on your single host machine your setup overhead is very small. You've probably already been through 28 | the configuration section and may have noticed the **queues** section under Global config. 29 | 30 | .. code-block:: yaml 31 | 32 | queues: 33 | qsv: 1 34 | cuda: 2 35 | 36 | This is optional and only needed to enable concurrency on your single (local) host. This configuration snippet reads "create a queue 37 | called *csv* that does 1 encode at a time and another called *cuda* which can do 2 at a time." They are simply a way of controlling the 38 | number of concurrent jobs. 39 | So, by themselves these settings do nothing. To make useful you need to associate various **profiles** with a queue. A good example is 40 | to assign all of your profiles that perform nVidia-based encoding to the *cuda* queue. When you run an encode and specifiy one of these 41 | profiles, or when a rule selects one, the jobs will be managed 2 at a time. 42 | 43 | A profile is assigned to a queue using the **queue:** directive, as seen in the sample profiles in the configuration section. 44 | If a profile has no queue:, it defaults to single, sequential encoding - i.e. one job at a time. 45 | 46 | So you've probably noticed the qsv: 1 queue above and are wondering why define a queue of 1 if the default is 1 anyhow. Well there's a 47 | good reason. Even though the qsv queue is set to 1, by defining it as another queue it can actually run concurrently with the other queue. 48 | Wait, what?? 49 | 50 | Consider this scenario. You have an 8th generation Intel i5 and an nVidia 970 GPU with 4gb. You have a bunch of videos to transcode and 51 | you want to max out your system to get it done. You can define one profile (my_qsv) assigned to the **qsv** queue and another (my_cuda) to the **cuda** queue. 52 | Depending on whether you are using the rules engine or commandline you can spread your videos across both profiles, this assigning them 53 | across 2 queues. You'll end up with 2 encodes running on your nVidia GPU and 1 on your CPU/QSV hardware. That's 3 concurrent encodes: 54 | 55 | .. code-block:: bash 56 | 57 | pytranscoder -p my_qsv /downloads/show.s01e0* -p my_cuda /downloads/show.s01e1* 58 | 59 | Only 2 concurrent jobs are known to work with nVidia 970 and nVidia 1050ti cards, but more may work on bigger more expensive cards. 60 | 61 | 62 | ----------------- 63 | Clustered 64 | ----------------- 65 | 66 | Clustering allows you to use available machines accessible on your network for encoding duties. You don't need to install anything on them 67 | other than **ffmpeg** and **ssh**, which is probably already there. See Cluster Configuration. 68 | 69 | 70 | -------------------------------------------------------------------------------- /docs/configuration/installation.rst: -------------------------------------------------------------------------------- 1 | ============ 2 | Installation 3 | ============ 4 | 5 | ############ 6 | Requirements 7 | ############ 8 | 9 | * Linux or MacOS, Windows 10, 11. For Windows, WSL (Ubuntu) recommended. 10 | * latest *ffmpeg* (3.4.3-2 or higher, lower versions may still work) 11 | * nVidia graphics card with latest nVidia CUDA drivers (*optional*) 12 | * Intel CPU with QSV enabled (*optional*) 13 | * Python 3 (3.6 or higher) 14 | 15 | 16 | ####### 17 | Support 18 | ####### 19 | Please log issues or questions via the github home page for now. 20 | 21 | Video Tutorials: `Part 1 - Linux Setup `_, `Part 2 - Usage `_ 22 | 23 | ############ 24 | Installation 25 | ############ 26 | 27 | There are a few possible ways to install a python app - one of these should work for you. 28 | 29 | **Linux (Ubuntu & others), Windows, MacOS** 30 | 31 | The confusion is due to the fact that not all distributions or OS's install pip3 by default. Either way, pytranscoder is available in the **pypi** repo. 32 | 33 | .. code-block:: bash 34 | 35 | pip3 install --user pytranscoder-ffmpeg 36 | # or... 37 | python3 -m pip install --user pytranscoder-ffmpeg 38 | 39 | After installing you will find documentation in $HOME/.local/shared/doc/pytranscoder (on Linux/MacOS) 40 | and in $HOME/AppData/Python/*pythonversion*/shared/doc/pytranscoder** (on Windows). Also available `online `_ 41 | 42 | ######### 43 | Upgrading 44 | ######### 45 | 46 | Whatever method above for installing works for you, just use the --upgrade option to update, ie: 47 | 48 | .. code-block:: bash 49 | 50 | pip3 install --upgrade pytranscoder-ffmpeg 51 | 52 | -------------------------------------------------------------------------------- /docs/configuration/quickstart.rst: -------------------------------------------------------------------------------- 1 | =============== 2 | Quick Start 3 | =============== 4 | 5 | For simple commandline help 6 | 7 | .. code-block:: bash 8 | 9 | pytranscoder -h 10 | 11 | 12 | To get started right away, start with this configuration: 13 | 14 | .. code-block:: yaml 15 | 16 | 17 | config: 18 | ffmpeg: '/usr/bin/ffmpeg' 19 | colorize: yes 20 | 21 | templates: 22 | qsv: # high quality h265 23 | cli: 24 | video-codec: "-c:v hevc_qsv -preset medium -qp 21 -b:v 7000K -f matroska -max_muxing_queue_size 1024" 25 | audio-codec: "-c:a copy" 26 | subtitles: "-c:s copy" 27 | audio-lang: eng 28 | subtitle-lang: eng 29 | threshold: 15 30 | threshold_check: 30 31 | extension: '.mkv' 32 | 33 | qsv_medium: # medium quality h265 34 | cli: 35 | video-codec: "-c:v hevc_qsv -preset medium -qp 21 -b:v 4000K -f matroska -max_muxing_queue_size 1024" 36 | audio-codec: "-c:a ac3 -b:a 768k" 37 | subtitles: "-c:s copy" 38 | audio-lang: eng 39 | subtitle-lang: eng 40 | threshold: 15 41 | threshold_check: 30 42 | extension: '.mkv' 43 | 44 | qsv_anime: # anime, medium quality h265 and keep both eng and jpn language tracks 45 | cli: 46 | video-codec: "-c:v hevc_qsv -preset medium -qp 21 -b:v 3000K -f matroska" 47 | audio-codec: "-c:a ac3 -b:a 768k" 48 | subtitles: "-c:s copy" 49 | audio-lang: "eng jpn" 50 | subtitle-lang: eng 51 | threshold: 15 52 | threshold_check: 30 53 | extension: '.mkv' 54 | 55 | 56 | 57 | Copy this file and save as **$HOME/.transcode.yml**, the default location pytranscoder will look for its configuration. 58 | Pick a video file to test with. Let's refer to it as "myvideo.mp4", using the "qsv" template defined above. 59 | 60 | .. code-block:: bash 61 | 62 | pytranscoder --dry-run -t qsv myvideo.mp4 63 | 64 | You will see details of the ffmpeg command pytranscoder will use when you run for real. 65 | 66 | Use the **--dry-run** flag whenever you change your configuration to test that things work the way you intend. To run for real, omit --dry-run. You'll see something like this: 67 | 68 | .. code-block:: bash 69 | 70 | myvideo.mkv: speed: 8.51x, comp: 81%, done: 8% 71 | myvideo.mkv: speed: 8.45x, comp: 81%, done: 16% 72 | myvideo.mkv: speed: 8.46x, comp: 82%, done: 25% 73 | myvideo.mkv: speed: 8.47x, comp: 81%, done: 33% 74 | myvideo.mkv: speed: 8.47x, comp: 82%, done: 42% 75 | myvideo.mkv: speed: 8.45x, comp: 81%, done: 50% 76 | myvideo.mkv: speed: 8.46x, comp: 82%, done: 59% 77 | myvideo.mkv: speed: 8.45x, comp: 82%, done: 68% 78 | myvideo.mkv: speed: 8.48x, comp: 82%, done: 76% 79 | myvideo.mkv: speed: 8.5x, comp: 82%, done: 85% 80 | myvideo.mkv: speed: 8.49x, comp: 82%, done: 94% 81 | Finished myvideo.mkv 82 | 83 | **Speed** is how fast your machine is encoding video, **comp** is the compression percentage, and **done** how much has been processed. 84 | Your original myvideo.mkv will be replaced with a new version. 85 | 86 | .. tip:: 87 | Should you wish to do test encodes without destroying the original, use the **-k** (keep) flag. The encode job will leave behind *myvideo.mkv.tmp*, for example. 88 | 89 | Now you are ready to tweak your configuration with profiles and rules to suit your needs. 90 | -------------------------------------------------------------------------------- /docs/index.rst: -------------------------------------------------------------------------------- 1 | .. PyTranscoder documentation master file, created by 2 | sphinx-quickstart on Wed May 22 19:38:42 2019. 3 | You can adapt this file completely to your liking, but it should at least 4 | contain the root `toctree` directive. 5 | 6 | Welcome to PyTranscoder 7 | ======================================== 8 | 9 | ======== 10 | Features 11 | ======== 12 | * Sequential or concurrent transcoding. 13 | * Concurrent mode allows you to make maximum use of your nVidia CUDA-enabled graphics card or Intel accelerated video (QSV) 14 | * Preserves all streams but allows for filtering by audio and subtitle languages. 15 | * Configurable transcoding profiles 16 | * Configurable rules and criteria to auto-match a video file to a transcoding profile 17 | * Transcode from a list of files (queue) or all on the command line 18 | * Cluster mode allows use of other machines See `Link Cluster.md `_ for details. 19 | * On-the-fly compression monitoring and optional early job termination if not compressing as expected. 20 | 21 | 22 | .. toctree:: 23 | :maxdepth: 2 24 | :caption: Contents: 25 | 26 | configuration/installation 27 | configuration/quickstart 28 | configuration/configuration 29 | configuration/concurrency 30 | configuration/cluster 31 | usage/running-local.rst 32 | usage/running-clustered.rst 33 | usage/includes.rst 34 | usage/mixins.rst 35 | 36 | Indices and tables 37 | 38 | ================== 39 | 40 | * :ref:`genindex` 41 | * :ref:`modindex` 42 | * :ref:`search` 43 | 44 | -------------------------------------------------------------------------------- /docs/make.bat: -------------------------------------------------------------------------------- 1 | @ECHO OFF 2 | 3 | pushd %~dp0 4 | 5 | REM Command file for Sphinx documentation 6 | 7 | if "%SPHINXBUILD%" == "" ( 8 | set SPHINXBUILD=sphinx-build 9 | ) 10 | set SOURCEDIR=. 11 | set BUILDDIR=_build 12 | 13 | if "%1" == "" goto help 14 | 15 | %SPHINXBUILD% >NUL 2>NUL 16 | if errorlevel 9009 ( 17 | echo. 18 | echo.The 'sphinx-build' command was not found. Make sure you have Sphinx 19 | echo.installed, then set the SPHINXBUILD environment variable to point 20 | echo.to the full path of the 'sphinx-build' executable. Alternatively you 21 | echo.may add the Sphinx directory to PATH. 22 | echo. 23 | echo.If you don't have Sphinx installed, grab it from 24 | echo.http://sphinx-doc.org/ 25 | exit /b 1 26 | ) 27 | 28 | %SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% 29 | goto end 30 | 31 | :help 32 | %SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% 33 | 34 | :end 35 | popd 36 | -------------------------------------------------------------------------------- /docs/simple.yml: -------------------------------------------------------------------------------- 1 | config: 2 | ffmpeg: '/usr/bin/ffmpeg' 3 | colorize: yes 4 | 5 | templates: 6 | qsv: 7 | cli: 8 | video-codec: "-c:v hevc_qsv -preset medium -qp 21 -b:v 7000K -f matroska -max_muxing_queue_size 1024" 9 | audio-codec: "-c:a copy" 10 | subtitles: "-c:s copy" 11 | audio-lang: eng 12 | subtitle-lang: eng 13 | threshold: 15 14 | threshold_check: 30 15 | extension: '.mkv' 16 | 17 | qsv_medium: 18 | cli: 19 | video-codec: "-c:v hevc_qsv -preset medium -qp 21 -b:v 4000K -f matroska -max_muxing_queue_size 1024" 20 | audio-codec: "-c:a ac3 -b:a 768k" 21 | subtitles: "-c:s copy" 22 | audio-lang: eng 23 | subtitle-lang: eng 24 | threshold: 15 25 | threshold_check: 30 26 | extension: '.mkv' 27 | 28 | qsv_anime: 29 | cli: 30 | video-codec: "-c:v hevc_qsv -preset medium -qp 21 -b:v 3000K -f matroska" 31 | audio-codec: "-c:a ac3 -b:a 768k" 32 | subtitles: "-c:s copy" 33 | audio-lang: "eng jpn" 34 | subtitle-lang: eng 35 | threshold: 15 36 | threshold_check: 30 37 | extension: '.mkv' 38 | 39 | 40 | -------------------------------------------------------------------------------- /docs/usage/includes.rst: -------------------------------------------------------------------------------- 1 | ============== 2 | Using includes 3 | ============== 4 | 5 | This feature requires a deeper familiarity with the YAML format. Essentially, you can define a partial profile or a full one and later "include" it into another profile. This facilitates reuse of definitions and simpler profiles. 6 | All options from all sections of the profile are combined. If there is a conflict, the descendant one wins. 7 | Profile sections input_options and output_options are combined. So all included profile options are combined with the descendant. 8 | Since the introduction of mixins you should only put general, non-encoding options in output_options. Audio, video, and subtitle options have their own sections now (see Mixins before proceeding). 9 | The rule is: input_options and output_options can be built-up via includes (inheritance), but mixins are replacements and the last most recent one wins. 10 | 11 | 12 | .. code-block:: yaml 13 | 14 | # 15 | # Merge-style example 16 | # 17 | profiles: 18 | # values universal to all my high-quality transcodes 19 | hq: 20 | output_options: # options using hyphens and separate lines are "lists" 21 | - "-crf 18" 22 | - "-preset slow" 23 | - "-c:a copy" 24 | - "-c:s copy" 25 | - "-f matroska" 26 | threshold: 20 27 | extension: ".mkv" 28 | 29 | hevc_cuda: 30 | include: hq # pull in everything defined above in "hq" 31 | output_options: # combine these options with those from "hq" 32 | - "-c:v hevc_nvenc" 33 | - "-profile:v main" 34 | threshold: 18 # replace "hq" threshold value with 18 35 | 36 | 37 | The above example is equivalent to: 38 | 39 | .. code-block:: yaml 40 | 41 | hevc_cuda: 42 | output_options: 43 | - "-crf 18" 44 | - "-preset slow" 45 | - "-c:a copy" 46 | - "-c:s copy" 47 | - "-f matroska" 48 | - "-c:v hevc_nvenc" 49 | - "-profile:v main" 50 | threshold : 18 51 | extension: ".mkv" 52 | 53 | The advantage is that now we have a base (parent) profile we can include into many others to avoid repetitive profile definitions. And, if we decide to change our base threshold, for example, we only need to change it in the base (parent). 54 | 55 | Note that the profiles "hq" and "hevc_cuda" were combined, and the value for threshold was overridden to 18. 56 | Lets refer to the first (base) profile as the parent, and the second as the child. So a child profile can include one or more parent profiles. All values in the child are retained. However, if input_options or output_options are lists instead of strings, the parent and child values will be combined. 57 | Here is the same example slightly reformatted: 58 | 59 | .. code-block:: yaml 60 | 61 | # 62 | # Replace-style example 63 | # 64 | profiles: 65 | hq: 66 | output_options: 67 | - "-crf 18" 68 | - "-preset slow" 69 | - "-c:a copy" 70 | - "-c:s copy" 71 | - "-f matroska" 72 | threshold: 20 73 | extension: ".mkv" 74 | 75 | hevc_cuda: 76 | include: hq 77 | output_options: 78 | - "-c:v hevc_nvenc" 79 | - "-profile:v main" 80 | threshold: 18 81 | 82 | This will produce a bad profile. Now I need to mention a feature of YAML only used in the **include** examples - lists. YAML-formatted data can be very complex but pytranscoder requirements are meager and simple. But to support the include feature in both _replace_ and _merge_ modes I needed another way to express input and output options. 83 | Note the difference in the Merge and Replace examples is that Merge uses hyphens and a separate line for the output_options sections. In Replace, all the options are on a single line. The former is an expression of a "list of arguments". The latter is just a "string of arguments" When a parent and child both have input_options or output_options that are lists, the two are combined. If either is not a list (just a string), then the child wins and the parent version is ignored. 84 | With this new information we can now see why the Replace example produces a bad profile. It will look like this: 85 | 86 | .. code-block:: yaml 87 | 88 | hevc_cuda: 89 | output_options: 90 | - "-c:v hevc_nvenc" 91 | - "-profile:v main" 92 | threshold: 18 93 | extension: ".mkv" 94 | 95 | Since _output_options_ is a simple string rather than list, pytranscoder doesn't know how to merge them so it doesn't try. The child values always wins. So this profile will produce undesirable results because the parent options weren't merged. Convert both back to lists and it will work again. 96 | -------------------------------------------------------------------------------- /docs/usage/mixins.rst: -------------------------------------------------------------------------------- 1 | ============ 2 | Using Mixins 3 | ============ 4 | 5 | Mixins are new as of version 2.2 and are a more flexible way of reusing profiles. You don't have to refit your transcode.yaml to use mixins - profiles are still backward-compatible. 6 | Simply put, a mixin is a profile fragment used, at runtime, to "mix-in" to an existing profiles. 7 | 8 | There are 2 requirements to use them: 9 | 10 | 1. Your profiles must be mixin-enabled, meaning you need to split your current output_options into output_options, output_options_video, output_options_audio, and output_options_subtitle (or whichever ones you intend to use for mixins). 11 | 2. The output_options section should be limited to general, non-encoding options. But you may continue to put, for example, subtitle options there. You just can't use a subtitle mixin later if you do. 12 | 3. Define mixin profiles. 13 | 4. Specify mixins from the command line as needed (using -m). 14 | 15 | So what's the difference between "includes" and "mixins". The short answer is, "much". 16 | An include pulls in options from other profiles and combines them. A mixin is exclusive - they are never combined. This means you can use includes and mixins at the same time. 17 | When you specify a mixin to use at runtime, whichever section is in that mixin will override the same section in the profile. This allows for each swapping out of specific output options as needed without defining new profiles. 18 | 19 | As an example, assume you have a 4G download of a 50 minute TV show. You think 4G is just too big so you use ffprobe to look at the details. Somebody encoded the audio in DTS at a very high bitrate and it's using 1/3 of the space just for audio. Well that's just silly, so you want to re-encode just the audio. Using just profiles, you would have to create a specific one just for this scenario. But if you have mixins defined you can do something like this: 20 | 21 | pytranscoder -p copy -m aac_hq my_large_video.mp4 22 | 23 | See example profiles below for how this would be setup. Note that the output_audio section of the mixin will *replace* the same section of the "copy" profile using the command line above. 24 | So now we have a generic copy file we can use for anything, such as changing video containers. Or we can use mixins to make selective changes to audio, video, or subtitle. 25 | Multiple mixins may be specified, separated with a comma (no spaces allowed). 26 | 27 | 28 | .. code-block:: yaml 29 | 30 | profiles: 31 | 32 | # generic, "copy" profiles 33 | copy: 34 | output_options: 35 | - "-f matroska" 36 | 37 | output_options_video: 38 | - "-c:v copy" 39 | 40 | output_options_audio: 41 | - "-c:a copy" 42 | 43 | output_options_subtitle: 44 | - "-c:s copy" 45 | 46 | threshold: 20 47 | extension: ".mkv" 48 | 49 | 50 | # this is a mixin 51 | aac_hq: 52 | output_options_audio: 53 | - "-c:a libfdk_aac" 54 | - "-b:a 384k" 55 | 56 | 57 | Another, more practical example. I do mostly cuda/hevc encoding but depending on content may want to vary the quality without resorting to a new profile: 58 | 59 | .. code-block:: yaml 60 | 61 | typical: 62 | output_options_video: # defaults to high quality 63 | - "-cq:v 21" # crf option passed to CUDA engine 64 | - "-rc vbr_hq" # variable bit-rate, high quality 65 | - "-rc-lookahead 20" 66 | - "-bufsize 5M" 67 | - "-b:v 7M" 68 | - "-profile:v main" 69 | - "-maxrate:v 7M" 70 | - "-c:v hevc_nvenc" 71 | - "-preset slow" 72 | - "-pix_fmt yuv420p" 73 | output_options_audio: 74 | - "-c:a copy" # copy all audio as-is 75 | output_options_subtitle: 76 | - "-c:s copy" # copy all subtitles as-is 77 | output_options: 78 | - "-f matroska" # mkv format 79 | - "-max_muxing_queue_size 1024" 80 | extension: '.mkv' 81 | threshold: 18 # minimum of 18% compression required 82 | threshold_check: 20 # start checking threshold at 20% complete 83 | 84 | # mixin to allow me to override the higher quality of the profile above, at runtime 85 | medium: 86 | output_options_video: 87 | - "-cq:v 23" 88 | - "-bufsize 3M" 89 | - "-b:v 5M" 90 | - "-profile:v main" 91 | - "-maxrate:v 5M" 92 | - "-preset medium" 93 | 94 | aac_hq: 95 | output_options_audio: 96 | - "-c:a libfdk_aac" 97 | - "-b:a 384k" 98 | 99 | 100 | So now I have the option to just use: 101 | pytranscoder -p code a_file.mp4 102 | 103 | or if I want a smaller file size: 104 | 105 | pytranscoder -p cuda -m medium,aac_hq a_file.mp4 106 | 107 | -------------------------------------------------------------------------------- /docs/usage/running-clustered.rst: -------------------------------------------------------------------------------- 1 | ===================== 2 | Running (Clustered) 3 | ===================== 4 | 5 | If you're here you are brave. It means you've configured multiple hosts and tested them for *ssh* access from your *cluster manager* machine. 6 | 7 | If you haven't already guessed, pytranscoder runs encodes on other hosts using *ssh* to "log-in" to that machine and run *ffmpeg*. This is 8 | fairly straightforward on Linux and MacOS but a bit of a pain under Windows, as you know if you've read the Windows Clustering setup. 9 | Still, it does work well and I use it regularly across multiple Linux and Windows 10 hosts. 10 | 11 | So really you can use any machine reachable with *ssh*, even outside your own network. Securing that connection with VPN or *ssh* certificates is 12 | a good idea though. Maybe setup a transcoding "co-op" with friends who have beefy rigs and fast internet using the *streaming* host type. 13 | 14 | In my home network I have 2 Ubuntu boxes each with decent nVidia cards and a Windows 10 box also with a good nVidia card. 15 | I push 2 jobs each - that's 6 videos encoding at once using GPUs. It's beautiful watching the scrolling 16 | progress indicators showing so much work being done on those 3 machines. I've seen a whole season of 12 50-minute HD videos transcoded in 14 minutes. 17 | 18 | The best part is you just set and forget. When you fire up a job it will use whatever machines in the defined cluster are awake and responsive. 19 | Of course, you don't always want to dominate every machine in your household. You may set some to just do single encodes at a time (non-concurrent) 20 | as to not impact whoever else is using the machine. How you configure your cluster is in your hands. 21 | 22 | 23 | ######### 24 | Examples 25 | ######### 26 | 27 | Encode everything in your default queue file on cluster "home": 28 | `pytranscoder -c home` 29 | 30 | If you configured a *default_queue_file* in your Global config section, it will be opened and read for a list of files to process. 31 | Each file that is successfully encoded will be removed from that file. Files are distributed across the cluster hosts based on profiles 32 | and queues. 33 | 34 | Encode some files on a specific host in the cluster: 35 | `pytranscoder -c home --host workstation /downloads/*.mp4` 36 | 37 | All other cluster hosts will be ignored and encodes sent only to host *workstation*. 38 | 39 | Encode files across 2 clusters: 40 | `pytranscoder -c home /downloads/series.s01*mp4 -c work /downloads/series.s02*mp4` 41 | 42 | You can also use --dry-run with a cluster: 43 | `pytranscoder -c home --dry-run /media/*.mp4` 44 | 45 | This will show all work to be done and perform a reachability test on each host 46 | 47 | .. note:: 48 | There is a small gotcha in cluster mode. If you **Ctrl-C** to kill pytranscoder the *ffmpeg* jobs running on the other hosts will 49 | continue to run. A solution is being pursued. 50 | 51 | -------------------------------------------------------------------------------- /docs/usage/running-local.rst: -------------------------------------------------------------------------------- 1 | =============== 2 | Running (Local) 3 | =============== 4 | 5 | This contain general information about running the tool locally on one host. For cluster support see Running (Clustered). 6 | 7 | pytranscoder is intended to be run in the foreground as it produces helpful progress output. You can still run in the background and redirect 8 | to /dev/null if you choose. 9 | 10 | Throughout this document all examples are showing using Linux-style invocations. However, it does work equally well under Windows 10, just obviously 11 | adjusting the way pathnames are expressed. 12 | 13 | 14 | 15 | Now with everything configured here are some tips for using the tool. 16 | 17 | ######## 18 | Examples 19 | ######## 20 | 21 | Get help: 22 | `pytranscoder -h` 23 | 24 | Try to encode everything listed in the default queue file: 25 | `pytranscoder` 26 | 27 | If you configured a *default_queue_file* in your Global config section, it will be opened and read for a list of files to process. 28 | Each file that is successfully encoded will be removed from that file. 29 | 30 | Test a profile match: 31 | `pytranscoder --dry-run /downloads/myvideo.mp4` 32 | 33 | The matching profile is display but no encoding is performed. 34 | 35 | Encode a file using a given profile: 36 | `pytranscoder -p h264_small /downloads/myvideo.mp4` 37 | 38 | Rules are not used since a profile was specified with **-p**. 39 | 40 | Encode several files using rules: 41 | `pytranscoder /downloads/*.mp4` 42 | 43 | No profile specified so use rules defined in ~/.transcode.yml to find an appropriate one. 44 | 45 | Encode using multiple different profiles, no rules: 46 | `pytranscoder -p h264_small /downloads/stuff_0*.mp4 -p h264_large /downloads/stuff_1*.mp4` 47 | 48 | Encode but keep the original: 49 | `pytranscoder -k /downloads/myvideo.mp4` 50 | 51 | The encoded file can be found as /downloads/myvideo.mp4.tmp 52 | 53 | Show rule-matched profiles for a bunch of files: 54 | `pytranscoder --dry-run /downloads/*.mp4` 55 | 56 | Each .mp4 file will be matched to a profile and displayed. 57 | 58 | Use an alternate (non-default) configuration file: 59 | `pytranscoder -y /tmp/sandbox.yml /downloads/myvideo.mp4` 60 | 61 | Encode all files listed in a text file: 62 | `pytranscoder --from-file /tmp/stuff_to_encode.txt` 63 | 64 | The file must contain a list of fully-qualified pathnames on separate lines. 65 | 66 | Force sequential (non-concurrent) mode, regardless of profile and queues: 67 | `pytranscoder -s /tmp/*.mp4` 68 | 69 | Verbose mode (for debugging and troubleshooting): 70 | `pytranscoder -v /tmp/*.mp4` 71 | 72 | -------------------------------------------------------------------------------- /main.py: -------------------------------------------------------------------------------- 1 | 2 | from pytranscoder import transcode 3 | 4 | # 5 | # stub for running pytranscoder as a package while developing and testing. 6 | # 7 | 8 | transcode.start() 9 | -------------------------------------------------------------------------------- /mixintests.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | 3 | from pytranscoder.config import ConfigFile 4 | 5 | 6 | class MixinTests(unittest.TestCase): 7 | def test_loaded(self): 8 | config = self.get_setup() 9 | profiles = config.find_mixins(['mixin1']) 10 | self.assertEqual(len(profiles), 1, "expected 1 profile") 11 | 12 | def test_mixin_enabled(self): 13 | config = self.get_setup() 14 | profile = config.get_directive('profile1') 15 | options = profile.output_options_audio 16 | self.assertIsNotNone(options, "expected output_options_audio in profile1 via include") 17 | self.assertEqual(['-c:a', 'copy'], options.as_shell_params(), "Invalid inherited audio options") 18 | 19 | def test_audio_mixin(self): 20 | config = self.get_setup() 21 | profile = config.get_directive('profile1') 22 | options = config.output_from_profile(profile, ['mixin1']) 23 | expect = ['-c:v', 'copy', '-f', 'matroska', '-threads', '4', '-c:a', 'mp3lame', '-b:a', '384k', '-c:s', 'copy'] 24 | self.assertEqual(expect, options, "Output options mismatch (audio)") 25 | 26 | def test_video_mixin(self): 27 | config = self.get_setup() 28 | profile = config.get_directive('profile2') 29 | options = config.output_from_profile(profile, ['mixin2']) 30 | expect = ['-c:s', 'copy', '-f', 'matroska', '-threads', '4', '-c:a', 'copy', 31 | '-aaa', 'bbb', '-ccc', 'ddd', '-eee', 'fff'] 32 | self.assertEqual(expect, options, "Output options mismatch (video)") 33 | 34 | def test_subtitle_mixin(self): 35 | config = self.get_setup() 36 | profile = config.get_directive('profile1') 37 | options = config.output_from_profile(profile, ['mixin3']) 38 | expect = ['-c:v', 'copy', '-f', 'matroska', '-threads', '4', '-c:a', 'copy', "-vf", "subtitles=subtitle.srt"] 39 | self.assertEqual(expect, options, "Output options mismatch (subtitle)") 40 | 41 | def test_multi_mixin(self): 42 | config = self.get_setup() 43 | profile = config.get_directive('profile2') 44 | options = config.output_from_profile(profile, ['mixin2', 'mixin1']) 45 | expect = ['-c:s', 'copy', '-f', 'matroska', '-threads', '4', '-c:a', 'mp3lame', '-b:a', '384k', 46 | '-aaa', 'bbb', '-ccc', 'ddd', '-eee', 'fff'] 47 | self.assertEqual(expect, options, "Output options mismatch (video)") 48 | 49 | def test_multi_mixin_all(self): 50 | config = self.get_setup() 51 | profile = config.get_directive('profile3') 52 | options = config.output_from_profile(profile, ['mixin2', 'mixin1', 'mixin3']) 53 | expect = ['-f', 'matroska', '-threads', '4', '-c:a', 'mp3lame', '-b:a', '384k', 54 | '-aaa', 'bbb', '-ccc', 'ddd', '-eee', 'fff', "-vf", "subtitles=subtitle.srt"] 55 | self.assertEqual(expect, options, "Output options mismatch (video)") 56 | 57 | def test_mixins_do_not_combine(self): 58 | config = self.get_setup() 59 | profile = config.get_directive('profile2') 60 | self.assertEqual(["-c:v", "hevc_x264"], profile.output_options_video.as_shell_params(), "mixin inheritance failed") 61 | 62 | @staticmethod 63 | def get_setup(): 64 | return ConfigFile('tests/mixinstest.yml') 65 | 66 | 67 | if __name__ == '__main__': 68 | unittest.main() 69 | -------------------------------------------------------------------------------- /pytranscoder/__init__.py: -------------------------------------------------------------------------------- 1 | __version__ = '2.2.7' 2 | __author__ = 'Marshall L Smith Jr ' 3 | __license__ = 'GPLv3' 4 | 5 | 6 | # 7 | # Global state indicators 8 | # 9 | from queue import Queue 10 | 11 | verbose = False 12 | keep_source = False 13 | dry_run = False 14 | 15 | status_queue = Queue() 16 | -------------------------------------------------------------------------------- /pytranscoder/__main__.py: -------------------------------------------------------------------------------- 1 | from . import transcode 2 | 3 | 4 | def main(): 5 | transcode.main() 6 | -------------------------------------------------------------------------------- /pytranscoder/agent.py: -------------------------------------------------------------------------------- 1 | import socket 2 | import os 3 | import subprocess 4 | import time 5 | 6 | 7 | class Agent: 8 | 9 | def run(self): 10 | s = socket.socket() 11 | port = 9567 12 | 13 | s.bind(("", port)) 14 | 15 | s.listen(1) 16 | 17 | while True: 18 | print(f"listening on port {port}...") 19 | c, addr = s.accept() 20 | try: 21 | print('got connection from addr', addr) 22 | hello = c.recv(2048).decode() 23 | print(hello) 24 | if hello.startswith("PING"): 25 | c.send(bytes("PONG".encode())) 26 | c.close() 27 | continue 28 | 29 | if hello.startswith("HELLO|"): 30 | 31 | parts = hello.split("|") 32 | if len(parts) < 5: 33 | print("No enough values in HELLO packet: " + hello) 34 | c.close() 35 | continue 36 | 37 | filesize = int(parts[1]) 38 | tempdir = parts[2] 39 | filename = parts[3] 40 | cli = parts[4] 41 | 42 | print(" echoing back hello") 43 | c.send(bytes(hello.encode())) 44 | 45 | print(f"receiving {filesize} bytes to {filename}...") 46 | output_filename = os.path.join(tempdir, filename) 47 | tmp_filename = os.path.join(tempdir, filename + ".tmp") 48 | 49 | with open(output_filename, "wb") as f: 50 | while filesize > 0: 51 | chunk = c.recv(min(1_000_000, filesize)) 52 | filesize -= len(chunk) 53 | f.write(chunk) 54 | 55 | cli = cli.replace(r"{FILENAME}", output_filename) 56 | cli_parts = cli.split(r"$") 57 | print("receive complete - executing " + " ".join(cli_parts)) 58 | cli_parts.append(tmp_filename) 59 | 60 | vetoed = False 61 | with subprocess.Popen(cli_parts, 62 | stdout=subprocess.PIPE, 63 | stderr=subprocess.STDOUT, 64 | universal_newlines=True, 65 | shell=False) as proc: 66 | while proc.poll() is None: 67 | line = proc.stdout.readline() 68 | if line.startswith("video:"): 69 | # transcode complete 70 | break 71 | 72 | c.send(bytes(line.encode())) 73 | 74 | response = c.recv(4) 75 | confirmation = response.decode() 76 | print(confirmation) 77 | if confirmation == "PING": 78 | # ping received out of context, ignore 79 | continue 80 | if confirmation == "STOP": 81 | proc.kill() 82 | print("Client stopped the transcode, cleaning up") 83 | vetoed = True 84 | break 85 | if confirmation == "VETO": 86 | proc.kill() 87 | print("Client vetoed the transcode, cleaning up") 88 | vetoed = True 89 | break 90 | elif confirmation != "ACK!": 91 | proc.kill() 92 | print(f"Protocol error - expected ACK from client, got {confirmation}") 93 | print("Cleaning up") 94 | vetoed = True 95 | break 96 | 97 | while proc.poll() is None: 98 | time.sleep(1) 99 | 100 | if not vetoed: 101 | if proc.returncode != 0: 102 | print("> ERR") 103 | c.send(bytes(f"ERR|{proc.returncode}".encode())) 104 | print("Cleaning up") 105 | else: 106 | print("> DONE") 107 | filesize = os.path.getsize(tmp_filename) 108 | c.send(bytes(f"DONE|{proc.returncode}|{filesize}".encode())) 109 | # wait for response, then send file 110 | response = c.recv(4).decode() 111 | if response == "ACK!": 112 | # send the file back 113 | print("sending transcoded file") 114 | with open(tmp_filename, "rb") as input_file: 115 | blk = input_file.read(1_000_000) 116 | while len(blk) > 0: 117 | c.send(blk) 118 | blk = input_file.read(1_000_000) 119 | print("done") 120 | os.remove(tmp_filename) 121 | os.remove(output_filename) 122 | 123 | except Exception as ex: 124 | print(str(ex)) 125 | 126 | c.close() 127 | -------------------------------------------------------------------------------- /pytranscoder/config.py: -------------------------------------------------------------------------------- 1 | 2 | import os 3 | from typing import Dict, Any, Optional, List 4 | 5 | import yaml 6 | 7 | from pytranscoder.media import MediaInfo 8 | from pytranscoder.profile import Profile, Directives 9 | from pytranscoder.rule import Rule 10 | from pytranscoder.template import Template 11 | 12 | 13 | class ConfigFile: 14 | settings: Dict 15 | queues: Dict 16 | directives: Dict[str, Directives] 17 | rules: Dict[str, Rule] 18 | 19 | def __init__(self, configuration: Any): 20 | """load configuration file (defaults to $HOME/.transcode.yml)""" 21 | 22 | self.directives = dict() 23 | self.rules = dict() 24 | if configuration is not None: 25 | if isinstance(configuration, Dict): 26 | yml = configuration 27 | else: 28 | if not os.path.exists(configuration): 29 | print(f'Configuration file "{configuration}" not found') 30 | exit(1) 31 | with open(configuration, 'r') as f: 32 | yml = yaml.load(f, Loader=yaml.Loader) 33 | self.settings = yml['config'] 34 | # 35 | # load profiles 36 | # 37 | if 'profiles' in yml: 38 | for name, profile in yml['profiles'].items(): 39 | p = Profile(name, profile) 40 | self.directives[name] = p 41 | parent_names = p.include_profiles 42 | for parent_name in parent_names: 43 | if parent_name not in self.directives: 44 | print(f'Profile error ({name}: included "{parent_name}" not defined') 45 | exit(1) 46 | p.include(self.directives[parent_name]) 47 | # 48 | # load templates 49 | # 50 | if "templates" in yml: 51 | for name, template in yml['templates'].items(): 52 | self.directives[name] = Template(name, template) 53 | 54 | if 'rules' in yml: 55 | for name, rule in yml['rules'].items(): 56 | self.rules[name] = Rule(name, rule) 57 | 58 | if 'queues' in self.settings: 59 | self.queues = self.settings['queues'] 60 | else: 61 | self.queues = dict() 62 | 63 | def fls_path(self) -> str: 64 | return self.settings.get('fls_path', None) 65 | 66 | def colorize(self) -> bool: 67 | return self.settings.get('colorize', 'no').lower() == 'yes' 68 | 69 | def has_queue(self, name) -> bool: 70 | return name in self.queues 71 | 72 | def has_directive(self, directive_name) -> bool: 73 | return directive_name in self.directives 74 | 75 | def get_directive(self, name) -> Directives: 76 | return self.directives.get(name, None) 77 | 78 | def find_mixins(self, mixins: List[str]) -> List[Profile]: 79 | profiles: List[Profile] = [] 80 | if mixins is None: 81 | return profiles 82 | for mixin in mixins: 83 | p = self.get_directive(mixin) 84 | if p: 85 | profiles.append(p) 86 | return profiles 87 | 88 | def match_rule(self, media_info: MediaInfo, restrict_profiles=None) -> Optional[Rule]: 89 | for rule in self.rules.values(): 90 | if restrict_profiles is not None and rule.profile not in restrict_profiles: 91 | continue 92 | if rule.match(media_info): 93 | if rule.is_skip(): 94 | return rule 95 | if not self.has_directive(rule.profile): 96 | print(f'profile "{rule.profile}" referenced from rule "{rule.name}" not found') 97 | exit(1) 98 | return rule 99 | return None 100 | 101 | @property 102 | def ffmpeg_path(self): 103 | return self.settings['ffmpeg'] 104 | 105 | @property 106 | def ssh_path(self): 107 | return self.settings.get('ssh', '/usr/bin/ssh') 108 | 109 | @property 110 | def default_queue_file(self): 111 | return self.settings.get('default_queue_file', None) 112 | 113 | def add_rule(self, name, rule: Rule): 114 | self.rules[name] = rule 115 | 116 | @property 117 | def automap(self) -> bool: 118 | return self.settings.get('automap', True) 119 | -------------------------------------------------------------------------------- /pytranscoder/ffmpeg.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | import os 3 | import re 4 | import subprocess 5 | import sys 6 | import threading 7 | from pathlib import PurePath 8 | from random import randint 9 | from tempfile import gettempdir 10 | from typing import Dict, Any, Optional 11 | import json 12 | 13 | from pytranscoder.media import MediaInfo 14 | from pytranscoder.processor import Processor 15 | 16 | status_re = re.compile( 17 | r'^.* fps=\s*(?P.+?) q=(?P.+\.\d) size=\s*(?P\d+?)kB time=(?P