├── utility.py ├── .gitignore ├── LICENSE ├── rd_average.sh ├── scheduler.py ├── ab_meta_compare.sh ├── README.md ├── awsremote.py ├── rd_tool.py ├── cfg └── vvc-vtm │ ├── encoder_intra_vtm.cfg │ ├── encoder_lowdelay_vtm.cfg │ ├── encoder_randomaccess_vtm.cfg │ └── encoder_randomaccess_vtm_gop16.cfg ├── sshslot.py ├── work.py ├── rd_server.py └── metrics_gather.sh /utility.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | from datetime import datetime 4 | import sys 5 | 6 | #our timestamping function, accurate to milliseconds 7 | #(remove [:-3] to display microseconds) 8 | def get_time(): 9 | return datetime.now().strftime("%Y-%m-%d %H:%M:%S.%f")[:-3] 10 | 11 | def rd_print(log, *args, **kwargs): 12 | print(get_time(), *args, flush=True, **kwargs) 13 | if log: 14 | try: 15 | print(get_time(), file=log, flush=True, *args, **kwargs) 16 | except: 17 | pass 18 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Byte-compiled / optimized / DLL files 2 | __pycache__/ 3 | *.py[cod] 4 | 5 | # C extensions 6 | *.so 7 | 8 | # Distribution / packaging 9 | .Python 10 | env/ 11 | bin/ 12 | build/ 13 | develop-eggs/ 14 | dist/ 15 | eggs/ 16 | lib/ 17 | lib64/ 18 | parts/ 19 | sdist/ 20 | var/ 21 | *.egg-info/ 22 | .installed.cfg 23 | *.egg 24 | 25 | # Installer logs 26 | pip-log.txt 27 | pip-delete-this-directory.txt 28 | 29 | # Unit test / coverage reports 30 | htmlcov/ 31 | .tox/ 32 | .coverage 33 | .cache 34 | nosetests.xml 35 | coverage.xml 36 | 37 | # Translations 38 | *.mo 39 | 40 | # Mr Developer 41 | .mr.developer.cfg 42 | .project 43 | .pydevproject 44 | 45 | # Rope 46 | .ropeproject 47 | 48 | # Django stuff: 49 | *.log 50 | *.pot 51 | 52 | # Sphinx documentation 53 | docs/_build/ 54 | 55 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2014, Thomas Daede 2 | All rights reserved. 3 | 4 | Redistribution and use in source and binary forms, with or without 5 | modification, are permitted provided that the following conditions are met: 6 | 7 | * Redistributions of source code must retain the above copyright notice, this 8 | list of conditions and the following disclaimer. 9 | 10 | * Redistributions in binary form must reproduce the above copyright notice, 11 | this list of conditions and the following disclaimer in the documentation 12 | and/or other materials provided with the distribution. 13 | 14 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 15 | AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16 | IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 17 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE 18 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 20 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 21 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 22 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 23 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 24 | -------------------------------------------------------------------------------- /rd_average.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | if [ $# == 0 ]; then 4 | echo "usage: OUTPUT= $0 *.out" 5 | exit 1 6 | fi 7 | 8 | TOTAL=total.out 9 | 10 | if [ -n "$OUTPUT" ]; then 11 | TOTAL="$OUTPUT.out" 12 | fi 13 | 14 | if [ -e "$TOTAL" ]; then 15 | echo "ERROR: $TOTAL already exists and will be included in average, please remove it first" 16 | exit 1 17 | fi 18 | 19 | awk '{size[$1]+=$2;bytes[$1]+=$3;psnr[$1]+=$2*$4;psnrhvs[$1]+=$2*$5;ssim[$1]+=$2*$6;fastssim[$1]+=$2*$7;ciede[$1]+=$2*$8;psnrcb[$1]+=$2*$9;psnrcr[$1]+=$2*$10;apsnr[$1]+=$2*$11;apsnrcb[$1]+=$2*$12;apsnrcr[$1]+=$2*$13;msssim[$1]+=$2*$14;encodetime[$1]+=$15;vmaf[$1]+=$2*$16;decodetime[$1]+=$17;psnr_libvmaf[$1]+=$2*$18;psnrcb_libvmaf[$1]+=$2*$19;psnrcr_libvmaf[$1]+=$2*$20;ciede2000_libvmaf[$1]+=$2*$21;ssim_libvmaf[$1]+=$2*$22;msssim_libvmaf[$1]+=$2*$23;psnrhvsy_libvmaf[$1]+=$2*$24;psnrhvscb_libvmaf[$1]+=$2*$25;psnrhvscr_libvmaf[$1]+=$2*$26;psnrhvs_libvmaf[$1]+=$2*$27;vmaf_libvmaf[$1]+=$2*$28;vmafneg_libvmaf[$1]+=$2*$29;apsnr_y_libvmaf[$1]+=$2*$30;apsnr_cb_libvmaf[$1]+=$2*$31;apsnr_cr_libvmaf[$1]+=$2*$32;}END{for(i in size)print i,size[i],bytes[i],psnr[i]/size[i],psnrhvs[i]/size[i],ssim[i]/size[i],fastssim[i]/size[i],ciede[i]/size[i],psnrcb[i]/size[i],psnrcr[i]/size[i],apsnr[i]/size[i],apsnrcb[i]/size[i],apsnrcr[i]/size[i],msssim[i]/size[i],encodetime[i],vmaf[i]/size[i],decodetime[i],psnr_libvmaf[i]/size[i],psnrcb_libvmaf[i]/size[i],psnrcr_libvmaf[i]/size[i],ciede2000_libvmaf[i]/size[i],ssim_libvmaf[i]/size[i],msssim_libvmaf[i]/size[i],psnrhvsy_libvmaf[i]/size[i],psnrhvscb_libvmaf[i]/size[i],psnrhvscr_libvmaf[i]/size[i],psnrhvs_libvmaf[i]/size[i],vmaf_libvmaf[i]/size[i],vmafneg_libvmaf[i]/size[i],apsnr_y_libvmaf[i]/size[i],apsnr_cb_libvmaf[i]/size[i],apsnr_cr_libvmaf[i]/size[i];}' $@ | sort -n > $TOTAL 20 | -------------------------------------------------------------------------------- /scheduler.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | from utility import get_time 4 | import threading 5 | from time import sleep 6 | import sys 7 | 8 | def run(work_items, slots): 9 | retries = 0 10 | max_retries = 5000 11 | free_slots = slots 12 | taken_slots = [] 13 | work_done = [] 14 | total_num_of_jobs = len(work_items) 15 | while(1): 16 | for slot in taken_slots: 17 | if slot.busy == False: 18 | if slot.work.failed == False: 19 | work_done.append(slot.work) 20 | print(get_time(),len(work_done),'out of',total_num_of_jobs,'finished.') 21 | elif retries >= max_retries: 22 | break 23 | else: 24 | retries = retries + 1 25 | print(get_time(),'Retrying work...',retries,'of',max_retries,'retries.') 26 | work_items.append(slot.work) 27 | slot.clear_work() 28 | taken_slots.remove(slot) 29 | free_slots.append(slot) 30 | 31 | #have we finished all the work? 32 | if len(work_items) == 0: 33 | if len(taken_slots) == 0: 34 | print(get_time(),'All work finished.') 35 | break 36 | elif retries >= max_retries: 37 | print(get_time(),'Max number of failed retries reached!') 38 | sys.exit(1) 39 | else: 40 | if len(free_slots) != 0: 41 | slot = free_slots.pop() 42 | work = work_items.pop() 43 | print(get_time(),'Encoding',work.get_name(),'on',slot.machine.host) 44 | slot.start_work(work) 45 | taken_slots.append(slot) 46 | sleep(0.2) 47 | return work_done 48 | -------------------------------------------------------------------------------- /ab_meta_compare.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | 4 | # script to generate one image or video file based on the given input 5 | # bpp, filename, and time. It is stored in `runs/$time/$bpp` 6 | 7 | bpp="$1" 8 | time="$2" 9 | workset="$3" # subset1 or video-1-short for example 10 | file_path="$4" # is a `*.y4m` file 11 | codec="$5" 12 | 13 | base_path="/home/ec2-user" 14 | daala_path="$base_path/daala" 15 | libvpx_path="$base_path/libvpx" 16 | x264_path="$base_path/x264" 17 | x265_path="$base_path/x265" 18 | tools_path="$daala_path/tools" 19 | output_path="$base_path/runs/$time/$workset/bpp_$bpp" 20 | 21 | options="-b $bpp 22 | -c $codec 23 | -l $libvpx_path 24 | -x $x264_path 25 | -X $x265_path 26 | -d $daala_path 27 | -a $tools_path" 28 | 29 | echo "Encoding $file_path with a bpp of $bpp" 30 | 31 | mkdir --parents "$output_path" 32 | cd "$output_path" 33 | 34 | "$tools_path/ab_compare.sh" $options "$file_path" 35 | 36 | basename=$(basename "$file_path") 37 | 38 | # Remove any generated .y4m files from $output_path 39 | rm *.y4m 40 | 41 | # Bash parameter expansion: `${foo%.*}` searches `foo` from the back looking to match `.*` 42 | # (find something starting with a period). If it finds it, it deletes it. A `%%` would delete 43 | # from the first period to the end. 44 | # This returns the filename without the `.y4m` extension. 45 | file=${basename%.*} 46 | 47 | # Remove newly generated files ending with .ogv 48 | rm "$file"*.ogv 49 | 50 | ext="png" 51 | 52 | # ab_compare renames `a.y4m` to something like `a-70.ogv.png` so 53 | # get `a` off the original filename and append `.png` to it. 54 | # This might have issues if the same file was output with different 55 | # bpp values in this folder. However, this script puts different bpp 56 | # values in different folders so it shouldn't have any issues. 57 | mv "$file"*".$ext" "$file.$ext" 58 | 59 | 60 | cd "$base_path" 61 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | rd_tool 2 | ======= 3 | 4 | rd_tool.py is a script for collecting rate-distortion curves across a series of either local or Amazon AWS nodes. 5 | 6 | This script is run by arewecompressedyet.com, but can also be run locally to produce the same data. 7 | 8 | Dependencies 9 | ============ 10 | 11 | You will need Python 3.4 or later, as well as [boto3](https://github.com/boto/boto3). 12 | 13 | On Ubuntu, install with: 14 | ``` 15 | sudo apt install python3-tornado python3-boto3 16 | ``` 17 | 18 | Node dependencies 19 | ================= 20 | 21 | Individual build machines do not need Python, but do need bash. Each machine 22 | should be configured with a user and work_root (such as that user's home 23 | directory). This work directory must be populated with folders called 24 | daalatool, dump\_ciede2000, and, optionally, dav1d and vmaf, which need to 25 | contain checkouts of their respective git repositories, each with tools built. 26 | 27 | To compile Daala tools: 28 | 29 | ``` 30 | sudo apt install build-essential autoconf libogg-dev libjpeg-dev libpng-dev check python3-numpy python3-scipy libtool pkg-config 31 | ``` 32 | 33 | ``` 34 | git clone https://git.xiph.org/daala.git daalatool 35 | cd daalatool 36 | ./autogen.sh 37 | ./configure --disable-player 38 | make tools -j4 39 | ``` 40 | 41 | For dump\_ciede2000: 42 | 43 | Install rust if you haven't already. You only need rust to compile the binary 44 | and don't need it on the individual machines. 45 | 46 | ``` 47 | curl -sf -L https://static.rust-lang.org/rustup.sh | sh 48 | ``` 49 | 50 | Build dump_ciede2000. 51 | 52 | ``` 53 | git clone https://github.com/KyleSiefring/dump_ciede2000 54 | cd dump_ciede2000 55 | cargo build --release 56 | ``` 57 | 58 | Exit that directory 59 | 60 | ``` 61 | cd ../ 62 | ``` 63 | 64 | For dav1d (optional): 65 | 66 | ``` 67 | sudo apt install meson 68 | ``` 69 | 70 | ``` 71 | git clone https://code.videolan.org/videolan/dav1d.git 72 | cd dav1d 73 | mkdir build && cd build 74 | meson .. 75 | ninja 76 | ``` 77 | 78 | For vmaf (optional): 79 | 80 | ``` 81 | git clone https://github.com/Netflix/vmaf.git 82 | cd vmaf 83 | cd ptools; make; cd ../wrapper; make; cd ..; 84 | ``` 85 | 86 | rd_tool will automatically create one slot directory per core, and upload 87 | codec binaries into that directory. 88 | 89 | Using AWS nodes 90 | =============== 91 | 92 | You will need a ~/.aws configuration for boto with your AWS information and credentials. 93 | 94 | Specify the autoscaling groukp to use with -awsgroup. 95 | 96 | Using local nodes 97 | ================= 98 | 99 | You can specify all of the machines you want to use in a JSON file: 100 | 101 | ```json 102 | [ 103 | { 104 | "host": "localhost", 105 | "user": "thomas", 106 | "cores": 4, 107 | "port": 22, 108 | "work_root": "/home/thomas/tmp", 109 | "media_path": "/home/thomas/sets" 110 | }, 111 | { 112 | ... 113 | } 114 | ] 115 | 116 | ``` 117 | 118 | Specify this configuration on the command line with -machineconf. 119 | 120 | Builds 121 | ====== 122 | 123 | Specify the path to the git checkout of thetested codec with the -bindir 124 | argument. rd_tool.py expects to find prebuilt binaries of the codec - it will 125 | not build the codec automatically. 126 | 127 | Output files 128 | ============ 129 | 130 | Output files are written into the current directory. The format of the .out 131 | files is one quantizer per line, of the format 132 | 133 | ... 134 | 135 | See RDRun.reduce() in work.py for an up-to-date list of metrics. 136 | -------------------------------------------------------------------------------- /awsremote.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import boto3 4 | from utility import * 5 | from time import sleep 6 | import subprocess 7 | import sys 8 | from sshslot import * 9 | 10 | # Returns true if an instance exists 11 | def instance_exists(instance, ec2): 12 | status = ec2.describe_instance_status(InstanceIds=[instance]) 13 | 14 | return len(status['InstanceStatuses']) > 0 15 | 16 | # These status inquiries return huge dictionaries which must be almost entirely ignored. 17 | # They provide a filter but I couldn't get it to work (and it required an extra 6 lines 18 | # to use it). 19 | def state_name_of(instance, ec2): 20 | status = ec2.describe_instance_status(InstanceIds=[instance]) 21 | 22 | # Returns one of the values in the previous call. 23 | return status['InstanceStatuses'][0]['InstanceState']['Name'] 24 | 25 | # This is similar to `state_name_of()` which also ignores almost all values from the 26 | # given dictionary. 27 | def status_of(instance, ec2): 28 | status = ec2.describe_instance_status(InstanceIds=[instance]) 29 | 30 | # Returns one of the values in the previous call. 31 | return status['InstanceStatuses'][0]['InstanceStatus']['Status'] 32 | 33 | # Again, similar to `state_name_of()` 34 | def ip_address_of(instance, ec2): 35 | address = ec2.describe_instances(InstanceIds=[instance]) 36 | 37 | if len(address['Reservations']) < 1 or len(address['Reservations'][0]['Instances']) < 1: 38 | print('There is no public IP address available for instance', instance) 39 | sys.exit(1) 40 | else: 41 | # Returns one of the values in the previous call. 42 | return address['Reservations'][0]['Instances'][0]['PublicIpAddress'] 43 | 44 | def get_instances_in_group(autoscale, aws_group_name): 45 | all_instances = autoscale.describe_auto_scaling_instances()['AutoScalingInstances'] 46 | instances = [] 47 | for instance in all_instances: 48 | if instance['AutoScalingGroupName'] == aws_group_name: 49 | instances.append(instance) 50 | return instances 51 | 52 | def stop_machines(aws_group_name): 53 | try: 54 | ec2 = boto3.client('ec2'); 55 | autoscale = boto3.client('autoscaling'); 56 | autoscale.set_desired_capacity( 57 | AutoScalingGroupName = aws_group_name, 58 | DesiredCapacity = 0 59 | ) 60 | except Exception as e: 61 | rd_print(None,e) 62 | 63 | def start_machines(num_instances_to_use, aws_group_name): 64 | #switch on more machines if we need them 65 | if num_instances < num_instances_to_use: 66 | rd_print(None,'Launching instances...') 67 | autoscale.set_desired_capacity( 68 | AutoScalingGroupName = aws_group_name, 69 | DesiredCapacity = num_instances_to_use 70 | ) 71 | 72 | #tell us status every few seconds 73 | while num_instances < num_instances_to_use: 74 | instances = get_instances_in_group(autoscale, aws_group_name) 75 | num_instances = len(instances) 76 | rd_print(None,'Number of instances online:', num_instances) 77 | sleep(3) 78 | 79 | def get_machines(num_instances_to_use, aws_group_name): 80 | machines = [] 81 | #connect to AWS 82 | ec2 = boto3.client('ec2'); 83 | autoscale = boto3.client('autoscaling'); 84 | 85 | #how many machines are currently running? 86 | instances = get_instances_in_group(autoscale, aws_group_name) 87 | num_instances = len(instances) 88 | rd_print(None,'Number of instances online:', num_instances) 89 | 90 | #grab instance IDs 91 | instance_ids = [i['InstanceId'] for i in instances] 92 | rd_print(None,"These instances are online:",instance_ids) 93 | running_instance_ids = [] 94 | for instance_id in instance_ids: 95 | try: 96 | state = state_name_of(instance_id, ec2) 97 | if state == 'running': 98 | rd_print(None,instance_id, 'is running!') 99 | running_instance_ids.append(instance_id) 100 | except IndexError: 101 | print(instance_id, 'not queryable yet') 102 | ok_instance_ids = [] 103 | for instance_id in running_instance_ids: 104 | try: 105 | if status_of(instance_id, ec2) == 'ok': 106 | rd_print(None,instance_id,'reported OK!') 107 | ok_instance_ids.append(instance_id) 108 | except IndexError: 109 | rd_print(None,'Instance',instance_id,'disappeared!') 110 | for instance_id in ok_instance_ids: 111 | machines.append(Machine(ip_address_of(instance_id, ec2))) 112 | return machines 113 | -------------------------------------------------------------------------------- /rd_tool.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | from utility import get_time, rd_print 4 | import argparse 5 | import os 6 | import sys 7 | import subprocess 8 | import json 9 | import codecs 10 | import awsremote 11 | import scheduler 12 | import sshslot 13 | from work import * 14 | 15 | 16 | config_dir = os.getenv("CONFIG_DIR", os.getcwd()) 17 | runs_dst_dir = os.getenv("RUNS_DST_DIR", os.path.join(os.getcwd(), "../runs")) 18 | codecs_src_dir = os.getenv("CODECS_SRC_DIR", os.path.join(os.getcwd(), "..")) 19 | 20 | if 'DAALA_ROOT' not in os.environ: 21 | rd_print(None,"Please specify the DAALA_ROOT environment variable to use this tool.") 22 | sys.exit(1) 23 | 24 | daala_root = os.environ['DAALA_ROOT'] 25 | 26 | extra_options = '' 27 | if 'EXTRA_OPTIONS' in os.environ: 28 | extra_options = os.environ['EXTRA_OPTIONS'] 29 | print(get_time(),'Passing extra command-line options:"%s"' % extra_options) 30 | 31 | work_items = [] 32 | 33 | #load all the different sets and their filenames 34 | video_sets_f = codecs.open(os.path.join(config_dir, 'sets.json'),'r',encoding='utf-8') 35 | video_sets = json.load(video_sets_f) 36 | 37 | parser = argparse.ArgumentParser(description='Collect RD curve data.') 38 | parser.add_argument('set',metavar='Video set name',nargs='+') 39 | parser.add_argument('-codec',default='daala') 40 | parser.add_argument('-bindir',default='./') 41 | parser.add_argument('-prefix',default='.') 42 | parser.add_argument('-awsgroup', default='Daala') 43 | parser.add_argument('-machines', default=14) 44 | parser.add_argument('-mode', default='metric') 45 | parser.add_argument('-runid', default=get_time()) 46 | parser.add_argument('-seed') 47 | parser.add_argument('-bpp') 48 | parser.add_argument('-qualities',nargs='+') 49 | parser.add_argument('-machineconf') 50 | parser.add_argument('-save-encode',action='store_true') 51 | 52 | args = parser.parse_args() 53 | 54 | aws_group_name = args.awsgroup 55 | 56 | #check we have the codec in our codec-qualities dictionary 57 | if args.codec not in quality_presets: 58 | rd_print(None,'Invalid codec. Valid codecs are:') 59 | for q in quality_presets: 60 | rd_print(None,q) 61 | sys.exit(1) 62 | 63 | #check we have the set name in our sets-filenames dictionary 64 | if args.set[0] not in video_sets: 65 | rd_print(None,'Specified invalid set '+args.set[0]+'. Available sets are:') 66 | for video_set in video_sets: 67 | rd_print(None,video_set) 68 | sys.exit(1) 69 | 70 | #Make a list of the bits of work we need to do. 71 | #We pack the stack ordered by filesize ASC, quality ASC (aka. -v DESC) 72 | #so we pop the hardest encodes first, 73 | #for more efficient use of the AWS machines' time. 74 | 75 | video_filenames = video_sets[args.set[0]]['sources'] 76 | 77 | if args.mode == 'metric': 78 | run = RDRun(args.codec) 79 | else: 80 | run = Run(args.codec) 81 | run.runid = str(args.runid) 82 | if args.qualities: 83 | run.quality = args.qualities 84 | run.set = args.set[0] 85 | run.bindir = args.bindir 86 | run.save_encode = args.save_encode 87 | run.extra_options = extra_options 88 | run.prefix = args.prefix 89 | 90 | if args.mode == 'metric': 91 | work_items = create_rdwork(run, video_filenames) 92 | elif args.mode == 'ab': 93 | if video_sets[args.set[0]]['type'] == 'video': 94 | print("mode `ab` isn't supported for videos. Skipping.") 95 | else: 96 | work_items = create_abwork(run, video_filenames) 97 | else: 98 | print('Unsupported -mode parameter.') 99 | sys.exit(1) 100 | run.work_items = list(work_items) 101 | 102 | total_num_of_jobs = len(video_sets[args.set[0]]['sources']) * len(run.quality) 103 | 104 | #a logging message just to get the regex progress bar on the AWCY site started... 105 | rd_print(None,'0 out of',total_num_of_jobs,'finished.') 106 | 107 | #how many AWS instances do we want to spin up? 108 | #The assumption is each machine can deal with 18 threads, 109 | #so up to 18 jobs, use 1 machine, then up to 64 use 2, etc... 110 | num_instances_to_use = (31 + total_num_of_jobs) // 18 111 | 112 | #...but lock AWS to a max number of instances 113 | max_num_instances_to_use = int(args.machines) 114 | 115 | if num_instances_to_use > max_num_instances_to_use: 116 | rd_print(None,'Ideally, we should use',num_instances_to_use, 117 | 'instances, but the max is',max_num_instances_to_use,'.') 118 | num_instances_to_use = max_num_instances_to_use 119 | 120 | machines = [] 121 | if args.machineconf: 122 | machineconf = json.load(open(args.machineconf, 'r')) 123 | for m in machineconf: 124 | machines.append(sshslot.Machine(m['host'],m['user'],m['cores'],m['work_root'],str(m['port']),m['media_path'])) 125 | else: 126 | while not machines: 127 | machines = awsremote.get_machines(num_instances_to_use, aws_group_name) 128 | 129 | slots = [] 130 | #set up our instances and their free job slots 131 | for machine in machines: 132 | slots.extend(machine.get_slots()) 133 | 134 | if len(slots) < 1: 135 | rd_print(None,'All AWS machines are down.') 136 | sys.exit(1) 137 | 138 | work_done = scheduler.run(work_items, slots) 139 | 140 | if args.mode == 'metric': 141 | run.reduce() 142 | 143 | rd_print(None,'Done!') 144 | -------------------------------------------------------------------------------- /cfg/vvc-vtm/encoder_intra_vtm.cfg: -------------------------------------------------------------------------------- 1 | #======== File I/O ===================== 2 | BitstreamFile : str.bin 3 | ReconFile : rec.yuv 4 | 5 | #======== Profile ================ 6 | Profile : auto 7 | 8 | #======== Unit definition ================ 9 | MaxCUWidth : 64 # Maximum coding unit width in pixel 10 | MaxCUHeight : 64 # Maximum coding unit height in pixel 11 | 12 | #======== Coding Structure ============= 13 | IntraPeriod : 1 # Period of I-Frame ( -1 = only first) 14 | DecodingRefreshType : 1 # Random Accesss 0:none, 1:CRA, 2:IDR, 3:Recovery Point SEI 15 | GOPSize : 1 # GOP Size (number of B slice = GOPSize-1) 16 | # Type POC QPoffset QPfactor tcOffsetDiv2 betaOffsetDiv2 temporal_id #ref_pics_active #ref_pics reference pictures 17 | 18 | #=========== Motion Search ============= 19 | FastSearch : 1 # 0:Full search 1:TZ search 20 | SearchRange : 64 # (0: Search range is a Full frame) 21 | HadamardME : 1 # Use of hadamard measure for fractional ME 22 | FEN : 1 # Fast encoder decision 23 | FDM : 1 # Fast Decision for Merge RD cost 24 | 25 | #======== Quantization ============= 26 | QP : 32 # Quantization parameter(0-51) 27 | MaxDeltaQP : 0 # CU-based multi-QP optimization 28 | MaxCuDQPSubdiv : 0 # Maximum subdiv for CU luma Qp adjustment 29 | DeltaQpRD : 0 # Slice-based multi-QP optimization 30 | RDOQ : 1 # RDOQ 31 | RDOQTS : 1 # RDOQ for transform skip 32 | 33 | #=========== Deblock Filter ============ 34 | DeblockingFilterOffsetInPPS : 1 # Dbl params: 0=varying params in SliceHeader, param = base_param + GOP_offset_param; 1 (default) =constant params in PPS, param = base_param) 35 | DeblockingFilterDisable : 0 # Disable deblocking filter (0=Filter, 1=No Filter) 36 | DeblockingFilterBetaOffset_div2 : -2 # base_param: -12 ~ 12 37 | DeblockingFilterTcOffset_div2 : -5 # base_param: -12 ~ 12 38 | DeblockingFilterCbBetaOffset_div2 : -2 # base_param: -12 ~ 12 39 | DeblockingFilterCbTcOffset_div2 : -5 # base_param: -12 ~ 12 40 | DeblockingFilterCrBetaOffset_div2 : -2 # base_param: -12 ~ 12 41 | DeblockingFilterCrTcOffset_div2 : -5 # base_param: -12 ~ 12 42 | DeblockingFilterMetric : 0 # blockiness metric (automatically configures deblocking parameters in bitstream). Applies slice-level loop filter offsets (DeblockingFilterOffsetInPPS and DeblockingFilterDisable must be 0) 43 | 44 | #=========== Misc. ============ 45 | InternalBitDepth : 10 # codec operating bit-depth 46 | 47 | #=========== Coding Tools ================= 48 | SAO : 1 # Sample adaptive offset (0: OFF, 1: ON) 49 | TransformSkip : 1 # Transform skipping (0: OFF, 1: ON) 50 | TransformSkipFast : 1 # Fast Transform skipping (0: OFF, 1: ON) 51 | TransformSkipLog2MaxSize : 5 52 | SAOLcuBoundary : 0 # SAOLcuBoundary using non-deblocked pixels (0: OFF, 1: ON) 53 | 54 | #============ VTM settings ====================== 55 | SEIDecodedPictureHash : 0 56 | CbQpOffset : 0 57 | CrQpOffset : 0 58 | SameCQPTablesForAllChroma : 1 59 | QpInValCb : 17 27 32 44 60 | QpOutValCb : 17 29 34 41 61 | TemporalSubsampleRatio : 8 62 | 63 | ReWriteParamSets : 1 64 | #============ NEXT ==================== 65 | 66 | # General 67 | CTUSize : 128 68 | LCTUFast : 1 69 | 70 | DualITree : 1 # separate partitioning of luma and chroma channels for I-slices 71 | MinQTLumaISlice : 8 72 | MinQTChromaISliceInChromaSamples: 4 # minimum QT size in chroma samples for chroma separate tree 73 | MinQTNonISlice : 8 74 | MaxMTTHierarchyDepth : 3 75 | MaxMTTHierarchyDepthISliceL : 3 76 | MaxMTTHierarchyDepthISliceC : 3 77 | 78 | MTS : 1 79 | MTSIntraMaxCand : 4 80 | MTSInterMaxCand : 4 81 | SBT : 1 82 | LFNST : 1 83 | ISP : 1 84 | Affine : 1 85 | SbTMVP : 1 86 | MaxNumMergeCand : 6 87 | LMChroma : 1 # use CCLM only 88 | DepQuant : 1 89 | IMV : 1 90 | ALF : 1 91 | IBC : 0 # turned off in CTC 92 | AllowDisFracMMVD : 1 93 | AffineAmvr : 0 94 | LMCSEnable : 1 # LMCS: 0: disable, 1:enable 95 | LMCSSignalType : 0 # Input signal type: 0:SDR, 1:HDR-PQ, 2:HDR-HLG 96 | LMCSUpdateCtrl : 1 # LMCS model update control: 0:RA, 1:AI, 2:LDB/LDP 97 | LMCSOffset : 2 # chroma residual scaling offset 98 | MRL : 1 99 | MIP : 1 100 | JointCbCr : 1 # joint coding of chroma residuals (if available): 0: disable, 1: enable 101 | ChromaTS : 1 102 | 103 | # Fast tools 104 | PBIntraFast : 1 105 | ISPFast : 1 106 | FastMrg : 1 107 | AMaxBT : 1 108 | FastMIP : 1 109 | FastLFNST : 1 110 | 111 | # Encoder optimization tools 112 | AffineAmvrEncOpt : 0 113 | ALFAllowPredefinedFilters : 1 114 | ALFStrengthTargetLuma : 1.0 115 | ALFStrengthTargetChroma : 1.0 116 | CCALFStrengthTarget : 1.0 117 | ### DO NOT ADD ANYTHING BELOW THIS LINE ### 118 | ### DO NOT DELETE THE EMPTY LINE BELOW ### 119 | 120 | 121 | -------------------------------------------------------------------------------- /sshslot.py: -------------------------------------------------------------------------------- 1 | from utility import * 2 | import subprocess 3 | import sys 4 | import os 5 | import threading 6 | import time 7 | 8 | ssh_privkey_file = os.getenv("SSH_PRIVKEY_FILE", "daala.pem") 9 | 10 | binaries = { 11 | 'daala':['examples/encoder_example','examples/dump_video'], 12 | 'x264': ['x264'], 13 | 'x264-rt': ['x264'], 14 | 'x265': ['build/linux/x265'], 15 | 'x265-rt': ['build/linux/x265'], 16 | 'xvc': ['build/app/xvcenc', 'build/app/xvcdec'], 17 | 'vp8': ['vpxenc','vpxdec'], 18 | 'vp9': ['vpxenc','vpxdec'], 19 | 'vp9-rt': ['vpxenc','vpxdec'], 20 | 'vp10': ['vpxenc','vpxdec'], 21 | 'vp10-rt': ['vpxenc','vpxdec'], 22 | 'av1': ['aomenc','aomdec'], 23 | 'av1-rt': ['aomenc','aomdec'], 24 | 'av2-ai': ['aomenc','aomdec'], 25 | 'av2-ra': ['aomenc','aomdec'], 26 | 'av2-ra-st': ['aomenc','aomdec'], 27 | 'av2-ld': ['aomenc','aomdec'], 28 | 'av2-as': ['aomenc','aomdec'], 29 | 'av2-as-st': ['aomenc','aomdec'], 30 | 'thor': ['build/Thorenc','build/Thordec','config_HDB16_high_efficiency.txt','config_LDB_high_efficiency.txt'], 31 | 'thor-rt': ['build/Thorenc','build/Thordec','config_HDB16_high_efficiency.txt','config_LDB_high_efficiency.txt'], 32 | 'rav1e': ['target/release/rav1e'], 33 | 'svt-av1': ['Bin/Release/SvtAv1EncApp'], 34 | "svt-av1-ra": ['Bin/Release/SvtAv1EncApp'], 35 | "svt-av1-ra-crf": ['Bin/Release/SvtAv1EncApp'], 36 | "svt-av1-ra-vbr": ['Bin/Release/SvtAv1EncApp'], 37 | "svt-av1-ra-vbr-2p": ['Bin/Release/SvtAv1EncApp'], 38 | "svt-av1-ld-cbr": ['Bin/Release/SvtAv1EncApp'], 39 | "svt-av1-ra-cq": ['Bin/Release/SvtAv1EncApp'], 40 | "svt-av1-as": ['Bin/Release/SvtAv1EncApp'], 41 | "svt-av1-as-ctc": ['Bin/Release/SvtAv1EncApp'], 42 | 'vvc-vtm': ['bin/EncoderAppStatic', 'bin/DecoderAppStatic', 'bin/parcatStatic'], 43 | 'vvc-vtm-ra': ['bin/EncoderAppStatic', 'bin/DecoderAppStatic', 'bin/parcatStatic'], 44 | 'vvc-vtm-ra-ctc': ['bin/EncoderAppStatic', 'bin/DecoderAppStatic', 'bin/parcatStatic'], 45 | 'vvc-vtm-as-ctc': ['bin/EncoderAppStatic', 'bin/DecoderAppStatic', 'bin/parcatStatic'], 46 | 'vvc-vtm-ra-st': ['bin/EncoderAppStatic', 'bin/DecoderAppStatic', 'bin/parcatStatic'], 47 | 'vvc-vtm-ld': ['bin/EncoderAppStatic', 'bin/DecoderAppStatic', 'bin/parcatStatic'], 48 | 'vvc-vtm-ai': ['bin/EncoderAppStatic', 'bin/DecoderAppStatic', 'bin/parcatStatic'] 49 | } 50 | 51 | # Finding files such as `this_(that)` requires `'` be placed on both 52 | # sides of the quote so the `()` are both captured. Files such as 53 | # `du_Parterre_d'Eau` must be converted into 54 | #`'du_Parterre_d'"'"'Eau' 55 | # ^^^ Required to make sure the `'` is captured. 56 | def shellquote(s): 57 | return "'" + s.replace("'", "'\"'\"'") + "'" 58 | 59 | class Machine: 60 | def __init__(self,host,user='ec2-user',cores=18,work_root='/home/ec2-user',port=22,media_path='/mnt/media'): 61 | self.host = host 62 | self.user = user 63 | self.cores = cores 64 | self.work_root = work_root 65 | self.port = str(port) 66 | self.media_path = media_path 67 | self.log = None 68 | self.slots = [] 69 | def rsync(self, local, remote): 70 | return subprocess.call(['rsync', '-r', '-e', "ssh -i "+ssh_privkey_file+" -o StrictHostKeyChecking=no -p "+str(self.port), local, self.user + '@' + self.host + ':' + remote]) 71 | def check_shell(self, command): 72 | return subprocess.check_output(['ssh','-i',ssh_privkey_file,'-p',self.port,'-o',' StrictHostKeyChecking=no', 73 | self.user+'@'+self.host, 74 | command.encode("utf-8")]) 75 | def get_slots(self): 76 | slots = [] 77 | #by doing the machines in the inner loop, 78 | #we end up with heavy jobs split across machines better 79 | for i in range(0,self.cores): 80 | slots.append(Slot(self, i, self.log)) 81 | self.slots = slots 82 | return slots 83 | def get_name(self): 84 | return self.host 85 | 86 | class SlotProcess: 87 | def __init__(self, log): 88 | self.p = None 89 | self.can_kill = threading.Event() 90 | self.log = log 91 | def kill(self): 92 | # wait until there is actually a process to kill 93 | success = self.can_kill.wait(40) 94 | if not success: 95 | rd_print(self.log,"Waited too long for process to kill.") 96 | if self.p: 97 | rd_print(self.log,"Will try to kill anyway.") 98 | else: 99 | rd_print(self.log,"Aborting kill.") 100 | return 101 | try: 102 | self.p.kill() 103 | except Exception as e: 104 | rd_print(self.log,"Couldn't cancel work item",e) 105 | def communicate(self): 106 | return self.p.communicate() 107 | def shell(self, args): 108 | self.p = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE) 109 | self.can_kill.set() 110 | 111 | #the job slots we can fill 112 | class Slot: 113 | def __init__(self, machine, num, log): 114 | self.machine = machine 115 | self.work_root = machine.work_root + '/slot' + str(num) 116 | self.p = None 117 | self.busy = False 118 | self.work = None 119 | self.log = log 120 | self.can_kill = None 121 | self.thread_hold = False # Would be useful in future to distinguish between thread holding and other reasons to hold a given slot 122 | def gather(self): 123 | return self.p.communicate() 124 | def start_work(self, work): 125 | self.work = work 126 | work.slot = self 127 | self.p = SlotProcess(self.log) 128 | work_thread = threading.Thread(target=self.execute) 129 | work_thread.daemon = True 130 | self.busy = True 131 | work_thread.start() 132 | # Mark a given slot as busy and hold it for parallel processing 133 | def hold_work(self, work): 134 | self.work = work 135 | work.slot = self 136 | self.p = SlotProcess(self.log) 137 | self.busy = True 138 | self.thread_hold = True 139 | def clear_work(self): 140 | if self.work: 141 | self.work.slot = None 142 | self.work = None 143 | def execute(self): 144 | try: 145 | self.work.execute() 146 | except Exception as e: 147 | rd_print(self.log, e) 148 | self.work.failed = True 149 | self.busy = False 150 | def setup(self,codec,bindir): 151 | time.sleep(1) 152 | try: 153 | self.check_shell('mkdir -p '+shellquote(self.work_root)) 154 | time.sleep(1) 155 | self.check_shell('rm -f '+shellquote(self.work_root)+'/*.y4m '+shellquote(self.work_root)+'/*.ivf') 156 | time.sleep(1) 157 | except subprocess.CalledProcessError as e: 158 | rd_print(self.log,e.output) 159 | rd_print(self.log,'Couldn\'t connect to machine '+self.machine.host) 160 | raise RuntimeError('This is a bug with AWCY. Likely this machine has gone unreachable.') 161 | if self.machine.rsync('./',self.work_root+'/rd_tool/') != 0: 162 | rd_print(self.log,'Couldn\'t set up machine '+self.machine.host) 163 | raise RuntimeError('Couldn\'t copy tools to machine (out of disk space?)') 164 | time.sleep(1) 165 | self.check_shell('rm -rf '+shellquote(self.work_root+'/'+codec)) 166 | for binary in binaries[codec]: 167 | time.sleep(1) 168 | self.check_shell('mkdir -p '+shellquote(self.work_root+'/'+codec+'/'+os.path.dirname(binary))); 169 | time.sleep(1) 170 | if self.machine.rsync(bindir+'/'+binary,self.work_root+'/'+codec+'/'+binary) != 0: 171 | rd_print(self.log,'Couldn\'t upload codec binary '+binary+'to '+self.machine.host) 172 | raise RuntimeError('Couldn\'t upload codec binary') 173 | def start_shell(self, command): 174 | self.p.shell(['ssh','-i',ssh_privkey_file,'-p',self.machine.port,'-o',' StrictHostKeyChecking=no', self.machine.user+'@'+self.machine.host, 175 | command.encode("utf-8")]) 176 | def kill(self): 177 | kill_thread = threading.Thread(target=self.p.kill) 178 | kill_thread.daemon = True 179 | kill_thread.start() 180 | def get_file(self, remote, local): 181 | return subprocess.call(['rsync', '-s', '-e', "ssh -i "+ssh_privkey_file+" -o StrictHostKeyChecking=no -p "+str(self.machine.port), self.machine.user+'@'+self.machine.host+':'+remote, local]) 182 | def check_shell(self, command): 183 | return subprocess.check_output(['ssh','-i',ssh_privkey_file,'-p',self.machine.port,'-o',' StrictHostKeyChecking=no', 184 | self.machine.user+'@'+self.machine.host, 185 | command.encode("utf-8")]) 186 | -------------------------------------------------------------------------------- /cfg/vvc-vtm/encoder_lowdelay_vtm.cfg: -------------------------------------------------------------------------------- 1 | #======== File I/O ===================== 2 | BitstreamFile : str.bin 3 | ReconFile : rec.yuv 4 | 5 | #======== Profile ================ 6 | Profile : auto 7 | 8 | #======== Unit definition ================ 9 | MaxCUWidth : 64 # Maximum coding unit width in pixel 10 | MaxCUHeight : 64 # Maximum coding unit height in pixel 11 | 12 | #======== Coding Structure ============= 13 | IntraPeriod : -1 # Period of I-Frame ( -1 = only first) 14 | DecodingRefreshType : 0 # Random Accesss 0:none, 1:CRA, 2:IDR, 3:Recovery Point SEI 15 | GOPSize : 8 # GOP Size (number of B slice = GOPSize-1) 16 | 17 | IntraQPOffset : -1 18 | LambdaFromQpEnable : 1 # see JCTVC-X0038 for suitable parameters for IntraQPOffset, QPoffset, QPOffsetModelOff, QPOffsetModelScale when enabled 19 | # Type POC QPoffset QPOffsetModelOff QPOffsetModelScale CbQPoffset CrQPoffset QPfactor tcOffsetDiv2 betaOffsetDiv2 CbTcOffsetDiv2 CbBetaOffsetDiv2 CrTcOffsetDiv2 CrBetaOffsetDiv2 temporal_id #ref_pics_active_L0 #ref_pics_L0 reference_pictures_L0 #ref_pics_active_L1 #ref_pics_L1 reference_pictures_L1 20 | Frame1: B 1 6 -6.5 0.2450 0 0 1.0 0 0 0 0 0 0 0 4 4 1 9 17 25 4 4 1 3 5 33 21 | Frame2: B 2 4 -6.5 0.2590 0 0 1.0 0 0 0 0 0 0 0 4 4 1 2 10 18 4 4 1 2 4 26 22 | Frame3: B 3 6 -6.5 0.2450 0 0 1.0 0 0 0 0 0 0 0 4 4 1 3 11 19 4 4 1 3 5 27 23 | Frame4: B 4 4 -6.5 0.2590 0 0 1.0 0 0 0 0 0 0 0 4 4 1 4 12 20 4 4 1 2 4 28 24 | Frame5: B 5 6 -6.5 0.2450 0 0 1.0 0 0 0 0 0 0 0 4 4 1 5 13 21 4 4 1 3 5 29 25 | Frame6: B 6 4 -6.5 0.2590 0 0 1.0 0 0 0 0 0 0 0 4 4 1 6 14 22 4 4 1 2 6 30 26 | Frame7: B 7 6 -6.5 0.2450 0 0 1.0 0 0 0 0 0 0 0 4 4 1 7 15 23 4 4 1 3 7 31 27 | Frame8: B 8 1 0.0 0.0 0 0 1.0 0 0 0 0 0 0 0 4 4 1 8 16 24 4 4 1 2 4 32 28 | 29 | #=========== Motion Search ============= 30 | FastSearch : 1 # 0:Full search 1:TZ search 31 | SearchRange : 64 # (0: Search range is a Full frame) 32 | BipredSearchRange : 4 # Search range for bi-prediction refinement 33 | HadamardME : 1 # Use of hadamard measure for fractional ME 34 | FEN : 1 # Fast encoder decision 35 | FDM : 1 # Fast Decision for Merge RD cost 36 | 37 | #======== Quantization ============= 38 | QP : 32 # Quantization parameter(0-51) 39 | MaxDeltaQP : 0 # CU-based multi-QP optimization 40 | MaxCuDQPSubdiv : 0 # Maximum subdiv for CU luma Qp adjustment 41 | DeltaQpRD : 0 # Slice-based multi-QP optimization 42 | RDOQ : 1 # RDOQ 43 | RDOQTS : 1 # RDOQ for transform skip 44 | 45 | #=========== Deblock Filter ============ 46 | DeblockingFilterOffsetInPPS : 1 # Dbl params: 0=varying params in SliceHeader, param = base_param + GOP_offset_param; 1 (default) =constant params in PPS, param = base_param) 47 | DeblockingFilterDisable : 0 # Disable deblocking filter (0=Filter, 1=No Filter) 48 | DeblockingFilterBetaOffset_div2 : -2 # base_param: -12 ~ 12 49 | DeblockingFilterTcOffset_div2 : 0 # base_param: -12 ~ 12 50 | DeblockingFilterCbBetaOffset_div2 : -2 # base_param: -12 ~ 12 51 | DeblockingFilterCbTcOffset_div2 : 0 # base_param: -12 ~ 12 52 | DeblockingFilterCrBetaOffset_div2 : -2 # base_param: -12 ~ 12 53 | DeblockingFilterCrTcOffset_div2 : 0 # base_param: -12 ~ 12 54 | DeblockingFilterMetric : 0 # blockiness metric (automatically configures deblocking parameters in bitstream). Applies slice-level loop filter offsets (DeblockingFilterOffsetInPPS and DeblockingFilterDisable must be 0) 55 | 56 | #=========== Misc. ============ 57 | InternalBitDepth : 10 # codec operating bit-depth 58 | 59 | #=========== Coding Tools ================= 60 | SAO : 1 # Sample adaptive offset (0: OFF, 1: ON) 61 | TransformSkip : 1 # Transform skipping (0: OFF, 1: ON) 62 | TransformSkipFast : 1 # Fast Transform skipping (0: OFF, 1: ON) 63 | TransformSkipLog2MaxSize : 5 64 | SAOLcuBoundary : 0 # SAOLcuBoundary using non-deblocked pixels (0: OFF, 1: ON) 65 | 66 | #=========== TemporalFilter ================= 67 | TemporalFilter : 1 68 | TemporalFilterPastRefs : 4 # Number of past references for temporal prefilter 69 | TemporalFilterFutureRefs : 0 # Number of future references for temporal prefilter 70 | TemporalFilterStrengthFrame8 : 0.2 # Enable filter at every 8th frame with strength 71 | 72 | #============ Rate Control ====================== 73 | RateControl : 0 # Rate control: enable rate control 74 | TargetBitrate : 1000000 # Rate control: target bitrate, in bps 75 | KeepHierarchicalBit : 2 # Rate control: 0: equal bit allocation; 1: fixed ratio bit allocation; 2: adaptive ratio bit allocation 76 | LCULevelRateControl : 1 # Rate control: 1: LCU level RC; 0: picture level RC 77 | RCLCUSeparateModel : 1 # Rate control: use LCU level separate R-lambda model 78 | InitialQP : 0 # Rate control: initial QP 79 | RCForceIntraQP : 0 # Rate control: force intra QP to be equal to initial QP 80 | 81 | #============ VTM settings ====================== 82 | SEIDecodedPictureHash : 0 83 | CbQpOffset : 0 84 | CrQpOffset : 0 85 | SameCQPTablesForAllChroma : 1 86 | QpInValCb : 17 22 34 42 87 | QpOutValCb : 17 23 35 39 88 | ReWriteParamSets : 1 89 | #============ NEXT ==================== 90 | 91 | # General 92 | CTUSize : 128 93 | LCTUFast : 1 94 | 95 | DualITree : 1 # separate partitioning of luma and chroma channels for I-slices 96 | MinQTLumaISlice : 8 97 | MinQTChromaISliceInChromaSamples: 4 # minimum QT size in chroma samples for chroma separate tree 98 | MinQTNonISlice : 8 99 | MaxMTTHierarchyDepth : 3 100 | MaxMTTHierarchyDepthISliceL : 3 101 | MaxMTTHierarchyDepthISliceC : 3 102 | 103 | MTS : 1 104 | MTSIntraMaxCand : 3 105 | MTSInterMaxCand : 4 106 | SBT : 1 107 | ISP : 1 108 | MMVD : 1 109 | Affine : 1 110 | SbTMVP : 1 111 | MaxNumMergeCand : 6 112 | LMChroma : 1 # use CCLM only 113 | DepQuant : 1 114 | IMV : 1 115 | ALF : 1 116 | BCW : 1 117 | BcwFast : 1 118 | CIIP : 1 119 | Geo : 1 120 | IBC : 0 # turned off in CTC 121 | AllowDisFracMMVD : 1 122 | AffineAmvr : 0 123 | LMCSEnable : 1 # LMCS: 0: disable, 1:enable 124 | LMCSSignalType : 0 # Input signal type: 0:SDR, 1:HDR-PQ, 2:HDR-HLG 125 | LMCSUpdateCtrl : 2 # LMCS model update control: 0:RA, 1:AI, 2:LDB/LDP 126 | LMCSOffset : 1 # chroma residual scaling offset 127 | MRL : 1 128 | MIP : 0 129 | JointCbCr : 1 # joint coding of chroma residuals (if available): 0: disable, 1: enable 130 | PROF : 1 131 | ChromaTS : 1 132 | 133 | # Fast tools 134 | PBIntraFast : 1 135 | ISPFast : 0 136 | FastMrg : 1 137 | AMaxBT : 1 138 | FastMIP : 0 139 | FastLocalDualTreeMode : 2 140 | 141 | # Encoder optimization tools 142 | AffineAmvrEncOpt : 0 143 | MmvdDisNum : 6 144 | ALFAllowPredefinedFilters : 1 145 | ALFStrengthTargetLuma : 1.0 146 | ALFStrengthTargetChroma : 1.0 147 | CCALFStrengthTarget : 1.0 148 | EncDbOpt : 1 # apply deblocking in RDO 149 | ### DO NOT ADD ANYTHING BELOW THIS LINE ### 150 | ### DO NOT DELETE THE EMPTY LINE BELOW ### 151 | 152 | 153 | 154 | -------------------------------------------------------------------------------- /cfg/vvc-vtm/encoder_randomaccess_vtm.cfg: -------------------------------------------------------------------------------- 1 | #======== File I/O ===================== 2 | BitstreamFile : str.bin 3 | ReconFile : rec.yuv 4 | 5 | #======== Profile ================ 6 | Profile : auto 7 | 8 | #======== Unit definition ================ 9 | MaxCUWidth : 64 # Maximum coding unit width in pixel 10 | MaxCUHeight : 64 # Maximum coding unit height in pixel 11 | 12 | #======== Coding Structure ============= 13 | IntraPeriod : 32 # Period of I-Frame ( -1 = only first) 14 | DecodingRefreshType : 1 # Random Accesss 0:none, 1:CRA, 2:IDR, 3:Recovery Point SEI 15 | GOPSize : 32 # GOP Size (number of B slice = GOPSize-1) 16 | 17 | IntraQPOffset : -3 18 | LambdaFromQpEnable : 1 # see JCTVC-X0038 for suitable parameters for IntraQPOffset, QPoffset, QPOffsetModelOff, QPOffsetModelScale when enabled 19 | # Type POC QPoffset QPOffsetModelOff QPOffsetModelScale CbQPoffset CrQPoffset QPfactor tcOffsetDiv2 betaOffsetDiv2 CbTcOffsetDiv2 CbBetaOffsetDiv2 CrTcOffsetDiv2 CrBetaOffsetDiv2 temporal_id #ref_pics_active_L0 #ref_pics_L0 reference_pictures_L0 #ref_pics_active_L1 #ref_pics_L1 reference_pictures_L1 20 | Frame1 : B 32 -1 0.0 0.0 0 0 1.0 0 0 0 0 0 0 0 2 5 32 64 48 40 36 1 2 32 48 21 | Frame2 : B 16 0 -4.9309 0.2265 0 0 1.0 0 0 0 0 0 0 1 3 5 16 32 48 24 20 1 1 -16 22 | Frame3 : B 8 1 -4.5000 0.1900 0 0 1.0 0 0 0 0 0 0 2 4 5 8 24 16 40 12 2 2 -8 -24 23 | Frame4 : B 4 3 -5.4095 0.2571 0 0 1.0 0 0 0 0 0 0 3 3 3 4 8 20 3 3 -4 -12 -28 24 | Frame5 : B 2 5 -4.4895 0.1947 0 0 1.0 0 0 0 0 0 0 4 3 3 2 6 18 4 4 -2 -6 -14 -30 25 | Frame6 : B 1 6 -5.4429 0.2429 0 0 1.0 0 0 0 0 0 0 5 1 1 1 2 5 -1 -3 -7 -15 -31 26 | Frame7 : B 3 6 -5.4429 0.2429 0 0 1.0 0 0 0 0 0 0 5 2 2 1 3 2 4 -1 -5 -13 -29 27 | Frame8 : B 6 5 -4.4895 0.1947 0 0 1.0 0 0 0 0 0 0 4 3 3 2 4 6 3 3 -2 -10 -26 28 | Frame9 : B 5 6 -5.4429 0.2429 0 0 1.0 0 0 0 0 0 0 5 2 2 1 5 2 4 -1 -3 -11 -27 29 | Frame10 : B 7 6 -5.4429 0.2429 0 0 1.0 0 0 0 0 0 0 5 2 3 1 3 7 2 3 -1 -9 -25 30 | Frame11 : B 12 3 -5.4095 0.2571 0 0 1.0 0 0 0 0 0 0 3 3 4 4 8 12 6 2 2 -4 -20 31 | Frame12 : B 10 5 -4.4895 0.1947 0 0 1.0 0 0 0 0 0 0 4 4 4 2 4 6 10 3 3 -2 -6 -22 32 | Frame13 : B 9 6 -5.4429 0.2429 0 0 1.0 0 0 0 0 0 0 5 2 3 1 5 9 2 4 -1 -3 -7 -23 33 | Frame14 : B 11 6 -5.4429 0.2429 0 0 1.0 0 0 0 0 0 0 5 2 3 1 3 11 2 3 -1 -5 -21 34 | Frame15 : B 14 5 -4.4895 0.1947 0 0 1.0 0 0 0 0 0 0 4 4 4 2 4 6 14 2 2 -2 -18 35 | Frame16 : B 13 6 -5.4429 0.2429 0 0 1.0 0 0 0 0 0 0 5 2 3 1 5 13 2 3 -1 -3 -19 36 | Frame17 : B 15 6 -5.4429 0.2429 0 0 1.0 0 0 0 0 0 0 5 2 4 1 3 7 15 2 2 -1 -17 37 | Frame18 : B 24 1 -4.5000 0.1900 0 0 1.0 0 0 0 0 0 0 2 3 3 8 16 24 1 1 -8 38 | Frame19 : B 20 3 -5.4095 0.2571 0 0 1.0 0 0 0 0 0 0 3 3 3 4 12 20 2 2 -4 -12 39 | Frame20 : B 18 5 -4.4895 0.1947 0 0 1.0 0 0 0 0 0 0 4 3 3 2 10 18 3 3 -2 -6 -14 40 | Frame21 : B 17 6 -5.4429 0.2429 0 0 1.0 0 0 0 0 0 0 5 2 3 1 9 17 2 4 -1 -3 -7 -15 41 | Frame22 : B 19 6 -5.4429 0.2429 0 0 1.0 0 0 0 0 0 0 5 2 3 1 3 19 2 3 -1 -5 -13 42 | Frame23 : B 22 5 -4.4895 0.1947 0 0 1.0 0 0 0 0 0 0 4 3 3 2 6 22 3 3 -2 -10 4 43 | Frame24 : B 21 6 -5.4429 0.2429 0 0 1.0 0 0 0 0 0 0 5 2 3 1 5 21 2 3 -1 -3 -11 44 | Frame25 : B 23 6 -5.4429 0.2429 0 0 1.0 0 0 0 0 0 0 5 2 4 1 3 7 23 2 2 -1 -9 45 | Frame26 : B 28 3 -5.4095 0.2571 0 0 1.0 0 0 0 0 0 0 3 4 4 4 8 12 28 1 1 -4 46 | Frame27 : B 26 5 -4.4895 0.1947 0 0 1.0 0 0 0 0 0 0 4 4 4 2 6 10 26 2 2 -2 -6 47 | Frame28 : B 25 6 -5.4429 0.2429 0 0 1.0 0 0 0 0 0 0 5 2 4 1 5 9 25 2 3 -1 -3 -7 48 | Frame29 : B 27 6 -5.4429 0.2429 0 0 1.0 0 0 0 0 0 0 5 2 4 1 3 11 27 2 2 -1 -5 49 | Frame30 : B 30 5 -4.4895 0.1947 0 0 1.0 0 0 0 0 0 0 4 4 4 2 6 14 30 1 1 -2 50 | Frame31 : B 29 6 -5.4429 0.2429 0 0 1.0 0 0 0 0 0 0 5 2 4 1 5 13 29 2 2 -1 -3 51 | Frame32 : B 31 6 -5.4429 0.2429 0 0 1.0 0 0 0 0 0 0 5 2 5 1 3 7 15 31 1 1 -1 52 | 53 | #=========== Motion Search ============= 54 | FastSearch : 1 # 0:Full search 1:TZ search 55 | SearchRange : 384 # (0: Search range is a Full frame) 56 | ASR : 1 # Adaptive motion search range 57 | MinSearchWindow : 96 # Minimum motion search window size for the adaptive window ME 58 | BipredSearchRange : 4 # Search range for bi-prediction refinement 59 | HadamardME : 1 # Use of hadamard measure for fractional ME 60 | FEN : 1 # Fast encoder decision 61 | FDM : 1 # Fast Decision for Merge RD cost 62 | 63 | #======== Quantization ============= 64 | QP : 32 # Quantization parameter(0-51) 65 | MaxDeltaQP : 0 # CU-based multi-QP optimization 66 | MaxCuDQPSubdiv : 0 # Maximum subdiv for CU luma Qp adjustment 67 | DeltaQpRD : 0 # Slice-based multi-QP optimization 68 | RDOQ : 1 # RDOQ 69 | RDOQTS : 1 # RDOQ for transform skip 70 | 71 | #=========== Deblock Filter ============ 72 | DeblockingFilterOffsetInPPS : 0 # Dbl params: 0=varying params in SliceHeader, param = base_param + GOP_offset_param; 1 (default) =constant params in PPS, param = base_param) 73 | DeblockingFilterDisable : 0 # Disable deblocking filter (0=Filter, 1=No Filter) 74 | DeblockingFilterBetaOffset_div2 : -2 # base_param: -12 ~ 12 75 | DeblockingFilterTcOffset_div2 : 0 # base_param: -12 ~ 12 76 | DeblockingFilterCbBetaOffset_div2 : -2 # base_param: -12 ~ 12 77 | DeblockingFilterCbTcOffset_div2 : 0 # base_param: -12 ~ 12 78 | DeblockingFilterCrBetaOffset_div2 : -2 # base_param: -12 ~ 12 79 | DeblockingFilterCrTcOffset_div2 : 0 # base_param: -12 ~ 12 80 | DeblockingFilterMetric : 0 # blockiness metric (automatically configures deblocking parameters in bitstream). Applies slice-level loop filter offsets (DeblockingFilterOffsetInPPS and DeblockingFilterDisable must be 0) 81 | 82 | #=========== Misc. ============ 83 | InternalBitDepth : 10 # codec operating bit-depth 84 | 85 | #=========== Coding Tools ================= 86 | SAO : 1 # Sample adaptive offset (0: OFF, 1: ON) 87 | TransformSkip : 1 # Transform skipping (0: OFF, 1: ON) 88 | TransformSkipFast : 1 # Fast Transform skipping (0: OFF, 1: ON) 89 | TransformSkipLog2MaxSize : 5 90 | SAOLcuBoundary : 0 # SAOLcuBoundary using non-deblocked pixels (0: OFF, 1: ON) 91 | 92 | #============ Rate Control ====================== 93 | RateControl : 0 # Rate control: enable rate control 94 | TargetBitrate : 1000000 # Rate control: target bitrate, in bps 95 | KeepHierarchicalBit : 2 # Rate control: 0: equal bit allocation; 1: fixed ratio bit allocation; 2: adaptive ratio bit allocation 96 | LCULevelRateControl : 1 # Rate control: 1: LCU level RC; 0: picture level RC 97 | RCLCUSeparateModel : 1 # Rate control: use LCU level separate R-lambda model 98 | InitialQP : 0 # Rate control: initial QP 99 | RCForceIntraQP : 0 # Rate control: force intra QP to be equal to initial QP 100 | 101 | #============ VTM settings ====================== 102 | SEIDecodedPictureHash : 0 103 | CbQpOffset : 0 104 | CrQpOffset : 0 105 | SameCQPTablesForAllChroma : 1 106 | QpInValCb : 17 22 34 42 107 | QpOutValCb : 17 23 35 39 108 | ReWriteParamSets : 1 109 | #============ NEXT ==================== 110 | 111 | # General 112 | CTUSize : 128 113 | LCTUFast : 1 114 | 115 | DualITree : 1 # separate partitioning of luma and chroma channels for I-slices 116 | MinQTLumaISlice : 8 117 | MinQTChromaISliceInChromaSamples: 4 # minimum QT size in chroma samples for chroma separate tree 118 | MinQTNonISlice : 8 119 | MaxMTTHierarchyDepth : 3 120 | MaxMTTHierarchyDepthISliceL : 3 121 | MaxMTTHierarchyDepthISliceC : 3 122 | 123 | MTS : 1 124 | MTSIntraMaxCand : 4 125 | MTSInterMaxCand : 4 126 | SBT : 1 127 | LFNST : 1 128 | ISP : 1 129 | MMVD : 1 130 | Affine : 1 131 | SbTMVP : 1 132 | MaxNumMergeCand : 6 133 | LMChroma : 1 # use CCLM only 134 | DepQuant : 1 135 | IMV : 1 136 | ALF : 1 137 | BCW : 1 138 | BcwFast : 1 139 | BIO : 1 140 | CIIP : 1 141 | Geo : 1 142 | IBC : 0 # turned off in CTC 143 | AllowDisFracMMVD : 1 144 | AffineAmvr : 1 145 | LMCSEnable : 1 # LMCS: 0: disable, 1:enable 146 | LMCSSignalType : 0 # Input signal type: 0:SDR, 1:HDR-PQ, 2:HDR-HLG 147 | LMCSUpdateCtrl : 0 # LMCS model update control: 0:RA, 1:AI, 2:LDB/LDP 148 | LMCSOffset : 6 # chroma residual scaling offset 149 | MRL : 1 150 | MIP : 1 151 | DMVR : 1 152 | SMVD : 1 153 | JointCbCr : 1 # joint coding of chroma residuals (if available): 0: disable, 1: enable 154 | PROF : 1 155 | 156 | # Fast tools 157 | PBIntraFast : 1 158 | ISPFast : 0 159 | FastMrg : 1 160 | AMaxBT : 1 161 | FastMIP : 0 162 | FastLFNST : 0 163 | FastLocalDualTreeMode : 1 164 | ChromaTS : 1 165 | 166 | # Encoder optimization tools 167 | AffineAmvrEncOpt : 1 168 | MmvdDisNum : 6 169 | ALFAllowPredefinedFilters : 1 170 | ALFStrengthTargetLuma : 1.0 171 | ALFStrengthTargetChroma : 1.0 172 | CCALFStrengthTarget : 1.0 173 | EncDbOpt : 1 # apply deblocking in RDO 174 | 175 | TemporalFilter : 1 176 | TemporalFilterPastRefs : 4 # Number of past references for temporal prefilter 177 | TemporalFilterFutureRefs : 4 # Number of future references for temporal prefilter 178 | TemporalFilterStrengthFrame8 : 0.95 # Enable filter at every 8th frame with given strength 179 | TemporalFilterStrengthFrame16 : 1.5 # Enable filter at every 16th frame with given strength, longer intervals has higher priority 180 | ### DO NOT ADD ANYTHING BELOW THIS LINE ### 181 | ### DO NOT DELETE THE EMPTY LINE BELOW ### 182 | 183 | 184 | 185 | -------------------------------------------------------------------------------- /cfg/vvc-vtm/encoder_randomaccess_vtm_gop16.cfg: -------------------------------------------------------------------------------- 1 | #======== File I/O ===================== 2 | BitstreamFile : str.bin 3 | ReconFile : rec.yuv 4 | 5 | #======== Profile ================ 6 | Profile : auto 7 | 8 | #======== Unit definition ================ 9 | MaxCUWidth : 64 # Maximum coding unit width in pixel 10 | MaxCUHeight : 64 # Maximum coding unit height in pixel 11 | 12 | #======== Coding Structure ============= 13 | IntraPeriod : 32 # Period of I-Frame ( -1 = only first) 14 | DecodingRefreshType : 1 # Random Accesss 0:none, 1:CRA, 2:IDR, 3:Recovery Point SEI 15 | GOPSize : 16 # GOP Size (number of B slice = GOPSize-1) 16 | 17 | IntraQPOffset : -3 18 | LambdaFromQpEnable : 1 # see JCTVC-X0038 for suitable parameters for IntraQPOffset, QPoffset, QPOffsetModelOff, QPOffsetModelScale when enabled 19 | # Type POC QPoffset QPOffsetModelOff QPOffsetModelScale CbQPoffset CrQPoffset QPfactor tcOffsetDiv2 betaOffsetDiv2 CbTcOffsetDiv2 CbBetaOffsetDiv2 CrTcOffsetDiv2 CrBetaOffsetDiv2 temporal_id #ref_pics_active_L0 #ref_pics_L0 reference_pictures_L0 #ref_pics_active_L1 #ref_pics_L1 reference_pictures_L1 20 | Frame1: B 16 1 0.0 0.0 0 0 1.0 0 0 0 0 0 0 0 2 3 16 32 24 2 2 16 32 21 | Frame2: B 8 1 -4.8848 0.2061 0 0 1.0 0 0 0 0 0 0 1 2 2 8 16 2 2 -8 8 22 | Frame3: B 4 4 -5.7476 0.2286 0 0 1.0 0 0 0 0 0 0 2 2 2 4 12 2 2 -4 -12 23 | Frame4: B 2 5 -5.90 0.2333 0 0 1.0 0 0 0 0 0 0 3 2 2 2 10 2 3 -2 -6 -14 24 | Frame5: B 1 6 -7.1444 0.3 0 0 1.0 0 0 0 0 0 0 4 2 2 1 -1 2 4 -1 -3 -7 -15 25 | Frame6: B 3 6 -7.1444 0.3 0 0 1.0 0 0 0 0 0 0 4 2 2 1 3 2 3 -1 -5 -13 26 | Frame7: B 6 5 -5.90 0.2333 0 0 1.0 0 0 0 0 0 0 3 2 2 2 6 2 2 -2 -10 27 | Frame8: B 5 6 -7.1444 0.3 0 0 1.0 0 0 0 0 0 0 4 2 2 1 5 2 3 -1 -3 -11 28 | Frame9: B 7 6 -7.1444 0.3 0 0 1.0 0 0 0 0 0 0 4 2 3 1 3 7 2 2 -1 -9 29 | Frame10: B 12 4 -5.7476 0.2286 0 0 1.0 0 0 0 0 0 0 2 2 2 4 12 2 2 -4 4 30 | Frame11: B 10 5 -5.90 0.2333 0 0 1.0 0 0 0 0 0 0 3 2 2 2 10 2 2 -2 -6 31 | Frame12: B 9 6 -7.1444 0.3 0 0 1.0 0 0 0 0 0 0 4 2 2 1 9 2 3 -1 -3 -7 32 | Frame13: B 11 6 -7.1444 0.3 0 0 1.0 0 0 0 0 0 0 4 2 3 1 3 11 2 2 -1 -5 33 | Frame14: B 14 5 -5.90 0.2333 0 0 1.0 0 0 0 0 0 0 3 2 3 2 6 14 2 2 -2 2 34 | Frame15: B 13 6 -7.1444 0.3 0 0 1.0 0 0 0 0 0 0 4 2 3 1 5 13 2 2 -1 -3 35 | Frame16: B 15 6 -7.1444 0.3 0 0 1.0 0 0 0 0 0 0 4 2 4 1 3 7 15 2 2 -1 1 36 | 37 | #=========== Motion Search ============= 38 | FastSearch : 1 # 0:Full search 1:TZ search 39 | SearchRange : 384 # (0: Search range is a Full frame) 40 | ASR : 1 # Adaptive motion search range 41 | MinSearchWindow : 96 # Minimum motion search window size for the adaptive window ME 42 | BipredSearchRange : 4 # Search range for bi-prediction refinement 43 | HadamardME : 1 # Use of hadamard measure for fractional ME 44 | FEN : 1 # Fast encoder decision 45 | FDM : 1 # Fast Decision for Merge RD cost 46 | 47 | #======== Quantization ============= 48 | QP : 32 # Quantization parameter(0-51) 49 | MaxDeltaQP : 0 # CU-based multi-QP optimization 50 | MaxCuDQPSubdiv : 0 # Maximum subdiv for CU luma Qp adjustment 51 | DeltaQpRD : 0 # Slice-based multi-QP optimization 52 | RDOQ : 1 # RDOQ 53 | RDOQTS : 1 # RDOQ for transform skip 54 | 55 | #=========== Deblock Filter ============ 56 | DeblockingFilterOffsetInPPS : 0 # Dbl params: 0=varying params in SliceHeader, param = base_param + GOP_offset_param; 1 (default) =constant params in PPS, param = base_param) 57 | DeblockingFilterDisable : 0 # Disable deblocking filter (0=Filter, 1=No Filter) 58 | DeblockingFilterBetaOffset_div2 : -2 # base_param: -12 ~ 12 59 | DeblockingFilterTcOffset_div2 : 0 # base_param: -12 ~ 12 60 | DeblockingFilterCbBetaOffset_div2 : -2 # base_param: -12 ~ 12 61 | DeblockingFilterCbTcOffset_div2 : 0 # base_param: -12 ~ 12 62 | DeblockingFilterCrBetaOffset_div2 : -2 # base_param: -12 ~ 12 63 | DeblockingFilterCrTcOffset_div2 : 0 # base_param: -12 ~ 12 64 | DeblockingFilterMetric : 0 # blockiness metric (automatically configures deblocking parameters in bitstream). Applies slice-level loop filter offsets (DeblockingFilterOffsetInPPS and DeblockingFilterDisable must be 0) 65 | 66 | #=========== Misc. ============ 67 | InternalBitDepth : 10 # codec operating bit-depth 68 | 69 | #=========== Coding Tools ================= 70 | SAO : 1 # Sample adaptive offset (0: OFF, 1: ON) 71 | TransformSkip : 1 # Transform skipping (0: OFF, 1: ON) 72 | TransformSkipFast : 1 # Fast Transform skipping (0: OFF, 1: ON) 73 | TransformSkipLog2MaxSize : 5 74 | SAOLcuBoundary : 0 # SAOLcuBoundary using non-deblocked pixels (0: OFF, 1: ON) 75 | 76 | #============ Rate Control ====================== 77 | RateControl : 0 # Rate control: enable rate control 78 | TargetBitrate : 1000000 # Rate control: target bitrate, in bps 79 | KeepHierarchicalBit : 2 # Rate control: 0: equal bit allocation; 1: fixed ratio bit allocation; 2: adaptive ratio bit allocation 80 | LCULevelRateControl : 1 # Rate control: 1: LCU level RC; 0: picture level RC 81 | RCLCUSeparateModel : 1 # Rate control: use LCU level separate R-lambda model 82 | InitialQP : 0 # Rate control: initial QP 83 | RCForceIntraQP : 0 # Rate control: force intra QP to be equal to initial QP 84 | 85 | #============ VTM settings ====================== 86 | SEIDecodedPictureHash : 0 87 | CbQpOffset : 0 88 | CrQpOffset : 0 89 | SameCQPTablesForAllChroma : 1 90 | QpInValCb : 17 22 34 42 91 | QpOutValCb : 17 23 35 39 92 | ReWriteParamSets : 1 93 | #============ NEXT ==================== 94 | 95 | # General 96 | CTUSize : 128 97 | LCTUFast : 1 98 | 99 | DualITree : 1 # separate partitioning of luma and chroma channels for I-slices 100 | MinQTLumaISlice : 8 101 | MinQTChromaISliceInChromaSamples: 4 # minimum QT size in chroma samples for chroma separate tree 102 | MinQTNonISlice : 8 103 | MaxMTTHierarchyDepth : 3 104 | MaxMTTHierarchyDepthISliceL : 3 105 | MaxMTTHierarchyDepthISliceC : 3 106 | 107 | MTS : 1 108 | MTSIntraMaxCand : 4 109 | MTSInterMaxCand : 4 110 | SBT : 1 111 | LFNST : 1 112 | ISP : 1 113 | MMVD : 1 114 | Affine : 1 115 | SbTMVP : 1 116 | MaxNumMergeCand : 6 117 | LMChroma : 1 # use CCLM only 118 | DepQuant : 1 119 | IMV : 1 120 | ALF : 1 121 | BCW : 1 122 | BcwFast : 1 123 | BIO : 1 124 | CIIP : 1 125 | Geo : 1 126 | IBC : 0 # turned off in CTC 127 | AllowDisFracMMVD : 1 128 | AffineAmvr : 1 129 | LMCSEnable : 1 # LMCS: 0: disable, 1:enable 130 | LMCSSignalType : 0 # Input signal type: 0:SDR, 1:HDR-PQ, 2:HDR-HLG 131 | LMCSUpdateCtrl : 0 # LMCS model update control: 0:RA, 1:AI, 2:LDB/LDP 132 | LMCSOffset : 6 # chroma residual scaling offset 133 | MRL : 1 134 | MIP : 1 135 | DMVR : 1 136 | SMVD : 1 137 | JointCbCr : 1 # joint coding of chroma residuals (if available): 0: disable, 1: enable 138 | PROF : 1 139 | 140 | # Fast tools 141 | PBIntraFast : 1 142 | ISPFast : 0 143 | FastMrg : 1 144 | AMaxBT : 1 145 | FastMIP : 0 146 | FastLFNST : 0 147 | FastLocalDualTreeMode : 1 148 | ChromaTS : 1 149 | 150 | # Encoder optimization tools 151 | AffineAmvrEncOpt : 1 152 | MmvdDisNum : 6 153 | ALFAllowPredefinedFilters : 1 154 | ALFStrengthTargetLuma : 1.0 155 | ALFStrengthTargetChroma : 1.0 156 | CCALFStrengthTarget : 1.0 157 | EncDbOpt : 1 # apply deblocking in RDO 158 | 159 | TemporalFilter : 1 160 | TemporalFilterPastRefs : 4 # Number of past references for temporal prefilter 161 | TemporalFilterFutureRefs : 4 # Number of future references for temporal prefilter 162 | TemporalFilterStrengthFrame8 : 0.95 # Enable filter at every 8th frame with given strength 163 | TemporalFilterStrengthFrame16 : 1.5 # Enable filter at every 16th frame with given strength, longer intervals has higher priority 164 | ### DO NOT ADD ANYTHING BELOW THIS LINE ### 165 | ### DO NOT DELETE THE EMPTY LINE BELOW ### 166 | 167 | 168 | 169 | -------------------------------------------------------------------------------- /work.py: -------------------------------------------------------------------------------- 1 | from utility import * 2 | import os 3 | import subprocess 4 | import sys 5 | import xml.etree.ElementTree as ET 6 | import time 7 | import re 8 | 9 | runs_dst_dir = os.getenv("RUNS_DST_DIR", os.path.join(os.getcwd(), "../runs")) 10 | media_src_dir = os.getenv("MEDIAS_SRC_DIR", os.path.join(os.getcwd(), "/media")) 11 | 12 | # Finding files such as `this_(that)` requires `'` be placed on both 13 | # sides of the quote so the `()` are both captured. Files such as 14 | # `du_Parterre_d'Eau` must be converted into 15 | #`'du_Parterre_d'"'"'Eau' 16 | # ^^^ Required to make sure the `'` is captured. 17 | def shellquote(s): 18 | return "'" + s.replace("'", "'\"'\"'") + "'" 19 | 20 | #set up Codec:QualityRange dictionary 21 | quality_presets = { 22 | "daala": [7,11,16,25,37], 23 | "x264": list(range(1,52,5)), 24 | "x265": list(range(5,52,5)), 25 | "x265-rt": list(range(5,52,5)), 26 | "xvc": [20,25,30,35,40], 27 | "vp8": list(range(12,64,5)), 28 | "vp9": [20,32,43,55,63], 29 | "vp9-rt": [20,32,43,55,63], 30 | "vp10": [8,20,32,43,55,63], 31 | "vp10-rt": [8,20,32,43,55,63], 32 | "av1": [20,32,43,55,63], 33 | "av1-rt": [20,32,43,55,63], 34 | "av2-ai": [85, 110, 135, 160, 185, 210], 35 | "av2-ra": [110, 135, 160, 185, 210, 235], 36 | "av2-ra-st": [110, 135, 160, 185, 210, 235], 37 | "av2-ld": [110, 135, 160, 185, 210, 235], 38 | "av2-as": [110, 135, 160, 185, 210, 235], 39 | "av2-as-st": [110, 135, 160, 185, 210, 235], 40 | "av2-f" : [60, 85, 110, 135, 160, 185], 41 | "thor": list(range(7,43,3)), 42 | "thor-rt": list(range(7,43,3)), 43 | "rav1e": [20*4,32*4,43*4,55*4,63*4], 44 | "svt-av1": [27, 33, 39, 46, 52, 58], 45 | "svt-av1-ra": [27, 33, 39, 46, 52, 58], 46 | "svt-av1-ra-crf": [27, 33, 39, 46, 52, 58], 47 | "svt-av1-ra-vbr": [250, 500, 1000, 4000, 8000, 12000], 48 | "svt-av1-ra-vbr-2p": [250, 500, 1000, 4000, 8000, 12000], 49 | "svt-av1-ld-cbr": [250, 500, 1000, 4000, 8000, 12000], 50 | "svt-av1-ra-cq": [27, 33, 39, 46, 52, 58], 51 | "svt-av1-as": [27, 33, 39, 46, 52, 58], 52 | "svt-av1-as-ctc": [23, 27, 31, 35, 39, 43, 47, 51, 55, 59, 63], 53 | "vvc-vtm": [22, 27, 32, 37], 54 | "vvc-vtm-ra": [22, 27, 32, 37], 55 | "vvc-vtm-ra-ctc": [22, 27, 32, 37, 42, 47], 56 | "vvc-vtm-as-ctc": [22, 27, 32, 37, 42, 47], 57 | "vvc-vtm-ra-st": [22, 27, 32, 37], 58 | "vvc-vtm-ld": [22, 27, 32, 37], 59 | "vvc-vtm-ai": [22, 27, 32, 37], 60 | } 61 | 62 | class Run: 63 | def __init__(self, codec): 64 | self.info = {} 65 | self.codec = codec 66 | self.quality = quality_presets[codec] 67 | self.runid = get_time() 68 | self.extra_options = '' 69 | self.save_encode = False 70 | self.work_items = [] 71 | self.prefix = './' 72 | self.log = None 73 | self.rundir = None 74 | self.status = 'running' 75 | self.work_items = [] 76 | self.cancelled = False 77 | self.nightly_run = False 78 | self.ctc_version = 7.0 79 | def write_status(self): 80 | f = open(self.rundir+'/status.txt','w') 81 | f.write(self.status) 82 | f.close() 83 | def cancel(self): 84 | self.status = 'cancelled' 85 | self.write_status() 86 | self.cancelled = True 87 | for work in self.work_items: 88 | work.cancel() 89 | def finish(self): 90 | if self.log: 91 | self.log.close() 92 | 93 | class RDRun(Run): 94 | def reduce(self): 95 | rd_print(self.log,'Logging results...') 96 | self.work_items.sort(key=lambda work: int(work.quality)) 97 | any_work_failed = False 98 | for work in self.work_items: 99 | if not work.failed: 100 | pass 101 | else: 102 | any_work_failed = True 103 | subprocess.call('OUTPUT="'+self.prefix+'/'+'total" "'+sys.path[0]+'/rd_average.sh" "'+self.prefix+'/*-daala.out"', 104 | shell=True) 105 | if any_work_failed: 106 | self.status = 'failed' 107 | else: 108 | self.status = 'completed' 109 | self.write_status() 110 | 111 | class ABRun(Run): 112 | def reduce(self): 113 | pass 114 | 115 | class Work: 116 | def __init__(self): 117 | self.run = None 118 | self.log = None 119 | self.retries = 0 120 | self.done = False 121 | self.failed = False 122 | self.runid = '' 123 | self.slot = None 124 | self.workid = '' 125 | self.nightly_run = False 126 | self.multislots = False 127 | self.thread_count = 1 128 | def cancel(self): 129 | self.failed = True 130 | self.done = True 131 | 132 | class RDWork(Work): 133 | def __init__(self): 134 | super().__init__() 135 | self.no_delete = False 136 | self.copy_back_files = ['-stdout.txt', '-enctime.out', '-dectime.out', '-encperf.out', '-decperf.out'] 137 | self.ctc_class = '' 138 | self.ctc_version = 7.0 139 | def parse(self, stdout, stderr): 140 | self.raw = stdout 141 | split = None 142 | try: 143 | split = self.raw.decode('utf-8').replace(')',' ').split(maxsplit=61) 144 | self.pixels = split[1] 145 | self.size = split[2] 146 | self.metric = {} 147 | self.metric['psnr'] = {} 148 | self.metric["psnr"][0] = split[6] 149 | self.metric["psnr"][1] = split[8] 150 | self.metric["psnr"][2] = split[10] 151 | self.metric['psnrhvs'] = {} 152 | self.metric["psnrhvs"][0] = split[14] 153 | self.metric["psnrhvs"][1] = split[16] 154 | self.metric["psnrhvs"][2] = split[18] 155 | self.metric['ssim'] = {} 156 | self.metric["ssim"][0] = split[22] 157 | self.metric["ssim"][1] = split[24] 158 | self.metric["ssim"][2] = split[26] 159 | self.metric['fastssim'] = {} 160 | self.metric["fastssim"][0] = split[30] 161 | self.metric["fastssim"][1] = split[32] 162 | self.metric["fastssim"][2] = split[34] 163 | self.metric['ciede2000'] = split[36] 164 | self.metric['apsnr'] = {} 165 | self.metric['apsnr'][0] = split[40] 166 | self.metric['apsnr'][1] = split[42] 167 | self.metric['apsnr'][2] = split[44] 168 | self.metric['msssim'] = {} 169 | self.metric['msssim'][0] = split[48] 170 | self.metric['msssim'][1] = split[50] 171 | self.metric['msssim'][2] = split[52] 172 | self.metric['encodetime'] = split[53] 173 | self.metric['vmaf_old'] = split[54] 174 | self.metric['decodetime'] = split[55] 175 | self.metric['enc_md5'] = split[56] 176 | self.metric['enc_instr_cnt'] = split[57] 177 | self.metric['enc_cycle_cnt'] = split[58] 178 | self.metric['dec_instr_cnt'] = split[59] 179 | self.metric['dec_cycle_cnt'] = split[60] 180 | self.vmaf_xml = split[61] 181 | root = ET.fromstring(self.vmaf_xml) 182 | for metric_name in ['psnr_y', 'psnr_cb', 'psnr_cr', 'ciede2000', 'float_ssim', 'float_ms_ssim', 'psnr_hvs_y', 'psnr_hvs_cb', 'psnr_hvs_cr', 'psnr_hvs', 'cambi']: 183 | self.metric['vmaf_'+metric_name] = root.find("pooled_metrics/metric[@name='"+metric_name+"']").get('mean') 184 | self.metric['vmaf'] = root.find("pooled_metrics/metric[@name='vmaf']").get('mean') 185 | self.metric['vmaf_neg'] = root.find("pooled_metrics/metric[@name='vmaf_neg']").get('mean') 186 | for metric_name in ['apsnr_y', 'apsnr_cb', 'apsnr_cr']: 187 | self.metric['vmaf_'+metric_name] = root.find("aggregate_metrics").get(metric_name) 188 | self.failed = False 189 | except (IndexError, ET.ParseError): 190 | rd_print(self.log,'Decoding result for '+self.filename+' at quality '+str(self.quality)+' failed!') 191 | rd_print(self.log,'stdout:') 192 | rd_print(self.log,stdout.decode('utf-8')) 193 | rd_print(self.log,'stderr:') 194 | rd_print(self.log,stderr.decode('utf-8')) 195 | self.failed = True 196 | def get_line(self): 197 | work = self 198 | f = '' 199 | f += (str(work.quality)+' ') 200 | f += (str(work.pixels)+' ') 201 | f += (str(work.size)+' ') 202 | f += (str(work.metric['psnr'][0])+' ') 203 | f += (str(work.metric['psnrhvs'][0])+' ') 204 | f += (str(work.metric['ssim'][0])+' ') 205 | f += (str(work.metric['fastssim'][0])+' ') 206 | f += (str(work.metric['ciede2000'])+' ') 207 | f += (str(work.metric['psnr'][1])+' ') 208 | f += (str(work.metric['psnr'][2])+' ') 209 | f += (str(work.metric['apsnr'][0])+' ') 210 | f += (str(work.metric['apsnr'][1])+' ') 211 | f += (str(work.metric['apsnr'][2])+' ') 212 | f += (str(work.metric['msssim'][0])+' ') 213 | f += (str(work.metric['encodetime'])+' ') 214 | f += (str(work.metric['vmaf_old'])+' ') 215 | f += (str(work.metric['decodetime'])+' ') 216 | f += (str(work.metric['vmaf_psnr_y'])+' ') 217 | f += (str(work.metric['vmaf_psnr_cb'])+' ') 218 | f += (str(work.metric['vmaf_psnr_cr'])+' ') 219 | f += (str(work.metric['vmaf_ciede2000'])+' ') 220 | f += (str(work.metric['vmaf_float_ssim'])+' ') 221 | f += (str(work.metric['vmaf_float_ms_ssim'])+' ') 222 | f += (str(work.metric['vmaf_psnr_hvs_y'])+' ') 223 | f += (str(work.metric['vmaf_psnr_hvs_cb'])+' ') 224 | f += (str(work.metric['vmaf_psnr_hvs_cr'])+' ') 225 | f += (str(work.metric['vmaf_psnr_hvs'])+' ') 226 | f += (str(work.metric['vmaf'])+' ') 227 | f += (str(work.metric['vmaf_neg'])+' ') 228 | f += (str(work.metric['vmaf_apsnr_y'])+' ') 229 | f += (str(work.metric['vmaf_apsnr_cb'])+' ') 230 | f += (str(work.metric['vmaf_apsnr_cr'])+' ') 231 | f += (str(work.metric['vmaf_cambi'])+' ') 232 | f += (str(work.metric['enc_md5'])+' ') 233 | f += (str(work.metric['enc_instr_cnt'])+' ') 234 | f += (str(work.metric['enc_cycle_cnt'])+' ') 235 | f += (str(work.metric['dec_instr_cnt'])+' ') 236 | f += (str(work.metric['dec_cycle_cnt'])+' ') 237 | f += ('\n') 238 | return f 239 | def execute(self): 240 | try: 241 | slot = self.slot 242 | slot.setup(self.codec,self.bindir) 243 | work = self 244 | # Unique identifer for a given work (encoder invocation) 245 | work.workid = '_'.join([str(work.filename), str(self.quality), str(self.codec), str(self.runid)]) 246 | input_path = slot.machine.media_path+'/'+work.set+'/'+work.filename 247 | daalatool_dir = os.getenv("DAALATOOL_DIR", os.path.join(slot.machine.work_root, "daalatool")) 248 | command = 'WORK_ROOT="'+slot.work_root+'" ' 249 | command += 'DAALATOOL_ROOT="'+daalatool_dir+'"' 250 | command += ' x="'+str(work.quality) + '" ' 251 | command += 'CODEC="'+work.codec+'" ' 252 | command += 'CTC_CLASS="'+work.ctc_class+'" ' 253 | command += 'CTC_VERSION="'+str(work.ctc_version)+'" ' 254 | command += 'EXTRA_OPTIONS="'+work.extra_options + '" ' 255 | if self.no_delete: 256 | command += 'NO_DELETE=1 ' 257 | command += slot.work_root + '/rd_tool/metrics_gather.sh '+shellquote(input_path) 258 | slot.start_shell(command) 259 | (stdout, stderr) = slot.gather() 260 | # We need some buffer before we try to copy 261 | time.sleep(2) 262 | for file in self.copy_back_files: 263 | dest_file_path = runs_dst_dir+'/'+work.runid+'/'+work.set+'/' 264 | if self.multicfg: 265 | dest_file_path = runs_dst_dir+'/'+work.runid+'/'+self.codec+'/'+ self.set+'/' 266 | if slot.get_file(slot.work_root+'/'+work.filename+'-'+str(work.quality)+file, dest_file_path) != 0: 267 | rd_print(self.log,'Failed to copy back '+work.filename+'-'+str(work.quality)+file+', continuing anyway') 268 | self.parse(stdout, stderr) 269 | except Exception as e: 270 | rd_print(self.log, 'Exception while running',self.get_name(),e) 271 | self.failed = True 272 | def write_results(self): 273 | base_filename = runs_dst_dir+'/'+self.runid+'/'+self.set+'/'+self.filename 274 | if self.multicfg: 275 | base_filename = runs_dst_dir+'/'+self.runid + \ 276 | '/'+self.codec+'/'+self.set+'/'+self.filename 277 | filename = (base_filename+'-daala.out').encode('utf-8') 278 | try: 279 | with open(filename,'r') as f: 280 | lines = f.readlines() 281 | except IOError: 282 | lines = [] 283 | new_line = self.get_line() 284 | for line in lines: 285 | if new_line.split()[0] == line.split()[0]: 286 | rd_print(self.log, 'Data already exists in out file, not writing',self.get_name()) 287 | return 288 | lines.append(new_line) 289 | lines.sort(key=lambda x: int(x.split()[0])) 290 | with open(filename,'w') as f: 291 | for line in lines: 292 | f.write(line) 293 | #write vmaf xml in separate files 294 | xml_filename = (base_filename+'-'+str(self.quality)+'-libvmaf.xml').encode('utf-8') 295 | with open(xml_filename, 'w') as f: 296 | f.write(self.vmaf_xml) 297 | def get_name(self): 298 | if len(self.ctc_class) > 0: 299 | return self.filename + ' with quality ' + str(self.quality) + ' of '+ str(self.ctc_class) + ' with config ' + str(self.codec) + ' for run ' + self.runid 300 | else: 301 | return self.filename + ' with quality ' + str(self.quality) + ' for run ' + self.runid 302 | def cancel(self): 303 | if self.slot: 304 | self.slot.kill() 305 | self.failed = True 306 | elif not self.done: 307 | self.failed = True 308 | self.done = True 309 | 310 | def get_thread_count(work): 311 | thread_count = 1 312 | if work.run.ctc_version >= 7.0: 313 | if work.codec in ['av2-ra', 'av2-ra-st']: 314 | if work.set in ["aomctc-a1-4k", "aomctc-e-nonpristine", "aomctc-g1-hdr-4k" ]: 315 | thread_count = 4 316 | elif work.set in ['aomctc-a2-2k', 'aomctc-b1-syn']: 317 | thread_count = 2 318 | if work.codec in ['av2-ld']: 319 | if work.set in ['aomctc-a2-2k', 'aomctc-b1-syn']: 320 | thread_count = 8 321 | elif work.set in ['aomctc-a3-720p']: 322 | thread_count = 2 323 | if work.codec in ['av2-as', 'av2-as-st']: 324 | target_resolution = str(work.width) + 'x' + str(work.height) 325 | if target_resolution in ['3840x2160', '2560x1440']: 326 | thread_count = 4 327 | elif target_resolution in ['1920x1080']: 328 | thread_count = 2 329 | if work.run.ctc_version == 6.0: 330 | if work.codec in ['av2-ra', 'av2-ra-st']: 331 | if work.set in ["aomctc-e-nonpristine", "aomctc-g1-hdr-4k"]: 332 | thread_count = 4 333 | if work.codec in ['av2-ld']: 334 | if work.set in ['aomctc-a2-2k', 'aomctc-b1-syn']: 335 | thread_count = 4 336 | if work.run.ctc_version == 5.0: 337 | if work.codec in ['av2-ld']: 338 | if work.set in ['aomctc-a2-2k', 'aomctc-b1-syn']: 339 | thread_count = 4 340 | return thread_count 341 | 342 | def create_rdwork(run, video_filenames): 343 | work_items = [] 344 | for filename in video_filenames: 345 | for q in sorted(run.quality, reverse = True): 346 | work = RDWork() 347 | work.run = run 348 | work.log = run.log 349 | work.quality = q 350 | work.runid = run.runid 351 | work.codec = run.codec 352 | work.bindir = run.bindir 353 | work.set = run.set 354 | work.multicfg = run.multicfg 355 | work.nightly_run = run.nightly_run 356 | work.ctc_version = run.ctc_version 357 | # Parse and Store the CTC class (A1..A5, E, F1/F2, G1/G2) 358 | if 'aomctc' in work.set: 359 | work.ctc_class = work.set.split('-')[1].upper() 360 | work.filename = filename 361 | video_input_path = media_src_dir + '/' + work.set + '/' + work.filename 362 | v = open(video_input_path, "rb") 363 | line = v.readline().decode("utf-8") 364 | work.fps_n, work.fps_d = re.search( 365 | r"F([0-9]*)\:([0-9]*)", line).group(1, 2) 366 | work.width = int(re.search(r"W([0-9]*)", line).group(1)) 367 | work.height = int(re.search(r"H([0-9]*)", line).group(1)) 368 | # Explictly signal multislots for AV2 and VVC jobs where video is of 369 | # 4K resolution and from AOM-CTC 370 | if 'aomctc' in work.set: 371 | if 'av2' in work.codec or 'vvc' in work.codec: 372 | if (int(work.width) >= 3840) and (int(work.height) >= 2160): 373 | work.multislots = True 374 | if (int(work.width) >= 1920) and (int(work.height) >= 1080): 375 | if work.set in ['aomctc-a2-2k', 'aomctc-b1-syn']: 376 | if work.nightly_run: 377 | work.multislots = True 378 | if work.codec in ['av2-ld']: 379 | work.multislots = True 380 | work.thread_count = get_thread_count(work) 381 | # Signal multislots for jobs which need more than 1 thread 382 | if work.thread_count > 1: 383 | work.multislots = True 384 | work.extra_options = run.extra_options 385 | if run.save_encode: 386 | work.no_delete = True 387 | if work.codec == 'av1' or work.codec == 'av1-rt' or work.codec == 'rav1e' or 'svt-av1' in work.codec: 388 | work.copy_back_files.append('.ivf') 389 | elif (len(work.codec) >= 3) and (work.codec[0:3] == 'av2'): 390 | work.copy_back_files.append('.obu') 391 | elif (len(work.codec) >= 3) and (work.codec[0:3] == 'vvc'): 392 | work.copy_back_files.append('.bin') 393 | elif work.codec == 'xvc': 394 | work.copy_back_files.append('.xvc') 395 | work_items.append(work) 396 | return work_items 397 | 398 | class ABWork(Work): 399 | def __init__(self): 400 | super().__init__() 401 | self.failed = False 402 | def execute(self): 403 | slot = self.slot 404 | work = self 405 | input_path = slot.machine.media_path +'/' + work.set + '/' + work.filename 406 | 407 | try: 408 | slot.setup(self.codec,self.bindir) 409 | slot.start_shell(slot.work_root+'/rd_tool/ab_meta_compare.sh ' + shellquote(str(self.bpp)) + ' ' + shellquote(self.runid) + ' ' + work.set + ' ' + shellquote(input_path) + ' ' + shellquote(self.codec)) 410 | (stdout, stderr) = slot.gather() 411 | 412 | # filename with extension 413 | if 'video' in work.set: 414 | filename = input_path.split('/')[-1].rsplit('.', 1)[0] + '.ogv' 415 | else: 416 | filename = input_path.split('/')[-1].rsplit('.', 1)[0] + '.png' 417 | 418 | middle = self.runid + '/' + work.set + '/bpp_' + str(self.bpp) 419 | 420 | remote_file = slot.work_root+'/runs/' + middle + '/' + shellquote(filename) 421 | local_folder = runs_dst_dir + '/' + middle 422 | local_file = runs_dst_dir + '/' + middle + '/' + filename 423 | 424 | subprocess.Popen(['mkdir', '--parents', local_folder]) 425 | slot.get_file(remote_file, local_file) 426 | self.failed = False 427 | except IndexError: 428 | rd_print(self.log,'Encoding and copying', filename, 'at bpp', str(self.bpp), 'failed') 429 | rd_print(self.log, 'stdout:') 430 | rd_print(self.log, stdout.decode('utf-8')) 431 | rd_print(self.log, 'stderr:') 432 | rd_print(self.log, stderr.decode('utf-8')) 433 | self.failed = True 434 | def cancel(self): 435 | self.failed = True 436 | self.done = True 437 | def get_name(self): 438 | return self.filename + ' with bpp ' + str(self.bpp) 439 | 440 | def create_abwork(run, video_filenames): 441 | work_items = [] 442 | bits_per_pixel = [x/10.0 for x in range(1, 11)] 443 | for filename in video_filenames: 444 | for bpp in bits_per_pixel: 445 | work = ABWork() 446 | work.run = run 447 | work.log = run.log 448 | work.bpp = bpp 449 | work.codec = run.codec 450 | work.bindir = run.bindir 451 | work.runid = run.runid 452 | work.set = run.set 453 | work.filename = filename 454 | work.extra_options = run.extra_options 455 | work_items.append(work) 456 | return work_items 457 | -------------------------------------------------------------------------------- /rd_server.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import tornado.ioloop 4 | import tornado.web 5 | import os 6 | import codecs 7 | import json 8 | import argparse 9 | import sshslot 10 | import threading 11 | import time 12 | import awsremote 13 | import queue 14 | import subprocess 15 | from work import * 16 | from utility import * 17 | from work import quality_presets 18 | from smtplib import SMTP_SSL as SMTP 19 | from email.mime.text import MIMEText 20 | 21 | config_dir = os.getenv("CONFIG_DIR", os.getcwd()) 22 | runs_dst_dir = os.getenv("RUNS_DST_DIR", os.path.join(os.getcwd(), "../runs")) 23 | codecs_src_dir = os.getenv("CODECS_SRC_DIR", os.path.join(os.getcwd(), "..")) 24 | 25 | video_sets_f = codecs.open(os.path.join(config_dir, 'sets.json'),'r',encoding='utf-8') 26 | video_sets = json.load(video_sets_f) 27 | 28 | try: 29 | smtp_config = json.load(open(os.path.join(config_dir, 'smtp_cfg.json'))) 30 | except Exception as e: 31 | smtp_config = {} 32 | print("W: STMP email config file not found ", e) 33 | print("W: Email based notification system will not work!") 34 | 35 | # CTC Configs 36 | # LD : ctc_sets_mandatory - A1-4K 37 | # RA: ctc_sets_mandatory + ctc_sets_optional 38 | # AI: ctc_sets_mandatory_ai + ctc_sets_optional 39 | # AS: A1 with Downsampling 40 | ctc_sets_mandatory = [ 41 | "aomctc-a1-4k", 42 | "aomctc-a2-2k", 43 | "aomctc-a3-720p", 44 | "aomctc-a4-360p", 45 | "aomctc-a5-270p", 46 | "aomctc-b1-syn", 47 | "aomctc-b2-syn"] 48 | ctc_sets_mandatory_ai = ctc_sets_mandatory + \ 49 | ["aomctc-f1-hires", "aomctc-f2-midres"] 50 | ctc_sets_mandatory_ld = [x for x in ctc_sets_mandatory if x != 'aomctc-a1-4k'] 51 | ctc_sets_optional = ["aomctc-g1-hdr-4k", 52 | "aomctc-g2-hdr-2k", "aomctc-e-nonpristine"] 53 | ctc_full_presets = ["av2-ra-st", 'av2-ld','av2-ai'] 54 | machines = [] 55 | slots = [] 56 | free_slots = [] 57 | work_list = [] 58 | run_list = [] 59 | run_set_list = [] 60 | run_preset_list = [] 61 | args = {} 62 | scheduler_tasks = queue.Queue() 63 | 64 | config = { 65 | 'runs': runs_dst_dir, 66 | 'codecs': codecs_src_dir, 67 | } 68 | 69 | def return_set_list(info_file, codec_id): 70 | if len(info_file['ctcSets']) > 0: 71 | if 'aomctc-all' in info_file['ctcSets']: 72 | if codec_id in ['av2-ai', 'vvc-vtm-ai']: 73 | run_set_list = ctc_sets_mandatory_ai + ctc_sets_optional 74 | elif codec_id in ['av2-ra-st', 'av2-ra', 'vvc-vtm-ra', 'vvc-vtm-ra-st', 'vvc-vtm-ra-ctc']: 75 | run_set_list = ctc_sets_mandatory + ctc_sets_optional 76 | elif codec_id in ['av2-ld', 'vvc-vtm-ld']: 77 | run_set_list = ctc_sets_mandatory_ld 78 | elif 'aomctc-mandatory' in info_file['ctcSets']: 79 | if codec_id in ['av2-ra-st', 'av2-ra', 'vvc-vtm-ra', 'vvc-vtm-ra-st', 'vvc-vtm-ra-ctc']: 80 | run_set_list = ctc_sets_mandatory 81 | elif codec_id in ['av2-ld', 'vvc-vtm-ld']: 82 | run_set_list = ctc_sets_mandatory_ld 83 | elif codec_id in ['av2-ai', 'vvc-vtm-ai']: 84 | run_set_list = ctc_sets_mandatory_ai 85 | # By default feed AOM-CTC Mandatory sets 86 | else: 87 | run_set_list = ctc_sets_mandatory 88 | else: 89 | run_set_list = info_file['ctcSets'] 90 | else: 91 | run_set_list = [info_file['task']] 92 | return run_set_list 93 | 94 | def lookup_run_by_id(run_id): 95 | for run in run_list: 96 | if run.runid == run_id: 97 | return run 98 | return None 99 | 100 | 101 | def generate_email_content(run, set_flag, cfg_flag, all_flag, fail_type): 102 | completed_jobs = 0 103 | total_jobs = 0 104 | server_cfg = json.load(open(os.path.join(config_dir, 'config.json'))) 105 | if 'base_url' in server_cfg.keys(): 106 | server_url = server_cfg['base_url'] 107 | else: 108 | server_url = 'http://' + \ 109 | os.getenv('EXTERNAL_ADDR', 'localhost') + \ 110 | ':' + str(server_cfg['port']) 111 | if isinstance(run, RDRun): 112 | this_run_id = run.runid 113 | elif isinstance(run, str): 114 | this_run_id = run 115 | try: 116 | for work in run.work_items: 117 | total_jobs += 1 118 | if work.done: 119 | completed_jobs += 1 120 | content_string = 'Run ID: %s, \nSet: %s, \nCodec: %s, \nCompleted Jobs: %d, \nTotal Jobs: %d, \nMetadata: %s ' % ( 121 | run.runid, run.set, run.codec, completed_jobs, total_jobs, run.info) 122 | if set_flag and len(fail_type) == 0: 123 | content_string += '\nFinished Encoding for %s set with %s config' % ( 124 | run.set, run.codec) 125 | if cfg_flag and len(fail_type) == 0: 126 | content_string += '\nFinished Encoding all sets for %s config' % ( 127 | run.codec) 128 | if all_flag and len(fail_type) == 0: 129 | content_string += '\nFinished Encoding all the given sets and configs for the RunID %s' % ( 130 | run.runid) 131 | if len(fail_type) > 0: 132 | content_string += '\nEncoding for all the given sets failed, \nfail_type:%s for the RunID %s' % ( 133 | fail_type, run.runid) 134 | except: 135 | content_string = 'Run ID: %s, \nFailed before starting the job, fail_type: %s' % ( 136 | this_run_id, fail_type) 137 | content_string += '\nCheck job for more information: ' + \ 138 | server_url + '/?job=' + this_run_id 139 | content_string += '\nJob logs: ' + \ 140 | server_url + '/runs/' + this_run_id + '/output.txt' 141 | return content_string 142 | 143 | 144 | def submit_email_notification(run, smtp_cfg, set_flag, cfg_flag, all_flag, fail_type, destination): 145 | text_subtype = 'plain' 146 | if isinstance(run, RDRun): 147 | subject = 'AWCY: Run status: %s' % (run.runid) 148 | destination = run.info['nick'] 149 | elif isinstance(run, str): 150 | subject = 'AWCY: Run status: %s' % (run) 151 | content = generate_email_content( 152 | run, set_flag, cfg_flag, all_flag, fail_type) 153 | # Do not attempt to do emailing if the Nick is not having email style ie. 154 | # containing @ symbol to keep it simple 155 | if '@' not in destination: 156 | return 157 | try: 158 | msg = MIMEText(content, text_subtype) 159 | msg['Subject'] = subject 160 | msg['From'] = smtp_cfg['sender'] 161 | msg['To'] = destination 162 | 163 | conn = SMTP(smtp_cfg['smtpserver']) 164 | conn.set_debuglevel(False) 165 | conn.login(smtp_cfg['username'], smtp_cfg['password']) 166 | try: 167 | conn.sendmail(smtp_cfg['sender'], destination, msg.as_string()) 168 | finally: 169 | conn.quit() 170 | except Exception as e: 171 | rd_print(run.log, "E: Failed to send email, error:", e) 172 | 173 | 174 | class SchedulerTask: 175 | def get(self): 176 | pass 177 | 178 | class CancelHandler(tornado.web.RequestHandler): 179 | def get(self): 180 | global scheduler_tasks 181 | run_id = self.get_query_argument('run_id') 182 | task = CancelTask() 183 | task.run_id = run_id 184 | scheduler_tasks.put(task) 185 | self.write('ok') 186 | 187 | class CancelTask(SchedulerTask): 188 | def __init__(self): 189 | self.run_id = None 190 | def run(self): 191 | global work_list 192 | run_id = self.run_id 193 | rd_print(None,'Cancelling '+run_id) 194 | run = lookup_run_by_id(run_id) 195 | if not run: 196 | rd_print(None,'Could not cancel '+run_id+'. run_id not found.') 197 | return 198 | run.cancel() 199 | # Explictly search through current runIDs and remove them 200 | for this_run in run_list: 201 | if this_run.runid == run_id: 202 | rd_print(this_run.log, "Cancelling sub-job of", 203 | this_run.set, "for", this_run.codec, ".") 204 | this_run.cancel() 205 | for work in work_list[:]: 206 | if work.runid == run_id: 207 | work_list.remove(work) 208 | else: 209 | rd_print(None, work.runid) 210 | rd_print(None, len(work_list)) 211 | if len(smtp_config) > 0: 212 | submit_email_notification(run, smtp_config, set_flag=False, cfg_flag=False, all_flag=False, fail_type='runCancel', destination='') 213 | 214 | class RunSubmitHandler(tornado.web.RequestHandler): 215 | def get(self): 216 | global scheduler_tasks 217 | run_id = self.get_query_argument('run_id') 218 | task = SubmitTask() 219 | task.run_id = run_id 220 | scheduler_tasks.put(task) 221 | self.write('ok') 222 | 223 | class SubmitTask(SchedulerTask): 224 | def __init__(self): 225 | self.run_id = None 226 | def run(self): 227 | global work_list 228 | global run_list 229 | global run_set_list 230 | global run_preset_list 231 | run_id = self.run_id 232 | rundir = config['runs'] + '/' + run_id 233 | info_file_path = rundir + '/info.json' 234 | log_file_path = rundir + '/output.txt' 235 | info_file = open(info_file_path, 'r') 236 | log_file = open(log_file_path, 'a') 237 | info = json.load(info_file) 238 | if len(info['ctcSets']) > 1: 239 | run_set_list = return_set_list(info, info['codec']) 240 | else: 241 | run_set_list = [info['task']] 242 | if len(info['ctcPresets']) > 1: 243 | run_preset_list = info['ctcPresets'] 244 | else: 245 | run_preset_list = [info['codec']] 246 | if 'av2-all' in info['ctcPresets']: 247 | run_preset_list = ctc_full_presets 248 | for this_preset in run_preset_list: 249 | run_set_list = return_set_list(info, this_preset) 250 | for this_video_set in sorted(run_set_list): 251 | run = RDRun(info['codec']) 252 | run.info = info 253 | run.runid = run_id 254 | run.rundir = config['runs'] + '/' + run_id 255 | run.log = log_file 256 | run.codec = this_preset 257 | run.set = this_video_set 258 | if len(run_preset_list) > 1: 259 | run.multicfg = True 260 | else: 261 | run.multicfg = False 262 | if this_video_set in ['aomctc-f1-hires','aomctc-f2-midres'] and this_preset in ['av2-ai']: 263 | run.quality = quality_presets['av2-f'] 264 | elif this_preset in ['av2-ai']: 265 | run.quality = quality_presets['av2-ai'] 266 | rd_print(run.log, "Starting encoding of ", this_video_set, "with", this_preset) 267 | if 'arch' in info: 268 | run.arch = info['arch'] 269 | else: 270 | run.arch = 'x86_64' 271 | run.bindir = run.rundir + '/' + run.arch + '/' 272 | run.prefix = run.rundir + '/' + run.set 273 | if len(run_preset_list) > 1: 274 | run.prefix = run.rundir + '/' + run.codec + '/' + run.set 275 | try: 276 | prefix_maker = subprocess.run(['mkdir','-p', run.prefix]) 277 | if prefix_maker.returncode == 0: 278 | pass 279 | else: 280 | print(prefix_maker) 281 | except FileExistsError: 282 | pass 283 | if 'qualities' in info: 284 | if info['qualities'] != '': 285 | run.quality = info['qualities'].split() 286 | if 'extra_options' in info: 287 | run.extra_options = info['extra_options'] 288 | if 'save_encode' in info: 289 | if info['save_encode']: 290 | run.save_encode = True 291 | if 'nightly_run' in info or 'nightly' in run.runid: 292 | # AVMNightly: Do only for AVM-RA 293 | if run.codec in ['av2-ra-st']: 294 | run.nightly_run = True 295 | # AVMNightly: 3 threads for A1, A2, B1 sets 296 | if run.set in ['aomctc-a1-4k', 'aomctc-a2-2k', 'aomctc-b1-syn']: 297 | run.extra_options += ' --tile-columns=2 --threads=3 --row-mt=0 ' 298 | # AVMNightly: Due to complexity reasons we can only do 17f:) 299 | run.extra_options += ' --limit=17 ' 300 | # Explictly signal Limit as 1 for still-images for AV2-AI. 301 | if run.set in ['aomctc-f1-hires','aomctc-f2-midres'] and 'av2-ai' in run.codec: 302 | run.extra_options += ' --limit=1 ' 303 | run.status = 'running' 304 | run.write_status() 305 | run_list.append(run) 306 | video_sets_f = codecs.open(os.path.join(config_dir, 'sets.json'),'r',encoding='utf-8') 307 | video_sets = json.load(video_sets_f) 308 | video_filenames = video_sets[run.set]['sources'] 309 | if 'ctcVersion' in info: 310 | run.ctc_version = float(info['ctcVersion']) 311 | if run.ctc_version < 5.0: 312 | if run.set in ['aomctc-a2-2k', 313 | 'aomctc-b2-syn', 'aomctc-e-nonpristine']: 314 | video_filenames = video_sets[run.set]['CTC_4.0'] 315 | run.set_type = video_sets[run.set].get('type', 'undef') 316 | run.work_items = create_rdwork(run, video_filenames) 317 | work_list.extend(run.work_items) 318 | if False: 319 | if 'ab_compare' in info: 320 | if info['ab_compare']: 321 | abrun = ABRun(info['codec']) 322 | abrun.runid = run_id 323 | abrun.rundir = config['runs'] + '/' + run_id 324 | abrun.log = log_file 325 | abrun.set = info['task'] 326 | abrun.bindir = config['codecs'] + '/' + info['codec'] 327 | abrun.prefix = run.rundir + '/' + run.set 328 | run_list.append(abrun) 329 | abrun.work_items.extend(create_abwork(abrun, video_filenames)) 330 | work_list.extend(abrun.work_items) 331 | pass 332 | 333 | class WorkListHandler(tornado.web.RequestHandler): 334 | def get(self): 335 | self.set_header("Content-Type", "application/json") 336 | self.write(json.dumps([w.get_name() for w in work_list])) 337 | 338 | class RunStatusHandler(tornado.web.RequestHandler): 339 | def get(self): 340 | self.set_header("Content-Type", "application/json") 341 | runs = [] 342 | for run in run_list: 343 | run_json = {} 344 | run_json['run_id'] = run.runid 345 | run_json['set'] = run.set 346 | run_json['config'] = run.codec 347 | run_json['completed'] = 0 348 | run_json['total'] = 0 349 | run_json['info'] = run.info 350 | for work in run.work_items: 351 | run_json['total'] += 1 352 | if work.done: 353 | run_json['completed'] += 1 354 | multi_config_completed = 0 355 | multi_config_total = 0 356 | # Iterate through all the `run_list` to count different 357 | # configurations and video-sets for reflecting true-running-count 358 | for this_run in run_list: 359 | if run.runid == this_run.runid: 360 | # This would be the current job, so skip that to avoid 361 | # double-counting 362 | if (run.runid == this_run.runid) and (run.set == this_run.set) and (run.codec == this_run.codec): 363 | continue 364 | else: 365 | for this_work in this_run.work_items: 366 | multi_config_total += 1 367 | if this_work.done: 368 | multi_config_completed += 1 369 | # Add the new total and completed to the run_json 370 | run_json['multi_completed'] = run_json['completed'] + \ 371 | multi_config_completed 372 | run_json['multi_total'] = run_json['total'] + multi_config_total 373 | runs.append(run_json) 374 | self.write(json.dumps(runs)) 375 | 376 | class MachineUsageHandler(tornado.web.RequestHandler): 377 | def get(self): 378 | global machines 379 | machine_usage = [] 380 | for machine in machines: 381 | machine_json = {} 382 | slot_in_use = [] 383 | for slot in machine.slots: 384 | if slot.work: 385 | slot_in_use.append(slot.work.get_name()) 386 | elif slot.busy: 387 | slot_in_use.append('busy with no work') 388 | else: 389 | slot_in_use.append('None') 390 | machine_json['name'] = machine.get_name() 391 | machine_json['slots'] = slot_in_use 392 | machine_usage.append(machine_json) 393 | self.write(json.dumps(machine_usage)) 394 | self.set_header("Content-Type", "application/json") 395 | 396 | class FreeSlotsHandler(tornado.web.RequestHandler): 397 | def get(self): 398 | global free_slots 399 | slot_text = [] 400 | for slot in free_slots: 401 | if slot.work: 402 | slot_text.append(slot.work.get_name()) 403 | elif slot.busy: 404 | slot_text.append('busy with no work') 405 | else: 406 | slot_text.append('None') 407 | self.write(json.dumps(slot_text)) 408 | 409 | class ExecuteTick(tornado.web.RequestHandler): 410 | def get(self): 411 | scheduler_tick() 412 | self.write('ok') 413 | 414 | 415 | # Handle Job Failed Email Notifications 416 | class FailNotifier(tornado.web.RequestHandler): 417 | def get(self): 418 | run_id = self.get_query_argument('run_id') 419 | fail_type = self.get_query_argument('failtype') 420 | nick = self.get_query_argument('nick') 421 | if len(smtp_config) > 0: 422 | submit_email_notification( 423 | run_id, smtp_config, set_flag=False, cfg_flag=False, all_flag=False, fail_type=fail_type, destination=nick) 424 | if '@' in nick: 425 | self.write("OK, sent email notification to " + nick) 426 | else: 427 | self.write( 428 | "ERR, no email found in nick for sending notification, nick: " + nick) 429 | 430 | def main(): 431 | global free_slots 432 | global machines 433 | global slots 434 | global args 435 | parser = argparse.ArgumentParser(description='Run AWCY scheduler daemon.') 436 | parser.add_argument('-machineconf') 437 | parser.add_argument('-port',default=4000) 438 | parser.add_argument('-awsgroup', default='nonexistent_group') 439 | parser.add_argument('-max-machines', default=3, type=int) 440 | args = parser.parse_args() 441 | if args.machineconf: 442 | machineconf = json.load(open(args.machineconf, 'r')) 443 | for m in machineconf: 444 | machines.append(sshslot.Machine(m['host'],m['user'],m['cores'],m['work_root'],str(m['port']),m['media_path'])) 445 | for machine in machines: 446 | slots.extend(machine.get_slots()) 447 | free_slots.extend(reversed(slots)) 448 | app = tornado.web.Application( 449 | [ 450 | (r"/work_list.json", WorkListHandler), 451 | (r"/run_status.json", RunStatusHandler), 452 | (r"/machine_usage.json", MachineUsageHandler), 453 | (r"/free_slots.json", FreeSlotsHandler), 454 | (r"/submit", RunSubmitHandler), 455 | (r"/cancel", CancelHandler), 456 | (r"/execute_tick",ExecuteTick), 457 | (r"/fail_job_notifier", FailNotifier) 458 | ], 459 | static_path=os.path.join(os.path.dirname(__file__), "static"), 460 | xsrf_cookies=True, 461 | debug=False, 462 | ) 463 | app.listen(args.port) 464 | ioloop = tornado.ioloop.IOLoop.current() 465 | if not args.machineconf: 466 | machine_allocator_tick() 467 | scheduler_tick() 468 | ioloop.start() 469 | 470 | def machine_allocator_tick(): 471 | global slots 472 | global free_slots 473 | global machines 474 | global work_list 475 | global run_list 476 | # start all machines if we don't have any but have work queued 477 | if len(work_list) and not len(machines): 478 | rd_print(None, "Starting machines.") 479 | #awsgroup.start_machines(args.max_machines, args.awsgroup) 480 | # stop all machines if nothing is running 481 | slots_busy = False 482 | for slot in slots: 483 | if slot.busy: 484 | slots_busy = True 485 | if not slots_busy and not len(work_list) and not len(run_list): 486 | rd_print(None, "Stopping all machines.") 487 | machines = [] 488 | slots = [] 489 | free_slots = [] 490 | #awsremote.stop_machines(args.awsgroup) 491 | try: 492 | updated_machines = awsremote.get_machines(args.max_machines, args.awsgroup) 493 | except: 494 | tornado.ioloop.IOLoop.current().call_later(60,machine_allocator_tick) 495 | return 496 | print(updated_machines) 497 | for m in machines: 498 | matching = [um for um in updated_machines if um.host == m.host] 499 | if len(matching) == 0: 500 | rd_print(None, "Machine disappeared: " + m.get_name()) 501 | for s in m.slots: 502 | slots.remove(s) 503 | try: 504 | free_slots.remove(s) 505 | except: 506 | pass 507 | machines.remove(m) 508 | for um in updated_machines: 509 | print(um, um.get_name()) 510 | matching = [m for m in machines if m.host == um.host] 511 | if len(matching) == 0: 512 | rd_print(None, "Machine appeared: " + um.get_name()) 513 | new_slots = um.get_slots() 514 | slots.extend(new_slots) 515 | free_slots.extend(new_slots) 516 | machines.append(um) 517 | tornado.ioloop.IOLoop.current().call_later(60,machine_allocator_tick) 518 | 519 | def find_image_work(items, default = None): 520 | for work in items: 521 | if work.run.set_type == 'image': 522 | return work 523 | return default 524 | 525 | def find_multislot_work(items, default = None): 526 | for work in items: 527 | # Work is of the class RDWork 528 | if work.multislots: 529 | return work 530 | return default 531 | 532 | def find_elfluente_work(items, default=None): 533 | for work in items: 534 | if work.set in ['elfuente-1080p-as', 'elfuente-1080p-as-1']: 535 | return work 536 | return default 537 | 538 | def find_not_elfluente_work(items, default=None): 539 | for work in items: 540 | if work.set not in ['elfuente-1080p-as', 'elfuente-1080p-as-1']: 541 | return work 542 | return default 543 | 544 | def check_elfluente_work(work): 545 | if work is None: 546 | return False 547 | if work.set in ['elfuente-1080p-as', 'elfuente-1080p-as']: 548 | return True 549 | else: 550 | return False 551 | 552 | def check_multislot_work(work): 553 | return work.multislots 554 | 555 | def check_nightly_work(work): 556 | return work.nightly_run 557 | 558 | def check_aomctc_ld(work): 559 | if work.run.ctc_version >= 5.0: 560 | if work.set in ['aomctc-a2-2k', 'aomctc-b1-syn']: 561 | if work.codec in ['av2-ld']: 562 | return True 563 | return False 564 | 565 | def check_aomctc_ra(work): 566 | if work.run.ctc_version >= 6.0: 567 | if work.set in ["aomctc-g1-hdr-4k", "aomctc-e-nonpristine"]: 568 | if work.codec in ['av2-ra', 'av2-ra-st']: 569 | return True 570 | return False 571 | 572 | def check_gop_parallel_work(work): 573 | if work.run.codec in ['av2-ra', 'av2-as', 'vvc-vtm-ra','vvc-vtm-ra-ctc','vvc-vtm-as-ctc']: 574 | return True 575 | else: 576 | return False 577 | 578 | def free_a_slot(this_free_slots, current_slot_host, current_slot_id, current_work): 579 | for slot_iterator in range(len(this_free_slots)-1, -1,-1): 580 | target_slot = this_free_slots[slot_iterator] 581 | target_slot_host = target_slot.machine.host 582 | target_slot_id = target_slot.work_root 583 | if current_slot_host == target_slot_host: 584 | if current_slot_id != target_slot_id: 585 | thread_slot = this_free_slots.pop(slot_iterator) 586 | thread_slot.hold_work(current_work) 587 | break 588 | return this_free_slots 589 | 590 | def scheduler_tick(): 591 | global slots 592 | global free_slots 593 | global work_list 594 | global run_list 595 | global run_set_list 596 | global scheduler_tasks 597 | max_retries = 5 598 | # run queued up tasks 599 | while not scheduler_tasks.empty(): 600 | task = scheduler_tasks.get() 601 | try: 602 | task.run() 603 | except Exception as e: 604 | rd_print(None,e) 605 | rd_print(None,'Task failed.') 606 | # look for completed work 607 | for slot in slots: 608 | current_work = slot.work 609 | if slot.busy == False and slot.work != None: 610 | if slot.work.failed == False: 611 | slot.work.done = True 612 | try: 613 | slot.work.write_results() 614 | except Exception as e: 615 | rd_print(None, e) 616 | rd_print('Failed to write results for work item',slot.work.get_name()) 617 | rd_print(slot.work.log,slot.work.get_name(),'finished.') 618 | elif slot.work.retries < max_retries and not slot.work.run.cancelled and (slot.p.p is None or not slot.p.p.returncode == 98): 619 | slot.work.retries += 1 620 | rd_print(slot.work.log,'Retrying work ',slot.work.get_name(),'...',slot.work.retries,'of',max_retries,'retries.') 621 | slot.work.failed = False 622 | work_list.insert(0, slot.work) 623 | else: 624 | slot.work.done = True 625 | rd_print(slot.work.log,slot.work.get_name(),'given up on.') 626 | slot.clear_work() 627 | free_slots.append(slot) 628 | # Free the holdout slots for multislot jobs 629 | if check_multislot_work(current_work) or check_gop_parallel_work(current_work): 630 | for cur_slot in slots: 631 | # When the work is done for a slot, it is cleared, so we 632 | # check for None. 633 | if cur_slot.work is None: 634 | continue 635 | # Check the unique work_id among all the slots to find slots 636 | # which current encoder invocation is using to free them. 637 | elif cur_slot.work.workid == current_work.workid: 638 | cur_slot.busy = False 639 | cur_slot.clear_work() 640 | free_slots.append(cur_slot) 641 | 642 | # fill empty slots with new work 643 | if len(work_list) != 0: 644 | if len(free_slots) != 0: 645 | slot = free_slots.pop() 646 | work = work_list[0] 647 | # Machine ID 648 | current_slot_host = slot.machine.host 649 | # Slot ID to be exlcuded 650 | current_slot_id = slot.work_root 651 | 652 | # search for image work if there is only one slot available 653 | # allows prioritizing image runs without making scheduler the bottleneck 654 | if len(free_slots) == 0: 655 | try: 656 | work = find_image_work(work_list, work) 657 | except Exception as e: 658 | rd_print(None, e) 659 | rd_print(None, 'Finding image work failed.') 660 | work = find_multislot_work(work_list, work) 661 | if check_multislot_work(work): 662 | # Multijob code-path 663 | # Free a slot 664 | free_slots = free_a_slot(free_slots, current_slot_host, current_slot_id, work) 665 | if check_nightly_work(work): 666 | free_slots = free_a_slot(free_slots, current_slot_host, current_slot_id, work) 667 | # CTCv5: Requires 4 threads for A2, B1 encodes in LD mode. 668 | # CTCv6: Requires 4 thresads for E, G1 encodes in RA mode. 669 | if check_aomctc_ld(work) or check_aomctc_ra(work): 670 | free_slots = free_a_slot(free_slots, current_slot_host, current_slot_id, work) 671 | free_slots = free_a_slot(free_slots, current_slot_host, current_slot_id, work) 672 | # At this point we freed 2 threads, check the thread count based 673 | # on the CTC version, and free additional slots if we want 674 | if work.thread_count > 2: 675 | thread_count = work.thread_count - 2 676 | for thread_idx in range(thread_count): 677 | free_slots = free_a_slot(free_slots, current_slot_host, current_slot_id, work) 678 | # GOP-Parallel: Multislot requires 2 free slots 679 | if check_gop_parallel_work(work): 680 | free_slots = free_a_slot(free_slots, current_slot_host, current_slot_id, work) 681 | free_slots = free_a_slot(free_slots, current_slot_host, current_slot_id, work) 682 | # Remove from the list and start encoding 683 | work_list.remove(work) 684 | rd_print(work.log, 'Encoding', work.get_name(), 685 | 'on', slot.work_root.split('/')[-1], 'of', slot.machine.host) 686 | slot.start_work(work) 687 | else: 688 | # Non-multijob code-path 689 | # GOP-Sequential: 1 Extra free slot for 2nd GOP 690 | if check_gop_parallel_work(work): 691 | free_slots = free_a_slot(free_slots, current_slot_host, current_slot_id, work) 692 | work_list.remove(work) 693 | rd_print(work.log, 'Encoding', work.get_name(), 694 | 'on', slot.work_root.split('/')[-1], 'of', slot.machine.host) 695 | slot.start_work(work) 696 | # As we have Work of Works with different sets for same RunID, 697 | # Create a mechanism to filter and store the results for unique jobs based 698 | # on sets. 699 | current_unique_run_list = [] 700 | for this_run in run_list: 701 | current_unique_run_list.append(this_run.runid) 702 | current_unique_run_list = list(dict.fromkeys(current_unique_run_list)) 703 | # Make a tracker for given a run 704 | run_tracker = {} 705 | # find runs/sub-runs where all work has been completed a given run_id 706 | for this_run in current_unique_run_list: 707 | # Create a set-based tracker for unique jobs. 708 | run_tracker[this_run] = {} 709 | run_tracker[this_run]['done'] = True 710 | run_tracker[this_run]['cfg'] = {} 711 | run_tracker[this_run]['status'] = {} 712 | #run_tracker[this_run]['sets'] = {} 713 | ## Part 1: Initialise config for different presets 714 | for run in run_list: 715 | if run.runid == this_run: 716 | run_tracker[this_run]['cfg'][run.codec] = {} 717 | ## Part 2: Update flags 718 | for run in run_list: 719 | if run.runid == this_run: 720 | run_tracker[this_run]['cfg'][run.codec][run.set] = True 721 | for work in run.work_items: 722 | if work.done == False: 723 | run_tracker[this_run]['cfg'][run.codec][run.set] = False 724 | run_tracker[this_run]['done'] = False 725 | run_tracker[this_run]['status'][run.codec] = False 726 | ## Part 3: Send updates and curate results. 727 | for run in run_list: 728 | if run.runid == this_run: 729 | if run_tracker[this_run]['cfg'][run.codec][run.set]: 730 | rd_print(run.log, "Finished Encoding ", run.set, "set for ", run.codec, "config.") 731 | run_list.remove(run) 732 | run_tracker[this_run]['done'] = False 733 | if ((len(run.info['ctcSets']) > 1) or ('aomctc-all' in run.info['ctcSets'])) and len(smtp_config) > 0: 734 | submit_email_notification( 735 | run, smtp_config, set_flag=True, cfg_flag=False, all_flag=False, fail_type='', destination='') 736 | if all(value == True for value in run_tracker[this_run]['cfg'][run.codec].values()): 737 | rd_print(run.log, "Finished Encoding", run.codec, "config.") 738 | run_tracker[this_run]['status'][run.codec] = True 739 | if ((len(run.info['ctcPresets']) > 1) or ('av2-all' in run.info['ctcPresets'])) and len(smtp_config) > 0: 740 | submit_email_notification( 741 | run, smtp_config, set_flag=False, cfg_flag=True, all_flag=False, fail_type='', destination='') 742 | if all(value == True for value in run_tracker[this_run]['status'].values()): 743 | run_tracker[this_run]['done'] = True 744 | rd_print(run.log, "Finished Encoding all sets for ", run.runid) 745 | if len(smtp_config) > 0: 746 | submit_email_notification( 747 | run, smtp_config, set_flag=False, cfg_flag=False, all_flag=True, fail_type='', destination='') 748 | try: 749 | # Use A2 set for mandatory/all CTC Class 750 | if 'aomctc-mandatory' in run.info['ctcSets'] or 'aomctc-all' in run.info['ctcSets']: 751 | run.prefix = run.rundir + '/aomctc-a2-2k' 752 | if len(run.info['ctcSets']) > 1 and len(run.info['ctcPresets']) <= 1: 753 | run.prefix = run.rundir + '/' + run.info['task'] 754 | if len(run.info['ctcPresets']) > 1: 755 | run.prefix = run.rundir + '/' + \ 756 | run.info['codec'] + '/' + sorted(run_set_list)[0] 757 | if 'aomctc-mandatory' in run.info['ctcSets'] or 'aomctc-all' in run.info['ctcSets']: 758 | run.prefix = run.rundir + '/' + run.info['codec'] + '/aomctc-a2-2k' 759 | elif 'av2-all' in run.info['ctcPresets']: 760 | run.prefix = run.rundir + '/' + run.info['codec'] + '/' + run.info['task'] 761 | run.reduce() 762 | except Exception as e: 763 | rd_print(run.log,e) 764 | rd_print(run.log,'Failed to run reduce step on '+run.runid) 765 | rd_print(run.log,'Finished '+run.runid) 766 | run.finish() 767 | tornado.ioloop.IOLoop.current().call_later(1,scheduler_tick) 768 | 769 | if __name__ == "__main__": 770 | main() 771 | -------------------------------------------------------------------------------- /metrics_gather.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | set -e 4 | 5 | export LD_LIBRARY_PATH=/usr/local/lib/ 6 | 7 | #3GB RAM limit 8 | ulimit -m 8000000 9 | 10 | #enable core dumps (warning - uses up to 3GB per slot!) 11 | #requires /proc/sys/kernel/core_pattern = core 12 | #switch to using gdb at some point 13 | #ulimit -S -c unlimited 14 | 15 | if [ -z "$WORK_ROOT" ]; then 16 | export WORK_ROOT=/home/ec2-user 17 | fi 18 | 19 | cd "$WORK_ROOT" 20 | 21 | if [ -z "$DAALATOOL_ROOT" ]; then 22 | export DAALATOOL_ROOT="$WORK_ROOT/daalatool/" 23 | fi 24 | export X264="$WORK_ROOT/x264/x264" 25 | export X265="$WORK_ROOT/x265/build/linux/x265" 26 | export XVCENC="$WORK_ROOT/xvc/build/app/xvcenc" 27 | export XVCDEC="$WORK_ROOT/xvc/build/app/xvcdec" 28 | export VPXENC="$WORK_ROOT/$CODEC/vpxenc" 29 | export VPXDEC="$WORK_ROOT/$CODEC/vpxdec" 30 | if [ -z "$AOMENC" ]; then 31 | export AOMENC="$WORK_ROOT/$CODEC/aomenc" 32 | fi 33 | if [ -z "$AOMDEC" ]; then 34 | export AOMDEC="$WORK_ROOT/$CODEC/aomdec" 35 | fi 36 | if [ -z "$THORENC" ]; then 37 | export THORENC="$WORK_ROOT/$CODEC/build/Thorenc" 38 | fi 39 | if [ -z "$THORDIR" ]; then 40 | export THORDIR="$(dirname $THORENC)/../" 41 | fi 42 | if [ -z "$THORDEC" ]; then 43 | export THORDEC="$(dirname $THORENC)/Thordec" 44 | fi 45 | if [ -z "$RAV1E" ]; then 46 | export RAV1E="$WORK_ROOT/$CODEC/target/release/rav1e" 47 | fi 48 | if [ -z "$SVTAV1" ]; then 49 | export SVTAV1="$WORK_ROOT/$CODEC/Bin/Release/SvtAv1EncApp" 50 | fi 51 | if [ -z "$VVCENC" ]; then 52 | export VVCENC="$WORK_ROOT/$CODEC/bin/EncoderAppStatic" 53 | fi 54 | if [ -z "$VVCDEC" ]; then 55 | export VVCDEC="$WORK_ROOT/$CODEC/bin/DecoderAppStatic" 56 | fi 57 | if [ -z "$VVCCAT" ]; then 58 | export VVCCAT="$WORK_ROOT/$CODEC/bin/parcatStatic" 59 | fi 60 | if [ -z "$ENCODER_EXAMPLE" ]; then 61 | export ENCODER_EXAMPLE="$WORK_ROOT/daala/examples/encoder_example" 62 | fi 63 | export YUV2YUV4MPEG="$DAALATOOL_ROOT/tools/yuv2yuv4mpeg" 64 | export Y4M2YUV="$DAALATOOL_ROOT/tools/y4m2yuv" 65 | if [ -z "$DUMP_VIDEO" ]; then 66 | export DUMP_VIDEO="$WORK_ROOT/daala/examples/dump_video" 67 | fi 68 | 69 | if [ -z "$DUMP_PSNR" ]; then 70 | export DUMP_PSNR="$DAALATOOL_ROOT/tools/dump_psnr" 71 | fi 72 | 73 | if [ -z "$DUMP_PSNRHVS" ]; then 74 | export DUMP_PSNRHVS="$DAALATOOL_ROOT/tools/dump_psnrhvs" 75 | fi 76 | 77 | if [ -z "$DUMP_SSIM" ]; then 78 | export DUMP_SSIM="$DAALATOOL_ROOT/tools/dump_ssim" 79 | fi 80 | 81 | if [ -z "$DUMP_FASTSSIM" ]; then 82 | export DUMP_FASTSSIM="$DAALATOOL_ROOT/tools/dump_fastssim" 83 | fi 84 | 85 | if [ -z "$DUMP_MSSSIM" ]; then 86 | export DUMP_MSSSIM="$DAALATOOL_ROOT/tools/dump_msssim" 87 | fi 88 | 89 | if [ -z "$DUMP_CIEDE" ]; then 90 | export DUMP_CIEDE="$DAALATOOL_ROOT/../dump_ciede2000/target/release/dump_ciede2000" 91 | fi 92 | 93 | if [ -z "$VMAF_ROOT" ]; then 94 | export VMAF_ROOT="$DAALATOOL_ROOT/../vmaf" 95 | fi 96 | 97 | if [ -z "$VMAF" ]; then 98 | export VMAF="$VMAF_ROOT/libvmaf/build/tools/vmaf" 99 | fi 100 | 101 | if [ -z "$VMAFMODEL" ]; then 102 | export VMAFMODEL="vmaf_v0.6.1.json" # File name in $VMAFROOT/model 103 | fi 104 | 105 | if [ -z "$YUV2YUV4MPEG" ]; then 106 | export YUV2YUV4MPEG="$DAALATOOL_ROOT/tools/yuv2yuv4mpeg" 107 | fi 108 | 109 | if [ -z "$HDRTOOLS_ROOT" ]; then 110 | export HDRTOOLS_ROOT="$DAALATOOL_ROOT/../hdrtools" 111 | fi 112 | 113 | if [ -z "$HDRCONVERT" ]; then 114 | export HDRCONVERT="$HDRTOOLS_ROOT/bin/HDRConvert" 115 | fi 116 | 117 | if [ -z "$CODEC" ]; then 118 | export CODEC=daala 119 | fi 120 | 121 | if [ -z "$x" ]; then 122 | echo Missing quality setting 123 | exit 1 124 | fi 125 | 126 | if [ -e pid ]; then 127 | kill -9 -$(cat pid) || true 128 | fi 129 | 130 | if [ ! -x "$(command -v perf)" ]; then 131 | echo "perf, perf-stat is not installed" 132 | exit 1 133 | fi 134 | 135 | echo $$ > pid 136 | 137 | FILE=$1 138 | 139 | BASENAME="$(basename $FILE)-$x" 140 | rm "$BASENAME.out" 2> /dev/null || true 141 | 142 | WIDTH=$(head -1 $FILE | tr ' ' '\n' | grep -E '\bW' | tr -d 'W') 143 | HEIGHT=$(head -1 $FILE | tr ' ' '\n' | grep -E '\bH' | tr -d 'H') 144 | CHROMA=$(head -1 $FILE | tr ' ' '\n' | grep -E '\bC') 145 | # YUV2Y4M requires to be just numbers 146 | YUV_CHROMA=$CHROMA 147 | DEPTH=8 148 | case $CHROMA in 149 | C444p10) 150 | DEPTH=10 151 | YUV_CHROMA=444 152 | ;; 153 | C420p10) 154 | DEPTH=10 155 | YUV_CHROMA=420 156 | ;; 157 | C420jpeg | C420mpeg2) 158 | YUV_CHROMA=420 159 | esac 160 | 161 | # used for libvpx vbr 162 | RATE=$(echo $x*$WIDTH*$HEIGHT*30/1000 | bc) 163 | 164 | KFINT=1000 165 | TIMEROUT=$BASENAME-enctime.out 166 | PERF_ENC_OUT=${BASENAME}'-encperf.out' 167 | PERF_DEC_OUT=${BASENAME}'-decperf.out' 168 | PERF_ENC_STAT='perf stat -o '${PERF_ENC_OUT}'' 169 | PERF_DEC_STAT='perf stat -o '${PERF_DEC_OUT}'' 170 | TIMERDECOUT=$BASENAME-dectime.out 171 | TIMER=$PERF_ENC_STAT' time -v --output='"$TIMEROUT" 172 | TIMERDEC=$PERF_DEC_STAT' time -v --output='"$TIMERDECOUT" 173 | AOMDEC_OPTS='-S' 174 | ENC_EXT='' 175 | 176 | case $CODEC in 177 | daala) 178 | $(OD_LOG_MODULES='encoder:10' OD_DUMP_IMAGES_SUFFIX="$BASENAME" $TIMER "$ENCODER_EXAMPLE" -k $KFINT -v "$x" $EXTRA_OPTIONS "$FILE" -o "$BASENAME.ogv" > "$BASENAME-stdout.txt" 2> "$BASENAME-enc.out") 179 | if [ ! -f "$BASENAME.ogv" ] 180 | then 181 | echo Failed to produce "$BASENAME.ogv" 182 | cat "$BASENAME-enc.out" 183 | exit 1 184 | fi 185 | SIZE=$(stat -c %s "$BASENAME.ogv") 186 | ENC_EXT='.ogv' 187 | #mv "00000000out-$BASENAME.y4m" "$BASENAME.y4m" 188 | rm -f "00000000out-$BASENAME.y4m" 189 | "$DUMP_VIDEO" "$BASENAME.ogv" -o "$BASENAME.y4m" 190 | ;; 191 | x264) 192 | $($TIMER $X264 --dump-yuv $BASENAME.yuv --preset placebo --min-keyint $KFINT --keyint $KFINT --no-scenecut --crf=$x -o $BASENAME.x264 $EXTRA_OPTIONS $FILE > "$BASENAME-stdout.txt") 193 | $YUV2YUV4MPEG $BASENAME -w$WIDTH -h$HEIGHT -an0 -ad0 -c420mpeg2 194 | SIZE=$(stat -c %s $BASENAME.x264) 195 | ENC_EXT='.x264' 196 | ;; 197 | x265) 198 | $($TIMER $X265 -r $BASENAME.y4m --preset slow --frame-threads 1 --min-keyint $KFINT --keyint $KFINT --no-scenecut --crf=$x -o $BASENAME.x265 $EXTRA_OPTIONS $FILE > "$BASENAME-stdout.txt") 199 | SIZE=$(stat -c %s $BASENAME.x265) 200 | ENC_EXT='.x265' 201 | ;; 202 | x265-rt) 203 | $($TIMER $X265 -r $BASENAME.y4m --preset slow --tune zerolatency --rc-lookahead 0 --bframes 0 --frame-threads 1 --min-keyint $KFINT --keyint $KFINT --no-scenecut --crf=$x --csv $BASENAME.csv -o $BASENAME.x265 $EXTRA_OPTIONS $FILE > "$BASENAME-stdout.txt") 204 | SIZE=$(stat -c %s $BASENAME.x265) 205 | ENC_EXT='.x265' 206 | ;; 207 | xvc) 208 | $($TIMER $XVCENC -max-keypic-distance $KFINT -qp $x -output-file $BASENAME.xvc $EXTRA_OPTIONS -input-file $FILE > "$BASENAME-stdout.txt") 209 | $($TIMERDEC $XVCDEC -output-bitdepth $DEPTH -dither 0 -output-file $BASENAME.yuv -bitstream-file $BASENAME.xvc >> $BASENAME-stdout.txt) 210 | $YUV2YUV4MPEG $BASENAME -w$WIDTH -h$HEIGHT -an0 -ad0 -c420mpeg2 211 | SIZE=$(stat -c %s $BASENAME.xvc) 212 | ENC_EXT='.x265' 213 | ;; 214 | vp8) 215 | $($TIMER $VPXENC --codec=$CODEC --threads=1 --cpu-used=0 --kf-min-dist=$KFINT --kf-max-dist=$KFINT --end-usage=cq --target-bitrate=100000 --cq-level=$x -o $BASENAME.vpx $EXTRA_OPTIONS $FILE > "$BASENAME-stdout.txt") 216 | $VPXDEC --codec=$CODEC -o $BASENAME.y4m $BASENAME.vpx 217 | SIZE=$(stat -c %s $BASENAME.vpx) 218 | ENC_EXT='.vpx' 219 | ;; 220 | vp9) 221 | $($TIMER $VPXENC --codec=$CODEC --ivf --frame-parallel=0 --tile-columns=0 --auto-alt-ref=2 --cpu-used=0 --passes=2 --threads=1 --kf-min-dist=$KFINT --kf-max-dist=$KFINT --lag-in-frames=25 --end-usage=q --cq-level=$x -o $BASENAME.vpx $EXTRA_OPTIONS $FILE > "$BASENAME-stdout.txt") 222 | $VPXDEC --codec=$CODEC -o $BASENAME.y4m $BASENAME.vpx 223 | SIZE=$(stat -c %s $BASENAME.vpx) 224 | ENC_EXT='.vpx' 225 | ;; 226 | vp9-rt) 227 | $($TIMER $VPXENC --codec=vp9 --ivf --frame-parallel=0 --tile-columns=0 --cpu-used=0 --threads=1 --kf-min-dist=$KFINT --kf-max-dist=$KFINT -p 1 --lag-in-frames=0 --end-usage=q --cq-level=$x -o $BASENAME.vpx $EXTRA_OPTIONS $FILE > "$BASENAME-stdout.txt") 228 | $VPXDEC --codec=vp9 -o $BASENAME.y4m $BASENAME.vpx 229 | SIZE=$(stat -c %s $BASENAME.vpx) 230 | ENC_EXT='.vpx' 231 | ;; 232 | vp10) 233 | $($TIMER $VPXENC --codec=$CODEC --ivf --frame-parallel=0 --tile-columns=0 --auto-alt-ref=2 --cpu-used=0 --passes=2 --threads=1 --kf-min-dist=$KFINT --kf-max-dist=$KFINT --lag-in-frames=25 --end-usage=q --cq-level=$x -o $BASENAME.vpx $EXTRA_OPTIONS $FILE > "$BASENAME-stdout.txt") 234 | $VPXDEC --codec=$CODEC -o $BASENAME.y4m $BASENAME.vpx 235 | SIZE=$(stat -c %s $BASENAME.vpx) 236 | ENC_EXT='.vpx' 237 | ;; 238 | vp10-rt) 239 | $($TIMER $VPXENC --codec=vp10 --ivf --frame-parallel=0 --tile-columns=0 --cpu-used=0 --passes=1 --threads=1 --kf-min-dist=$KFINT --kf-max-dist=$KFINT --lag-in-frames=0 --end-usage=q --cq-level=$x -o $BASENAME.vpx $EXTRA_OPTIONS $FILE > "$BASENAME-stdout.txt") 240 | $VPXDEC --codec=vp10 -o $BASENAME.y4m $BASENAME.vpx 241 | SIZE=$(stat -c %s $BASENAME.vpx) 242 | ENC_EXT='.vpx' 243 | ;; 244 | av1) 245 | $($TIMER $AOMENC --codec=$CODEC --ivf --frame-parallel=0 --tile-columns=0 --auto-alt-ref=2 --cpu-used=0 --passes=2 --threads=1 --kf-min-dist=$KFINT --kf-max-dist=$KFINT --lag-in-frames=25 --end-usage=q --cq-level=$x --test-decode=fatal -o $BASENAME.ivf $EXTRA_OPTIONS $FILE > "$BASENAME-stdout.txt") 246 | if $AOMDEC --help 2>&1 | grep output-bit-depth > /dev/null; then 247 | AOMDEC_OPTS+=" --output-bit-depth=$DEPTH" 248 | fi 249 | $($TIMERDEC $AOMDEC --codec=$CODEC $AOMDEC_OPTS -o $BASENAME.y4m $BASENAME.ivf) 250 | SIZE=$(stat -c %s $BASENAME.ivf) 251 | ENC_EXT='.ivf' 252 | ;; 253 | av1-rt) 254 | $($TIMER $AOMENC --codec=av1 --ivf --frame-parallel=0 --tile-columns=0 --cpu-used=0 --passes=1 --threads=1 --kf-min-dist=$KFINT --kf-max-dist=$KFINT --lag-in-frames=0 --end-usage=q --cq-level=$x --test-decode=fatal -o $BASENAME.ivf $EXTRA_OPTIONS $FILE > "$BASENAME-stdout.txt") 255 | if $AOMDEC --help 2>&1 | grep output-bit-depth > /dev/null; then 256 | AOMDEC_OPTS+=" --output-bit-depth=$DEPTH" 257 | fi 258 | $($TIMERDEC $AOMDEC --codec=av1 $AOMDEC_OPTS -o $BASENAME.y4m $BASENAME.ivf) 259 | SIZE=$(stat -c %s $BASENAME.ivf) 260 | ENC_EXT='.ivf' 261 | ;; 262 | av2 | av2-ai | av2-ra | av2-ra-st | av2-ld | av2-as | av2-as-st) 263 | case $CODEC in 264 | av2-ai) 265 | CTC_PROFILE_OPTS="--cpu-used=0 --passes=1 --end-usage=q --kf-min-dist=0 --kf-max-dist=0 --use-fixed-qp-offsets=1 --limit=30 --deltaq-mode=0 --enable-tpl-model=0 --enable-keyframe-filtering=0 --obu" 266 | ;; 267 | av2-ra | av2-ra-st | av2-as | av2-as | av2-as-st) 268 | CTC_PROFILE_OPTS="--cpu-used=0 --passes=1 --lag-in-frames=19 --auto-alt-ref=1 --min-gf-interval=16 --max-gf-interval=16 --gf-min-pyr-height=4 --gf-max-pyr-height=4 --limit=130 --kf-min-dist=65 --kf-max-dist=65 --use-fixed-qp-offsets=1 --deltaq-mode=0 --enable-tpl-model=0 --end-usage=q --enable-keyframe-filtering=0 --obu" 269 | ;; 270 | av2-ld) 271 | CTC_PROFILE_OPTS="--cpu-used=0 --passes=1 --lag-in-frames=0 --min-gf-interval=16 --max-gf-interval=16 --gf-min-pyr-height=4 --gf-max-pyr-height=4 --limit=130 --kf-min-dist=9999 --kf-max-dist=9999 --use-fixed-qp-offsets=1 --deltaq-mode=0 --enable-tpl-model=0 --end-usage=q --subgop-config-str=ld --enable-keyframe-filtering=0 --obu" 272 | ;; 273 | av2) 274 | # generic, not currently used 275 | CTC_PROFILE_OPTS="" 276 | ;; 277 | esac 278 | if [ $((WIDTH)) -ge 3840 ] && [ $((HEIGHT)) -ge 2160 ]; then 279 | CTC_PROFILE_OPTS+=" --tile-columns=1 --threads=2 --row-mt=0" 280 | else 281 | CTC_PROFILE_OPTS+=" --tile-columns=0 --threads=1" 282 | fi 283 | # CTCv6: 1. 2x2 tiling for E/G1 in RA 284 | # 2. 10bit for A2/A4/B1 285 | # 3. SCC tune for B2; SC detector on for others 286 | # CTCv7: 1. 4 Column tiles for A1, E, G1 in RA 287 | # 2. 2 Column tiles for A2, B1 in RA 288 | # 3. 2 Column, 1 Row tiles (8) for A2, B1 in LD 289 | # 4. 2 Column tiles for A3 in LD 290 | # 5. 15 Frames in AI 291 | # 6. AS is Optional 292 | # 6.1 4 tiles for 2K, 4K (3840x2160, 2560x1440) 293 | # 6.2 2 tiles for 1080p (1920x1080) 294 | # 6.3 1 tiles for 720p and below (1280x720, 960x540, 640x360) 295 | case $CTC_VERSION in 296 | 7.0) 297 | case $CODEC in 298 | av2-ra | av2-ra-st) 299 | case $CTC_CLASS in 300 | A1 | E | G1) 301 | CTC_PROFILE_OPTS+=" --row-mt=0 --threads=4 --tile-rows=0 --tile-columns=2" 302 | ;; 303 | A2 | B1) 304 | CTC_PROFILE_OPTS+=" --row-mt=0 --threads=2 --tile-rows=0 --tile-columns=1" 305 | esac 306 | ;; 307 | av2-ld) 308 | case $CTC_CLASS in 309 | A2 | B1) 310 | CTC_PROFILE_OPTS+=" --row-mt=0 --threads=8 --tile-rows=1 --tile-columns=2" 311 | ;; 312 | A3) 313 | CTC_PROFILE_OPTS+=" --row-mt=0 --threads=2 --tile-rows=0 --tile-columns=1" 314 | esac 315 | ;; 316 | av2-ai) 317 | CTC_PROFILE_OPTS+=" --limit-=15" 318 | ;; 319 | av2-as | av2-as-st) 320 | case "$width"x"$height" in 321 | 3840x2160|2560x1440) 322 | CTC_PROFILE_OPTS+=" --row-mt=0 --threads=4 --tile-rows=0 --tile-columns=2" 323 | ;; 324 | 1920x1080) 325 | CTC_PROFILE_OPTS+=" --row-mt=0 --threads=2 --tile-rows=0 --tile-columns=1" 326 | ;; 327 | 1280x720|960x540|640x360) 328 | CTC_PROFILE_OPTS+=" --row-mt=0 --threads=1 --tile-rows=0 --tile-columns=0" 329 | ;; 330 | esac 331 | esac 332 | ;; 333 | 6.0) 334 | case $CTC_CLASS in 335 | E | G1) 336 | case $CODEC in 337 | av2-ra | av2-ra-st) 338 | CTC_PROFILE_OPTS+=" --row-mt=0 --threads=4 --tile-rows=1 --tile-columns=1" 339 | ;; 340 | esac 341 | esac 342 | ;; 343 | esac 344 | 345 | case $CTC_VERSION in 346 | 6.0 | 7.0) 347 | case $CTC_CLASS in 348 | A2 | A4 | B1) 349 | CTC_PROFILE_OPTS+=" --bit-depth=10" 350 | ;; 351 | esac 352 | esac 353 | 354 | case $CTC_VERSION in 355 | 6.0 | 7.0) 356 | case $CTC_CLASS in 357 | B2) 358 | CTC_PROFILE_OPTS+=" --tune-content=screen --enable-intrabc-ext=1" 359 | ;; 360 | *) 361 | CTC_PROFILE_OPTS+=" --enable-intrabc-ext=2" 362 | ;; 363 | esac 364 | ;; 365 | esac 366 | case $CTC_CLASS in 367 | G1 | G2) 368 | CTC_PROFILE_OPTS+=" --color-primaries=bt2020 --transfer-characteristics=smpte2084 --matrix-coefficients=bt2020ncl --chroma-sample-position=colocated" 369 | ;; 370 | F1 | F2) 371 | CTC_PROFILE_OPTS+=" --limit=1 " 372 | ;; 373 | # CTCv4: Suggests to have multhreading with tiling for A2 and B1 in LD 374 | # CTCv5/v6: Suggests to have column tiling along with row-tiling:) 375 | A2 | B1) 376 | case $CODEC in 377 | av2-ld) 378 | case $CTC_VERSION in 379 | 5.0 | 6.0) 380 | CTC_PROFILE_OPTS+=" --row-mt=0 --threads=4 --tile-rows=1 --tile-columns=1" 381 | ;; 382 | 4.0) 383 | CTC_PROFILE_OPTS+=" --row-mt=0 --threads=2 --tile-rows=1" 384 | ;; 385 | esac 386 | ;; 387 | esac 388 | ;; 389 | esac 390 | # threading options for the A1 test set must be overriden via EXTRA_OPTIONS at a higher level 391 | case $CODEC in 392 | av2-ra | av2-as) 393 | # this is intentionally not a separate script as only metrics_gather.sh is sent to workers 394 | echo "#!/bin/bash" > /tmp/enc$$.sh 395 | echo "PERF_ENC_OUT='${BASENAME}'-encperf-\$1.out" >> /tmp/enc$$.sh 396 | echo "PERF_ENC_STAT='perf stat -o '\${PERF_ENC_OUT}''" >> /tmp/enc$$.sh 397 | echo "TIMER=''\${PERF_ENC_STAT}' time -v --output='enctime$$-\$1.out" >> /tmp/enc$$.sh 398 | echo "RUN='$AOMENC --qp=$x --test-decode=fatal $CTC_PROFILE_OPTS -o $BASENAME-'\$1'.obu --limit=130 --'\$1'=65 $EXTRA_OPTIONS $FILE'" >> /tmp/enc$$.sh 399 | echo "\$(\$TIMER \$RUN > $BASENAME$$-\$1-stdout.txt)" >> /tmp/enc$$.sh 400 | chmod +x /tmp/enc$$.sh 401 | for s in {limit,skip}; do printf "$s\0"; done | xargs -0 -n1 -P2 /tmp/enc$$.sh 402 | $(cat $BASENAME-limit.obu $BASENAME-skip.obu > $BASENAME.obu) 403 | $(cat $BASENAME$$-limit-stdout.txt $BASENAME$$-skip-stdout.txt > $BASENAME-stdout.txt) 404 | $(cat enctime$$-limit.out enctime$$-skip.out > $BASENAME-enctime.out) 405 | $(cat ${BASENAME}-encperf-limit.out ${BASENAME}-encperf-skip.out > $BASENAME-encperf.out) 406 | TIME1=$(cat enctime$$-limit.out | grep User | cut -d\ -f4) 407 | TIME2=$(cat enctime$$-skip.out | grep User | cut -d\ -f4) 408 | PERF_ENC_INSTR_CNT1=$(awk '/instructions/ { s=$1 } END { gsub(",", "", s) ; print s }' ${BASENAME}-encperf-limit.out) 409 | PERF_ENC_INSTR_CNT2=$(awk '/instructions/ { s=$1 } END { gsub(",", "", s) ; print s }' ${BASENAME}-encperf-skip.out) 410 | PERF_ENC_CYCLE_CNT1=$(awk '/cycles/ { s=$1 } END { gsub(",", "", s) ; print s }' ${BASENAME}-encperf-limit.out) 411 | PERF_ENC_CYCLE_CNT2=$(awk '/cycles/ { s=$1 } END { gsub(",", "", s) ; print s }' ${BASENAME}-encperf-skip.out) 412 | ENCTIME=$(awk "BEGIN {print $TIME1+$TIME2; exit}") 413 | PERF_ENC_INSTR_CNT=$(awk "BEGIN {print $PERF_ENC_INSTR_CNT1+$PERF_ENC_INSTR_CNT2; exit}") 414 | PERF_ENC_CYCLE_CNT=$(awk "BEGIN {print $PERF_ENC_CYCLE_CNT1+$PERF_ENC_CYCLE_CNT2; exit}") 415 | rm -f /tmp/enc$$.sh enctime$$-limit.out enctime$$-skip.out $BASENAME-limit.obu $BASENAME-skip.obu 416 | ;; 417 | *) 418 | $($TIMER $AOMENC --qp=$x --test-decode=fatal $CTC_PROFILE_OPTS -o $BASENAME.obu $EXTRA_OPTIONS $FILE > "$BASENAME-stdout.txt") 419 | ;; 420 | esac 421 | # decode the OBU to Y4M 422 | if $AOMDEC --help 2>&1 | grep output-bit-depth > /dev/null; then 423 | AOMDEC_OPTS+=" --output-bit-depth=$DEPTH" 424 | fi 425 | $($TIMERDEC $AOMDEC $AOMDEC_OPTS -o $BASENAME.y4m $BASENAME.obu) 426 | SIZE=$(stat -c %s $BASENAME.obu) 427 | ENC_EXT='.obu' 428 | case $CODEC in 429 | av2-as | av2-as-st) 430 | if [ $((WIDTH)) -ne 3840 ] && [ $((HEIGHT)) -ne 2160 ]; then 431 | # change the reference to 3840x2160 432 | FILE=$(sed -e 's/$640x360\|960x540\|1280x720\|1920x1080\|2560x1440$/3840x2160/' <<< $FILE) 433 | # hack to force input Y4M file to F30:1 because HDRConvert requires specifying an output frame rate, and if they do not match it will resample temporaly 434 | echo "YUV4MPEG2 W$WIDTH H$HEIGHT F30:1 $CHROMA" > $BASENAME-$$.y4m 435 | $(tail -n+2 $BASENAME.y4m >> $BASENAME-$$.y4m) 436 | # upsample decoded output to 3840x2160 437 | $HDRCONVERT -p SourceFile=$BASENAME-$$.y4m -p OutputFile=$BASENAME-$$-out.y4m -p OutputWidth=3840 -p OutputHeight=2160 -p OutputChromaFormat=1 -p OutputBitDepthCmp0=10 -p OutputBitDepthCmp1=10 -p OutputBitDepthCmp2=10 -p OutputColorSpace=0 -p OutputColorPrimaries=0 -p OutputTransferFunction=12 -p SilentMode=1 -p ScaleOnly=1 -p ScalingMode=12 -p OutputRate=30 -p NumberOfFrames=130 1>&2 438 | # replace decoded output with upsampled file using Y4M header from the reference 439 | $(head -n 1 $FILE > $BASENAME.y4m) 440 | $(tail -n+2 $BASENAME-$$-out.y4m >> $BASENAME.y4m) 441 | rm $BASENAME-$$.y4m $BASENAME-$$-out.y4m 442 | fi 443 | ;; 444 | esac 445 | ;; 446 | vvc-vtm | vvc-vtm-ra | vvc-vtm-ra-ctc | vvc-vtm-ra-st | vvc-vtm-as-ctc | vvc-vtm-ld | vvc-vtm-ai) 447 | case $CODEC in 448 | vvc-vtm-ra | vvc-vtm-ra-st) 449 | # VVC_CTC says IntraPeriod should be different for different FPS, *shrug* 450 | VVC_CFG=$WORK_ROOT/rd_tool/cfg/vvc-vtm/encoder_randomaccess_vtm.cfg 451 | INTRA_PERIOD=32 452 | case $FPS in 453 | 20 | 24 | 30) 454 | INTRA_PERIOD=32 455 | ;; 456 | 50 | 60) 457 | INTRA_PERIOD=64 458 | ;; 459 | 100) 460 | INTRA_PERIOD=100 461 | ;; 462 | esac 463 | CTC_PROFILE_OPTS+=" --IntraPeriod=$INTRA_PERIOD " 464 | ;; 465 | vvc-vtm-ra-ctc | vvc-vtm-as-ctc) 466 | VVC_CFG=$WORK_ROOT/rd_tool/cfg/vvc-vtm/encoder_randomaccess_vtm_gop16.cfg 467 | ;; 468 | vvc-vtm-ld) 469 | VVC_CFG=$WORK_ROOT/rd_tool/cfg/vvc-vtm/encoder_lowdelay_vtm.cfg 470 | ;; 471 | vvc-vtm-ai) 472 | VVC_CFG=$WORK_ROOT/rd_tool/cfg/vvc-vtm/encoder_intra_vtm.cfg 473 | ;; 474 | vvc-vtm) 475 | VVC_CFG='' 476 | ;; 477 | esac 478 | FPS_NUM=$(grep -o -a -m 1 -P "(?<=F)([0-9]+)(?=:[0-9]+)" "$FILE") 479 | FPS_DEN=$(grep -o -a -m 1 -P "(?<=F$FPS_NUM:)([0-9]+)" "$FILE") 480 | FPS=$(bc <<< 'scale=3; '$FPS_NUM' / '$FPS_DEN'') 481 | # Enbale Tiling explictly for >=4K 482 | if [ $((WIDTH)) -ge 3840 ] && [ $((HEIGHT)) -ge 2160 ]; then 483 | CTC_PROFILE_OPTS+=" --EnablePicPartitioning=1 --TileColumnWidthArray=15" 484 | else 485 | CTC_PROFILE_OPTS+=" " 486 | fi 487 | # CTCv6: 1. 2x2 tiling for E/G1 in RA 488 | # 2. 10bit for A2/A4/B1 489 | case $CTC_VERSION in 490 | 6.0) 491 | case $CTC_CLASS in 492 | E | G1) 493 | case $CODEC in 494 | vvc-vtm-ra | vvc-vtm-ra-st | vvc-vtm-ra-ctc) 495 | # Manually count the tiles based on MAX_CTU which is currently 128 as 496 | # there are vertical videos 497 | TILES_W_CNT=$(awk -v num=$(echo "scale=2; $WIDTH/128/2" | bc) 'BEGIN { print int(num+(num>int(num))) }') 498 | TILES_H_CNT=$(awk -v num=$(echo "scale=2; $HEIGHT/128/2" | bc) 'BEGIN { print int(num+(num>int(num))) }') 499 | CTC_PROFILE_OPTS+=" --EnablePicPartitioning=1 --TileRowHeightArray=${TILES_H_CNT} --TileColumnWidthArray=${TILES_W_CNT}" 500 | ;; 501 | esac 502 | ;; 503 | A2 | A4 | B1) 504 | CTC_PROFILE_OPTS+=" --InternalBitDepth=10 --InputBitDepth=10" 505 | ;; 506 | esac 507 | ;; 508 | esac 509 | case $CTC_CLASS in 510 | A2 | B1) 511 | case $CODEC in 512 | vvc-vtm-ld) 513 | case $CTC_VERSION in 514 | 5.0 | 6.0) 515 | TILES_W_CNT=$(awk -v num=$(echo "scale=2; $WIDTH/128/2" | bc) 'BEGIN { print int(num+(num>int(num))) }') 516 | TILES_H_CNT=$(awk -v num=$(echo "scale=2; $HEIGHT/128/2" | bc) 'BEGIN { print int(num+(num>int(num))) }') 517 | CTC_PROFILE_OPTS+=" --EnablePicPartitioning=1 --TileRowHeightArray=${TILES_H_CNT} --TileColumnWidthArray=${TILES_W_CNT}" 518 | ;; 519 | esac 520 | ;; 521 | esac 522 | ;; 523 | esac 524 | 525 | INPUT_SRC_VID=$FILE 526 | 527 | # Encode video 528 | case $CODEC in 529 | vvc-vtm-ra) 530 | # this is intentionally not a separate script as only metrics_gather.sh is 531 | # sent to workers 532 | # VVC parCat requires perfect constructed GOP to concat, so we are always 533 | # overriding the $EXTRA_OPTIONS wrt GOP Struct. 534 | echo "#!/bin/bash" > /tmp/enc$$.sh 535 | echo "TIMER='time -v --output='enctime$$-\$1.out" >> /tmp/enc$$.sh 536 | echo "case \$1 in FramesToBeEncoded) GOP_PARAMS=\"--\$1=65\";; FrameSkip) GOP_PARAMS=\" --FramesToBeEncoded=65 --\$1=65 \" ;; esac" >> /tmp/enc$$.sh 537 | echo "RUN='$VVCENC -i $INPUT_SRC_VID -c $VVC_CFG --SourceWidth=$WIDTH --SourceHeight=$HEIGHT --FrameRate=$FPS --InputBitDepth=$DEPTH --InternalBitDepth=$DEPTH --FramesToBeEncoded=130 --QP=$x $CTC_PROFILE_OPTS --ReconFile=${BASENAME}-'\$1'-rec.yuv -b $BASENAME-'\$1'.bin $cd '" >> /tmp/enc$$.sh 538 | echo "\$(\$TIMER \$RUN \$GOP_PARAMS > $BASENAME$$-stdout.txt)" >> /tmp/enc$$.sh 539 | chmod +x /tmp/enc$$.sh 540 | for s in {FramesToBeEncoded,FrameSkip}; do printf "$s\0"; done | xargs -0 -n1 -P2 /tmp/enc$$.sh 541 | # parcat is tool by VTM for concating files as per JVET-B0036 542 | $($VVCCAT $BASENAME-FramesToBeEncoded.bin $BASENAME-FrameSkip.bin $BASENAME.bin) 543 | TIME1=$(cat enctime$$-FramesToBeEncoded.out | grep User | cut -d\ -f4) 544 | TIME2=$(cat enctime$$-FrameSkip.out | grep User | cut -d\ -f4) 545 | ENCTIME=$(awk "BEGIN {print $TIME1+$TIME2; exit}") 546 | rm -f /tmp/enc$$.sh enctime$$-FramesToBeEncoded.out enctime$$-FrameSkip.out $BASENAME-FramesToBeEncoded.bin $BASENAME-FrameSkip.bin 547 | ;; 548 | vvc-vtm-ra-ctc | vvc-vtm-as-ctc) 549 | # this is intentionally not a separate script as only metrics_gather.sh is 550 | # sent to workers 551 | echo "#!/bin/bash" > /tmp/enc$$.sh 552 | echo "PERF_ENC_OUT='${BASENAME}'-encperf-\$1.out" >> /tmp/enc$$.sh 553 | echo "PERF_ENC_STAT='perf stat -o '\${PERF_ENC_OUT}''" >> /tmp/enc$$.sh 554 | echo "TIMER=''\${PERF_ENC_STAT}' time -v --output='enctime$$-\$1.out" >> /tmp/enc$$.sh 555 | echo "case \$1 in FramesToBeEncoded) GOP_PARAMS=\"--\$1=65\";; FrameSkip) GOP_PARAMS=\" --FramesToBeEncoded=65 --\$1=65 \" ;; esac" >> /tmp/enc$$.sh 556 | echo "RUN='$VVCENC -i $INPUT_SRC_VID -c $VVC_CFG --SourceWidth=$WIDTH --SourceHeight=$HEIGHT --FrameRate=$FPS --InputBitDepth=$DEPTH --InternalBitDepth=$DEPTH --FramesToBeEncoded=130 --QP=$x $CTC_PROFILE_OPTS --ReconFile=${BASENAME}-'\$1'-rec.yuv -b $BASENAME-'\$1'.bin $EXTRA_OPTIONS '" >> /tmp/enc$$.sh 557 | # Force config to be Closed-GOP (IDR) with only 1 I Frame 558 | CTC_PARAMS="--DecodingRefreshType=2 --IntraPeriod=-1" 559 | echo "CTC_PARAMS=\" $CTC_PARAMS \" " >> /tmp/enc$$.sh 560 | echo "\$(\$TIMER \$RUN \$GOP_PARAMS \$CTC_PARAMS > $BASENAME$$-\$1-stdout.txt) " >> /tmp/enc$$.sh 561 | chmod +x /tmp/enc$$.sh 562 | for s in {FramesToBeEncoded,FrameSkip}; do printf "$s\0"; done | xargs -0 -n1 -P2 /tmp/enc$$.sh 563 | # do classic concat as it is IDR 564 | $(cat $BASENAME-FramesToBeEncoded.bin $BASENAME-FrameSkip.bin > $BASENAME.bin) 565 | $(cat $BASENAME$$-FramesToBeEncoded-stdout.txt $BASENAME$$-FrameSkip-stdout.txt > $BASENAME-stdout.txt) 566 | $(cat enctime$$-FramesToBeEncoded.out enctime$$-FrameSkip.out > $BASENAME-enctime.out) 567 | $(cat ${BASENAME}-encperf-FramesToBeEncoded.out ${BASENAME}-encperf-FrameSkip.out > $BASENAME-encperf.out) 568 | TIME1=$(cat enctime$$-FramesToBeEncoded.out | grep User | cut -d\ -f4) 569 | TIME2=$(cat enctime$$-FrameSkip.out | grep User | cut -d\ -f4) 570 | PERF_ENC_INSTR_CNT1=$(awk '/instructions/ { s=$1 } END { gsub(",", "", s) ; print s }' ${BASENAME}-encperf-FramesToBeEncoded.out) 571 | PERF_ENC_INSTR_CNT2=$(awk '/instructions/ { s=$1 } END { gsub(",", "", s) ; print s }' ${BASENAME}-encperf-FrameSkip.out) 572 | PERF_ENC_CYCLE_CNT1=$(awk '/cycles/ { s=$1 } END { gsub(",", "", s) ; print s }' ${BASENAME}-encperf-FramesToBeEncoded.out) 573 | PERF_ENC_CYCLE_CNT2=$(awk '/cycles/ { s=$1 } END { gsub(",", "", s) ; print s }' ${BASENAME}-encperf-FrameSkip.out) 574 | ENCTIME=$(awk "BEGIN {print $TIME1+$TIME2; exit}") 575 | PERF_ENC_INSTR_CNT=$(awk "BEGIN {print $PERF_ENC_INSTR_CNT1+$PERF_ENC_INSTR_CNT2; exit}") 576 | PERF_ENC_CYCLE_CNT=$(awk "BEGIN {print $PERF_ENC_CYCLE_CNT1+$PERF_ENC_CYCLE_CNT2; exit}") 577 | rm -f /tmp/enc$$.sh enctime$$-FramesToBeEncoded.out enctime$$-FrameSkip.out $BASENAME-FramesToBeEncoded.bin $BASENAME-FrameSkip.bin 578 | ;; 579 | *) 580 | $($TIMER $VVCENC -i $INPUT_SRC_VID -c $VVC_CFG --SourceWidth=$WIDTH --SourceHeight=$HEIGHT --FrameRate=$FPS --InputBitDepth=$DEPTH --InternalBitDepth=$DEPTH --FramesToBeEncoded=130 --QP=$x $CTC_PROFILE_OPTS -b $BASENAME.bin --ReconFile=${BASENAME}-rec.yuv $EXTRA_OPTIONS > "$BASENAME-stdout.txt") 581 | ;; 582 | esac 583 | # Decode the video 584 | $($TIMERDEC $VVCDEC -b $BASENAME.bin -d $DEPTH -o $BASENAME.yuv > "$BASENAME-dec.txt") 585 | # Convert the YUV file to Y4M file. 586 | # YUV2YUV4MPEG of daala_tool takes headers as args and filename without *.yuv* 587 | # extension 588 | $YUV2YUV4MPEG $BASENAME -an1 -ad1 -w$WIDTH -h$HEIGHT -fn$FPS_NUM -fd$FPS_DEN -c$YUV_CHROMA -b$DEPTH 589 | SIZE=$(stat -c %s $BASENAME.bin) 590 | ENC_EXT='.bin' 591 | case $CODEC in 592 | vvc-vtm-as-ctc) 593 | if [ $((WIDTH)) -ne 3840 ] && [ $((HEIGHT)) -ne 2160 ]; then 594 | # change the reference to 3840x2160 595 | FILE=$(sed -e 's/$640x360\|960x540\|1280x720\|1920x1080\|2560x1440$/3840x2160/' <<< $FILE) 596 | # hack to force input Y4M file to F30:1 because HDRConvert requires specifying an output frame rate, and if they do not match it will resample temporaly 597 | echo "YUV4MPEG2 W$WIDTH H$HEIGHT F30:1 $CHROMA" > $BASENAME-$$.y4m 598 | $(tail -n+2 $BASENAME.y4m >> $BASENAME-$$.y4m) 599 | # upsample decoded output to 3840x2160 600 | $HDRCONVERT -p SourceFile=$BASENAME-$$.y4m -p OutputFile=$BASENAME-$$-out.y4m -p OutputWidth=3840 -p OutputHeight=2160 -p OutputChromaFormat=1 -p OutputBitDepthCmp0=10 -p OutputBitDepthCmp1=10 -p OutputBitDepthCmp2=10 -p OutputColorSpace=0 -p OutputColorPrimaries=0 -p OutputTransferFunction=12 -p SilentMode=1 -p ScaleOnly=1 -p ScalingMode=12 -p OutputRate=30 -p NumberOfFrames=130 1>&2 601 | # replace decoded output with upsampled file using Y4M header from the reference 602 | $(head -n 1 $FILE > $BASENAME.y4m) 603 | $(tail -n+2 $BASENAME-$$-out.y4m >> $BASENAME.y4m) 604 | rm $BASENAME-$$.y4m $BASENAME-$$-out.y4m 605 | fi 606 | ;; 607 | esac 608 | ;; 609 | thor) 610 | $($TIMER $THORENC -qp $x -cf "$THORDIR/config_HDB16_high_efficiency.txt" -if $FILE -of $BASENAME.thor $EXTRA_OPTIONS > $BASENAME-enc.out) 611 | SIZE=$(stat -c %s $BASENAME.thor) 612 | ENC_EXT='.thor' 613 | # using reconstruction is currently broken with HDB 614 | $THORDEC $BASENAME.thor $BASENAME.yuv > "$BASENAME-stdout.txt" 615 | $YUV2YUV4MPEG $BASENAME -an1 -ad1 -w$WIDTH -h$HEIGHT 616 | ;; 617 | thor-rt) 618 | $($TIMER $THORENC -qp $x -cf "$THORDIR/config_LDB_high_efficiency.txt" -if $FILE -of $BASENAME.thor -rf $BASENAME.y4m $EXTRA_OPTIONS > $BASENAME-enc.out) 619 | SIZE=$(stat -c %s $BASENAME.thor) 620 | ENC_EXT='.thor' 621 | ;; 622 | rav1e) 623 | $($TIMER $RAV1E $FILE --quantizer $x -o $BASENAME.ivf -r $BASENAME-rec.y4m --threads 1 $EXTRA_OPTIONS > $BASENAME-enc.out) 624 | if hash dav1d 2>/dev/null; then 625 | $($TIMERDEC dav1d -q -i $BASENAME.ivf -o $BASENAME.y4m) || (echo "Corrupt bitstream detected!"; exit 98) 626 | elif hash aomdec 2>/dev/null; then 627 | $($TIMERDEC aomdec --codec=av1 $AOMDEC_OPTS -o $BASENAME.y4m $BASENAME.ivf) || (echo "Corrupt bitstream detected!"; exit 98) 628 | else 629 | echo "AV1 decoder not found, desync/corruption detection disabled." >&2 630 | fi 631 | 632 | if [ -f $BASENAME.y4m ]; then 633 | "$Y4M2YUV" "$BASENAME-rec.y4m" -o rec.yuv 634 | "$Y4M2YUV" "$BASENAME.y4m" -o enc.yuv 635 | cmp --silent rec.yuv enc.yuv || (echo "Reconstruction differs from output!"; rm -f rec.yuv enc.yuv "$BASENAME-rec.y4m"; exit 98) 636 | rm -f rec.yuv enc.yuv "$BASENAME-rec.y4m" 637 | else 638 | mv "$BASENAME-rec.y4m" "$BASENAME.y4m" 639 | fi 640 | SIZE=$(stat -c %s $BASENAME.ivf) 641 | ENC_EXT='.ivf' 642 | ;; 643 | svt-av1 | svt-av1-ra | svt-av1-ra-crf | svt-av1-ra-vbr | svt-av1-ra-vbr-2p | svt-av1-ld-cbr | svt-av1-ra-cq | svt-av1-as | svt-av1-as-ctc) 644 | case $CODEC in 645 | # 1-pass CQ, CTC Style Preset 646 | svt-av1-ra) 647 | SVT_PROFILE_OPTS="--preset 0 --passes 1 --hierarchical-levels 4 --frames 130 --keyint 65 --aq-mode 0 --rc 0 --qp $x" 648 | ;; 649 | # 1-pass CRF RA 650 | svt-av1-ra-crf) 651 | SVT_PROFILE_OPTS="--lp 1 --passes 1 --rc 0 --crf $x --pred-struct 2" 652 | ;; 653 | # 1-pass VBR RA 654 | svt-av1-ra-vbr) 655 | SVT_PROFILE_OPTS="--lp 1 --passes 1 --rc 1 --tbr $x --pred-struct 2" 656 | ;; 657 | # 2-pass VBR RA 658 | svt-av1-ra-vbr-2p) 659 | SVT_PROFILE_OPTS="--lp 1 --passes 2 --rc 1 --tbr $x --pred-struct 2" 660 | ;; 661 | # 1-pass CBR LD, RTC Mode 662 | svt-av1-ld-cbr) 663 | SVT_PROFILE_OPTS="--lp 1 --passes 1 --rc 2 --tbr $x --pred-struct 1" 664 | ;; 665 | # 1-pass CQP RA 666 | svt-av1-ra-cq) 667 | SVT_PROFILE_OPTS="--lp 1 --passes 1 --rc 0 --aq-mode 0 --crf $x --pred-struct 2 --keyint 999" 668 | ;; 669 | # 1-pass SIWG-CTC AS 670 | svt-av1-as | svt-av1-as-ctc) 671 | SVT_PROFILE_OPTS="--lp 1 --passes 1 --keyint -1 --crf $x" 672 | ;; 673 | svt-av1) 674 | # Always define CRF points for the given QPs 675 | SVT_PROFILE_OPTS="--lp 1 --crf $x " 676 | ;; 677 | esac 678 | # Encode the video 679 | $($TIMER $SVTAV1 -i $FILE $SVT_PROFILE_OPTS -b $BASENAME.ivf $EXTRA_OPTIONS > $BASENAME-stdout.txt 2>&1) 680 | # Decode the video 681 | if hash dav1d 2>/dev/null; then 682 | $($TIMERDEC dav1d -q -i $BASENAME.ivf -o $BASENAME.y4m) || (echo "Corrupt bitstream detected!"; exit 98) 683 | elif hash aomdec 2>/dev/null; then 684 | $($TIMERDEC aomdec --codec=av1 $AOMDEC_OPTS -o $BASENAME.y4m $BASENAME.ivf) || (echo "Corrupt bitstream detected!"; exit 98) 685 | else 686 | echo "AV1 decoder not found, desync/corruption detection disabled." >&2 687 | fi 688 | SIZE=$(stat -c %s $BASENAME.ivf) 689 | ENC_EXT='.ivf' 690 | # Upscale to 1080p and compute metrics for svt-av1-as 691 | # Depends on custom-ffmpeg binary 692 | case $CODEC in 693 | svt-av1-as | svt-av1-as-ctc) 694 | if [ $((WIDTH)) -ne 1920 ] && [ $((HEIGHT)) -ne 1080 ]; then 695 | # Change source file to be 1080p 696 | FILE=$(sed -e 's/$to256x144_lanc\|to384x216_lanc\|to512x288_lanc\|to640x360_lanc\|to768x432_lanc\|to960x540_lanc\|to1280x720_lanc$//' <<< $FILE) 697 | fi 698 | # Use FFmpeg to upscale and compute reduced-libvmaf as they are fast 699 | # When measured the process can be very fast, we are talking ~3fps over 700 | # ~0.8fps 701 | # Reduced-libvmaf: LIBVMAF - {CIEDE2000, MS-SSIM, CAMBI, PSNR_HVS} 702 | ffmpeg -hide_banner -loglevel error -threads 1 -y -nostdin -r 25 -i $BASENAME.y4m -r 25 -i $FILE -lavfi '[0:v]scale=1920x1080:flags=lanczos+accurate_rnd+full_chroma_int:sws_dither=none:param0=5[main];[main][1:v]libvmaf=aom_ctc=1:log_path='$BASENAME-vmaf.xml':log_fmt=xml' -f null - 703 | ;; 704 | esac 705 | ;; 706 | esac 707 | 708 | #rename core dumps to prevent more than 1 per slot on disk 709 | mv core.* core 2>/dev/null || true 710 | 711 | if [ -f "$VMAF" ]; then 712 | case $CODEC in 713 | svt-av1-as | svt-av1-as-ctc) 714 | # Reduced-Libvmaf: Handle missing metrics at the computation side to make 715 | # life easier. 716 | line_to_append+='\ ' 717 | line_to_append+='' 718 | line_to_append+='\n ' 719 | line_to_append+='' 720 | line_to_append+='\n ' 721 | line_to_append+='' 722 | line_to_append+='\n ' 723 | line_to_append+='' 724 | line_to_append+='\n ' 725 | line_to_append+='' 726 | line_to_append+='\n ' 727 | line_to_append+='' 728 | line_to_append+='\n ' 729 | line_to_append+='' 730 | 731 | # Count lines and insert from 5 rows from end of the file using sed 732 | total_lines=$(wc -l < "$BASENAME-vmaf.xml") 733 | line_number=$((total_lines - 5)) 734 | sed -i "${line_number}a${line_to_append}" "$BASENAME-vmaf.xml" 735 | ;; 736 | *) 737 | "$VMAF" -r "$FILE" -d "$BASENAME.y4m" --aom_ctc v6.0 --xml -o "$BASENAME-vmaf.xml" --thread 1 | tail -n 1 738 | ;; 739 | esac 740 | FRAMES=$(cat "$BASENAME-vmaf.xml" | grep "frame frameNum" | wc -l) 741 | PSNR="Total: 0 (Y': 0 Cb: 0 Cr: 0 )" 742 | else 743 | "$DUMP_PSNR" -a "$FILE" "$BASENAME.y4m" > "$BASENAME-psnr.out" 744 | FRAMES=$(cat "$BASENAME-psnr.out" | grep ^0 | wc -l) 745 | PSNR=$(cat "$BASENAME-psnr.out" | grep Total) 746 | fi 747 | 748 | PIXELS=$(($WIDTH*$HEIGHT*$FRAMES)) 749 | 750 | echo "$x" "$PIXELS" "$SIZE" 751 | 752 | 753 | echo "$PSNR" 754 | 755 | APSNR="Frame-averaged: 0 (Y': 0 Cb: 0 Cr: 0 )" #$(cat "$BASENAME-psnr.out" | grep Frame-averaged) 756 | PSNRHVS="Total: 0 (Y': 0 Cb: 0 Cr: 0 )" #$("$DUMP_PSNRHVS" "$FILE" "$BASENAME.y4m" 2> /dev/null | grep Total) 757 | 758 | echo "$PSNRHVS" 759 | 760 | SSIM="Total: 0 (Y': 0 Cb: 0 Cr: 0 )" #$("$DUMP_SSIM" "$FILE" "$BASENAME.y4m" 2> /dev/null | grep Total) 761 | 762 | echo "$SSIM" 763 | 764 | FASTSSIM="Total: 0 (Y': 0 Cb: 0 Cr: 0 )" #$("$DUMP_FASTSSIM" -c "$FILE" "$BASENAME.y4m" 2> /dev/null | grep Total) 765 | 766 | echo "$FASTSSIM" 767 | 768 | CIEDE="0 0" #$("$DUMP_CIEDE" --threads 1 "$FILE" "$BASENAME.y4m" 2> /dev/null | grep Total) 769 | 770 | echo "$CIEDE" 771 | 772 | echo "$APSNR" 773 | 774 | MSSSIM="Total: 0 (Y': 0 Cb: 0 Cr: 0 )" #$("$DUMP_MSSSIM" "$FILE" "$BASENAME.y4m" 2> /dev/null | grep Total) 775 | 776 | echo "$MSSSIM" 777 | 778 | if [ -e "$TIMEROUT" ]; then 779 | ENCTIME=$(awk '/User/ { s=$4 } END { printf "%.2f", s }' "$TIMEROUT") 780 | else 781 | if [ -z "$ENCTIME" ]; then 782 | ENCTIME=0 783 | fi 784 | fi 785 | 786 | echo "$ENCTIME" 787 | 788 | echo "0" # Dummy placeholder for vmaf_old in the output 789 | 790 | if [ -e "$TIMERDECOUT" ]; then 791 | DECTIME=$(awk '/User/ { s=$4 } END { printf "%.2f", s }' "$TIMERDECOUT") 792 | else 793 | DECTIME=0 794 | fi 795 | 796 | echo "$DECTIME" 797 | 798 | # Extract MD5 of encoded file 799 | ENC_FILE=${BASENAME}${ENC_EXT} 800 | MD5SUM=($(md5sum $ENC_FILE)) 801 | echo $MD5SUM 802 | 803 | # Extract Encoding Instruction count and cycles for Parallel-GOP cases 804 | if ! [[ "$CODEC" =~ ^(av2-ra|av2-as|vvc-vtm-ra|vvc-vtm-as|vvc-vtm-ra-ctc| vvc-vtm-as-ctc)$ ]]; then 805 | if [ -e "$PERF_ENC_OUT" ]; then 806 | PERF_ENC_INSTR_CNT=$(awk '/instructions/ { s=$1 } END { gsub(",", "", s) ; print s }' "$PERF_ENC_OUT") 807 | PERF_ENC_CYCLE_CNT=$(awk '/cycles/ { s=$1 } END { gsub(",", "", s) ; print s }' "$PERF_ENC_OUT") 808 | else 809 | if [ -z "$PERF_ENC_INSTR_CNT" ]; then 810 | PERF_ENC_INSTR_CNT=0 811 | fi 812 | if [ -z "$PERF_ENC_CYCLE_CNT" ]; then 813 | PERF_ENC_CYCLE_CNT=0 814 | fi 815 | fi 816 | # Default to 0 if instr/cycle count is not available for Parallel-GOP cases 817 | else 818 | if [ -z "$PERF_ENC_INSTR_CNT" ]; then 819 | PERF_ENC_INSTR_CNT=0 820 | fi 821 | if [ -z "$PERF_ENC_CYCLE_CNT" ]; then 822 | PERF_ENC_CYCLE_CNT=0 823 | fi 824 | fi 825 | 826 | # Extract Decoding Instruction count and cycles 827 | if [ -e "$PERF_DEC_OUT" ]; then 828 | PERF_DEC_INSTR_CNT=$(awk '/instructions/ { s=$1 } END { gsub(",", "", s) ; print s }' "$PERF_DEC_OUT") 829 | PERF_DEC_CYCLE_CNT=$(awk '/cycles/ { s=$1 } END { gsub(",", "", s) ; print s }' "$PERF_DEC_OUT") 830 | else 831 | if [ -z "$PERF_DEC_INSTR_CNT" ]; then 832 | PERF_DEC_INSTR_CNT=0 833 | fi 834 | if [ -z "$PERF_DEC_CYCLE_CNT" ]; then 835 | PERF_DEC_CYCLE_CNT=0 836 | fi 837 | fi 838 | 839 | echo $PERF_ENC_INSTR_CNT 840 | echo $PERF_ENC_CYCLE_CNT 841 | echo $PERF_DEC_INSTR_CNT 842 | echo $PERF_DEC_CYCLE_CNT 843 | 844 | if [ -f "$VMAF" ]; then 845 | cat "$BASENAME-vmaf.xml" 846 | rm "$BASENAME-vmaf.xml" 847 | fi 848 | 849 | if [ ! "$NO_DELETE" ]; then 850 | rm -f "$BASENAME.ogv" "$BASENAME.x264" "$BASENAME.x265" "$BASENAME.xvc" "$BASENAME.vpx" "$BASENAME.ivf" "$TIMEROUT" "$BASENAME-enc.out" "$BASENAME-psnr.out" "$BASENAME.thor" 2> /dev/null 851 | fi 852 | 853 | rm -f "$BASENAME.y4m" "$BASENAME.yuv" "$BASENAME-rec.y4m" ${BASENAME}_src.yuv ${BASENAME}-rec.yuv ${BASENAME}-FrameSkip-rec.yuv ${BASENAME}-FramesToBeEncoded-rec.yuv 2> /dev/null 854 | 855 | rm -f pid 856 | --------------------------------------------------------------------------------