├── .gitignore
├── LICENSE_BSD.txt
├── Makefile
├── README.md
├── edge264.c
├── edge264.h
├── edge264_bitstream.c
├── edge264_deblock.c
├── edge264_headers.c
├── edge264_inter.c
├── edge264_internal.h
├── edge264_intra.c
├── edge264_mvpred.c
├── edge264_residual.c
├── edge264_sei.c
├── edge264_slice.c
├── edge264_test.c
└── tools
    ├── gen_avc.py
    ├── split_gops.c
    └── unit_tests.c


/.gitignore:
--------------------------------------------------------------------------------
1 | *
2 | !.gitignore
3 | !edge264*.c
4 | !edge264*.h
5 | !LICENSE_BSD.txt
6 | !Makefile
7 | !*.md
8 | 


--------------------------------------------------------------------------------
/LICENSE_BSD.txt:
--------------------------------------------------------------------------------
 1 | Copyright (c) 2013-2014, Celticom / TVLabs
 2 | Copyright (c) 2014-2024 Thibault Raffaillac <traf@kth.se>
 3 | All rights reserved.
 4 | 
 5 | Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
 6 | 1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
 7 | 2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
 8 | 3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission.
 9 | 
10 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.


--------------------------------------------------------------------------------
/Makefile:
--------------------------------------------------------------------------------
 1 | ARCH ?= native
 2 | OS ?= $(shell uname)
 3 | BUILD_TEST ?= yes
 4 | VARIANTS ?= logs
 5 | 
 6 | VERSION := 1.0.0
 7 | MAJOR := 1
 8 | TARGETCC := $(CC)
 9 | LDLINUX := -Wl,-soname,libedge264.so.$(MAJOR) -Wl,-rpath,'$$ORIGIN'
10 | override CFLAGS := -std=gnu11 -O3 -flax-vector-conversions -w $(if $(findstring Windows,$(OS)),,-fpic) $(CFLAGS)
11 | override LDFLAGS := -pthread $(if $(findstring Linux,$(OS)),$(LDLINUX),) $(LDFLAGS)
12 | RUNTIME_TESTS := $(if $(findstring x86-64-v2,$(VARIANTS)),-DHAS_X86_64_V2,) $(if $(findstring x86-64-v3,$(VARIANTS)),-DHAS_X86_64_V3,) $(if $(findstring logs,$(VARIANTS)),-DHAS_LOGS,)
13 | OBJ := edge264.o $(if $(findstring x86-64-v2,$(VARIANTS)),edge264_headers_v2.o,) $(if $(findstring x86-64-v3,$(VARIANTS)),edge264_headers_v3.o,) $(if $(findstring logs,$(VARIANTS)),edge264_headers_log.o,)
14 | LIB := $(if $(findstring Windows,$(OS)),edge264.$(MAJOR).dll,$(if $(findstring Linux,$(OS)),libedge264.so.$(VERSION),libedge264.$(VERSION).dylib))
15 | EXE := $(if $(findstring Windows,$(OS)),edge264_test.exe,edge264_test)
16 | .DEFAULT_GOAL := $(if $(findstring yes,$(BUILD_TEST)),$(EXE),$(LIB))
17 | 
18 | 
19 | # check existence of compiler executable
20 | WHICH := $(if $(findstring Windows,$(OS)),where 2>nul,which 2>/dev/null)
21 | ifeq (,$(shell $(WHICH) $(CC)))
22 |   $(error CC=$(CC) does not point to a valid command, consider passing CC=gcc or CC=clang and having any of these compilers installed)
23 | endif
24 | 
25 | 
26 | # rules
27 | $(EXE): edge264_test.c edge264.h $(LIB)
28 | 	$(TARGETCC) edge264_test.c $(LIB) -march=$(ARCH) -O3 $(LDFLAGS) -o $(EXE)
29 | 
30 | $(LIB): $(OBJ)
31 | 	$(TARGETCC) -shared $(OBJ) $(LDFLAGS) -o $(LIB)
32 | 
33 | edge264.o: edge264.h edge264_internal.h edge264.c edge264_bitstream.c edge264_deblock.c edge264_headers.c edge264_inter.c edge264_intra.c edge264_mvpred.c edge264_residual.c edge264_slice.c
34 | 	$(CC) edge264.c -c -march=$(ARCH) $(CFLAGS) $(RUNTIME_TESTS) -o edge264.o
35 | 
36 | edge264_headers_v2.o: edge264.h edge264_internal.h edge264_bitstream.c edge264_deblock.c edge264_headers.c edge264_inter.c edge264_intra.c edge264_mvpred.c edge264_residual.c edge264_slice.c
37 | 	$(CC) edge264_headers.c -c -march=x86-64-v2 $(CFLAGS) "-DADD_VARIANT(f)=f##_v2" -o edge264_headers_v2.o
38 | 
39 | edge264_headers_v3.o: edge264.h edge264_internal.h edge264_bitstream.c edge264_deblock.c edge264_headers.c edge264_inter.c edge264_intra.c edge264_mvpred.c edge264_residual.c edge264_slice.c
40 | 	$(CC) edge264_headers.c -c -march=x86-64-v3 $(CFLAGS) "-DADD_VARIANT(f)=f##_v3" -o edge264_headers_v3.o
41 | 
42 | edge264_headers_log.o: edge264.h edge264_internal.h edge264_bitstream.c edge264_deblock.c edge264_headers.c edge264_inter.c edge264_intra.c edge264_mvpred.c edge264_residual.c edge264_sei.c edge264_slice.c
43 | 	$(CC) edge264_headers.c -c -march=$(ARCH) $(CFLAGS) -DLOGS "-DADD_VARIANT(f)=f##_log" -o edge264_headers_log.o
44 | 
45 | .PHONY: clean clear
46 | clean clear:
47 | 	rm -f release/* $(EXE) edge264*.o libedge264*.dylib libedge264.so.* edge264*.dll
48 | 
49 | 
50 | # hard-coded for my machine (mac-x64), edit to make them work on your own
51 | LINUX_GNU_X86_SYSROOT := i686-unknown-linux-gnu/i686-unknown-linux-gnu/sysroot
52 | LINUX_GNU_X86_GCC := i686-unknown-linux-gnu/bin/i686-linux-gnu-gcc
53 | LINUX_GNU_X64_SYSROOT := x86_64-unknown-linux-gnu/x86_64-unknown-linux-gnu/sysroot
54 | LINUX_GNU_X64_GCC := x86_64-unknown-linux-gnu/bin/x86_64-linux-gnu-gcc
55 | LINUX_MUSL_ARM64_SYSROOT := aarch64-unknown-linux-musl/aarch64-unknown-linux-musl/sysroot
56 | LINUX_MUSL_ARM64_GCC := aarch64-unknown-linux-musl/bin/aarch64-unknown-linux-musl-gcc
57 | WINDOWS_MINGW_X86_TOOLCHAIN := /usr/local/Cellar/mingw-w64/12.0.0_1/toolchain-i686
58 | WINDOWS_MINGW_X64_TOOLCHAIN := /usr/local/Cellar/mingw-w64/12.0.0_1/toolchain-x86_64
59 | .PHONY: release
60 | release: edge264*.c edge264*.h Makefile
61 | 	mkdir -p release
62 | 	# x86_64-apple-darwinXX
63 | 	$(MAKE) ARCH=core2 VARIANTS=x86-64-v3,logs BUILD_TEST=no
64 | 	zip -m release/edge264-$(VERSION)-mac-x64.zip libedge264.$(VERSION).dylib
65 | 	# i686-linux-gnu
66 | 	$(MAKE) ARCH=pentium-m OS=Linux VARIANTS=x86-64-v2,x86-64-v3,logs BUILD_TEST=no CFLAGS="--target=i686-linux-gnu --sysroot=$(LINUX_GNU_X86_SYSROOT)" TARGETCC=$(LINUX_GNU_X86_GCC)
67 | 	zip -m release/edge264-$(VERSION)-linux-gnu-x86.zip libedge264.so.$(VERSION)
68 | 	# x86_64-linux-gnu
69 | 	$(MAKE) ARCH=x86-64 OS=Linux VARIANTS=x86-64-v2,x86-64-v3,logs BUILD_TEST=no CFLAGS="--target=x86_64-linux-gnu --sysroot=$(LINUX_GNU_X64_SYSROOT)" TARGETCC=$(LINUX_GNU_X64_GCC)
70 | 	zip -m release/edge264-$(VERSION)-linux-gnu-x64.zip libedge264.so.$(VERSION)
71 | 	# i686-w64-mingw32
72 | 	$(MAKE) ARCH=pentium-m OS=Windows VARIANTS=x86-64-v2,x86-64-v3,logs BUILD_TEST=no CFLAGS="--target=i686-w64-mingw32 --sysroot=$(WINDOWS_MINGW_X86_TOOLCHAIN)" TARGETCC=$(WINDOWS_MINGW_X86_TOOLCHAIN)/bin/i686-w64-mingw32-gcc
73 | 	zip -m release/edge264-$(VERSION)-windows-mingw-x86.zip edge264.$(MAJOR).dll
74 | 	# x86_64-w64-mingw32
75 | 	$(MAKE) ARCH=x86-64 OS=Windows VARIANTS=x86-64-v2,x86-64-v3,logs BUILD_TEST=no CFLAGS="--target=x86_64-w64-mingw32 --sysroot=$(WINDOWS_MINGW_X64_TOOLCHAIN)" TARGETCC=$(WINDOWS_MINGW_X64_TOOLCHAIN)/bin/x86_64-w64-mingw32-gcc
76 | 	zip -m release/edge264-$(VERSION)-windows-mingw-x64.zip edge264.$(MAJOR).dll
77 | 	# aarch64-linux-musl
78 | 	$(MAKE) ARCH=armv8-a+simd OS=Linux VARIANTS=logs BUILD_TEST=no CFLAGS="--target=aarch64-linux-musl --sysroot=$(LINUX_MUSL_ARM64_SYSROOT)" TARGETCC=$(LINUX_MUSL_ARM64_GCC)
79 | 	zip -m release/edge264-$(VERSION)-linux-musl-arm64.zip libedge264.so.$(VERSION)
80 | 
81 | 
82 | # cross-compiling edge264_test for aarch64-linux
83 | # make ARCH=armv8-a+simd OS=Linux VARIANTS=logs CFLAGS="--target=aarch64-linux-musl --sysroot=aarch64-unknown-linux-musl/aarch64-unknown-linux-musl/sysroot" TARGETCC=aarch64-unknown-linux-musl/bin/aarch64-unknown-linux-musl-gcc LDFLAGS="-Wl,-rpath '-Wl,\$\$ORIGIN'"
84 | 


--------------------------------------------------------------------------------
/README.md:
--------------------------------------------------------------------------------
  1 | edge264
  2 | =======
  3 | 
  4 | Minimalist software decoder with state-of-the-art performance for the H.264/AVC video format.
  5 | 
  6 | *Please note this is a work in progress and will be ready for use after making GStreamer/VLC plugins.*
  7 | 
  8 | Features
  9 | --------
 10 | 
 11 | * Supports **Progressive High** and **MVC 3D** profiles, up to level 6.2
 12 | * Any resolution up to 8K UHD
 13 | * 8-bit 4:2:0 planar YUV output
 14 | * Slices and Arbitrary Slice Order
 15 | * Slice and frame multi-threading
 16 | * Per-slice reference picture list
 17 | * Memory Management Control Operations
 18 | * Long-term reference frames
 19 | 
 20 | 
 21 | Supported platforms
 22 | -------------------
 23 | 
 24 | * Windows: x86, x64
 25 | * Linux: x86, x64, ARM64
 26 | * Mac OS: x64
 27 | 
 28 | 
 29 | Compiling and testing
 30 | ---------------------
 31 | 
 32 | edge264 is entirely developed in C using 128-bit [vector extensions](https://gcc.gnu.org/onlinedocs/gcc/Vector-Extensions.html) and vector intrinsics, and can be compiled with GNU GCC or LLVM Clang. [SDL2](https://www.libsdl.org/) runtime library may be used (optional) to enable display with `edge264_test`.
 33 | 
 34 | Here are the `make` options for tuning the compiled library file:
 35 | 
 36 | * `CC` - C compiler used to convert source file to object files (default `cc`)
 37 | * `CFLAGS` - additional compilation flags passed to `CC`
 38 | * `ARCH` - target architecture that will be passed to -march (default `native`)
 39 | * `OS` - target operating system (defaults to host)
 40 | * `TARGETCC` - C compiler used to link object files into library file (defaults to `CC`)
 41 | * `LDFLAGS` - additional compilation flags passed to `TARGETCC`
 42 | * `VARIANTS` - comma-separated list of additional variants included in the library and selected at runtime (default `logs`)
 43 | 	* `x86-64-v2` - variant compiled for x86-64 microarchitecture level 2 (SSSE3, SSE4.1 and POPCOUNT)
 44 | 	* `x86-64-v3` - variant compiled for x86-64 microarchitecture level 3 (AVX2, BMI, LZCNT, MOVBE)
 45 | 	* `logs` - variant compiled with logging support in YAML format (headers and slices)
 46 | * `BUILD_TEST` - toggles compilation of `edge264_test` (default `yes`)
 47 | 
 48 | ```sh
 49 | $ make ARCH=x86-64 VARIANTS=x86-64-v2,x86-64-v3 BUILD_TEST=no # example release build
 50 | ```
 51 | 
 52 | The automated test program `edge264_test` can browse files in a given directory, decoding each `<video>.264` file and comparing its output with each sibling file `<video>.yuv` if found. On the set of AVCv1, FRExt and MVC [conformance bitstreams](https://www.itu.int/wftp3/av-arch/jvt-site/draft_conformance/), 109/224 files are decoded without errors, the rest using yet unsupported features.
 53 | 
 54 | ```sh
 55 | $ make
 56 | $ ./edge264_test --help # prints all options available
 57 | $ ffmpeg -i vid.mp4 -vcodec copy -bsf h264_mp4toannexb -an vid.264 # optional, converts from MP4 format
 58 | $ ./edge264_test -d vid.264 # replace -d with -b to benchmark instead of display
 59 | ```
 60 | 
 61 | 
 62 | Example code
 63 | ------------
 64 | 
 65 | Here is a complete example that opens an input file in Annex B format from command line, and dumps its decoded frames in planar YUV order to standard output. See [edge264_test.c](edge264_test.c) for a more complete example which can also display frames.
 66 | 
 67 | ```c
 68 | #include <fcntl.h>
 69 | #include <unistd.h>
 70 | #include <sys/mman.h>
 71 | #include <sys/stat.h>
 72 | #include <sys/types.h>
 73 | 
 74 | #include "edge264.h"
 75 | 
 76 | int main(int argc, char *argv[]) {
 77 | 	int fd = open(argv[1], O_RDONLY);
 78 | 	struct stat st;
 79 | 	fstat(fd, &st);
 80 | 	uint8_t *buf = mmap(NULL, st.st_size, PROT_READ, MAP_SHARED, fd, 0);
 81 | 	const uint8_t *nal = buf + 3 + (buf[2] == 0); // skip the [0]001 delimiter
 82 | 	const uint8_t *end = buf + st.st_size;
 83 | 	Edge264Decoder *dec = edge264_alloc(-1, NULL, NULL, NULL, NULL); // auto number of threads, no logging
 84 | 	Edge264Frame frm;
 85 | 	int res;
 86 | 	do {
 87 | 		res = edge264_decode_NAL(dec, nal, end, 0, NULL, NULL, &nal);
 88 | 		while (!edge264_get_frame(dec, &frm, 0)) {
 89 | 			for (int y = 0; y < frm.height_Y; y++)
 90 | 				write(1, frm.samples[0] + y * frm.stride_Y, frm.width_Y);
 91 | 			for (int y = 0; y < frm.height_C; y++)
 92 | 				write(1, frm.samples[1] + y * frm.stride_C, frm.width_C);
 93 | 			for (int y = 0; y < frm.height_C; y++)
 94 | 				write(1, frm.samples[2] + y * frm.stride_C, frm.width_C);
 95 | 		}
 96 | 	} while (res == 0 || res == ENOBUFS);
 97 | 	edge264_free(&dec);
 98 | 	munmap(buf, st.st_size);
 99 | 	close(fd);
100 | 	return 0;
101 | }
102 | ```
103 | 
104 | 
105 | API reference
106 | -------------
107 | 
108 | <code>const uint8_t * <b>edge264_find_start_code(buf, end, four_byte)</b></code>
109 | 
110 | Return a pointer to the next three or four byte (0)001 start code prefix, or `end` if not found.
111 | 
112 | * `const uint8_t * buf` - first byte of buffer to search into
113 | * `const uint8_t * end` - first invalid byte past the buffer that stops the search
114 | * `int four_byte` - if 0 seek a 001 prefix, otherwise seek a 0001
115 | 
116 | ---
117 | 
118 | <code>Edge264Decoder * <b>edge264_alloc(n_threads, log_cb, log_header, log_sei, log_mb)</b></code>
119 | 
120 | Allocate and initialize a decoding context.
121 | 
122 | * `int n_threads` - number of background worker threads, with 0 to disable multithreading and -1 to detect the number of logical cores at runtime
123 | * `void (* log_cb)(const char * str, void * log_arg)` - if not NULL, a `fputs`-compatible function pointer that will be called to log every header, SEI or macroblock
124 | * `void * log_header_arg` - if not NULL, the argument passed to `log_cb` when `edge264_decode_NAL` logs a header (⚠️ *large*, always called from the same thread, enabling it requires the `logs` variant otherwise the function will fail at runtime)
125 | * `void * log_sei_arg` - if not NULL, the argument passed to `log_cb` when `edge264_decode_NAL` logs a Supplemental Enhancement Information message (see [H.264 annex D](https://www.itu.int/rec/T-REC-H.264), always called from the same thread, requires `logs` too)
126 | * `void * log_mb_arg` - if not NULL, the argument passed to `log_cb` when `edge264_decode_NAL` logs a macroblock (⚠️ *very large*, may be called from another thread if multithreaded, requires `logs`too)
127 | 
128 | ---
129 | 
130 | <code>void <b>edge264_flush(dec)</b></code>
131 | 
132 | For use when seeking, stop all background processing and clear all delayed frames. The parameter sets are kept, thus do not need to be sent again if they did not change.
133 | 
134 | * `Edge264Decoder * dec` - initialized decoding context
135 | 
136 | ---
137 | 
138 | <code>void <b>edge264_free(pdec)</b></code>
139 | 
140 | Deallocate the entire decoding context, and unset the pointer.
141 | 
142 | * `Edge264Decoder ** pdec` - pointer to a decoding context, initialized or not
143 | 
144 | ---
145 | 
146 | <code>int <b>edge264_decode_NAL(dec, buf, end, non_blocking, free_cb, free_arg, next_NAL)</b></code>
147 | 
148 | Decode a single NAL unit containing any parameter set or slice.
149 | 
150 | * `Edge264Decoder * dec` - initialized decoding context
151 | * `const uint8_t * buf` - first byte of NAL unit (containing `nal_unit_type`)
152 | * `const uint8_t * end` - first byte past the buffer (max buffer size is 2<sup>31</sup>-1 on 32-bit and 2<sup>63</sup>-1 on 64-bit)
153 | * `int non_blocking` - set to 1 if the current thread has other processing thus cannot block here
154 | * `void (* free_cb)(void * free_arg, int ret)` - callback that may be called from another thread when multithreaded, to signal the end of parsing and release the NAL buffer
155 | * `void * free_arg` - custom value that will be passed to `free_cb`
156 | * `const uint8_t ** next_NAL` - if not NULL and the return code is `0`|`ENOTSUP`|`EBADMSG`, will receive a pointer to the next NAL unit after the next start code in an Annex B stream
157 | 
158 | Return codes are:
159 | 
160 | * `0` on success
161 | * `ENOTSUP` on unsupported stream (decoding may proceed but could return zero frames)
162 | * `EBADMSG` on invalid stream (decoding may proceed but could show visual artefacts, if you can check with another decoder that the stream is actually flawless, please consider filling a bug report 🙏)
163 | * `EINVAL` if the function was called with `dec == NULL` or `dec->buf == NULL`
164 | * `ENODATA` if the function was called while `dec->buf >= dec->end`
165 | * `ENOMEM` if `malloc` failed to allocate memory
166 | * `ENOBUFS` if more frames should be consumed with `edge264_get_frame` to release a picture slot
167 | * `EWOULDBLOCK` if the non-blocking function would have to wait before a picture slot is available
168 | 
169 | ---
170 | 
171 | <code>int <b>edge264_get_frame(dec, out, borrow)</b></code>
172 | 
173 | Fetch the next frame ready for output.
174 | 
175 | * `Edge264Decoder * dec` - initialized decoding context
176 | * `Edge264Frame *out` - a structure that will be filled with data for the frame returned
177 | * `int borrow` - if 0 the frame may be accessed until the next call to `edge264_decode_NAL`, otherwise the frame should be explicitly returned with `edge264_return_frame`. Note that access is not exclusive, it may be used concurrently as reference for other frames.
178 | 
179 | Return codes are:
180 | 
181 | * `0` on success (one frame is returned)
182 | * `EINVAL` if the function was called with `dec == NULL` or `out == NULL`
183 | * `ENOMSG` if there is no frame to output at the moment
184 | 
185 | While reference frames may be decoded ahead of their actual display (ex. B-Pyramid technique), all frames are buffered for reordering before being released for display:
186 | 
187 | * Decoding a non-reference frame releases it and all frames set to be displayed before it.
188 | * Decoding a key frame releases all stored frames (but not the key frame itself which might be reordered later).
189 | * Exceeding the maximum number of frames held for reordering releases the next frame in display order.
190 | * Lacking an available frame buffer releases the next non-reference frame in display order (to salvage its buffer) and all reference frames displayed before it.
191 | 
192 | ---
193 | 
194 | <code>void <b>edge264_return_frame(dec, return_arg)</b></code>
195 | 
196 | Give back ownership of the frame if it was borrowed from a previous call to `edge264_get_frame`.
197 | 
198 | * `Edge264Decoder * dec` - initialized decoding context
199 | * `void * return_arg` - the value stored inside the frame to return
200 | 
201 | ```c
202 | typedef struct Edge264Frame {
203 | 	const uint8_t *samples[3]; // Y/Cb/Cr planes
204 | 	const uint8_t *samples_mvc[3]; // second view
205 | 	const uint8_t *mb_errors; // probabilities (0..100) for each macroblock to be erroneous, NULL if there are no errors, values are spaced by stride_mb in memory
206 | 	int8_t pixel_depth_Y; // 0 for 8-bit, 1 for 16-bit
207 | 	int8_t pixel_depth_C;
208 | 	int16_t width_Y;
209 | 	int16_t width_C;
210 | 	int16_t height_Y;
211 | 	int16_t height_C;
212 | 	int16_t stride_Y;
213 | 	int16_t stride_C;
214 | 	int16_t stride_mb;
215 | 	int32_t TopFieldOrderCnt;
216 | 	int32_t BottomFieldOrderCnt;
217 | 	int16_t frame_crop_offsets[4]; // {top,right,bottom,left}, useful to derive the original frame with 16x16 macroblocks
218 | 	void *return_arg;
219 | } Edge264Frame;
220 | ```
221 | 
222 | 
223 | Error recovery
224 | --------------
225 | 
226 | * Any invalid or corrupted header is ignored by edge264, i.e. if an invalid parameter set is received then the previous one will still be kept.
227 | * Any invalid or corrupted frame will be signaled by setting `mb_errors` in `Edge264Frame`. Since edge264 cannot detect exactly where a corruption occurs, it returns a 0-100% integer probability for each macroblock to contain errors caused by the corruption. This probability is rounded upward, such that all macroblocks inside a corrupted slice will at least have the value 1. edge264 does a basic job at reconstructing the corrupted macroblocks with neighboring frames, so media players are encouraged to use `mb_errors` to provide a better reconstruction.
228 | 
229 | 
230 | Roadmap
231 | -------
232 | 
233 | * Multithreading (in progress)
234 | * Error recovery (in progress)
235 | * Integration in VLC/ffmpeg/GStreamer
236 | * ARM32
237 | * PAFF and MBAFF
238 | * 4:0:0, 4:2:2 and 4:4:4
239 | * 9-14 bit depths with possibility of different luma/chroma depths
240 | * Transform-bypass for macroblocks with QP==0
241 | * SEI messages
242 | * AVX-2 optimizations
243 | 
244 | 
245 | Programming techniques
246 | ----------------------
247 | 
248 | I use edge264 to experiment on new programming techniques to improve performance and code size over existing decoders, and presented a few of these techniques at [FOSDEM'24](https://fosdem.org/2024/schedule/event/fosdem-2024-2931-innovations-in-h-264-avc-software-decoding-architecture-and-optimization-of-a-block-based-video-decoder-to-reach-10-faster-speed-and-3x-code-reduction-over-the-state-of-the-art-/) and [FOSDEM'25](https://fosdem.org/2025/schedule/event/fosdem-2025-5455-more-innovations-in-h-264-avc-software-decoding/).
249 | 
250 | 1. [Single header file](edge264_internal.h) - It contains all struct definitions, common constants and enums, SIMD aliases, inline functions and macros, and exported functions for each source file. To understand the code base you should look at this file first.
251 | 2. [Code blocks instead of functions](edge264_slice.c) - The main decoding loop is a forward pipeline designed as a DAG loosely resembling hardware decoders, with nodes being non-inlined functions and edges being tail calls. It helps mutualize code branches wherever possible, thus reduces code size to help fit in L1 cache.
252 | 3. [Tree branching](edge264_intra.c) - Directional intra modes are implemented with a jump table to the leaves of a tree then unconditional jumps down to the trunk. It allows sharing the bottom code among directional modes, to reduce code size.
253 | 4. ~~Global context register - The pointer to the main structure holding context data is assigned to a register when supported by the compiler (GCC).~~ This technique was dropped as Clang eventually reached on-par performance, so there is little incentive to maintain this hack.
254 | 5. [Default neighboring values](edge264_internal.h) (search `unavail_mb`) - Tests for availability of neighbors are replaced with fake neighboring macroblocks around each frame. It reduces the number of conditional tests inside the main decoding loop, thus reduces code size and branch predictor pressure.
255 | 6. [Relative neighboring offsets](edge264_internal.h) (look for `A4x4_int8` and related variables) - Access to left/top macroblock values is done with direct offsets in memory instead of copying their values to a buffer beforehand. It helps to reduce the reads and writes in the main decoding loop.
256 | 7. [Parsing uneven block shapes](edge264_slice.c) (look at function `parse_P_sub_mb`) - Each Inter macroblock paving specified with mb_type and sub_mb_type is first converted to a bitmask, then iterated on set bits to fetch the correct number of reference indices and motion vectors. This helps to reduce code size and number of conditional blocks.
257 | 8. [Using vector extensions](edge264_internal.h) - GCC's vector extensions are used along vector intrinsics to write more compact code. All intrinsics from Intel are aliased with shorter names, which also provides an enumeration of all SIMD instructions used in the decoder.
258 | 9. [Register-saturating SIMD](edge264_deblock.c) - Some critical SIMD algorithms use more simultaneous vectors than available registers, effectively saturating the register bank and generating stack spills on purpose. In some cases this is more efficient than splitting the algorithm into smaller bits, and has the additional benefit of scaling well with later CPUs.
259 | 10. [Piston cached bitstream reader](edge264_bitstream.c) - The bitstream bits are read in a size_t\[2\] intermediate cache with a trailing set bit to keep track of the number of cached bits, giving access to 32/64 bits per read from the cache, and allowing wide refills from memory.
260 | 11. [On-the-fly SIMD unescaping](edge264_bitstream.c) - The input bitstream is unescaped on the fly using vector code, avoiding a full preprocessing pass to remove escape sequences, and thus reducing memory reads/writes.
261 | 12. [Multiarch SIMD programming](edge264_internal.h) - Using vector extensions along with aliased intrinsics allows supporting both Intel SSE and ARM NEON with around 80% common code and few #if #else blocks, while keeping state-of-the-art performance for both architectures.
262 | 13. [The Structure of Arrays pattern](edge264_internal.h) - The frame buffer is stored with arrays for each distinct field rather than an array of structures, to express operations on frames with bitwise and vector operators (see [AoS and SoA](https://en.wikipedia.org/wiki/AoS_and_SoA)). The task buffer for multithreading also relies on it partially.
263 | 14. [Deferred error checking](edge264_headers.c) - Error detection is performed once in each type of NAL unit (search for `return` statements), by clamping all input values to their expected ranges, then expecting `rbsp_trailing_bit` afterwards (with _very high_ probability of catching an error if the stream is corrupted). This design choice is discussed in [A case about parsing errors](https://traffaillac.github.io/parsing.html).
264 | 
265 | Other yet-to-be-presented bits:
266 | 
267 | * [Minimalistic API](edge264.h) with FFI-friendly design (7 functions and 1 structure).
268 | * [The bitstream caches](edge264_internal.h) for CAVLC and CABAC (search for `rbsp_reg`) are stored in two size_t variables each, which may be mapped to Global Register Variables in the future.
269 | * [The decoding of input symbols](edge264_slice.c) is interspersed with their parsing (instead of parsing to a `struct` then decoding the data). It deduplicates branches and loops that are present in both parsing and decoding, and even eliminates the need to store some symbols (e.g. mb_type, sub_mb_type, mb_qp_delta).
270 | 


--------------------------------------------------------------------------------
/edge264.h:
--------------------------------------------------------------------------------
 1 | /**
 2 |  * Copyright (c) 2013-2014, Celticom / TVLabs
 3 |  * Copyright (c) 2014-2024 Thibault Raffaillac <traf@kth.se>
 4 |  * All rights reserved.
 5 |  *
 6 |  * Redistribution and use in source and binary forms, with or without
 7 |  * modification, are permitted provided that the following conditions are met:
 8 |  * 1. Redistributions of source code must retain the above copyright
 9 |  *    notice, this list of conditions and the following disclaimer.
10 |  * 2. Redistributions in binary form must reproduce the above copyright
11 |  *    notice, this list of conditions and the following disclaimer in the
12 |  *    documentation and/or other materials provided with the distribution.
13 |  * 3. Neither the name of the copyright holders nor the names of their
14 |  *    contributors may be used to endorse or promote products derived from this
15 |  *    software without specific prior written permission.
16 |  *
17 |  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
18 |  * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19 |  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20 |  * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS BE LIABLE FOR ANY
21 |  * DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
22 |  * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
23 |  * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
24 |  * ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25 |  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
26 |  * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27 |  */
28 | #ifndef edge264_H
29 | #define edge264_H
30 | 
31 | #include <errno.h>
32 | #include <stdint.h>
33 | 
34 | #ifdef __cplusplus
35 | extern "C" {
36 | #endif
37 | 
38 | typedef struct Edge264Decoder Edge264Decoder;
39 | 
40 | typedef struct Edge264Frame {
41 |    const uint8_t *samples[3]; // Y/Cb/Cr planes
42 |    const uint8_t *samples_mvc[3]; // second view
43 |    const uint8_t *mb_errors; // probabilities (0..100) for each macroblock to be erroneous, NULL if there are no errors, values are spaced by stride_mb in memory
44 |    int8_t pixel_depth_Y; // 0 for 8-bit, 1 for 16-bit
45 |    int8_t pixel_depth_C;
46 |    int16_t width_Y;
47 |    int16_t width_C;
48 |    int16_t height_Y;
49 |    int16_t height_C;
50 |    int16_t stride_Y;
51 |    int16_t stride_C;
52 |    int16_t stride_mb;
53 |    int32_t TopFieldOrderCnt;
54 |    int32_t BottomFieldOrderCnt;
55 |    int16_t frame_crop_offsets[4]; // {top,right,bottom,left}, useful to derive the original frame with 16x16 macroblocks
56 |    void *return_arg;
57 | } Edge264Frame;
58 | 
59 | const uint8_t *edge264_find_start_code(const uint8_t *buf, const uint8_t *end, int four_byte);
60 | Edge264Decoder *edge264_alloc(int n_threads, void (*log_cb)(const char *str, void *log_arg), void *log_header_arg, void *log_sei_arg, void *log_mb_arg);
61 | void edge264_flush(Edge264Decoder *dec);
62 | void edge264_free(Edge264Decoder **pdec);
63 | int edge264_decode_NAL(Edge264Decoder *dec, const uint8_t *buf, const uint8_t *end, int non_blocking, void (*free_cb)(void *free_arg, int ret), void *free_arg, const uint8_t **next_NAL);
64 | int edge264_get_frame(Edge264Decoder *dec, Edge264Frame *out, int borrow);
65 | void edge264_return_frame(Edge264Decoder *dec, void *return_arg);
66 | 
67 | #ifdef __cplusplus
68 | }
69 | #endif
70 | 
71 | #endif
72 | 


--------------------------------------------------------------------------------
/edge264_bitstream.c:
--------------------------------------------------------------------------------
  1 | /**
  2 |  * Parsing CAVLC and CABAC values (for 32 or 64 bit machines).
  3 |  */
  4 | #include "edge264_internal.h"
  5 | 
  6 | 
  7 | /**
  8 |  * Extract nbytes from the bitstream and return them as big endian.
  9 |  * 
 10 |  * The process reads an unaligned 16-byte chunk and will not read past the last
 11 |  * aligned 16-byte chunk containing the last bytes before ctx->t._gb.end.
 12 |  */
 13 | static inline size_t get_bytes(Edge264GetBits *gb, int nbytes)
 14 | {
 15 | 	static const i8x16 shuf[8] = {
 16 | 		{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, -1},
 17 | 		{0, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, -1},
 18 | 		{0, 1, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, -1},
 19 | 		{0, 1, 2, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, -1},
 20 | 		{0, 1, 2, 3, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, -1},
 21 | 		{0, 1, 2, 3, 4, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, -1},
 22 | 		{0, 1, 2, 3, 4, 5, 7, 8, 9, 10, 11, 12, 13, 14, 15, -1},
 23 | 		{0, 1, 2, 3, 4, 5, 6, 8, 9, 10, 11, 12, 13, 14, 15, -1},
 24 | 	};
 25 | 	
 26 | 	// load 16 bytes without reading past aligned buffer boundaries
 27 | 	const uint8_t *CPB = gb->CPB;
 28 | 	const uint8_t *end = gb->end;
 29 | 	intptr_t diff = CPB - end;
 30 | 	u8x16 v;
 31 | 	if (__builtin_expect(diff <= -14, 1)) {
 32 | 		v = load128(CPB - 2);
 33 | 	} else if (diff < 0) {
 34 | 		const uint8_t *p = minp(CPB - 2, (uint8_t *)((uintptr_t)(end - 1) & -16));
 35 | 		v = shrv128(shlv128(load128(p), p + 16 - end), min(14 + diff, 16));
 36 | 	} else {
 37 | 		gb->CPB = CPB + nbytes;
 38 | 		return 0;
 39 | 	}
 40 | 	
 41 | 	// make a bitmask for the positions of 00n escape sequences and iterate on it
 42 | 	u8x16 eq0 = v == 0;
 43 | 	u8x16 x = shr128(v, 2);
 44 | 	#if defined(__SSE2__)
 45 | 		unsigned esc = movemask(eq0 & shr128(eq0, 1) & (x <= 3));
 46 | 		if (__builtin_expect(esc, 0)) {
 47 | 			while (esc & ((1 << nbytes) - 1)) {
 48 | 				int i = __builtin_ctz(esc);
 49 | 				esc = (esc & (esc - 1)) >> 1;
 50 | 				if (CPB[i] <3) { // delimiter sequence, set end to its first byte
 51 | 					gb->end = CPB + i - 2;
 52 | 					x &= ~shlv128(set8(-1), i);
 53 | 					break; // we cannot iterate more since now x differs from esc
 54 | 				}
 55 | 				// otherwise this is an emulation_prevention_three_byte -> remove it
 56 | 				x = shuffle(x, shuf[i]);
 57 | 				CPB++;
 58 | 			}
 59 | 		}
 60 | 	#elif defined(__ARM_NEON)
 61 | 		uint64_t esc = (uint64_t)vshrn_n_u16(eq0 & shr128(eq0, 1) & (x <= 3), 4);
 62 | 		if (__builtin_expect(esc, 0)) {
 63 | 			esc &= 0x8888888888888888ULL;
 64 | 			while (esc & ((1ULL << (nbytes << 2)) - 1)) {
 65 | 				int i = __builtin_ctzll(esc) >> 2;
 66 | 				esc = (esc & (esc - 1)) >> 4;
 67 | 				if (CPB[i] <3) { // delimiter sequence, set end to its first byte
 68 | 					gb->end = CPB + i - 2;
 69 | 					x &= ~shlv128(set8(-1), i);
 70 | 					break; // we cannot iterate more since now x differs from esc
 71 | 				}
 72 | 				// otherwise this is an emulation_prevention_three_byte -> remove it
 73 | 				x = shuffle(x, shuf[i]);
 74 | 				CPB++;
 75 | 			}
 76 | 		}
 77 | 	#endif
 78 | 	
 79 | 	// increment CPB and return the requested bytes in upper part of the result
 80 | 	gb->CPB = CPB + nbytes;
 81 | 	#if SIZE_BIT == 32
 82 | 		return big_endian32(((i32x4)x)[0]);
 83 | 	#elif SIZE_BIT == 64
 84 | 		return big_endian64(((i64x2)x)[0]);
 85 | 	#endif
 86 | }
 87 | 
 88 | 
 89 | 
 90 | /**
 91 |  * Read Exp-Golomb codes and bit sequences.
 92 |  *
 93 |  * History of versions and updates:
 94 |  * _ Removing all emulation_prevention_three_bytes beforehand to a buffer, then
 95 |  *   making 1~3 aligned reads depending on expected code size. The code for
 96 |  *   reading Exp-Golomb values is branchless, although the functions for long
 97 |  *   codes are quite complex.
 98 |  * _ Same as above but making 1~2 unaligned reads. It makes the code shorter
 99 |  *   and probably faster, but more complex to understand.
100 |  * _ Removing emulation_prevention_three_bytes on the fly to a uint64_t cache
101 |  *   with a shift count. This adds a test for refill each time a value is read
102 |  *   (predicted not taken), but spares a full read/write pass on memory and
103 |  *   needs zero buffer allocation. However it incurs some complexity to
104 |  *   reassemble codes bigger than 32 bits.
105 |  * _ Same as above but with a size_t[2] cache with shift count. It doubles the
106 |  *   storage for 64 bit machines, and allows them to read any code size without
107 |  *   an intermediate refill (pushing it as a tail call).
108 |  * _ Same as above but using a trailing set bit instead of a shift variable.
109 |  *   The main context then fits in 2 variables, which are easier to store in
110 |  *   Global Register Variables.
111 |  */
112 | static noinline int refill(Edge264GetBits *gb, int ret) {
113 | 	size_t bytes = get_bytes(gb, SIZE_BIT >> 3);
114 | 	int trailing_bit = ctz(gb->msb_cache); // [0..SIZE_BIT-1]
115 | 	gb->msb_cache = (gb->msb_cache ^ (size_t)1 << trailing_bit) | bytes >> (SIZE_BIT - 1 - trailing_bit);
116 | 	gb->lsb_cache = (bytes * 2 + 1) << trailing_bit;
117 | 	return ret;
118 | }
119 | 
120 | static noinline int get_u1(Edge264GetBits *gb) {
121 | 	int ret = gb->msb_cache >> (SIZE_BIT - 1);
122 | 	gb->msb_cache = shld(gb->lsb_cache, gb->msb_cache, 1);
123 | 	if (gb->lsb_cache <<= 1)
124 | 		return ret;
125 | 	return refill(gb, ret);
126 | }
127 | 
128 | // Parses a 1~32-bit fixed size code
129 | static noinline unsigned get_uv(Edge264GetBits *gb, unsigned v) {
130 | 	unsigned ret = gb->msb_cache >> (SIZE_BIT - v);
131 | 	if (SIZE_BIT == 32 && __builtin_expect(v == 32, 0)) {
132 | 		gb->msb_cache = gb->lsb_cache;
133 | 		gb->lsb_cache = 0;
134 | 	} else {
135 | 		gb->msb_cache = shld(gb->lsb_cache, gb->msb_cache, v);
136 | 		gb->lsb_cache <<= v;
137 | 	}
138 | 	if (gb->lsb_cache)
139 | 		return ret;
140 | 	return refill(gb, ret);
141 | }
142 | 
143 | // Parses a Exp-Golomb code in one read, up to 2^16-2 (2^32-2 on 64-bit machines)
144 | static noinline unsigned get_ue16(Edge264GetBits *gb, unsigned upper) {
145 | 	unsigned v = clz(gb->msb_cache | (size_t)1 << (SIZE_BIT / 2)) * 2 + 1; // [1..SIZE_BIT-1]
146 | 	unsigned ret = minu((gb->msb_cache >> (SIZE_BIT - v)) - 1, upper);
147 | 	gb->msb_cache = shld(gb->lsb_cache, gb->msb_cache, v);
148 | 	if (gb->lsb_cache <<= v)
149 | 		return ret;
150 | 	return refill(gb, ret);
151 | }
152 | 
153 | // Parses a signed Exp-Golomb code in one read, from -2^15+1 to 2^15-1 (-2^31+1 to 2^31-1 on 64-bit machines)
154 | static noinline int get_se16(Edge264GetBits *gb, int lower, int upper) {
155 | 	unsigned v = clz(gb->msb_cache | (size_t)1 << (SIZE_BIT / 2)) * 2 + 1; // [1..SIZE_BIT-1]
156 | 	unsigned ue = (gb->msb_cache >> (SIZE_BIT - v)) - 1;
157 | 	int ret = min(max((ue & 1) ? ue / 2 + 1 : -(ue / 2), lower), upper);
158 | 	gb->msb_cache = shld(gb->lsb_cache, gb->msb_cache, v);
159 | 	if (gb->lsb_cache <<= v)
160 | 		return ret;
161 | 	return refill(gb, ret);
162 | }
163 | 
164 | // Extensions to [0,2^32-2] and [-2^31+1,2^31-1] for 32-bit machines
165 | #if SIZE_BIT == 32
166 | 	static noinline unsigned get_ue32(Edge264GetBits *gb, unsigned upper) {
167 | 		unsigned leadingZeroBits = clz(gb->msb_cache | 1); // [0..31]
168 | 		gb->msb_cache = shld(gb->lsb_cache, gb->msb_cache, leadingZeroBits);
169 | 		if (!(gb->lsb_cache <<= leadingZeroBits))
170 | 			refill(gb, 0);
171 | 		return minu(get_uv(gb, leadingZeroBits + 1) - 1, upper);
172 | 	}
173 | 
174 | 	static noinline int get_se32(Edge264GetBits *gb, int lower, int upper) {
175 | 		unsigned leadingZeroBits = clz(gb->msb_cache | 1); // [0..31]
176 | 		gb->msb_cache = shld(gb->lsb_cache, gb->msb_cache, leadingZeroBits);
177 | 		if (!(gb->lsb_cache <<= leadingZeroBits))
178 | 			refill(gb, 0);
179 | 		unsigned ue = get_uv(gb, leadingZeroBits + 1) - 1;
180 | 		return min(max(ue & 1 ? ue / 2 + 1 : -(ue / 2), lower), upper);
181 | 	}
182 | #endif
183 | 
184 | 
185 | 
186 | /**
187 |  * Read CABAC bins (9.3.3.2).
188 |  * 
189 |  * History of versions and updates:
190 |  * _ Storing codIOffset and codIRange in two size_t variables, along with a
191 |  *   counter to indicate how many extra bits are present in codIOffset.
192 |  *   codIRange corresponds with the spec, and codIOffset is shifted with extra
193 |  *   least significant bits. It dramatically reduces the number of memory reads
194 |  *   to refill codIOffset compared to the specification, at the expense of more
195 |  *   complex shift math.
196 |  * _ Same as above but using the leading set bit of codIRange instead of a
197 |  *   variable to count the remaining bits in codIOffset. codIRange is now
198 |  *   shifted along with codIOffset. It requires one less memory load and
199 |  *   shortens the dependency chain of critical function get_ae, thus improving
200 |  *   overall performance.
201 |  * _ Using a division instruction to get bypass bits ahead of time, since the
202 |  *   process described in 9.3.3.2.3 is actually equivalent with a binary
203 |  *   division. It is unclear whether it actually improves performance, but it
204 |  *   looks very cool!
205 |  * _ Storing codIOffset and codIRange in Global Register Variables (possible
206 |  *   with GCC), since they account for a lot of reads/writes otherwise. It
207 |  *   improves performance overall, and was later followed by the storage of
208 |  *   CAVLC's context in GRVs. However the renormalization is now quite complex
209 |  *   as it involves swapping CABAC and CAVLC in GRVs to fetch bits before
210 |  *   inserting them into codIOffset.
211 |  * _ Increasing the codIRange lower bound to trigger a refill from 256 to 512,
212 |  *   to spare the refill test when parsing coeff_sign_flag. It was later
213 |  *   abandoned since it required additional comments to explain divergence from
214 |  *   specification.
215 |  * _ Removing the dependency on CAVLC for renormalization, such that it can
216 |  *   fetch bytes without restoring CAVLC's context. It spared a test for
217 |  *   calling CAVLC's refill (not well predicted), thus improving performance.
218 |  *   However the bypass division trick is now unavailable on 32 bit machines,
219 |  *   since renormalization will now leave 25~32 bits in codIOffset, but the
220 |  *   algorithm needs at least 29 to remain simple.
221 |  */
222 | static noinline int get_ae(Edge264Context *ctx, int ctxIdx)
223 | {
224 | 	static const uint8_t rangeTabLPS[64 * 4] = {
225 | 		128, 176, 208, 240, 128, 167, 197, 227, 128, 158, 187, 216, 123, 150, 178, 205,
226 | 		116, 142, 169, 195, 111, 135, 160, 185, 105, 128, 152, 175, 100, 122, 144, 166,
227 | 		 95, 116, 137, 158,  90, 110, 130, 150,  85, 104, 123, 142,  81,  99, 117, 135,
228 | 		 77,  94, 111, 128,  73,  89, 105, 122,  69,  85, 100, 116,  66,  80,  95, 110,
229 | 		 62,  76,  90, 104,  59,  72,  86,  99,  56,  69,  81,  94,  53,  65,  77,  89,
230 | 		 51,  62,  73,  85,  48,  59,  69,  80,  46,  56,  66,  76,  43,  53,  63,  72,
231 | 		 41,  50,  59,  69,  39,  48,  56,  65,  37,  45,  54,  62,  35,  43,  51,  59,
232 | 		 33,  41,  48,  56,  32,  39,  46,  53,  30,  37,  43,  50,  29,  35,  41,  48,
233 | 		 27,  33,  39,  45,  26,  31,  37,  43,  24,  30,  35,  41,  23,  28,  33,  39,
234 | 		 22,  27,  32,  37,  21,  26,  30,  35,  20,  24,  29,  33,  19,  23,  27,  31,
235 | 		 18,  22,  26,  30,  17,  21,  25,  28,  16,  20,  23,  27,  15,  19,  22,  25,
236 | 		 14,  18,  21,  24,  14,  17,  20,  23,  13,  16,  19,  22,  12,  15,  18,  21,
237 | 		 12,  14,  17,  20,  11,  14,  16,  19,  11,  13,  15,  18,  10,  12,  15,  17,
238 | 		 10,  12,  14,  16,   9,  11,  13,  15,   9,  11,  12,  14,   8,  10,  12,  14,
239 | 		  8,   9,  11,  13,   7,   9,  11,  12,   7,   9,  10,  12,   7,   8,  10,  11,
240 | 		  6,   8,   9,  11,   6,   7,   9,  10,   6,   7,   8,   9,   2,   2,   2,   2,
241 | 	};
242 | 	static const uint8_t transIdx[256] = {
243 | 		  4,   5, 253, 252,   8,   9, 153, 152,  12,  13, 153, 152,  16,  17, 149, 148,
244 | 		 20,  21, 149, 148,  24,  25, 149, 148,  28,  29, 145, 144,  32,  33, 145, 144,
245 | 		 36,  37, 145, 144,  40,  41, 141, 140,  44,  45, 141, 140,  48,  49, 141, 140,
246 | 		 52,  53, 137, 136,  56,  57, 137, 136,  60,  61, 133, 132,  64,  65, 133, 132,
247 | 		 68,  69, 133, 132,  72,  73, 129, 128,  76,  77, 129, 128,  80,  81, 125, 124,
248 | 		 84,  85, 121, 120,  88,  89, 121, 120,  92,  93, 121, 120,  96,  97, 117, 116,
249 | 		100, 101, 117, 116, 104, 105, 113, 112, 108, 109, 109, 108, 112, 113, 109, 108,
250 | 		116, 117, 105, 104, 120, 121, 105, 104, 124, 125, 101, 100, 128, 129,  97,  96,
251 | 		132, 133,  97,  96, 136, 137,  93,  92, 140, 141,  89,  88, 144, 145,  89,  88,
252 | 		148, 149,  85,  84, 152, 153,  85,  84, 156, 157,  77,  76, 160, 161,  77,  76,
253 | 		164, 165,  73,  72, 168, 169,  73,  72, 172, 173,  65,  64, 176, 177,  65,  64,
254 | 		180, 181,  61,  60, 184, 185,  61,  60, 188, 189,  53,  52, 192, 193,  53,  52,
255 | 		196, 197,  49,  48, 200, 201,  45,  44, 204, 205,  45,  44, 208, 209,  37,  36,
256 | 		212, 213,  37,  36, 216, 217,  33,  32, 220, 221,  29,  28, 224, 225,  25,  24,
257 | 		228, 229,  21,  20, 232, 233,  17,  16, 236, 237,  17,  16, 240, 241,   9,   8,
258 | 		244, 245,   9,   8, 248, 249,   5,   4, 248, 249,   1,   0, 252, 253,   0,   1,
259 | 	};
260 | 	size_t state = ctx->cabac[ctxIdx];
261 | 	size_t shift = SIZE_BIT - 3 - clz(ctx->t._gb.codIRange); // [6..SIZE_BIT-3]
262 | 	size_t idx = (state & -4) + (ctx->t._gb.codIRange >> shift);
263 | 	size_t codIRangeLPS = (size_t)((uint8_t *)rangeTabLPS - 4)[idx] << (shift - 6);
264 | 	ctx->t._gb.codIRange -= codIRangeLPS;
265 | 	if (ctx->t._gb.codIOffset >= ctx->t._gb.codIRange) {
266 | 		state ^= 255;
267 | 		ctx->t._gb.codIOffset -= ctx->t._gb.codIRange;
268 | 		ctx->t._gb.codIRange = codIRangeLPS;
269 | 	}
270 | 	ctx->cabac[ctxIdx] = transIdx[state];
271 | 	int binVal = state & 1;
272 | 	if (__builtin_expect(ctx->t._gb.codIRange < 256, 0)) {
273 | 		ctx->t._gb.codIOffset = shld(get_bytes(&ctx->t._gb, SIZE_BIT / 8 - 1), ctx->t._gb.codIOffset, SIZE_BIT - 8);
274 | 		ctx->t._gb.codIRange <<= SIZE_BIT - 8;
275 | 	}
276 | 	return binVal;
277 | }
278 | 
279 | static inline int get_bypass(Edge264Context *ctx) {
280 | 	if (ctx->t._gb.codIRange < 512) {
281 | 		ctx->t._gb.codIOffset = shld(get_bytes(&ctx->t._gb, SIZE_BIT / 8 - 2), ctx->t._gb.codIOffset, SIZE_BIT - 16);
282 | 		ctx->t._gb.codIRange <<= SIZE_BIT - 16;
283 | 	}
284 | 	ctx->t._gb.codIRange >>= 1;
285 | 	size_t binVal = ctx->t._gb.codIOffset >= ctx->t._gb.codIRange;
286 | 	ctx->t._gb.codIOffset = binVal ? ctx->t._gb.codIOffset - ctx->t._gb.codIRange : ctx->t._gb.codIOffset;
287 | 	return binVal;
288 | }
289 | 
290 | static int cabac_start(Edge264Context *ctx) {
291 | 	// reclaim bits from cache while realigning with CPB on a byte boundary
292 | 	int extra_bits = SIZE_BIT - 1 - ctz(ctx->t._gb.lsb_cache);
293 | 	int shift = extra_bits & 7;
294 | 	int ret = shift > 0 && (ssize_t)ctx->t._gb.msb_cache >> (SIZE_BIT - shift) != -1; // return 1 if not all alignment bits are ones
295 | 	ctx->t._gb.codIOffset = shld(ctx->t._gb.lsb_cache, ctx->t._gb.msb_cache, shift); // codIOffset and msb_cache are the same memory slot
296 | 	ctx->t._gb.lsb_cache >>= (SIZE_BIT - extra_bits) & (SIZE_BIT - 1);
297 | 	while (extra_bits >= 8) {
298 | 		int32_t i = 0;
299 | 		if ((intptr_t)(ctx->t._gb.end - ctx->t._gb.CPB) >= 0)
300 | 			memcpy(&i, ctx->t._gb.CPB - 4, 4);
301 | 		ctx->t._gb.CPB -= 1 + ((big_endian32(i) & 0xffffff) == 3);
302 | 		ctx->t._gb.lsb_cache >>= 8;
303 | 		extra_bits -= 8;
304 | 	}
305 | 	ctx->t._gb.codIRange = (size_t)510 << (SIZE_BIT - 9);
306 | 	ctx->t._gb.codIOffset = (ctx->t._gb.codIOffset < ctx->t._gb.codIRange) ? ctx->t._gb.codIOffset : ctx->t._gb.codIRange - 1; // protection against invalid bitstream
307 | 	return ret;
308 | }
309 | 
310 | static int cabac_terminate(Edge264Context *ctx) {
311 | 	int extra = SIZE_BIT - 9 - clz(ctx->t._gb.codIRange); // [0..SIZE_BIT-9]
312 | 	ctx->t._gb.codIRange -= (size_t)2 << extra;
313 | 	if (ctx->t._gb.codIOffset >= ctx->t._gb.codIRange) {
314 | 		// reclaim the extra bits minus alignment bits, then refill the cache
315 | 		ctx->t._gb.msb_cache = (ctx->t._gb.codIOffset * 2 + 1) << (SIZE_BIT - 1 - (extra & -8));
316 | 		return refill(&ctx->t._gb, 1);
317 | 	}
318 | 	if (__builtin_expect(ctx->t._gb.codIRange < 256, 0)) {
319 | 		ctx->t._gb.codIOffset = shld(get_bytes(&ctx->t._gb, SIZE_BIT / 8 - 1), ctx->t._gb.codIOffset, SIZE_BIT - 8);
320 | 		ctx->t._gb.codIRange <<= SIZE_BIT - 8;
321 | 	}
322 | 	return 0;
323 | }
324 | 
325 | 
326 | 
327 | /**
328 |  * Initialise CABAC's context variables using vector code (9.3.1.1).
329 |  *
330 |  * Considering the bottleneck of memory access, this is probably just as fast
331 |  * as copying from precomputed values. Please refrain from providing a default,
332 |  * unoptimised version.
333 |  */
334 | static void cabac_init(Edge264Context *ctx) {
335 | 	i8x16 mul = set16(max(ctx->t.QP[0], 0) + 4096);
336 | 	i8x16 c1 = set8(1), c64 = set8(64), c126 = set8(126);
337 | 	const i8x16 *src = (i8x16 *)cabac_context_init[ctx->t.cabac_init_idc];
338 | 	for (i8x16 *dst = ctx->cabac_v; dst < ctx->cabac_v + 64; dst++, src += 2) {
339 | 		#if defined(__SSE2__)
340 | 			i16x8 sum0 = maddubs(mul, src[0]) >> 4;
341 | 			i16x8 sum1 = maddubs(mul, src[1]) >> 4;
342 | 		#elif defined(__ARM_NEON)
343 | 			i16x8 l0 = vmull_s8(vget_low_s8(mul), vget_low_s8(src[0]));
344 | 			i16x8 h0 = vmull_high_s8(mul, src[0]);
345 | 			i16x8 l1 = vmull_s8(vget_low_s8(mul), vget_low_s8(src[1]));
346 | 			i16x8 h1 = vmull_high_s8(mul, src[1]);
347 | 			i16x8 sum0 = (i16x8)vpaddq_s16(l0, h0) >> 4;
348 | 			i16x8 sum1 = (i16x8)vpaddq_s16(l1, h1) >> 4;
349 | 		#endif
350 | 		i8x16 min = minu8(packus16(sum0, sum1), c126);
351 | 		i8x16 mask = c64 > min;
352 | 		i8x16 preCtxState = maxu8(min, c1);
353 | 		i8x16 pStateIdx = preCtxState ^ mask;
354 | 		i8x16 shift = pStateIdx + pStateIdx;
355 | 		*dst = shift + shift + mask + c1; // pStateIdx << 2 | valMPS (middle bit is for transIdxLPS/MPS)
356 | 	}
357 | 	ctx->cabac[276] = 252;
358 | }
359 | 


--------------------------------------------------------------------------------
/edge264_deblock.c:
--------------------------------------------------------------------------------
   1 | #include "edge264_internal.h"
   2 | 
   3 | #if defined(__SSE2__)
   4 | 	#define packabd16(a, b, c, d) abs8(packs16(subs16(a, b), subs16(c, d)))
   5 | 	#define trnlo32(a) shuffle32(a, 0, 0, 2, 2)
   6 | #elif defined(__ARM_NEON)
   7 | 	#define packabd16(a, b, c, d) vqmovn_high_u16(vqmovn_u16(vabdq_s16(a, b)), vabdq_s16(c, d))
   8 | 	static always_inline i32x4 trnlo32(i32x4 a) {return vtrn1q_s32(a, a);}
   9 | #endif
  10 | static always_inline i8x16 expand4(int32_t a) {
  11 | 	i32x4 x0 = {a};
  12 | 	i8x16 x1 = ziplo8(x0, x0);
  13 | 	return ziplo8(x1, x1);
  14 | }
  15 | static always_inline i8x16 expand2(int64_t a) {
  16 | 	i64x2 x0 = {a};
  17 | 	return ziplo8(x0, x0);
  18 | }
  19 | 
  20 | 
  21 | 
  22 | /**
  23 |  * Filter a single edge in place for bS in [0..3].
  24 |  * tC0 should equal -1 for each position where bS=0 or beta=0.
  25 |  * 
  26 |  * I must acknowledge that I looked a LOT at ffmpeg's filters while developing
  27 |  * these filters. They taught me many tricks to handle 9~10 bit operations on
  28 |  * 8 bit hardware, so I owe them a lot of time saved. Thanks!
  29 |  * 
  30 |  * For 8-pixel chroma edges we filter both Cb and Cr in lower and upper halves
  31 |  * of registers. In these cases note that alpha and beta may contain zero and
  32 |  * non-zero values, so we cannot use alpha-1 or beta-1.
  33 |  */
  34 | #if defined(__SSE2__)
  35 | 	#define DEBLOCK_LUMA_SOFT(p2, p1, p0, q0, q1, q2, ialpha, ibeta, itC0) {\
  36 | 		/* compute the opposite of filterSamplesFlags as a mask, and transfer the mask bS=0 from tC0 */\
  37 | 		i8x16 pq0 = subu8(p0, q0);\
  38 | 		i8x16 qp0 = subu8(q0, p0);\
  39 | 		i8x16 sub0 = subu8(addu8(qp0, set8(-128)), pq0); /* save 128+q0-p0 for later */\
  40 | 		i8x16 abs0 = pq0 | qp0;\
  41 | 		i8x16 abs1 = subu8(p1, p0) | subu8(p0, p1);\
  42 | 		i8x16 abs2 = subu8(q1, q0) | subu8(q0, q1);\
  43 | 		i8x16 beta = set8(ibeta);\
  44 | 		i8x16 and = minu8(subu8(set8(ialpha), abs0), subu8(beta, maxu8(abs1, abs2)));\
  45 | 		i8x16 tC0 = expand4(itC0);\
  46 | 		i8x16 ignoreSamplesFlags = (and == 0) | (tC0 < 0);\
  47 | 		i8x16 ftC0 = tC0 & ~ignoreSamplesFlags;\
  48 | 		/* filter p1 and q1 (same as ffmpeg, I couldn't find better) */\
  49 | 		i8x16 c1 = set8(1);\
  50 | 		i8x16 x0 = avgu8(p0, q0); /* (p0+q0+1)>>1 */\
  51 | 		i8x16 x1 = avgu8(p2, x0) - ((p2 ^ x0) & c1); /* (p2+((p0+q0+1)>>1))>>1 */\
  52 | 		i8x16 x2 = avgu8(q2, x0) - ((q2 ^ x0) & c1); /* (q2+((p0+q0+1)>>1))>>1 */\
  53 | 		i8x16 pp1 = minu8(maxu8(x1, subu8(p1, ftC0)), addu8(p1, ftC0));\
  54 | 		i8x16 qp1 = minu8(maxu8(x2, subu8(q1, ftC0)), addu8(q1, ftC0));\
  55 | 		i8x16 cm1 = set8(-1);\
  56 | 		i8x16 bm1 = (i8x16)beta + cm1;\
  57 | 		i8x16 sub1 = avgu8(p1, q1 ^ cm1); /* save 128+((p1-q1)>>1) for later */\
  58 | 		i8x16 apltb = subu8(subu8(p2, p0), bm1) == subu8(subu8(p0, p2), bm1);\
  59 | 		i8x16 aqltb = subu8(subu8(q2, q0), bm1) == subu8(subu8(q0, q2), bm1);\
  60 | 		p1 = ifelse_mask(apltb, pp1, p1);\
  61 | 		q1 = ifelse_mask(aqltb, qp1, q1);\
  62 | 		/* filter p0 and q0 (by offsetting signed to unsigned to apply pavg) */\
  63 | 		i8x16 ftC = (ftC0 - apltb - aqltb) & ~ignoreSamplesFlags;\
  64 | 		i8x16 x3 = avgu8(sub0, avgu8(sub1, set8(127))); /* 128+((q0-p0+((p1-q1)>>2)+1)>>1) */\
  65 | 		i8x16 c128 = set8(-128);\
  66 | 		i8x16 delta = minu8(subu8(x3, c128), ftC); /* delta if delta>0 */\
  67 | 		i8x16 ndelta = minu8(subu8(c128, x3), ftC); /* -delta if delta<0 */\
  68 | 		p0 = subu8(addu8(p0, delta), ndelta);\
  69 | 		q0 = subu8(addu8(q0, ndelta), delta);}
  70 | 	#define DEBLOCK_CHROMA_SOFT(p1, p0, q0, q1, ialpha, ibeta, itC0) {\
  71 | 		/* compute the opposite of filterSamplesFlags and apply if to tC */\
  72 | 		i8x16 c128 = set8(-128);\
  73 | 		i8x16 pq0 = subu8(p0, q0);\
  74 | 		i8x16 qp0 = subu8(q0, p0);\
  75 | 		i8x16 sub0 = subu8(addu8(qp0, c128), pq0); /* save 128+q0-p0 for later */\
  76 | 		i8x16 abs0 = pq0 | qp0;\
  77 | 		i8x16 abs1 = subu8(p1, p0) | subu8(p0, p1);\
  78 | 		i8x16 abs2 = subu8(q1, q0) | subu8(q0, q1);\
  79 | 		i8x16 shufab = {1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2};\
  80 | 		i8x16 alpha = shuffle((i32x4){ialpha}, shufab);\
  81 | 		i8x16 beta = shuffle((i32x4){ibeta}, shufab);\
  82 | 		i8x16 and = minu8(subu8(alpha, abs0), subu8(beta, maxu8(abs1, abs2)));\
  83 | 		i8x16 ignoreSamplesFlags = and == 0;\
  84 | 		i8x16 cm1 = set8(-1);\
  85 | 		i8x16 ftC = (expand2(itC0) - cm1) & ~ignoreSamplesFlags;\
  86 | 		/* filter p0 and q0 (by offsetting signed to unsigned to apply pavg) */\
  87 | 		i8x16 sub1 = avgu8(p1, q1 ^ cm1); /* 128+((p1-q1)>>1) */\
  88 | 		i8x16 x3 = avgu8(sub0, avgu8(sub1, set8(127))); /* 128+((q0-p0+((p1-q1)>>2)+1)>>1) */\
  89 | 		i8x16 delta = minu8(subu8(x3, c128), ftC); /* delta if delta>0 */\
  90 | 		i8x16 ndelta = minu8(subu8(c128, x3), ftC); /* -delta if delta<0 */\
  91 | 		p0 = subu8(addu8(p0, delta), ndelta);\
  92 | 		q0 = subu8(addu8(q0, ndelta), delta);}
  93 | #elif defined(__ARM_NEON)
  94 | 	#define DEBLOCK_LUMA_SOFT(p2, p1, p0, q0, q1, q2, ialpha, ibeta, itC0) {\
  95 | 		/* compute all common masks */\
  96 | 		u8x16 alpha = set8(ialpha);\
  97 | 		u8x16 beta = set8(ibeta);\
  98 | 		i8x16 tC0 = expand4(itC0);\
  99 | 		u8x16 abs0 = vabdq_u8(p0, q0);\
 100 | 		u8x16 abs1 = vmaxq_u8(vabdq_u8(p1, p0), vabdq_u8(q1, q0));\
 101 | 		i8x16 filterSamplesFlag = (tC0 >= 0) & (abs0 < alpha) & (abs1 < beta);\
 102 | 		i8x16 ftC0 = tC0 & filterSamplesFlag;\
 103 | 		i8x16 apltb = (u8x16)vabdq_u8(p2, p0) < beta;\
 104 | 		i8x16 aqltb = (u8x16)vabdq_u8(q2, q0) < beta;\
 105 | 		/* filter p1 and q1 (same as ffmpeg, I couldn't find better) */\
 106 | 		u8x16 pq0 = vrhaddq_u8(p0, q0);\
 107 | 		i8x16 pq1 = vhsubq_u8(p1, q1); /* save (p1-q1)>>1 for later */\
 108 | 		u8x16 lop1 = vqsubq_u8(p1, ftC0);\
 109 | 		u8x16 hip1 = vqaddq_u8(p1, ftC0);\
 110 | 		u8x16 loq1 = vqsubq_u8(q1, ftC0);\
 111 | 		u8x16 hiq1 = vqaddq_u8(q1, ftC0);\
 112 | 		u8x16 pp1 = vminq_u8(vmaxq_u8(vhaddq_u8(p2, pq0), lop1), hip1);\
 113 | 		u8x16 qp1 = vminq_u8(vmaxq_u8(vhaddq_u8(q2, pq0), loq1), hiq1);\
 114 | 		p1 = ifelse_mask(apltb, pp1, p1);\
 115 | 		q1 = ifelse_mask(aqltb, qp1, q1);\
 116 | 		/* filter p0 and q0 (by offsetting unsigned to signed to properly saturate q0-p0) */\
 117 | 		i8x16 tC = (ftC0 - apltb - aqltb) & filterSamplesFlag;\
 118 | 		i8x16 x0 = vrhaddq_s8(vqsubq_s8(q0 + -128, p0 + -128), pq1 >> 1);\
 119 | 		i8x16 delta = vmaxq_s8(vminq_s8(x0, tC), -tC);\
 120 | 		p0 = vsqaddq_u8(p0, delta);\
 121 | 		q0 = vsqaddq_u8(q0, -delta);}
 122 | 	#define DEBLOCK_CHROMA_SOFT(p1, p0, q0, q1, ialpha, ibeta, itC0) {\
 123 | 		/* compute all common masks */\
 124 | 		i8x16 shufab = {1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2};\
 125 | 		i8x16 alpha = shuffle((i32x4){ialpha}, shufab);\
 126 | 		i8x16 beta = shuffle((i32x4){ibeta}, shufab);\
 127 | 		i8x16 tC0 = expand2(itC0);\
 128 | 		u8x16 abs0 = vabdq_u8(p0, q0);\
 129 | 		u8x16 abs1 = vmaxq_u8(vabdq_u8(p1, p0), vabdq_u8(q1, q0));\
 130 | 		i8x16 filterSamplesFlag = (tC0 >= 0) & (abs0 < alpha) & (abs1 < beta);\
 131 | 		/* filter p0 and q0 (by offsetting unsigned to signed to properly saturate q0-p0) */\
 132 | 		i8x16 c128 = set8(-128);\
 133 | 		i8x16 pq1 = vhsubq_u8(p1, q1);\
 134 | 		i8x16 tC = (tC0 - -1) & filterSamplesFlag;\
 135 | 		i8x16 x0 = vrhaddq_s8(vqsubq_s8(q0 + c128, p0 + c128), pq1 >> 1);\
 136 | 		i8x16 delta = vmaxq_s8(vminq_s8(x0, tC), -tC);\
 137 | 		p0 = vsqaddq_u8(p0, delta);\
 138 | 		q0 = vsqaddq_u8(q0, -delta);}
 139 | #endif
 140 | 
 141 | 
 142 | 
 143 | /**
 144 |  * Filter a single edge in place for bS=4.
 145 |  * 
 146 |  * The luma filter uses beta-1 thus should not be used with beta=0.
 147 |  */
 148 | #if defined(__SSE2__)
 149 | 	#define DEBLOCK_LUMA_HARD(p3, p2, p1, p0, q0, q1, q2, q3, ialpha, ibeta) {\
 150 | 		/* compute the opposite of filterSamplesFlags, and condition masks for filtering modes */\
 151 | 		i8x16 alpha = set8(ialpha);\
 152 | 		i8x16 beta = set8(ibeta);\
 153 | 		i8x16 abs0 = subu8(p0, q0) | subu8(q0, p0);\
 154 | 		i8x16 abs1 = subu8(p1, p0) | subu8(p0, p1);\
 155 | 		i8x16 abs2 = subu8(q1, q0) | subu8(q0, q1);\
 156 | 		i8x16 ignoreSamplesFlags = minu8(subu8(alpha, abs0), subu8(beta, maxu8(abs1, abs2))) == 0;\
 157 | 		i8x16 c1 = set8(1);\
 158 | 		i8x16 zero = {};\
 159 | 		i8x16 condpq = subu8(abs0, avgu8(avgu8(alpha, c1), zero)); /* abs0-((alpha>>2)+1) */\
 160 | 		i8x16 bm1 = (i8x16)beta - c1;\
 161 | 		i8x16 condp = (subu8(subu8(p2, p0) | subu8(p0, p2), bm1) | condpq) == zero;\
 162 | 		i8x16 condq = (subu8(subu8(q2, q0) | subu8(q0, q2), bm1) | condpq) == zero;\
 163 | 		/* compute p'0 and q'0 */\
 164 | 		i8x16 fix0 = (p0 ^ q0) & c1;\
 165 | 		i8x16 pq0 = avgu8(p0, q0) - fix0;\
 166 | 		i8x16 and0 = fix0 ^ c1;\
 167 | 		i8x16 p2q1 = avgu8(p2, q1) - ((p2 ^ q1) & c1); /* (p2+q1)/2 */\
 168 | 		i8x16 q2p1 = avgu8(q2, p1) - ((q2 ^ p1) & c1); /* (q2+p1)/2 */\
 169 | 		i8x16 p21q1 = avgu8(p2q1, p1) - ((p2q1 ^ p1) & and0); /* p21q1+pq0 == (p2q1+p1+p0+q0)/2 */\
 170 | 		i8x16 q21p1 = avgu8(q2p1, q1) - ((q2p1 ^ q1) & and0); /* q21p1+pq0 == (q2p1+q1+p0+q0)/2 */\
 171 | 		i8x16 pp0a = avgu8(p21q1, pq0); /* p'0 (first formula) */\
 172 | 		i8x16 qp0a = avgu8(q21p1, pq0); /* q'0 (first formula) */\
 173 | 		i8x16 pp0b = avgu8(p1, avgu8(p0, q1) - ((p0 ^ q1) & c1)); /* p'0 (second formula) */\
 174 | 		i8x16 qp0b = avgu8(q1, avgu8(q0, p1) - ((q0 ^ p1) & c1)); /* q'0 (second formula) */\
 175 | 		p0 = ifelse_mask(ignoreSamplesFlags, p0, ifelse_mask(condp, pp0a, pp0b));\
 176 | 		q0 = ifelse_mask(ignoreSamplesFlags, q0, ifelse_mask(condq, qp0a, qp0b));\
 177 | 		/* compute p'1 and q'1 */\
 178 | 		i8x16 fcondp = condp & ~ignoreSamplesFlags;\
 179 | 		i8x16 fcondq = condq & ~ignoreSamplesFlags;\
 180 | 		i8x16 p21 = avgu8(p2, p1) - ((p2 ^ p1) & and0); /* p21+pq0 == (p2+p1+p0+q0)/2 */\
 181 | 		i8x16 q21 = avgu8(q2, q1) - ((q2 ^ q1) & and0); /* q21+pq0 == (q2+q1+q0+p1)/2 */\
 182 | 		i8x16 pp1 = avgu8(p21, pq0); /* p'1 */\
 183 | 		i8x16 qp1 = avgu8(q21, pq0); /* q'1 */\
 184 | 		p1 = ifelse_mask(fcondp, pp1, p1);\
 185 | 		q1 = ifelse_mask(fcondq, qp1, q1);\
 186 | 		/* compute p'2 and q'2 */\
 187 | 		i8x16 fix1 = ((p21 ^ pq0) & c1);\
 188 | 		i8x16 fix2 = ((q21 ^ pq0) & c1);\
 189 | 		i8x16 p210q0 = pp1 - fix1; /* (p2+p1+p0+q0)/4 */\
 190 | 		i8x16 q210p0 = qp1 - fix2; /* (q2+q1+q0+p0)/4 */\
 191 | 		i8x16 p3p2 = avgu8(p3, p2) - ((p3 ^ p2) & (fix1 ^ c1)); /* p3p2+p210q0 == (p3+p2+(p2+p1+p0+q0)/2)/2 */\
 192 | 		i8x16 q3q2 = avgu8(q3, q2) - ((q3 ^ q2) & (fix2 ^ c1)); /* q3q2+q210p0 == (q3+q2+(q2+q1+p0+q0)/2)/2 */\
 193 | 		i8x16 pp2 = avgu8(p3p2, p210q0); /* p'2 */\
 194 | 		i8x16 qp2 = avgu8(q3q2, q210p0); /* q'2 */\
 195 | 		p2 = ifelse_mask(fcondp, pp2, p2);\
 196 | 		q2 = ifelse_mask(fcondq, qp2, q2);}
 197 | 	#define DEBLOCK_CHROMA_HARD(p1, p0, q0, q1, ialpha, ibeta) {\
 198 | 		/* compute the opposite of filterSamplesFlags */\
 199 | 		i8x16 abs0 = subu8(p0, q0) | subu8(q0, p0);\
 200 | 		i8x16 abs1 = subu8(p1, p0) | subu8(p0, p1);\
 201 | 		i8x16 abs2 = subu8(q1, q0) | subu8(q0, q1);\
 202 | 		i8x16 shufab = {1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2};\
 203 | 		i8x16 alpha = shuffle((i32x4){ialpha}, shufab);\
 204 | 		i8x16 beta = shuffle((i32x4){ibeta}, shufab);\
 205 | 		i8x16 and = minu8(subu8(alpha, abs0), subu8(beta, maxu8(abs1, abs2)));\
 206 | 		i8x16 ignoreSamplesFlags = and == 0;\
 207 | 		/* compute p'0 and q'0 */\
 208 | 		i8x16 c1 = set8(1);\
 209 | 		i8x16 pp0b = avgu8(p1, avgu8(p0, q1) - ((p0 ^ q1) & c1)); /* p'0 (second formula) */\
 210 | 		i8x16 qp0b = avgu8(q1, avgu8(q0, p1) - ((q0 ^ p1) & c1)); /* q'0 (second formula) */\
 211 | 		p0 = ifelse_mask(ignoreSamplesFlags, p0, pp0b);\
 212 | 		q0 = ifelse_mask(ignoreSamplesFlags, q0, qp0b);}
 213 | #elif defined(__ARM_NEON)
 214 | 	#define DEBLOCK_LUMA_HARD(p3, p2, p1, p0, q0, q1, q2, q3, ialpha, ibeta) {\
 215 | 		/* common masks */\
 216 | 		u8x16 alpha = set8(ialpha);\
 217 | 		u8x16 beta = set8(ibeta);\
 218 | 		u8x16 abs0 = vabdq_u8(p0, q0);\
 219 | 		u8x16 abs1 = vmaxq_u8(vabdq_u8(p1, p0), vabdq_u8(q1, q0));\
 220 | 		i8x16 filterSamplesFlag = (abs0 < alpha) & (abs1 < beta);\
 221 | 		i8x16 condpq = abs0 < (alpha >> 2) + 2;\
 222 | 		i8x16 condp = ((u8x16)vabdq_u8(p2, p0) < beta) & condpq;\
 223 | 		i8x16 condq = ((u8x16)vabdq_u8(q2, q0) < beta) & condpq;\
 224 | 		/* common wide sums */\
 225 | 		u16x8 p10l = vaddl_u8(vget_low_u8(p1), vget_low_u8(p0));\
 226 | 		u16x8 p10h = vaddl_high_u8(p1, p0);\
 227 | 		u16x8 q01l = vaddl_u8(vget_low_u8(q0), vget_low_u8(q1));\
 228 | 		u16x8 q01h = vaddl_high_u8(q0, q1);\
 229 | 		u16x8 p210q0l = p10l + (u16x8)vaddl_u8(vget_low_u8(p2), vget_low_u8(q0));\
 230 | 		u16x8 p210q0h = p10h + (u16x8)vaddl_high_u8(p2, q0);\
 231 | 		u16x8 p0q012l = q01l + (u16x8)vaddl_u8(vget_low_u8(p0), vget_low_u8(q2));\
 232 | 		u16x8 p0q012h = q01h + (u16x8)vaddl_high_u8(p0, q2);\
 233 | 		u16x8 p10q01l = p10l + q01l;\
 234 | 		u16x8 p10q01h = p10h + q01h;\
 235 | 		/* compute p'0 and q'0 */\
 236 | 		u8x16 pp0a = shrrpu16(p210q0l + p10q01l, p210q0h + p10q01h, 3); /* p'0 (first formula) */\
 237 | 		u8x16 qp0a = shrrpu16(p10q01l + p0q012l, p10q01h + p0q012h, 3); /* q'0 (first formula) */\
 238 | 		u8x16 pp0b = vrhaddq_u8(p1, vhaddq_u8(p0, q1)); /* p'0 (second formula) */\
 239 | 		u8x16 qp0b = vrhaddq_u8(q1, vhaddq_u8(q0, p1)); /* q'0 (second formula) */\
 240 | 		p0 = ifelse_mask(filterSamplesFlag, ifelse_mask(condp, pp0a, pp0b), p0);\
 241 | 		q0 = ifelse_mask(filterSamplesFlag, ifelse_mask(condq, qp0a, qp0b), q0);\
 242 | 		/* compute p'1 and q'1 */\
 243 | 		i8x16 fcondp = condp & filterSamplesFlag;\
 244 | 		i8x16 fcondq = condq & filterSamplesFlag;\
 245 | 		p1 = ifelse_mask(fcondp, shrrpu16(p210q0l, p210q0h, 2), p1);\
 246 | 		q1 = ifelse_mask(fcondq, shrrpu16(p0q012l, p0q012h, 2), q1);\
 247 | 		/* compute p'2 and q'2 */\
 248 | 		u16x8 p32l = vaddl_u8(vget_low_u8(p3), vget_low_u8(p2));\
 249 | 		u16x8 p32h = vaddl_high_u8(p3, p2);\
 250 | 		u16x8 q23l = vaddl_u8(vget_low_u8(q2), vget_low_u8(q3));\
 251 | 		u16x8 q23h = vaddl_high_u8(q2, q3);\
 252 | 		p2 = ifelse_mask(fcondp, shrrpu16((p32l << 1) + p210q0l, (p32h << 1) + p210q0h, 3), p2);\
 253 | 		q2 = ifelse_mask(fcondq, shrrpu16((q23l << 1) + p0q012l, (q23h << 1) + p0q012h, 3), q2);}
 254 | 	#define DEBLOCK_CHROMA_HARD(p1, p0, q0, q1, ialpha, ibeta) {\
 255 | 		i8x16 shufab = {1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2};\
 256 | 		i8x16 alpha = shuffle((i32x4){ialpha}, shufab);\
 257 | 		i8x16 beta = shuffle((i32x4){ibeta}, shufab);\
 258 | 		u8x16 abs0 = vabdq_u8(p0, q0);\
 259 | 		u8x16 abs1 = vmaxq_u8(vabdq_u8(p1, p0), vabdq_u8(q1, q0));\
 260 | 		i8x16 filterSamplesFlag = (abs0 < alpha) & (abs1 < beta);\
 261 | 		p0 = ifelse_mask(filterSamplesFlag, vrhaddq_u8(p1, vhaddq_u8(p0, q1)), p0);\
 262 | 		q0 = ifelse_mask(filterSamplesFlag, vrhaddq_u8(q1, vhaddq_u8(q0, p1)), q0);}
 263 | #endif
 264 | 
 265 | 
 266 | 
 267 | /**
 268 |  * Efficient addressing for both Intel and ARM architectures.
 269 |  * Call INIT_PX(p, stride) once to compute anchor addresses, then PX(x,y) will
 270 |  * return p + x + y * stride.
 271 |  */
 272 | #if defined(__SSE2__)
 273 | 	#define INIT_PX(p, stride) size_t _stride = (stride); ssize_t _nstride = -_stride; uint8_t * restrict _p0 = (p), * restrict _p7 = _p0 + _stride * 8 + _nstride, * restrict _pE = _p7 + _stride * 8 + _nstride
 274 | 	#define PX(x, y) (__builtin_choose_expr(y <= 2 || y == 4, _p0, __builtin_choose_expr(y <= 9 || y == 11, _p7, _pE)) +\
 275 | 		__builtin_choose_expr(y < 0, x - y * _nstride, __builtin_choose_expr(y == 0 || y == 7 || y == 14, x, __builtin_choose_expr(y == 1 || y == 8 || y == 15, x + _stride, __builtin_choose_expr(y == 2 || y == 9, x + _stride * 2, __builtin_choose_expr(y == 3 || y == 10, x + _nstride * 4, __builtin_choose_expr(y == 4 || y == 11, x + _stride * 4, __builtin_choose_expr(y == 5 || y == 12, x + _nstride * 2, x + _nstride))))))))
 276 | #elif defined(__ARM_NEON)
 277 | 	#define INIT_PX(p, stride) size_t _stride = (stride), _stridem2 = _stride - 2, _stridem4 = _stride - 4, _stridem8 = _stride - 8; uint8_t * restrict _p0 = (p), * restrict _p2 = _p0 + _stride * 2, * restrict _p4 = _p0 + _stride * 4, * restrict _p6 = _p2 + _stride * 4, * restrict _p8 = _p0 + _stride * 8, * restrict _pA = _p2 + _stride * 8, * restrict _pC = _p4 + _stride * 8, * restrict _pE = _p6 + _stride * 8
 278 | 	#define PX(x, y) (__builtin_choose_expr(y < 2, _p0, __builtin_choose_expr(y < 4, _p2, __builtin_choose_expr(y < 6, _p4, __builtin_choose_expr(y < 8, _p6, __builtin_choose_expr(y < 10, _p8, __builtin_choose_expr(y < 12, _pA, __builtin_choose_expr(y < 14, _pC, _pE))))))) +\
 279 | 		__builtin_choose_expr(y < 0 || y > 15, x + y * _stride, __builtin_choose_expr((y & 1) == 0, x, __builtin_choose_expr(x == 0, _stride, __builtin_choose_expr(x == -2, _stridem2, __builtin_choose_expr(x == -4, _stridem4, __builtin_choose_expr(x == -8, _stridem8, x + _stride)))))))
 280 | #endif
 281 | 
 282 | 
 283 | 
 284 | /**
 285 |  * Deblock both chroma planes of the current macroblock in place.
 286 |  */
 287 | static void deblock_CbCr_8bit(Edge264Context *ctx) {
 288 | 	INIT_PX(ctx->samples_mb[1], ctx->t.stride[1] >> 1);
 289 | 	i8x16 v0, v1, v2, v3, v4, v5, v6, v7;
 290 | 	if (mb->filter_edges & 1) {
 291 | 		// load and transpose both 12x8 matrices (with left macroblock)
 292 | 		i8x16 xa0 = load128(PX(-8, 0));
 293 | 		i8x16 xa8 = load128(PX(-8, 1));
 294 | 		i8x16 xa1 = load128(PX(-8, 2));
 295 | 		i8x16 xa9 = load128(PX(-8, 3));
 296 | 		i8x16 xa2 = load128(PX(-8, 4));
 297 | 		i8x16 xaA = load128(PX(-8, 5));
 298 | 		i8x16 xa3 = load128(PX(-8, 6));
 299 | 		i8x16 xaB = load128(PX(-8, 7));
 300 | 		i8x16 xb0 = ziplo8(xa0, xa1);
 301 | 		i8x16 xb1 = ziphi8(xa0, xa1);
 302 | 		i8x16 xb2 = ziplo8(xa2, xa3);
 303 | 		i8x16 xb3 = ziphi8(xa2, xa3);
 304 | 		i8x16 xb8 = ziplo8(xa8, xa9);
 305 | 		i8x16 xb9 = ziphi8(xa8, xa9);
 306 | 		i8x16 xbA = ziplo8(xaA, xaB);
 307 | 		i8x16 xbB = ziphi8(xaA, xaB);
 308 | 		i8x16 xc0 = ziphi16(xb0, xb2);
 309 | 		i8x16 xc1 = ziplo16(xb1, xb3);
 310 | 		i8x16 xc2 = ziphi16(xb1, xb3);
 311 | 		i8x16 xc6 = ziphi16(xb8, xbA);
 312 | 		i8x16 xc7 = ziplo16(xb9, xbB);
 313 | 		i8x16 xc8 = ziphi16(xb9, xbB);
 314 | 		i8x16 xa4 = load128(PX(-8, 8));
 315 | 		i8x16 xaC = load128(PX(-8, 9));
 316 | 		i8x16 xa5 = load128(PX(-8, 10));
 317 | 		i8x16 xaD = load128(PX(-8, 11));
 318 | 		i8x16 xa6 = load128(PX(-8, 12));
 319 | 		i8x16 xaE = load128(PX(-8, 13));
 320 | 		i8x16 xa7 = load128(PX(-8, 14));
 321 | 		i8x16 xaF = load128(PX(-8, 15));
 322 | 		i8x16 xb4 = ziplo8(xa4, xa5);
 323 | 		i8x16 xb5 = ziphi8(xa4, xa5);
 324 | 		i8x16 xb6 = ziplo8(xa6, xa7);
 325 | 		i8x16 xb7 = ziphi8(xa6, xa7);
 326 | 		i8x16 xc3 = ziphi16(xb4, xb6);
 327 | 		i8x16 xc4 = ziplo16(xb5, xb7);
 328 | 		i8x16 xc5 = ziphi16(xb5, xb7);
 329 | 		i8x16 xbC = ziplo8(xaC, xaD);
 330 | 		i8x16 xbD = ziphi8(xaC, xaD);
 331 | 		i8x16 xbE = ziplo8(xaE, xaF);
 332 | 		i8x16 xbF = ziphi8(xaE, xaF);
 333 | 		i8x16 xc9 = ziphi16(xbC, xbE);
 334 | 		i8x16 xcA = ziplo16(xbD, xbF);
 335 | 		i8x16 xcB = ziphi16(xbD, xbF);
 336 | 		i8x16 xd0 = ziphi32(xc0, xc3);
 337 | 		i8x16 xd1 = ziplo32(xc1, xc4);
 338 | 		i8x16 xd2 = ziphi32(xc1, xc4);
 339 | 		i8x16 xd3 = ziplo32(xc2, xc5);
 340 | 		i8x16 xd4 = ziphi32(xc2, xc5);
 341 | 		i8x16 xd5 = ziphi32(xc6, xc9);
 342 | 		i8x16 xd6 = ziplo32(xc7, xcA);
 343 | 		i8x16 xd7 = ziphi32(xc7, xcA);
 344 | 		i8x16 xd8 = ziplo32(xc8, xcB);
 345 | 		i8x16 xd9 = ziphi32(xc8, xcB);
 346 | 		i8x16 vY = ziplo64(xd0, xd5);
 347 | 		i8x16 vZ = ziphi64(xd0, xd5);
 348 | 		v0 = ziplo64(xd1, xd6);
 349 | 		v1 = ziphi64(xd1, xd6);
 350 | 		v2 = ziplo64(xd2, xd7);
 351 | 		v3 = ziphi64(xd2, xd7);
 352 | 		v4 = ziplo64(xd3, xd8);
 353 | 		v5 = ziphi64(xd3, xd8);
 354 | 		v6 = ziplo64(xd4, xd9);
 355 | 		v7 = ziphi64(xd4, xd9);
 356 | 		
 357 | 		// first vertical edge
 358 | 		if (mbA->mbIsInterFlag & mb->mbIsInterFlag) {
 359 | 			int64_t tC0a = ctx->tC0_l[4];
 360 | 			if (tC0a != -1)
 361 | 				DEBLOCK_CHROMA_SOFT(vY, vZ, v0, v1, ctx->alpha_s[2], ctx->beta_s[2], tC0a);
 362 | 		} else {
 363 | 			DEBLOCK_CHROMA_HARD(vY, vZ, v0, v1, ctx->alpha_s[2], ctx->beta_s[2]);
 364 | 		}
 365 | 		
 366 | 		// store vY/vZ into the left macroblock
 367 | 		i16x8 xf0 = ziplo8(vY, vZ);
 368 | 		i16x8 xf1 = ziphi8(vY, vZ);
 369 | 		*(int16_t *)PX(-2, 0) = xf0[0];
 370 | 		*(int16_t *)PX(-2, 1) = xf1[0];
 371 | 		*(int16_t *)PX(-2, 2) = xf0[1];
 372 | 		*(int16_t *)PX(-2, 3) = xf1[1];
 373 | 		*(int16_t *)PX(-2, 4) = xf0[2];
 374 | 		*(int16_t *)PX(-2, 5) = xf1[2];
 375 | 		*(int16_t *)PX(-2, 6) = xf0[3];
 376 | 		*(int16_t *)PX(-2, 7) = xf1[3];
 377 | 		*(int16_t *)PX(-2, 8) = xf0[4];
 378 | 		*(int16_t *)PX(-2, 9) = xf1[4];
 379 | 		*(int16_t *)PX(-2, 10) = xf0[5];
 380 | 		*(int16_t *)PX(-2, 11) = xf1[5];
 381 | 		*(int16_t *)PX(-2, 12) = xf0[6];
 382 | 		*(int16_t *)PX(-2, 13) = xf1[6];
 383 | 		*(int16_t *)PX(-2, 14) = xf0[7];
 384 | 		*(int16_t *)PX(-2, 15) = xf1[7];
 385 | 	} else {
 386 | 		// load and transpose both 8x8 matrices
 387 | 		i8x16 xa0 = load64(PX(0, 0));
 388 | 		i8x16 xa8 = load64(PX(0, 1));
 389 | 		i8x16 xa1 = load64(PX(0, 2));
 390 | 		i8x16 xa9 = load64(PX(0, 3));
 391 | 		i8x16 xa2 = load64(PX(0, 4));
 392 | 		i8x16 xaA = load64(PX(0, 5));
 393 | 		i8x16 xa3 = load64(PX(0, 6));
 394 | 		i8x16 xaB = load64(PX(0, 7));
 395 | 		i8x16 xb0 = ziplo8(xa0, xa1);
 396 | 		i8x16 xb1 = ziplo8(xa2, xa3);
 397 | 		i8x16 xb4 = ziplo8(xa8, xa9);
 398 | 		i8x16 xb5 = ziplo8(xaA, xaB);
 399 | 		i8x16 xc0 = ziplo16(xb0, xb1);
 400 | 		i8x16 xc1 = ziphi16(xb0, xb1);
 401 | 		i8x16 xc4 = ziplo16(xb4, xb5);
 402 | 		i8x16 xc5 = ziphi16(xb4, xb5);
 403 | 		i8x16 xa4 = load64(PX(0, 8));
 404 | 		i8x16 xaC = load64(PX(0, 9));
 405 | 		i8x16 xa5 = load64(PX(0, 10));
 406 | 		i8x16 xaD = load64(PX(0, 11));
 407 | 		i8x16 xa6 = load64(PX(0, 12));
 408 | 		i8x16 xaE = load64(PX(0, 13));
 409 | 		i8x16 xa7 = load64(PX(0, 14));
 410 | 		i8x16 xaF = load64(PX(0, 15));
 411 | 		i8x16 xb2 = ziplo8(xa4, xa5);
 412 | 		i8x16 xb3 = ziplo8(xa6, xa7);
 413 | 		i8x16 xb6 = ziplo8(xaC, xaD);
 414 | 		i8x16 xb7 = ziplo8(xaE, xaF);
 415 | 		i8x16 xc2 = ziplo16(xb2, xb3);
 416 | 		i8x16 xc3 = ziphi16(xb2, xb3);
 417 | 		i8x16 xc6 = ziplo16(xb6, xb7);
 418 | 		i8x16 xc7 = ziphi16(xb6, xb7);
 419 | 		i8x16 xd0 = ziplo32(xc0, xc2);
 420 | 		i8x16 xd1 = ziphi32(xc0, xc2);
 421 | 		i8x16 xd2 = ziplo32(xc1, xc3);
 422 | 		i8x16 xd3 = ziphi32(xc1, xc3);
 423 | 		i8x16 xd4 = ziplo32(xc4, xc6);
 424 | 		i8x16 xd5 = ziphi32(xc4, xc6);
 425 | 		i8x16 xd6 = ziplo32(xc5, xc7);
 426 | 		i8x16 xd7 = ziphi32(xc5, xc7);
 427 | 		v0 = ziplo64(xd0, xd4);
 428 | 		v1 = ziphi64(xd0, xd4);
 429 | 		v2 = ziplo64(xd1, xd5);
 430 | 		v3 = ziphi64(xd1, xd5);
 431 | 		v4 = ziplo64(xd2, xd6);
 432 | 		v5 = ziphi64(xd2, xd6);
 433 | 		v6 = ziplo64(xd3, xd7);
 434 | 		v7 = ziphi64(xd3, xd7);
 435 | 	}
 436 | 	
 437 | 	// second vertical edge
 438 | 	int64_t tC0c = ctx->tC0_l[5];
 439 | 	if (tC0c != -1)
 440 | 		DEBLOCK_CHROMA_SOFT(v2, v3, v4, v5, ctx->alpha_s[0], ctx->beta_s[0], tC0c);
 441 | 	
 442 | 	// transpose both 8x8 matrices
 443 | 	i8x16 xa0 = ziplo8(v0, v1);
 444 | 	i8x16 xa1 = ziphi8(v0, v1);
 445 | 	i8x16 xa2 = ziplo8(v2, v3);
 446 | 	i8x16 xa3 = ziphi8(v2, v3);
 447 | 	i8x16 xa4 = ziplo8(v4, v5);
 448 | 	i8x16 xa5 = ziphi8(v4, v5);
 449 | 	i8x16 xa6 = ziplo8(v6, v7);
 450 | 	i8x16 xa7 = ziphi8(v6, v7);
 451 | 	i8x16 xb0 = ziplo16(xa0, xa2);
 452 | 	i8x16 xb1 = ziphi16(xa0, xa2);
 453 | 	i8x16 xb2 = ziplo16(xa1, xa3);
 454 | 	i8x16 xb3 = ziphi16(xa1, xa3);
 455 | 	i8x16 xb4 = ziplo16(xa4, xa6);
 456 | 	i8x16 xb5 = ziphi16(xa4, xa6);
 457 | 	i8x16 xb6 = ziplo16(xa5, xa7);
 458 | 	i8x16 xb7 = ziphi16(xa5, xa7);
 459 | 	i8x16 xc0 = ziplo32(xb0, xb4);
 460 | 	i8x16 xc1 = ziphi32(xb0, xb4);
 461 | 	i8x16 xc2 = ziplo32(xb1, xb5);
 462 | 	i8x16 xc3 = ziphi32(xb1, xb5);
 463 | 	i8x16 xc4 = ziplo32(xb2, xb6);
 464 | 	i8x16 xc5 = ziphi32(xb2, xb6);
 465 | 	i8x16 xc6 = ziplo32(xb3, xb7);
 466 | 	i8x16 xc7 = ziphi32(xb3, xb7);
 467 | 	i8x16 h0 = ziplo64(xc0, xc4);
 468 | 	i8x16 h1 = ziphi64(xc0, xc4);
 469 | 	i8x16 h2 = ziplo64(xc1, xc5);
 470 | 	i8x16 h3 = ziphi64(xc1, xc5);
 471 | 	i8x16 h4 = ziplo64(xc2, xc6);
 472 | 	i8x16 h5 = ziphi64(xc2, xc6);
 473 | 	i8x16 h6 = ziplo64(xc3, xc7);
 474 | 	i8x16 h7 = ziphi64(xc3, xc7);
 475 | 	
 476 | 	// first horizontal edge
 477 | 	if (mb->filter_edges & 2) {
 478 | 		i8x16 hY = (i64x2){*(int64_t *)(PX(0, -4)), *(int64_t *)(PX(0, -3))};
 479 | 		i8x16 hZ = (i64x2){*(int64_t *)(PX(0, -2)), *(int64_t *)(PX(0, -1))};
 480 | 		if (mbB->mbIsInterFlag & mb->mbIsInterFlag) {
 481 | 			int64_t tC0e = ctx->tC0_l[6];
 482 | 			if (tC0e != -1)
 483 | 				DEBLOCK_CHROMA_SOFT(hY, hZ, h0, h1, ctx->alpha_s[3], ctx->beta_s[3], tC0e);
 484 | 		} else {
 485 | 			DEBLOCK_CHROMA_HARD(hY, hZ, h0, h1, ctx->alpha_s[3], ctx->beta_s[3]);
 486 | 		}
 487 | 		*(int64_t *)PX(0, -2) = ((i64x2)hZ)[0];
 488 | 		*(int64_t *)PX(0, -1) = ((i64x2)hZ)[1];
 489 | 	}
 490 | 	mb->filter_edges = 0; // prevent redundant deblocking with deblock_idc==2 and ASO
 491 | 	*(int64_t *)PX(0, 0) = ((i64x2)h0)[0];
 492 | 	*(int64_t *)PX(0, 1) = ((i64x2)h0)[1];
 493 | 	*(int64_t *)PX(0, 2) = ((i64x2)h1)[0];
 494 | 	*(int64_t *)PX(0, 3) = ((i64x2)h1)[1];
 495 | 	
 496 | 	// second horizontal edge
 497 | 	int64_t tC0g = ctx->tC0_l[7];
 498 | 	if (tC0g != -1)
 499 | 		DEBLOCK_CHROMA_SOFT(h2, h3, h4, h5, ctx->alpha_s[0], ctx->beta_s[0], tC0g);
 500 | 	*(int64_t *)PX(0, 4) = ((i64x2)h2)[0];
 501 | 	*(int64_t *)PX(0, 5) = ((i64x2)h2)[1];
 502 | 	*(int64_t *)PX(0, 6) = ((i64x2)h3)[0];
 503 | 	*(int64_t *)PX(0, 7) = ((i64x2)h3)[1];
 504 | 	*(int64_t *)PX(0, 8) = ((i64x2)h4)[0];
 505 | 	*(int64_t *)PX(0, 9) = ((i64x2)h4)[1];
 506 | 	*(int64_t *)PX(0, 10) = ((i64x2)h5)[0];
 507 | 	*(int64_t *)PX(0, 11) = ((i64x2)h5)[1];
 508 | 	*(int64_t *)PX(0, 12) = ((i64x2)h6)[0];
 509 | 	*(int64_t *)PX(0, 13) = ((i64x2)h6)[1];
 510 | 	*(int64_t *)PX(0, 14) = ((i64x2)h7)[0];
 511 | 	*(int64_t *)PX(0, 15) = ((i64x2)h7)[1];
 512 | }
 513 | 
 514 | 
 515 | 
 516 | /**
 517 |  * Deblock the luma plane of the current macroblock in place, then tail call to
 518 |  * chroma deblocking.
 519 |  */
 520 | static void deblock_Y_8bit(Edge264Context *ctx) {
 521 | 	INIT_PX(ctx->samples_mb[0], ctx->t.stride[0]);
 522 | 	i8x16 v0, v1, v2, v3, v4, v5, v6, v7;
 523 | 	if (mb->filter_edges & 1) {
 524 | 		// load and transpose the left 12x16 matrix
 525 | 		i8x16 xa0 = load128(PX(-8, 0));
 526 | 		i8x16 xa1 = load128(PX(-8, 1));
 527 | 		i8x16 xa2 = load128(PX(-8, 2));
 528 | 		i8x16 xa3 = load128(PX(-8, 3));
 529 | 		i8x16 xa4 = load128(PX(-8, 4));
 530 | 		i8x16 xa5 = load128(PX(-8, 5));
 531 | 		i8x16 xa6 = load128(PX(-8, 6));
 532 | 		i8x16 xa7 = load128(PX(-8, 7));
 533 | 		i8x16 xb0 = ziplo8(xa0, xa1);
 534 | 		i8x16 xb1 = ziphi8(xa0, xa1);
 535 | 		i8x16 xb2 = ziplo8(xa2, xa3);
 536 | 		i8x16 xb3 = ziphi8(xa2, xa3);
 537 | 		i8x16 xb4 = ziplo8(xa4, xa5);
 538 | 		i8x16 xb5 = ziphi8(xa4, xa5);
 539 | 		i8x16 xb6 = ziplo8(xa6, xa7);
 540 | 		i8x16 xb7 = ziphi8(xa6, xa7);
 541 | 		i8x16 xc0 = ziphi16(xb0, xb2);
 542 | 		i8x16 xc1 = ziplo16(xb1, xb3);
 543 | 		i8x16 xc2 = ziphi16(xb1, xb3);
 544 | 		i8x16 xc3 = ziphi16(xb4, xb6);
 545 | 		i8x16 xc4 = ziplo16(xb5, xb7);
 546 | 		i8x16 xc5 = ziphi16(xb5, xb7);
 547 | 		i8x16 xa8 = load128(PX(-8, 8));
 548 | 		i8x16 xa9 = load128(PX(-8, 9));
 549 | 		i8x16 xaA = load128(PX(-8, 10));
 550 | 		i8x16 xaB = load128(PX(-8, 11));
 551 | 		i8x16 xaC = load128(PX(-8, 12));
 552 | 		i8x16 xaD = load128(PX(-8, 13));
 553 | 		i8x16 xaE = load128(PX(-8, 14));
 554 | 		i8x16 xaF = load128(PX(-8, 15));
 555 | 		i8x16 xb8 = ziplo8(xa8, xa9);
 556 | 		i8x16 xb9 = ziphi8(xa8, xa9);
 557 | 		i8x16 xbA = ziplo8(xaA, xaB);
 558 | 		i8x16 xbB = ziphi8(xaA, xaB);
 559 | 		i8x16 xbC = ziplo8(xaC, xaD);
 560 | 		i8x16 xbD = ziphi8(xaC, xaD);
 561 | 		i8x16 xbE = ziplo8(xaE, xaF);
 562 | 		i8x16 xbF = ziphi8(xaE, xaF);
 563 | 		i8x16 xc6 = ziphi16(xb8, xbA);
 564 | 		i8x16 xc7 = ziplo16(xb9, xbB);
 565 | 		i8x16 xc8 = ziphi16(xb9, xbB);
 566 | 		i8x16 xc9 = ziphi16(xbC, xbE);
 567 | 		i8x16 xcA = ziplo16(xbD, xbF);
 568 | 		i8x16 xcB = ziphi16(xbD, xbF);
 569 | 		i8x16 xd0 = ziplo32(xc0, xc3);
 570 | 		i8x16 xd1 = ziphi32(xc0, xc3);
 571 | 		i8x16 xd2 = ziplo32(xc1, xc4);
 572 | 		i8x16 xd3 = ziphi32(xc1, xc4);
 573 | 		i8x16 xd4 = ziplo32(xc2, xc5);
 574 | 		i8x16 xd5 = ziphi32(xc2, xc5);
 575 | 		i8x16 xd6 = ziplo32(xc6, xc9);
 576 | 		i8x16 xd7 = ziphi32(xc6, xc9);
 577 | 		i8x16 xd8 = ziplo32(xc7, xcA);
 578 | 		i8x16 xd9 = ziphi32(xc7, xcA);
 579 | 		i8x16 xdA = ziplo32(xc8, xcB);
 580 | 		i8x16 xdB = ziphi32(xc8, xcB);
 581 | 		i8x16 vW = ziplo64(xd0, xd6);
 582 | 		i8x16 vX = ziphi64(xd0, xd6);
 583 | 		i8x16 vY = ziplo64(xd1, xd7);
 584 | 		i8x16 vZ = ziphi64(xd1, xd7);
 585 | 		v0 = ziplo64(xd2, xd8);
 586 | 		v1 = ziphi64(xd2, xd8);
 587 | 		v2 = ziplo64(xd3, xd9);
 588 | 		v3 = ziphi64(xd3, xd9);
 589 | 		v4 = ziplo64(xd4, xdA);
 590 | 		v5 = ziphi64(xd4, xdA);
 591 | 		v6 = ziplo64(xd5, xdB);
 592 | 		v7 = ziphi64(xd5, xdB);
 593 | 		
 594 | 		// first vertical edge
 595 | 		if (mbA->mbIsInterFlag & mb->mbIsInterFlag) {
 596 | 			int tC0a = ctx->tC0_s[0];
 597 | 			if (tC0a != -1)
 598 | 				DEBLOCK_LUMA_SOFT(vX, vY, vZ, v0, v1, v2, ctx->alpha[8], ctx->beta[8], tC0a);
 599 | 		} else if (ctx->alpha[8] != 0) {
 600 | 			DEBLOCK_LUMA_HARD(vW, vX, vY, vZ, v0, v1, v2, v3, ctx->alpha[8], ctx->beta[8]);
 601 | 		}
 602 | 		
 603 | 		// store vW/vX/vY/vZ into the left macroblock
 604 | 		i8x16 xe0 = ziplo8(vW, vX);
 605 | 		i8x16 xe1 = ziphi8(vW, vX);
 606 | 		i8x16 xe2 = ziplo8(vY, vZ);
 607 | 		i8x16 xe3 = ziphi8(vY, vZ);
 608 | 		i32x4 xe4 = ziplo16(xe0, xe2);
 609 | 		i32x4 xe5 = ziphi16(xe0, xe2);
 610 | 		i32x4 xe6 = ziplo16(xe1, xe3);
 611 | 		i32x4 xe7 = ziphi16(xe1, xe3);
 612 | 		*(int32_t *)PX(-4, 0) = xe4[0];
 613 | 		*(int32_t *)PX(-4, 1) = xe4[1];
 614 | 		*(int32_t *)PX(-4, 2) = xe4[2];
 615 | 		*(int32_t *)PX(-4, 3) = xe4[3];
 616 | 		*(int32_t *)PX(-4, 4) = xe5[0];
 617 | 		*(int32_t *)PX(-4, 5) = xe5[1];
 618 | 		*(int32_t *)PX(-4, 6) = xe5[2];
 619 | 		*(int32_t *)PX(-4, 7) = xe5[3];
 620 | 		*(int32_t *)PX(-4, 8) = xe6[0];
 621 | 		*(int32_t *)PX(-4, 9) = xe6[1];
 622 | 		*(int32_t *)PX(-4, 10) = xe6[2];
 623 | 		*(int32_t *)PX(-4, 11) = xe6[3];
 624 | 		*(int32_t *)PX(-4, 12) = xe7[0];
 625 | 		*(int32_t *)PX(-4, 13) = xe7[1];
 626 | 		*(int32_t *)PX(-4, 14) = xe7[2];
 627 | 		*(int32_t *)PX(-4, 15) = xe7[3];
 628 | 	} else {
 629 | 		// load and transpose the left 8x16 matrix
 630 | 		i8x16 xa0 = load64(PX(0, 0));
 631 | 		i8x16 xa1 = load64(PX(0, 1));
 632 | 		i8x16 xa2 = load64(PX(0, 2));
 633 | 		i8x16 xa3 = load64(PX(0, 3));
 634 | 		i8x16 xa4 = load64(PX(0, 4));
 635 | 		i8x16 xa5 = load64(PX(0, 5));
 636 | 		i8x16 xa6 = load64(PX(0, 6));
 637 | 		i8x16 xa7 = load64(PX(0, 7));
 638 | 		i8x16 xb0 = ziplo8(xa0, xa1);
 639 | 		i8x16 xb1 = ziplo8(xa2, xa3);
 640 | 		i8x16 xb2 = ziplo8(xa4, xa5);
 641 | 		i8x16 xb3 = ziplo8(xa6, xa7);
 642 | 		i8x16 xc0 = ziplo16(xb0, xb1);
 643 | 		i8x16 xc1 = ziphi16(xb0, xb1);
 644 | 		i8x16 xc2 = ziplo16(xb2, xb3);
 645 | 		i8x16 xc3 = ziphi16(xb2, xb3);
 646 | 		i8x16 xa8 = load64(PX(0, 8));
 647 | 		i8x16 xa9 = load64(PX(0, 9));
 648 | 		i8x16 xaA = load64(PX(0, 10));
 649 | 		i8x16 xaB = load64(PX(0, 11));
 650 | 		i8x16 xaC = load64(PX(0, 12));
 651 | 		i8x16 xaD = load64(PX(0, 13));
 652 | 		i8x16 xaE = load64(PX(0, 14));
 653 | 		i8x16 xaF = load64(PX(0, 15));
 654 | 		i8x16 xb4 = ziplo8(xa8, xa9);
 655 | 		i8x16 xb5 = ziplo8(xaA, xaB);
 656 | 		i8x16 xb6 = ziplo8(xaC, xaD);
 657 | 		i8x16 xb7 = ziplo8(xaE, xaF);
 658 | 		i8x16 xc4 = ziplo16(xb4, xb5);
 659 | 		i8x16 xc5 = ziphi16(xb4, xb5);
 660 | 		i8x16 xc6 = ziplo16(xb6, xb7);
 661 | 		i8x16 xc7 = ziphi16(xb6, xb7);
 662 | 		i8x16 xd0 = ziplo32(xc0, xc2);
 663 | 		i8x16 xd1 = ziphi32(xc0, xc2);
 664 | 		i8x16 xd2 = ziplo32(xc1, xc3);
 665 | 		i8x16 xd3 = ziphi32(xc1, xc3);
 666 | 		i8x16 xd4 = ziplo32(xc4, xc6);
 667 | 		i8x16 xd5 = ziphi32(xc4, xc6);
 668 | 		i8x16 xd6 = ziplo32(xc5, xc7);
 669 | 		i8x16 xd7 = ziphi32(xc5, xc7);
 670 | 		v0 = ziplo64(xd0, xd4);
 671 | 		v1 = ziphi64(xd0, xd4);
 672 | 		v2 = ziplo64(xd1, xd5);
 673 | 		v3 = ziphi64(xd1, xd5);
 674 | 		v4 = ziplo64(xd2, xd6);
 675 | 		v5 = ziphi64(xd2, xd6);
 676 | 		v6 = ziplo64(xd3, xd7);
 677 | 		v7 = ziphi64(xd3, xd7);
 678 | 	}
 679 | 	
 680 | 	// second vertical edge
 681 | 	if (!mb->f.transform_size_8x8_flag) {
 682 | 		int tC0b = ctx->tC0_s[1];
 683 | 		if (tC0b != -1)
 684 | 			DEBLOCK_LUMA_SOFT(v1, v2, v3, v4, v5, v6, ctx->alpha[0], ctx->beta[0], tC0b);
 685 | 	}
 686 | 	
 687 | 	// load and transpose the right 8x16 matrix
 688 | 	i8x16 xa0 = *(i8x16 *)PX(0, 0);
 689 | 	i8x16 xa1 = *(i8x16 *)PX(0, 1);
 690 | 	i8x16 xa2 = *(i8x16 *)PX(0, 2);
 691 | 	i8x16 xa3 = *(i8x16 *)PX(0, 3);
 692 | 	i8x16 xa4 = *(i8x16 *)PX(0, 4);
 693 | 	i8x16 xa5 = *(i8x16 *)PX(0, 5);
 694 | 	i8x16 xa6 = *(i8x16 *)PX(0, 6);
 695 | 	i8x16 xa7 = *(i8x16 *)PX(0, 7);
 696 | 	i8x16 xb0 = ziphi8(xa0, xa1);
 697 | 	i8x16 xb1 = ziphi8(xa2, xa3);
 698 | 	i8x16 xb2 = ziphi8(xa4, xa5);
 699 | 	i8x16 xb3 = ziphi8(xa6, xa7);
 700 | 	i8x16 xc0 = ziplo16(xb0, xb1);
 701 | 	i8x16 xc1 = ziphi16(xb0, xb1);
 702 | 	i8x16 xc2 = ziplo16(xb2, xb3);
 703 | 	i8x16 xc3 = ziphi16(xb2, xb3);
 704 | 	i8x16 xa8 = *(i8x16 *)PX(0, 8);
 705 | 	i8x16 xa9 = *(i8x16 *)PX(0, 9);
 706 | 	i8x16 xaA = *(i8x16 *)PX(0, 10);
 707 | 	i8x16 xaB = *(i8x16 *)PX(0, 11);
 708 | 	i8x16 xaC = *(i8x16 *)PX(0, 12);
 709 | 	i8x16 xaD = *(i8x16 *)PX(0, 13);
 710 | 	i8x16 xaE = *(i8x16 *)PX(0, 14);
 711 | 	i8x16 xaF = *(i8x16 *)PX(0, 15);
 712 | 	i8x16 xb4 = ziphi8(xa8, xa9);
 713 | 	i8x16 xb5 = ziphi8(xaA, xaB);
 714 | 	i8x16 xb6 = ziphi8(xaC, xaD);
 715 | 	i8x16 xb7 = ziphi8(xaE, xaF);
 716 | 	i8x16 xc4 = ziplo16(xb4, xb5);
 717 | 	i8x16 xc5 = ziphi16(xb4, xb5);
 718 | 	i8x16 xc6 = ziplo16(xb6, xb7);
 719 | 	i8x16 xc7 = ziphi16(xb6, xb7);
 720 | 	i8x16 xd0 = ziplo32(xc0, xc2);
 721 | 	i8x16 xd1 = ziphi32(xc0, xc2);
 722 | 	i8x16 xd2 = ziplo32(xc1, xc3);
 723 | 	i8x16 xd3 = ziphi32(xc1, xc3);
 724 | 	i8x16 xd4 = ziplo32(xc4, xc6);
 725 | 	i8x16 xd5 = ziphi32(xc4, xc6);
 726 | 	i8x16 xd6 = ziplo32(xc5, xc7);
 727 | 	i8x16 xd7 = ziphi32(xc5, xc7);
 728 | 	i8x16 v8 = ziplo64(xd0, xd4);
 729 | 	i8x16 v9 = ziphi64(xd0, xd4);
 730 | 	i8x16 vA = ziplo64(xd1, xd5);
 731 | 	i8x16 vB = ziphi64(xd1, xd5);
 732 | 	i8x16 vC = ziplo64(xd2, xd6);
 733 | 	i8x16 vD = ziphi64(xd2, xd6);
 734 | 	i8x16 vE = ziplo64(xd3, xd7);
 735 | 	i8x16 vF = ziphi64(xd3, xd7);
 736 | 	
 737 | 	// third vertical edge
 738 | 	int tC0c = ctx->tC0_s[2];
 739 | 	if (tC0c != -1)
 740 | 		DEBLOCK_LUMA_SOFT(v5, v6, v7, v8, v9, vA, ctx->alpha[0], ctx->beta[0], tC0c);
 741 | 	
 742 | 	// fourth vertical edge
 743 | 	if (!mb->f.transform_size_8x8_flag) {
 744 | 		int tC0d = ctx->tC0_s[3];
 745 | 		if (tC0d != -1)
 746 | 			DEBLOCK_LUMA_SOFT(v9, vA, vB, vC, vD, vE, ctx->alpha[0], ctx->beta[0], tC0d);
 747 | 	}
 748 | 	
 749 | 	// transpose the top 16x8 matrix
 750 | 	i8x16 xe0 = ziplo8(v0, v1);
 751 | 	i8x16 xe1 = ziplo8(v2, v3);
 752 | 	i8x16 xe2 = ziplo8(v4, v5);
 753 | 	i8x16 xe3 = ziplo8(v6, v7);
 754 | 	i8x16 xe4 = ziplo8(v8, v9);
 755 | 	i8x16 xe5 = ziplo8(vA, vB);
 756 | 	i8x16 xe6 = ziplo8(vC, vD);
 757 | 	i8x16 xe7 = ziplo8(vE, vF);
 758 | 	i8x16 xf0 = ziplo16(xe0, xe1);
 759 | 	i8x16 xf1 = ziphi16(xe0, xe1);
 760 | 	i8x16 xf2 = ziplo16(xe2, xe3);
 761 | 	i8x16 xf3 = ziphi16(xe2, xe3);
 762 | 	i8x16 xf4 = ziplo16(xe4, xe5);
 763 | 	i8x16 xf5 = ziphi16(xe4, xe5);
 764 | 	i8x16 xf6 = ziplo16(xe6, xe7);
 765 | 	i8x16 xf7 = ziphi16(xe6, xe7);
 766 | 	i8x16 xg0 = ziplo32(xf0, xf2);
 767 | 	i8x16 xg1 = ziphi32(xf0, xf2);
 768 | 	i8x16 xg2 = ziplo32(xf1, xf3);
 769 | 	i8x16 xg3 = ziphi32(xf1, xf3);
 770 | 	i8x16 xg4 = ziplo32(xf4, xf6);
 771 | 	i8x16 xg5 = ziphi32(xf4, xf6);
 772 | 	i8x16 xg6 = ziplo32(xf5, xf7);
 773 | 	i8x16 xg7 = ziphi32(xf5, xf7);
 774 | 	i8x16 h0 = ziplo64(xg0, xg4);
 775 | 	i8x16 h1 = ziphi64(xg0, xg4);
 776 | 	i8x16 h2 = ziplo64(xg1, xg5);
 777 | 	i8x16 h3 = ziphi64(xg1, xg5);
 778 | 	i8x16 h4 = ziplo64(xg2, xg6);
 779 | 	i8x16 h5 = ziphi64(xg2, xg6);
 780 | 	i8x16 h6 = ziplo64(xg3, xg7);
 781 | 	i8x16 h7 = ziphi64(xg3, xg7);
 782 | 	
 783 | 	// first horizontal edge
 784 | 	if (mb->filter_edges & 2) {
 785 | 		i8x16 hx = *(i8x16 *)PX(0, -3);
 786 | 		i8x16 hy = *(i8x16 *)PX(0, -2);
 787 | 		i8x16 hz = *(i8x16 *)PX(0, -1);
 788 | 		if (mbB->mbIsInterFlag & mb->mbIsInterFlag) {
 789 | 			int tC0e = ctx->tC0_s[4];
 790 | 			if (tC0e != -1)
 791 | 				DEBLOCK_LUMA_SOFT(hx, hy, hz, h0, h1, h2, ctx->alpha[12], ctx->beta[12], tC0e);
 792 | 		} else if (ctx->alpha[12] != 0) {
 793 | 			i8x16 hw = *(i8x16 *)PX(0, -4);
 794 | 			DEBLOCK_LUMA_HARD(hw, hx, hy, hz, h0, h1, h2, h3, ctx->alpha[12], ctx->beta[12]);
 795 | 			*(i8x16 *)PX(0, -3) = hx;
 796 | 		}
 797 | 		*(i8x16 *)PX(0, -2) = hy;
 798 | 		*(i8x16 *)PX(0, -1) = hz;
 799 | 	}
 800 | 	*(i8x16 *)PX(0, 0) = h0;
 801 | 	*(i8x16 *)PX(0, 1) = h1;
 802 | 	
 803 | 	// second horizontal edge
 804 | 	if (!mb->f.transform_size_8x8_flag) {
 805 | 		int tC0f = ctx->tC0_s[5];
 806 | 		if (tC0f != -1)
 807 | 			DEBLOCK_LUMA_SOFT(h1, h2, h3, h4, h5, h6, ctx->alpha[0], ctx->beta[0], tC0f);
 808 | 	}
 809 | 	*(i8x16 *)PX(0, 2) = h2;
 810 | 	*(i8x16 *)PX(0, 3) = h3;
 811 | 	*(i8x16 *)PX(0, 4) = h4;
 812 | 	*(i8x16 *)PX(0, 5) = h5;
 813 | 	
 814 | 	// transpose the bottom 16x8 matrix
 815 | 	i8x16 xh0 = ziphi8(v0, v1);
 816 | 	i8x16 xh1 = ziphi8(v2, v3);
 817 | 	i8x16 xh2 = ziphi8(v4, v5);
 818 | 	i8x16 xh3 = ziphi8(v6, v7);
 819 | 	i8x16 xh4 = ziphi8(v8, v9);
 820 | 	i8x16 xh5 = ziphi8(vA, vB);
 821 | 	i8x16 xh6 = ziphi8(vC, vD);
 822 | 	i8x16 xh7 = ziphi8(vE, vF);
 823 | 	i8x16 xi0 = ziplo16(xh0, xh1);
 824 | 	i8x16 xi1 = ziphi16(xh0, xh1);
 825 | 	i8x16 xi2 = ziplo16(xh2, xh3);
 826 | 	i8x16 xi3 = ziphi16(xh2, xh3);
 827 | 	i8x16 xi4 = ziplo16(xh4, xh5);
 828 | 	i8x16 xi5 = ziphi16(xh4, xh5);
 829 | 	i8x16 xi6 = ziplo16(xh6, xh7);
 830 | 	i8x16 xi7 = ziphi16(xh6, xh7);
 831 | 	i8x16 xj0 = ziplo32(xi0, xi2);
 832 | 	i8x16 xj1 = ziphi32(xi0, xi2);
 833 | 	i8x16 xj2 = ziplo32(xi1, xi3);
 834 | 	i8x16 xj3 = ziphi32(xi1, xi3);
 835 | 	i8x16 xj4 = ziplo32(xi4, xi6);
 836 | 	i8x16 xj5 = ziphi32(xi4, xi6);
 837 | 	i8x16 xj6 = ziplo32(xi5, xi7);
 838 | 	i8x16 xj7 = ziphi32(xi5, xi7);
 839 | 	i8x16 h8 = ziplo64(xj0, xj4);
 840 | 	i8x16 h9 = ziphi64(xj0, xj4);
 841 | 	i8x16 hA = ziplo64(xj1, xj5);
 842 | 	i8x16 hB = ziphi64(xj1, xj5);
 843 | 	i8x16 hC = ziplo64(xj2, xj6);
 844 | 	i8x16 hD = ziphi64(xj2, xj6);
 845 | 	i8x16 hE = ziplo64(xj3, xj7);
 846 | 	i8x16 hF = ziphi64(xj3, xj7);
 847 | 	
 848 | 	// third horizontal edge
 849 | 	int tC0g = ctx->tC0_s[6];
 850 | 	if (tC0g != -1)
 851 | 		DEBLOCK_LUMA_SOFT(h5, h6, h7, h8, h9, hA, ctx->alpha[0], ctx->beta[0], tC0g);
 852 | 	*(i8x16 *)PX(0, 6) = h6;
 853 | 	*(i8x16 *)PX(0, 7) = h7;
 854 | 	*(i8x16 *)PX(0, 8) = h8;
 855 | 	*(i8x16 *)PX(0, 9) = h9;
 856 | 	
 857 | 	// fourth horizontal edge
 858 | 	if (!mb->f.transform_size_8x8_flag) {
 859 | 		int tC0h = ctx->tC0_s[7];
 860 | 		if (tC0h != -1)
 861 | 			DEBLOCK_LUMA_SOFT(h9, hA, hB, hC, hD, hE, ctx->alpha[0], ctx->beta[0], tC0h);
 862 | 	}
 863 | 	*(i8x16 *)PX(0, 10) = hA;
 864 | 	*(i8x16 *)PX(0, 11) = hB;
 865 | 	*(i8x16 *)PX(0, 12) = hC;
 866 | 	*(i8x16 *)PX(0, 13) = hD;
 867 | 	*(i8x16 *)PX(0, 14) = hE;
 868 | 	*(i8x16 *)PX(0, 15) = hF;
 869 | 	
 870 | 	// jump to chroma deblocking filter
 871 | 	deblock_CbCr_8bit(ctx);
 872 | }
 873 | 
 874 | 
 875 | 
 876 | /**
 877 |  * Compute alpha, beta and tC0 for all planes and all edges (labeled a to h in
 878 |  * deblocking order) of the current macroblock, then tail-call to luma filter.
 879 |  * 
 880 |  * One of the hard parts here is computing a mask for bS=1. Edges a/c/e/g and
 881 |  * b/d/f/h are handled separately, and we calculate 4 values per edge:
 882 |  * _ differences of references in parallel (l0 vs l0, l1 vs l1) -> refs_p
 883 |  *   (zero on b/d/f/h edges)
 884 |  * _ differences of references in cross (l0 vs l1, l1 vs l0) -> refs_c
 885 |  *   (zero on P macroblocks)
 886 |  * _ differences of motion vectors in parallel -> mvs_p
 887 |  * _ differences of motion vectors in cross -> mvs_c
 888 |  *   (zero on P macroblocks)
 889 |  * 
 890 |  * Then bS is inferred with the help of a Karnaugh map, where the AND operation
 891 |  * is achieved using umin8:
 892 |  *                 |      refs_p=0       |      refs_p>0       |
 893 |  *                 | refs_c=0 | refs_c>0 | refs_c>0 | refs_c=0 |
 894 |  * ----------------+----------+----------+----------+----------+
 895 |  *         mvs_c=0 |    0     |    0     |    1     |    0     |
 896 |  * mvs_p=0 --------+----------+----------+----------+----------+
 897 |  *         mvs_c>0 |    0     |    0     |    1     |    1     |
 898 |  * ----------------+----------+----------+----------+----------+
 899 |  *         mvs_c>0 |    1     |    1     |    1     |    1     |
 900 |  * mvs_p>0 --------+----------+----------+----------+----------+
 901 |  *         mvs_c=0 |    0     |    1     |    1     |    0     |
 902 |  * ----------------+----------+----------+----------+----------+
 903 |  */
 904 | static noinline void deblock_mb(Edge264Context *ctx)
 905 | {
 906 | 	static const u8x16 idx2alpha[3] =
 907 | 		{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, 5, 6, 7, 8, 9, 10, 12, 13, 15, 17, 20, 22, 25, 28, 32, 36, 40, 45, 50, 56, 63, 71, 80, 90, 101, 113, 127, 144, 162, 182, 203, 226, 255, 255};
 908 | 	static const i8x16 idx2beta[3] =
 909 | 		{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 2, 2, 2, 3, 3, 3, 3, 4, 4, 4, 6, 6, 7, 7, 8, 8, 9, 9, 10, 10, 11, 11, 12, 12, 13, 13, 14, 14, 15, 15, 16, 16, 17, 17, 18, 18};
 910 | 	static const i8x16 idx2tC0[3][3] = { // modified to disable deblocking when alpha or beta is zero
 911 | 		{-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 4, 4, 4, 5, 6, 6, 7, 8, 9, 10, 11, 13},
 912 | 		{-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 4, 4, 5, 5, 6, 7, 8, 8, 10, 11, 12, 13, 15, 17},
 913 | 		{-1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 4, 4, 4, 5, 6, 6, 7, 8, 9, 10, 11, 13, 14, 16, 18, 20, 23, 25},
 914 | 	};
 915 | 	if (!mb->filter_edges)
 916 | 		return;
 917 | 	
 918 | 	// compute all values of alpha and beta for each of the color planes first
 919 | 	mbA = mb - 1;
 920 | 	mbB = mbA - ctx->t.pic_width_in_mbs;
 921 | 	i8x16 zero = {};
 922 | 	i8x16 qP = set32((int32_t)mb->QP_s);
 923 | 	i32x4 qPAB = {(int32_t)mbA->QP_s, (int32_t)mbB->QP_s};
 924 | 	i8x16 qPav = avgu8(qP, ziplo64(qP, qPAB)); // mid/mid/A/B
 925 | 	i8x16 c51 = set8(51);
 926 | 	#if defined(__SSE2__)
 927 | 		i8x16 indexA = minu8(max8(qPav + set8(ctx->t.FilterOffsetA), zero), c51);
 928 | 		i8x16 indexB = minu8(max8(qPav + set8(ctx->t.FilterOffsetB), zero), c51);
 929 | 	#elif defined(__ARM_NEON)
 930 | 		i8x16 indexA = minu8(vsqaddq_u8(qPav, set8(ctx->t.FilterOffsetA)), c51);
 931 | 		i8x16 indexB = minu8(vsqaddq_u8(qPav, set8(ctx->t.FilterOffsetB)), c51);
 932 | 	#endif
 933 | 	i8x16 c4 = set8(4);
 934 | 	i8x16 Am4 = subu8(indexA, c4);
 935 | 	ctx->alpha_v = shuffle3((const i8x16 *)idx2alpha, Am4);
 936 | 	ctx->beta_v = shuffle3(idx2beta, subu8(indexB, c4));
 937 | 	
 938 | 	// initialize tC0 for bS=1/2/3
 939 | 	if (!mb->mbIsInterFlag) {
 940 | 		i8x16 tC03 = shuffle3(idx2tC0[2], Am4);
 941 | 		ctx->tC0_v[0] = ctx->tC0_v[1] = broadcast8(tC03, 0);
 942 | 		ctx->tC0_v[2] = ctx->tC0_v[3] = shuffle(tC03, ((i8x16){-1, -1, -1, -1, -1, -1, -1, -1, 1, 1, 1, 1, 2, 2, 2, 2}));
 943 | 	} else {
 944 | 		i8x16 tC01 = shuffle3(idx2tC0[0], Am4);
 945 | 		i8x16 tC02 = shuffle3(idx2tC0[1], Am4);
 946 | 		
 947 | 		// compute masks for bS!=1 based on equality of references and motion vectors
 948 | 		i8x16 bS0aceg, bS0bdfh;
 949 | 		i8x16 c3 = set8(3);
 950 | 		static const i8x16 shufVHAB = {0, 2, 1, 3, 0, 1, 2, 3, 9, 11, 0, 2, 14, 15, 0, 1};
 951 | 		if ((mb->refIdx_s[1] & mbA->refIdx_s[1] & mbB->refIdx_s[1]) == -1) { // P macroblocks
 952 | 			i16x8 mvsv0 = unziphi32(mbA->mvs_v[1], mbA->mvs_v[3]);
 953 | 			i16x8 mvsv1 = unziplo32(mb->mvs_v[0], mb->mvs_v[2]);
 954 | 			i16x8 mvsv2 = unziphi32(mb->mvs_v[0], mb->mvs_v[2]);
 955 | 			i16x8 mvsv3 = unziplo32(mb->mvs_v[1], mb->mvs_v[3]);
 956 | 			i16x8 mvsv4 = unziphi32(mb->mvs_v[1], mb->mvs_v[3]);
 957 | 			i16x8 mvsh0 = ziphi64(mbB->mvs_v[2], mbB->mvs_v[3]);
 958 | 			i16x8 mvsh1 = ziplo64(mb->mvs_v[0], mb->mvs_v[1]);
 959 | 			i16x8 mvsh2 = ziphi64(mb->mvs_v[0], mb->mvs_v[1]);
 960 | 			i16x8 mvsh3 = ziplo64(mb->mvs_v[2], mb->mvs_v[3]);
 961 | 			i16x8 mvsh4 = ziphi64(mb->mvs_v[2], mb->mvs_v[3]);
 962 | 			i8x16 mvsac = packabd16(mvsv0, mvsv1, mvsv2, mvsv3);
 963 | 			i8x16 mvsbd = packabd16(mvsv1, mvsv2, mvsv3, mvsv4);
 964 | 			i8x16 mvseg = packabd16(mvsh0, mvsh1, mvsh2, mvsh3);
 965 | 			i8x16 mvsfh = packabd16(mvsh1, mvsh2, mvsh3, mvsh4);
 966 | 			i8x16 mvsaceg = packs16(subu8(mvsac, c3), subu8(mvseg, c3));
 967 | 			i8x16 mvsbdfh = packs16(subu8(mvsbd, c3), subu8(mvsfh, c3));
 968 | 			i8x16 refs = shuffle(((i32x4){mb->refPic_s[0], 0, mbA->refPic_s[0], mbB->refPic_s[0]}), shufVHAB); // (v0,h0,A0,B0)
 969 | 			i8x16 neq = refs ^ shr128(refs, 8); // (v0^A0,h0^B0,0,0)
 970 | 			i8x16 refsaceg = ziplo8(neq, neq);
 971 | 			bS0aceg = (refsaceg | mvsaceg) == zero;
 972 | 			bS0bdfh = mvsbdfh == zero;
 973 | 		} else if (mb->f.inter_eqs_s == little_endian32(0x1b5fbbff)) { // 16x16 B macroblock
 974 | 			i16x8 mvsv0l0 = unziphi32(mbA->mvs_v[1], mbA->mvs_v[3]);
 975 | 			i16x8 mvsv1l0 = unziplo32(mb->mvs_v[0], mb->mvs_v[2]);
 976 | 			i16x8 mvsv0l1 = unziphi32(mbA->mvs_v[5], mbA->mvs_v[7]);
 977 | 			i16x8 mvsv1l1 = unziplo32(mb->mvs_v[4], mb->mvs_v[6]);
 978 | 			i16x8 mvsh0l0 = ziphi64(mbB->mvs_v[2], mbB->mvs_v[3]);
 979 | 			i16x8 mvsh1l0 = ziplo64(mb->mvs_v[0], mb->mvs_v[1]);
 980 | 			i16x8 mvsh0l1 = ziphi64(mbB->mvs_v[6], mbB->mvs_v[7]);
 981 | 			i16x8 mvsh1l1 = ziplo64(mb->mvs_v[4], mb->mvs_v[5]);
 982 | 			i8x16 mvsael00 = packabd16(mvsv0l0, mvsv1l0, mvsh0l0, mvsh1l0);
 983 | 			i8x16 mvsael01 = packabd16(mvsv0l0, mvsv1l1, mvsh0l0, mvsh1l1);
 984 | 			i8x16 mvsael10 = packabd16(mvsv0l1, mvsv1l0, mvsh0l1, mvsh1l0);
 985 | 			i8x16 mvsael11 = packabd16(mvsv0l1, mvsv1l1, mvsh0l1, mvsh1l1);
 986 | 			i8x16 mvsaep = subu8(maxu8(mvsael00, mvsael11), c3);
 987 | 			i8x16 mvsaec = subu8(maxu8(mvsael01, mvsael10), c3);
 988 | 			i8x16 mvsacegp = ziplo32(packs16(mvsaep, zero), zero);
 989 | 			i8x16 mvsacegc = ziplo32(packs16(mvsaec, zero), zero);
 990 | 			i64x2 refPic = {mb->refPic_l};
 991 | 			i64x2 refPicAB = {mbA->refPic_l, mbB->refPic_l};
 992 | 			i8x16 refs0 = shuffle(unziplo32(refPic, refPicAB), shufVHAB); // (v0,h0,A0,B0)
 993 | 			i8x16 refs1 = shuffle(unziphi32(refPic, refPicAB), shufVHAB); // (v1,h1,A1,B1)
 994 | 			i8x16 neq0 = refs0 ^ (i8x16)shrd128(refs1, refs0, 8); // (v0^A1,h0^B1,A0^v0,B0^h0)
 995 | 			i8x16 neq1 = refs1 ^ (i8x16)shrd128(refs0, refs1, 8); // (v1^A0,h1^B0,A1^v1,B1^h1)
 996 | 			i8x16 refsaceg = neq0 | neq1; // low=cross, high=parallel
 997 | 			i8x16 refsacegc = ziplo8(refsaceg, refsaceg);
 998 | 			i8x16 refsacegp = ziphi8(refsaceg, refsaceg);
 999 | 			i8x16 neq3 = minu8(refsacegp, refsacegc) | minu8(mvsacegp, mvsacegc);
1000 | 			i8x16 neq4 = minu8(refsacegp, mvsacegc) | minu8(mvsacegp, refsacegc);
1001 | 			bS0aceg = (neq3 | neq4) == zero;
1002 | 			bS0bdfh = set8(-1);
1003 | 		} else { // B macroblocks
1004 | 			i16x8 mvsv0l0 = unziphi32(mbA->mvs_v[1], mbA->mvs_v[3]);
1005 | 			i16x8 mvsv1l0 = unziplo32(mb->mvs_v[0], mb->mvs_v[2]);
1006 | 			i16x8 mvsv2l0 = unziphi32(mb->mvs_v[0], mb->mvs_v[2]);
1007 | 			i16x8 mvsv3l0 = unziplo32(mb->mvs_v[1], mb->mvs_v[3]);
1008 | 			i16x8 mvsv4l0 = unziphi32(mb->mvs_v[1], mb->mvs_v[3]);
1009 | 			i16x8 mvsv0l1 = unziphi32(mbA->mvs_v[5], mbA->mvs_v[7]);
1010 | 			i16x8 mvsv1l1 = unziplo32(mb->mvs_v[4], mb->mvs_v[6]);
1011 | 			i16x8 mvsv2l1 = unziphi32(mb->mvs_v[4], mb->mvs_v[6]);
1012 | 			i16x8 mvsv3l1 = unziplo32(mb->mvs_v[5], mb->mvs_v[7]);
1013 | 			i16x8 mvsv4l1 = unziphi32(mb->mvs_v[5], mb->mvs_v[7]);
1014 | 			i8x16 mvsacl00 = packabd16(mvsv0l0, mvsv1l0, mvsv2l0, mvsv3l0);
1015 | 			i8x16 mvsbdl00 = packabd16(mvsv1l0, mvsv2l0, mvsv3l0, mvsv4l0);
1016 | 			i8x16 mvsacl01 = packabd16(mvsv0l0, mvsv1l1, mvsv2l0, mvsv3l1);
1017 | 			i8x16 mvsbdl01 = packabd16(mvsv1l0, mvsv2l1, mvsv3l0, mvsv4l1);
1018 | 			i8x16 mvsacl10 = packabd16(mvsv0l1, mvsv1l0, mvsv2l1, mvsv3l0);
1019 | 			i8x16 mvsbdl10 = packabd16(mvsv1l1, mvsv2l0, mvsv3l1, mvsv4l0);
1020 | 			i8x16 mvsacl11 = packabd16(mvsv0l1, mvsv1l1, mvsv2l1, mvsv3l1);
1021 | 			i8x16 mvsbdl11 = packabd16(mvsv1l1, mvsv2l1, mvsv3l1, mvsv4l1);
1022 | 			i8x16 mvsacp = subu8(maxu8(mvsacl00, mvsacl11), c3);
1023 | 			i8x16 mvsbdp = subu8(maxu8(mvsbdl00, mvsbdl11), c3);
1024 | 			i8x16 mvsacc = subu8(maxu8(mvsacl01, mvsacl10), c3);
1025 | 			i8x16 mvsbdc = subu8(maxu8(mvsbdl01, mvsbdl10), c3);
1026 | 			i16x8 mvsh0l0 = ziphi64(mbB->mvs_v[2], mbB->mvs_v[3]);
1027 | 			i16x8 mvsh1l0 = ziplo64(mb->mvs_v[0], mb->mvs_v[1]);
1028 | 			i16x8 mvsh2l0 = ziphi64(mb->mvs_v[0], mb->mvs_v[1]);
1029 | 			i16x8 mvsh3l0 = ziplo64(mb->mvs_v[2], mb->mvs_v[3]);
1030 | 			i16x8 mvsh4l0 = ziphi64(mb->mvs_v[2], mb->mvs_v[3]);
1031 | 			i16x8 mvsh0l1 = ziphi64(mbB->mvs_v[6], mbB->mvs_v[7]);
1032 | 			i16x8 mvsh1l1 = ziplo64(mb->mvs_v[4], mb->mvs_v[5]);
1033 | 			i16x8 mvsh2l1 = ziphi64(mb->mvs_v[4], mb->mvs_v[5]);
1034 | 			i16x8 mvsh3l1 = ziplo64(mb->mvs_v[6], mb->mvs_v[7]);
1035 | 			i16x8 mvsh4l1 = ziphi64(mb->mvs_v[6], mb->mvs_v[7]);
1036 | 			i8x16 mvsegl00 = packabd16(mvsh0l0, mvsh1l0, mvsh2l0, mvsh3l0);
1037 | 			i8x16 mvsfhl00 = packabd16(mvsh1l0, mvsh2l0, mvsh3l0, mvsh4l0);
1038 | 			i8x16 mvsegl01 = packabd16(mvsh0l0, mvsh1l1, mvsh2l0, mvsh3l1);
1039 | 			i8x16 mvsfhl01 = packabd16(mvsh1l0, mvsh2l1, mvsh3l0, mvsh4l1);
1040 | 			i8x16 mvsegl10 = packabd16(mvsh0l1, mvsh1l0, mvsh2l1, mvsh3l0);
1041 | 			i8x16 mvsfhl10 = packabd16(mvsh1l1, mvsh2l0, mvsh3l1, mvsh4l0);
1042 | 			i8x16 mvsegl11 = packabd16(mvsh0l1, mvsh1l1, mvsh2l1, mvsh3l1);
1043 | 			i8x16 mvsfhl11 = packabd16(mvsh1l1, mvsh2l1, mvsh3l1, mvsh4l1);
1044 | 			i8x16 mvsegp = subu8(maxu8(mvsegl00, mvsegl11), c3);
1045 | 			i8x16 mvsfhp = subu8(maxu8(mvsfhl00, mvsfhl11), c3);
1046 | 			i8x16 mvsegc = subu8(maxu8(mvsegl01, mvsegl10), c3);
1047 | 			i8x16 mvsfhc = subu8(maxu8(mvsfhl01, mvsfhl10), c3);
1048 | 			i8x16 mvsacegp = packs16(mvsacp, mvsegp);
1049 | 			i8x16 mvsbdfhp = packs16(mvsbdp, mvsfhp);
1050 | 			i8x16 mvsacegc = packs16(mvsacc, mvsegc);
1051 | 			i8x16 mvsbdfhc = packs16(mvsbdc, mvsfhc);
1052 | 			i64x2 refPic = {mb->refPic_l};
1053 | 			i64x2 refPicAB = {mbA->refPic_l, mbB->refPic_l};
1054 | 			i8x16 refs0 = shuffle(unziplo32(refPic, refPicAB), shufVHAB); // (v0,h0,A0,B0)
1055 | 			i8x16 refs1 = shuffle(unziphi32(refPic, refPicAB), shufVHAB); // (v1,h1,A1,B1)
1056 | 			i8x16 neq0 = refs0 ^ shrd128(refs1, refs0, 8); // (v0^A1,h0^B1,A0^v0,B0^h0)
1057 | 			i8x16 neq1 = refs1 ^ shrd128(refs0, refs1, 8); // (v1^A0,h1^B0,A1^v1,B1^h1)
1058 | 			i8x16 neq2 = refs0 ^ refs1;
1059 | 			i8x16 refsaceg = neq0 | neq1; // low=cross, high=parallel
1060 | 			i8x16 refsacegc = ziplo8(refsaceg, refsaceg);
1061 | 			i8x16 refsacegp = ziphi8(refsaceg, refsaceg);
1062 | 			i8x16 refsbdfhc = ziplo8(neq2, neq2);
1063 | 			i8x16 neq3 = minu8(refsacegp, refsacegc) | minu8(mvsacegp, mvsacegc);
1064 | 			i8x16 neq4 = minu8(refsacegp, mvsacegc) | minu8(mvsacegp, refsacegc);
1065 | 			bS0aceg = (neq3 | neq4) == zero;
1066 | 			bS0bdfh = minu8(mvsbdfhp, refsbdfhc | mvsbdfhc) == zero;
1067 | 		}
1068 | 		i8x16 bS0abcd = ziplo32(bS0aceg, bS0bdfh);
1069 | 		i8x16 bS0efgh = ziphi32(bS0aceg, bS0bdfh);
1070 | 		i8x16 bS0aacc = ziplo32(bS0aceg, bS0aceg);
1071 | 		i8x16 bS0eegg = ziphi32(bS0aceg, bS0aceg);
1072 | 		
1073 | 		// for 8x8 blocks with CAVLC, broadcast transform tokens beforehand
1074 | 		i8x16 nC = mb->nC_v[0];
1075 | 		if (!ctx->t.pps.entropy_coding_mode_flag && mb->f.transform_size_8x8_flag) {
1076 | 			mb->nC_v[0] = nC = (i8x16)((i32x4)nC == 0) - -1;
1077 | 		}
1078 | 		
1079 | 		// compute masks for edges with bS=2
1080 | 		static const i8x16 shufV = {0, 2, 8, 10, 1, 3, 9, 11, 4, 6, 12, 14, 5, 7, 13, 15};
1081 | 		static const i8x16 shufH = {0, 1, 4, 5, 2, 3, 6, 7, 8, 9, 12, 13, 10, 11, 14, 15};
1082 | 		i8x16 nnzv = shuffle(nC, shufV);
1083 | 		i8x16 nnzl = shuffle(mbA->nC_v[0], shufV);
1084 | 		i8x16 nnzh = shuffle(nC, shufH);
1085 | 		i8x16 nnzt = shuffle(mbB->nC_v[0], shufH);
1086 | 		i8x16 bS2abcd = (nnzv | shrd128(nnzl, nnzv, 12)) > zero;
1087 | 		i8x16 bS2efgh = (nnzh | shrd128(nnzt, nnzh, 12)) > zero;
1088 | 		i8x16 bS2aacc = trnlo32(bS2abcd);
1089 | 		i8x16 bS2eegg = trnlo32(bS2efgh);
1090 | 		
1091 | 		// shuffle, blend and store tC0 values (tC00 is -1)
1092 | 		static const i8x16 shuf0 = {8, 8, 8, 8, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
1093 | 		static const i8x16 shuf1 = {12, 12, 12, 12, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0};
1094 | 		static const i8x16 shuf2 = {9, 9, 9, 9, 10, 10, 10, 10, 1, 1, 1, 1, 2, 2, 2, 2};
1095 | 		static const i8x16 shuf3 = {13, 13, 13, 13, 14, 14, 14, 14, 1, 1, 1, 1, 2, 2, 2, 2};
1096 | 		ctx->tC0_v[0] = ifelse_mask(bS2abcd, shuffle(tC02, shuf0), bS0abcd | shuffle(tC01, shuf0));
1097 | 		ctx->tC0_v[1] = ifelse_mask(bS2efgh, shuffle(tC02, shuf1), bS0efgh | shuffle(tC01, shuf1));
1098 | 		ctx->tC0_v[2] = ifelse_mask(bS2aacc, shuffle(tC02, shuf2), bS0aacc | shuffle(tC01, shuf2));
1099 | 		ctx->tC0_v[3] = ifelse_mask(bS2eegg, shuffle(tC02, shuf3), bS0eegg | shuffle(tC01, shuf3));
1100 | 	}
1101 | 	
1102 | 	// jump to luma deblocking filter
1103 | 	deblock_Y_8bit(ctx);
1104 | }
1105 | 


--------------------------------------------------------------------------------
/edge264_intra.c:
--------------------------------------------------------------------------------
  1 | /**
  2 |  * Intra decoding involves a lot of suffling for which I use compatible
  3 |  * instructions from both SSE and NEON, with a few exceptions where a common
  4 |  * ground would be too suboptimal.
  5 |  * 
  6 |  * Each decoding function is a large switch with a downward tree structure for
  7 |  * sharing code. Cases start at the leaves and branch towards the exit root.
  8 |  * While there is no way to make compilers generate proper code with vanilla
  9 |  * switches or functions, goto instructions are used for internal branches.
 10 |  * 
 11 |  * Choosing between the different possibilities of a same function is tricky,
 12 |  * in general I favor in order:
 13 |  * _ the fastest code, obviously (http://www.agner.org/optimize/#manual_instr_tab),
 14 |  * _ a short dependency chain (more freedom for compilers to reorder),
 15 |  * _ smaller code+data (avoid excessive use of pshufb),
 16 |  * _ readable code.
 17 |  * 
 18 |  * My thumb rules:
 19 |  * _ Aligned reads are favored if they incur no additional instructions.
 20 |  * _ pshufb is used iff doing otherwise would require 3+ instructions.
 21 |  * _ Favor vector over scalar code to avoid callee-save conventions.
 22 |  */
 23 | 
 24 | #include "edge264_internal.h"
 25 | 
 26 | #if defined(__SSE2__)
 27 | 	#define ldedge4x4() shrd128(ziplo16(ziplo8(load32(P(-4, 3)), load32(P(-4, 2))), ziplo8(load32(P(-4, 1)), load32(P(-4, 0)))), load64(P(-1, -1)), 12)
 28 | 	#define ldedge8x8lo() ({i8x16 _v7 = load64(P(-8, 7)); shuffleps(_v7, ziphi32(ziphi16(ziplo8(_v7, load64(P(-8, 6))), ziplo8(load64(P(-8, 5)), load64(P(-8, 4)))), ziplo16(ziplo8(load32(P(-4, 3)), load32(P(-4, 2))), ziplo8(load32(P(-4, 1)), load32(P(-4, 0))))), 0, 1, 2, 3);})
 29 | 	#define ldleft3(v0, y1, y2, y3) shr128(ziplo16(ziplo8((u32x4)(v0) << 24, load32(P(-4, y1))), ziplo8(load32(P(-4, y2)), load32(P(-4, y3)))), 12)
 30 | 	#define ldleft4(y0, y1, y2, y3) shr128(ziplo16(ziplo8(load32(P(-4, y0)), load32(P(-4, y1))), ziplo8(load32(P(-4, y2)), load32(P(-4, y3)))), 12)
 31 | 	#define ldleft7(v0, y1, y2, y3, y4, y5, y6, y7) shr128(ziphi32(ziplo16(ziplo8((u32x4)(v0) << 24, load32(P(-4, y1))), ziplo8(load32(P(-4, y2)), load32(P(-4, y3)))), ziplo16(ziplo8(load32(P(-4, y4)), load32(P(-4, y5))), ziplo8(load32(P(-4, y6)), load32(P(-4, y7))))), 8)
 32 | 	#define ldleft8(y0, y1, y2, y3, y4, y5, y6, y7) shr128(ziphi32(ziplo16(ziplo8(load32(P(-4, y0)), load32(P(-4, y1))), ziplo8(load32(P(-4, y2)), load32(P(-4, y3)))), ziplo16(ziplo8(load32(P(-4, y4)), load32(P(-4, y5))), ziplo8(load32(P(-4, y6)), load32(P(-4, y7))))), 8)
 33 | 	#define ldleft16(y0, y1, y2, y3, y4, y5, y6, y7, y8, y9, yA, yB, yC, yD, yE, yF) ziphi64(ziphi32(ziplo16(ziplo8(load32(P(-4, y0)), load32(P(-4, y1))), ziplo8(load32(P(-4, y2)), load32(P(-4, y3)))), ziplo16(ziplo8(load32(P(-4, y4)), load32(P(-4, y5))), ziplo8(load32(P(-4, y6)), load32(P(-4, y7))))), ziphi32(ziplo16(ziplo8(load32(P(-4, y8)), load32(P(-4, y9))), ziplo8(load32(P(-4, yA)), load32(P(-4, yB)))), ziplo16(ziplo8(load32(P(-4, yC)), load32(P(-4, yD))), ziplo8(load32(P(-4, yE)), load32(P(-4, yF))))))
 34 | 	#define spreadh8(a) shuffle(a, (i8x16){0, 1, 2, 3, 4, 5, 6, 7, 7, 7, 7, 7, 7, 7, 7, 7})
 35 | 	#define spreadq8(a) shuffle(a, (i8x16){0, 1, 2, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3})
 36 | 	static always_inline i8x16 lowpass8(i8x16 l, i8x16 m, i8x16 r) {return avgu8(subu8(avgu8(l, r), (l ^ r) & set8(1)), m);}
 37 | #elif defined(__ARM_NEON)
 38 | 	#define addlou8(a, b) (i16x8)vaddl_u8(vget_low_s8(a), vget_low_s8(b))
 39 | 	#define ldedge4x4() ({i8x16 _v = load128(P(-5, -1)); _v[3] = *P(-1, 0); _v[2] = *P(-1, 1); _v[1] = *P(-1, 2); _v[0] = *P(-1, 3); _v;})
 40 | 	#define ldedge8x8lo() ({i8x16 _v = set8(*P(-1, 7)); _v[15] = *P(-1, 0); _v[14] = *P(-1, 1); _v[13] = *P(-1, 2); _v[12] = *P(-1, 3); _v[11] = *P(-1, 4); _v[10] = *P(-1, 5); _v[9] = *P(-1, 6); _v;})
 41 | 	#define ldleft3(v0, y1, y2, y3) ({i8x16 _v = v0; _v[1] = *P(-1, y1), _v[2] = *P(-1, y2), _v[3] = *P(-1, y3); _v;})
 42 | 	#define ldleft4(y0, y1, y2, y3) (i8x16){*P(-1, y0), *P(-1, y1), *P(-1, y2), *P(-1, y3)}
 43 | 	#define ldleft7(v0, y1, y2, y3, y4, y5, y6, y7) ({i8x16 _v = v0; _v[1] = *P(-1, y1), _v[2] = *P(-1, y2), _v[3] = *P(-1, y3), _v[4] = *P(-1, y4), _v[5] = *P(-1, y5), _v[6] = *P(-1, y6), _v[7] = *P(-1, y7); _v;})
 44 | 	#define ldleft8(y0, y1, y2, y3, y4, y5, y6, y7) (i8x16){*P(-1, y0), *P(-1, y1), *P(-1, y2), *P(-1, y3), *P(-1, y4), *P(-1, y5), *P(-1, y6), *P(-1, y7)}
 45 | 	#define ldleft16(y0, y1, y2, y3, y4, y5, y6, y7, y8, y9, yA, yB, yC, yD, yE, yF) (i8x16){*P(-1, y0), *P(-1, y1), *P(-1, y2), *P(-1, y3), *P(-1, y4), *P(-1, y5), *P(-1, y6), *P(-1, y7), *P(-1, y8), *P(-1, y9), *P(-1, yA), *P(-1, yB), *P(-1, yC), *P(-1, yD), *P(-1, yE), *P(-1, yF)}
 46 | 	#define lowpass8(l, m, r) (i8x16)vrhaddq_u8(vhaddq_u8(l, r), m)
 47 | 	#define sublou8(a, b) (i16x8)vsubl_u8(vget_low_s8(a), vget_low_s8(b))
 48 | 	static always_inline i8x16 spreadh8(i8x16 a) {return vzip1q_s64(a, vdupq_laneq_s8(a, 7));}
 49 | 	static always_inline i8x16 spreadq8(i8x16 a) {return vextq_s8(vextq_s8(a, a, 4), vdupq_laneq_s8(a, 3), 12);}
 50 | #endif
 51 | 
 52 | 
 53 | 
 54 | /**
 55 |  * Intra 4x4
 56 |  */
 57 | static void decode_intra4x4(int mode, uint8_t * restrict p, size_t stride, i16x8 clip) {
 58 | 	INIT_P();
 59 | 	i8x16 v, shuf;
 60 | 	i32x4 pred;
 61 | 	switch (mode) {
 62 | 	default: __builtin_unreachable();
 63 | 	
 64 | 	case I4x4_V_8:
 65 | 		pred = set32(*(int32_t *)P(0, -1));
 66 | 		goto store_4x4;
 67 | 	
 68 | 	case I4x4_H_8:
 69 | 		*(int32_t *)P(0, 0) = ((i32x4)set8(*P(-1, 0)))[0];
 70 | 		*(int32_t *)P(0, 1) = ((i32x4)set8(*P(-1, 1)))[0];
 71 | 		*(int32_t *)P(0, 2) = ((i32x4)set8(*P(-1, 2)))[0];
 72 | 		*(int32_t *)P(0, 3) = ((i32x4)set8(*P(-1, 3)))[0];
 73 | 		return;
 74 | 	
 75 | 	case I4x4_DC_8:
 76 | 		v = ziplo32(ldleft4(0, 1, 2, 3), load32(P(0, -1)));
 77 | 	dc_4x4:
 78 | 		pred = broadcast8(shrru16(sumh8(v), 3), __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__);
 79 | 		goto store_4x4;
 80 | 	case I4x4_DCA_8: {
 81 | 		i8x16 v0 = load32(P(0, -1));
 82 | 		v = ziplo32(v0, v0);
 83 | 		} goto dc_4x4;
 84 | 	case I4x4_DCB_8: {
 85 | 		i8x16 v0 = ldleft4(0, 1, 2, 3);
 86 | 		v = ziplo32(v0, v0);
 87 | 		} goto dc_4x4;
 88 | 	case I4x4_DCAB_8:
 89 | 		pred = set8(-128);
 90 | 		goto store_4x4;
 91 | 	
 92 | 	case I4x4_DDL_8:
 93 | 		v = spreadh8(load64(P(0, -1)));
 94 | 		shuf = (i8x16){0, 1, 2, 3, 1, 2, 3, 4, 2, 3, 4, 5, 3, 4, 5, 6};
 95 | 		goto lowpass_4x4;
 96 | 	case I4x4_DDLC_8:
 97 | 		v = spreadq8(load32(P(0, -1)));
 98 | 		shuf = (i8x16){0, 1, 2, 3, 1, 2, 3, 4, 2, 3, 4, 5, 3, 4, 5, 6};
 99 | 		goto lowpass_4x4;
100 | 	case I4x4_DDR_8:
101 | 		shuf = (i8x16){3, 4, 5, 6, 2, 3, 4, 5, 1, 2, 3, 4, 0, 1, 2, 3};
102 | 		goto down_right_4x4;
103 | 	case I4x4_VR_8:
104 | 		shuf = (i8x16){12, 13, 14, 15, 3, 4, 5, 6, 2, 12, 13, 14, 1, 3, 4, 5};
105 | 		goto down_right_4x4;
106 | 	case I4x4_HD_8:
107 | 		shuf = (i8x16){11, 3, 4, 5, 10, 2, 11, 3, 9, 1, 10, 2, 8, 0, 9, 1};
108 | 		goto down_right_4x4;
109 | 	case I4x4_VL_8:
110 | 		v = load64(P(0, -1));
111 | 		shuf = (i8x16){8, 9, 10, 11, 0, 1, 2, 3, 9, 10, 11, 12, 1, 2, 3, 4};
112 | 		goto lowpass_4x4;
113 | 	case I4x4_VLC_8:
114 | 		v = spreadq8(load32(P(0, -1)));
115 | 		shuf = (i8x16){8, 9, 10, 11, 0, 1, 2, 3, 9, 10, 11, 12, 1, 2, 3, 4};
116 | 		goto lowpass_4x4;
117 | 	case I4x4_HU_8:
118 | 		v = shlc128(ldleft4(3, 2, 1, 0), 1);
119 | 		shuf = (i8x16){11, 2, 10, 1, 10, 1, 9, 0, 9, 0, 8, 8, 8, 8, 8, 8};
120 | 		goto lowpass_4x4;
121 | 	down_right_4x4:
122 | 		v = ldedge4x4();
123 | 	lowpass_4x4: {
124 | 		i8x16 w = shr128(v, 1);
125 | 		i8x16 x = shr128(v, 2);
126 | 		pred = shuffle(lowpass8(ziplo64(v, v), ziplo64(w, w), ziplo64(x, v)), shuf);
127 | 	} store_4x4:
128 | 		*(int32_t *)P(0, 0) = pred[0];
129 | 		*(int32_t *)P(0, 1) = pred[1];
130 | 		*(int32_t *)P(0, 2) = pred[2];
131 | 		*(int32_t *)P(0, 3) = pred[3];
132 | 		return;
133 | 	}
134 | }
135 | 
136 | 
137 | 
138 | /**
139 |  * Intra 8x8
140 |  * 
141 |  * Neighbouring samples are named a to z from bottom left to top right, with
142 |  * i being p[-1,-1] or p[-1,0] if unavailable, and j being p[-1,-1] or p[0,-1].
143 |  */
144 | static void decode_intra8x8(int mode, uint8_t * restrict p, size_t stride, i16x8 clip) {
145 | 	INIT_P();
146 | 	i8x16 i2a, j2s, j2y, k2z, j2q, k2r, l2s;
147 | 	i64x2 pred, p0, p1, p2, p3, p4, p5, p6, p7;
148 | 	switch (mode) {
149 | 	default: __builtin_unreachable();
150 | 	
151 | 	case I8x8_V_8:
152 | 		j2q = load128(P(-1, -1));
153 | 		l2s = shr128(j2q, 2);
154 | 		k2r = shr128(j2q, 1);
155 | 	vertical_8x8_lowpass:
156 | 		pred = lowpass8(j2q, k2r, l2s);
157 | 	store1_8x8:
158 | 		*(int64_t *)P(0, 0) = pred[0];
159 | 		*(int64_t *)P(0, 1) = pred[0];
160 | 		*(int64_t *)P(0, 2) = pred[0];
161 | 		*(int64_t *)P(0, 3) = pred[0];
162 | 		*(int64_t *)P(0, 4) = pred[0];
163 | 		*(int64_t *)P(0, 5) = pred[0];
164 | 		*(int64_t *)P(0, 6) = pred[0];
165 | 		*(int64_t *)P(0, 7) = pred[0];
166 | 		return;
167 | 	case I8x8_V_C_8: {
168 | 		i8x16 v0 = load128(P(-8, -1));
169 | 		l2s = shrc128(v0, 9);
170 | 		j2q = shr128(v0, 7);
171 | 		k2r = shr128(v0, 8);
172 | 		} goto vertical_8x8_lowpass;
173 | 	case I8x8_V_D_8:
174 | 		k2r = load128(P(0, -1));
175 | 		j2q = shlc128(k2r, 1);
176 | 		l2s = shr128(k2r, 1);
177 | 		goto vertical_8x8_lowpass;
178 | 	case I8x8_V_CD_8:
179 | 		k2r = spreadh8(load64(P(0, -1)));
180 | 		j2q = shlc128(k2r, 1);
181 | 		l2s = shr128(k2r, 1);
182 | 		goto vertical_8x8_lowpass;
183 | 	
184 | 	case I8x8_H_8:
185 | 		i2a = (i8x16){*P(-1, -1)};
186 | 	horizontal_8x8_load_left: {
187 | 		i2a = ziplo64(ldleft7(i2a, 0, 1, 2, 3, 4, 5, 6), set8(*P(-1, 7)));
188 | 		i8x16 v0 = lowpass8(shr128(i2a, 2), shr128(i2a, 1), i2a);
189 | 		p0 = broadcast8(v0, 0);
190 | 		p1 = broadcast8(v0, 1);
191 | 		p2 = broadcast8(v0, 2);
192 | 		p3 = broadcast8(v0, 3);
193 | 		p4 = broadcast8(v0, 4);
194 | 		p5 = broadcast8(v0, 5);
195 | 		p6 = broadcast8(v0, 6);
196 | 		p7 = broadcast8(v0, 7);
197 | 	} store8_8x8:
198 | 		*(int64_t *)P(0, 0) = p0[0];
199 | 		*(int64_t *)P(0, 1) = p1[0];
200 | 		*(int64_t *)P(0, 2) = p2[0];
201 | 		*(int64_t *)P(0, 3) = p3[0];
202 | 		*(int64_t *)P(0, 4) = p4[0];
203 | 		*(int64_t *)P(0, 5) = p5[0];
204 | 		*(int64_t *)P(0, 6) = p6[0];
205 | 		*(int64_t *)P(0, 7) = p7[0];
206 | 		return;
207 | 	case I8x8_H_D_8:
208 | 		i2a = (i8x16){*P(-1, 0)};
209 | 		goto horizontal_8x8_load_left;
210 | 	
211 | 	case I8x8_DC_8:
212 | 		i2a = j2s = load128(P(-1, -1));
213 | 	dc_8x8_load_left:
214 | 		i2a = ziplo64(ldleft7(i2a, 0, 1, 2, 3, 4, 5, 6), set8(*P(-1, 7)));
215 | 	dc_8x8_sum: {
216 | 		i8x16 v0 = ziplo64(j2s, i2a);
217 | 		i8x16 v1 = ziplo64(shr128(j2s, 1), shr128(i2a, 1));
218 | 		i8x16 v2 = ziplo64(shr128(j2s, 2), shr128(i2a, 2));
219 | 		pred = broadcast8(shrru16(sum8(lowpass8(v0, v1, v2)), 4), __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__);
220 | 		} goto store1_8x8;
221 | 	case I8x8_DC_C_8:
222 | 		i2a = j2s = shrc128(load128(P(-8, -1)), 7);
223 | 		goto dc_8x8_load_left;
224 | 	case I8x8_DC_D_8:
225 | 		j2s = shlc128(load128(P(0, -1)), 1);
226 | 		i2a = (i8x16){*P(-1, 0)};
227 | 		goto dc_8x8_load_left;
228 | 	case I8x8_DC_CD_8:
229 | 		j2s = shlc128(spreadh8(load64(P(0, -1))), 1);
230 | 		i2a = (i8x16){*P(-1, 0)};
231 | 		goto dc_8x8_load_left;
232 | 	case I8x8_DC_A_8:
233 | 		i2a = j2s = load128(P(-1, -1));
234 | 		goto dc_8x8_sum;
235 | 	case I8x8_DC_AC_8:
236 | 		i2a = j2s = shrc128(load128(P(-8, -1)), 7);
237 | 		goto dc_8x8_sum;
238 | 	case I8x8_DC_AD_8:
239 | 		i2a = j2s = shlc128(load128(P(0, -1)), 1);
240 | 		goto dc_8x8_sum;
241 | 	case I8x8_DC_ACD_8:
242 | 		i2a = j2s = shlc128(spreadh8(load64(P(0, -1))), 1);
243 | 		goto dc_8x8_sum;
244 | 	case I8x8_DC_B_8:
245 | 		i2a = (i8x16){*P(-1, -1)};
246 | 	dc_8x8_dup_left:
247 | 		j2s = i2a = ziplo64(ldleft7(i2a, 0, 1, 2, 3, 4, 5, 6), set8(*P(-1, 7)));
248 | 		goto dc_8x8_sum;
249 | 	case I8x8_DC_BD_8:
250 | 		i2a = (i8x16){*P(-1, 0)};
251 | 		goto dc_8x8_dup_left;
252 | 	case I8x8_DC_AB_8:
253 | 		pred = set8(-128);
254 | 		goto store1_8x8;
255 | 	
256 | 	case I8x8_DDL_8:
257 | 		j2y = load128(P(-1, -1));
258 | 		k2z = load128(P(0, -1));
259 | 	diagonal_down_left_8x8_lowpass: {
260 | 		i8x16 v0 = lowpass8(j2y, k2z, shrc128(k2z, 1));
261 | 		p0 = lowpass8(v0, shr128(v0, 1), shrc128(v0, 2));
262 | 		p1 = shr128(p0, 1);
263 | 		p2 = shr128(p0, 2);
264 | 		p3 = shr128(p0, 3);
265 | 		p4 = shr128(p0, 4);
266 | 		p5 = shr128(p0, 5);
267 | 		p6 = shr128(p0, 6);
268 | 		p7 = shr128(p0, 7);
269 | 		} goto store8_8x8;
270 | 	case I8x8_DDL_C_8: {
271 | 		i8x16 j2r = load128(P(-8, -1));
272 | 		j2y = shrc128(j2r, 7);
273 | 		k2z = shrc128(j2r, 8);
274 | 		} goto diagonal_down_left_8x8_lowpass;
275 | 	case I8x8_DDL_D_8:
276 | 		k2z = load128(P(0, -1));
277 | 		j2y = shlc128(k2z, 1);
278 | 		goto diagonal_down_left_8x8_lowpass;
279 | 	case I8x8_DDL_CD_8:
280 | 		k2z = spreadh8(load64(P(0, -1)));
281 | 		j2y = shlc128(k2z, 1);
282 | 		goto diagonal_down_left_8x8_lowpass;
283 | 	
284 | 	case I8x8_DDR_8:
285 | 		j2s = load128(P(-1, -1));
286 | 	diagonal_down_right_8x8_load_left: {
287 | 		i8x16 a2h = ldedge8x8lo();
288 | 		i8x16 a2q = shrd128(a2h, j2s, 8);
289 | 		i8x16 b2r = shrd128(a2h, j2s, 9);
290 | 		i8x16 x0 = lowpass8(shrd128(a2h, j2s, 7), a2q, b2r);
291 | 		i8x16 x1 = lowpass8(a2q, b2r, shrd128(a2h, j2s, 10));
292 | 		p7 = lowpass8(x0, x1, shr128(x1, 1));
293 | 		p6 = shr128(p7, 1);
294 | 		p5 = shr128(p7, 2);
295 | 		p4 = shr128(p7, 3);
296 | 		p3 = shr128(p7, 4);
297 | 		p2 = shr128(p7, 5);
298 | 		p1 = shr128(p7, 6);
299 | 		p0 = shr128(p7, 7);
300 | 		} goto store8_8x8;
301 | 	case I8x8_DDR_C_8:
302 | 		j2s = shrc128(load128(P(-8, -1)), 7);
303 | 		goto diagonal_down_right_8x8_load_left;
304 | 	
305 | 	case I8x8_VR_8:
306 | 		j2s = load128(P(-1, -1));
307 | 	vertical_right_8x8_load_left: {
308 | 		i8x16 a2h = ldedge8x8lo();
309 | 		i8x16 a2q = shrd128(a2h, j2s, 8);
310 | 		i8x16 b2r = shrd128(a2h, j2s, 9);
311 | 		i8x16 c2s = shrd128(a2h, j2s, 10);
312 | 		i8x16 v0 = lowpass8(a2q, b2r, c2s);
313 | 		i8x16 v1 = shl128(v0, 1);
314 | 		i8x16 v2 = lowpass8(v0, v1, shl128(v0, 2));
315 | 		p0 = shr128(avgu8(v0, v1), 8);
316 | 		p1 = shr128(v2, 8);
317 | 		p2 = shrd128(shl128(v2, 8), p0, 15);
318 | 		p3 = shrd128(shl128(v2, 9), p1, 15);
319 | 		p4 = shrd128(shl128(v2, 10), p2, 15);
320 | 		p5 = shrd128(shl128(v2, 11), p3, 15);
321 | 		p6 = shrd128(shl128(v2, 12), p4, 15);
322 | 		p7 = shrd128(shl128(v2, 13), p5, 15);
323 | 		} goto store8_8x8;
324 | 	case I8x8_VR_C_8:
325 | 		j2s = shrc128(load128(P(-8, -1)), 7);
326 | 		goto vertical_right_8x8_load_left;
327 | 	
328 | 	case I8x8_HD_8: {
329 | 		j2s = load128(P(-1, -1));
330 | 		i8x16 a2h = ldedge8x8lo();
331 | 		i8x16 a2p = shrd128(a2h, j2s, 7);
332 | 		i8x16 a2q = shrd128(a2h, j2s, 8);
333 | 		i8x16 b2r = shrd128(a2h, j2s, 9);
334 | 		i8x16 v0 = lowpass8(a2p, a2q, b2r);
335 | 		i8x16 v1 = shr128(v0, 1);
336 | 		i8x16 v2 = lowpass8(v0, v1, shr128(v0, 2));
337 | 		p7 = ziplo8(avgu8(v0, v1), v2);
338 | 		p6 = shr128(p7, 2);
339 | 		p5 = shr128(p7, 4);
340 | 		p4 = shr128(p7, 6);
341 | 		p3 = ziphi64(p7, v2);
342 | 		p2 = shr128(p3, 2);
343 | 		p1 = shr128(p3, 4);
344 | 		p0 = shr128(p3, 6);
345 | 		} goto store8_8x8;
346 | 	
347 | 	case I8x8_VL_8:
348 | 		j2y = load128(P(-1, -1));
349 | 		k2z = load128(P(0, -1));
350 | 	vertical_left_8x8_lowpass: {
351 | 		i8x16 v0 = lowpass8(j2y, k2z, shr128(k2z, 1));
352 | 		i8x16 v1 = shr128(v0, 1);
353 | 		p0 = avgu8(v0, v1);
354 | 		p1 = lowpass8(v0, v1, shr128(v0, 2));
355 | 		p2 = shr128(p0, 1);
356 | 		p3 = shr128(p1, 1);
357 | 		p4 = shr128(p2, 1);
358 | 		p5 = shr128(p3, 1);
359 | 		p6 = shr128(p4, 1);
360 | 		p7 = shr128(p5, 1);
361 | 		} goto store8_8x8;
362 | 	case I8x8_VL_C_8: {
363 | 		i8x16 j2r = load128(P(-8, -1));
364 | 		j2y = shrc128(j2r, 7);
365 | 		k2z = shrc128(j2r, 8);
366 | 		} goto vertical_left_8x8_lowpass;
367 | 	case I8x8_VL_D_8:
368 | 		k2z = load128(P(0, -1));
369 | 		j2y = shlc128(k2z, 1);
370 | 		goto vertical_left_8x8_lowpass;
371 | 	case I8x8_VL_CD_8:
372 | 		k2z = spreadh8(load64(P(0, -1)));
373 | 		j2y = shlc128(k2z, 1);
374 | 		goto vertical_left_8x8_lowpass;
375 | 	
376 | 	case I8x8_HU_8:
377 | 		i2a = (i8x16){*P(-1, -1)};
378 | 	horizontal_up_8x8_load_left: {
379 | 		i2a = ziplo64(ldleft7(i2a, 0, 1, 2, 3, 4, 5, 6), set8(*P(-1, 7)));
380 | 		i8x16 v0 = spreadh8(lowpass8(shr128(i2a, 2), shr128(i2a, 1), i2a));
381 | 		i8x16 v1 = shr128(v0, 1);
382 | 		p0 = ziplo8(avgu8(v0, v1), lowpass8(v0, v1, shr128(v0, 2)));
383 | 		p1 = shr128(p0, 2);
384 | 		p2 = shr128(p0, 4);
385 | 		p3 = shr128(p0, 6);
386 | 		p4 = ziphi64(p0, v0);
387 | 		p5 = shr128(p4, 2);
388 | 		p6 = shr128(p4, 4);
389 | 		p7 = shr128(p4, 6);
390 | 		} goto store8_8x8;
391 | 	case I8x8_HU_D_8:
392 | 		i2a = (i8x16){*P(-1, 0)};
393 | 		goto horizontal_up_8x8_load_left;
394 | 	}
395 | }
396 | 
397 | 
398 | 
399 | /**
400 |  * Intra 16x16
401 |  */
402 | static void decode_intra16x16(int mode, uint8_t * restrict p, size_t stride, i16x8 clip) {
403 | 	INIT_P();
404 | 	i8x16 pred, top;
405 | 	switch (mode) {
406 | 	default: __builtin_unreachable();
407 | 	
408 | 	case I16x16_V_8:
409 | 		pred = load128(P(0, -1));
410 | 		break;
411 | 	
412 | 	case I16x16_H_8:
413 | 		*(i8x16 *)P(0, 0) = set8(*P(-1, 0));
414 | 		*(i8x16 *)P(0, 1) = set8(*P(-1, 1));
415 | 		*(i8x16 *)P(0, 2) = set8(*P(-1, 2));
416 | 		*(i8x16 *)P(0, 3) = set8(*P(-1, 3));
417 | 		*(i8x16 *)P(0, 4) = set8(*P(-1, 4));
418 | 		*(i8x16 *)P(0, 5) = set8(*P(-1, 5));
419 | 		*(i8x16 *)P(0, 6) = set8(*P(-1, 6));
420 | 		*(i8x16 *)P(0, 7) = set8(*P(-1, 7));
421 | 		*(i8x16 *)P(0, 8) = set8(*P(-1, 8));
422 | 		*(i8x16 *)P(0, 9) = set8(*P(-1, 9));
423 | 		*(i8x16 *)P(0, 10) = set8(*P(-1, 10));
424 | 		*(i8x16 *)P(0, 11) = set8(*P(-1, 11));
425 | 		*(i8x16 *)P(0, 12) = set8(*P(-1, 12));
426 | 		*(i8x16 *)P(0, 13) = set8(*P(-1, 13));
427 | 		*(i8x16 *)P(0, 14) = set8(*P(-1, 14));
428 | 		*(i8x16 *)P(0, 15) = set8(*P(-1, 15));
429 | 		return;
430 | 	
431 | 	case I16x16_DC_8: {
432 | 		i8x16 t = load128(P(0, -1));
433 | 		i8x16 l = ldleft16(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15);
434 | 		pred = broadcast8(shrru16(sumd8(t, l), 5), __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__);
435 | 		} break;
436 | 	case I16x16_DCA_8:
437 | 		top = load128(P(0, -1));
438 | 	dca_16x16_sum:
439 | 		pred = broadcast8(shrru16(sum8(top), 4), __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__);
440 | 		break;
441 | 	case I16x16_DCB_8:
442 | 		top = ldleft16(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15);
443 | 		goto dca_16x16_sum;
444 | 	case I16x16_DCAB_8:
445 | 		pred = set8(-128);
446 | 		break;
447 | 	
448 | 	case I16x16_P_8: {
449 | 		i8x16 tl = load64(P(-1, -1));
450 | 		i8x16 tr = load64(P(8, -1));
451 | 		i8x16 lt = ldleft7(tl, 0, 1, 2, 3, 4, 5, 6);
452 | 		i8x16 lb = ldleft8(8, 9, 10, 11, 12, 13, 14, 15);
453 | 		#if defined(__SSE2__)
454 | 			i8x16 m = {-8, -7, -6, -5, -4, -3, -2, -1, 1, 2, 3, 4, 5, 6, 7, 8};
455 | 			i16x8 mul = ziphi8(m, (i8x16){});
456 | 			i16x8 v0 = maddubs(ziplo64(tl, tr), m);
457 | 			i16x8 v1 = maddubs(ziplo64(lt, lb), m);
458 | 			i16x8 v2 = (i16x8)ziplo32(v0, v1) + (i16x8)ziphi32(v0, v1);
459 | 			i16x8 v3 = v2 + (i16x8)shr128(v2, 8);
460 | 			i16x8 HV = v3 + shufflelo(v3, 1, 0, 3, 2); // H, H, V, V
461 | 			i16x8 a = (broadcast16((i16x8)shr128(tr, 7) + (i16x8)shr128(lb, 7), 0) - -1) << 4;
462 | 		#elif defined(__ARM_NEON)
463 | 			i16x8 mul = {1, 2, 3, 4, 5, 6, 7, 8};
464 | 			i16x8 v0 = sublou8(tr, vrev64q_s8(tl)) * mul;
465 | 			i16x8 v1 = sublou8(lb, vrev64q_s8(lt)) * mul;
466 | 			i16x8 v2 = vpaddq_s16(v0, v1);
467 | 			i16x8 v3 = vpaddq_s16(v2, v2);
468 | 			i16x8 HV = (i16x8)vtrn1q_s16(v3, v3) + (i16x8)vtrn2q_s16(v3, v3); // {H, H, V, V, 0, 0, 0, 0}, -9180..9180
469 | 			i16x8 a = (broadcast16((i16x8)addlou8(tr, lb) - -1, 7)) << 4;
470 | 		#endif
471 | 		i16x8 v4 = shrrs16(HV + (HV >> 2), 4); // (5 * HV + 32) >> 6, -717..717
472 | 		i16x8 b = broadcast32(v4, 0);
473 | 		i16x8 c = broadcast32(v4, 1);
474 | 		i16x8 p1 = (a + c) - (c << 3) + (b * mul);
475 | 		i16x8 p0 = p1 - (b << 3);
476 | 		for (int i = 16; i--; p0 += c, p1 += c, p += stride)
477 | 			*(i8x16 *)p = shrpus16(p0, p1, 5);
478 | 		} return;
479 | 	}
480 | 	*(i8x16 *)P(0, 0) = pred;
481 | 	*(i8x16 *)P(0, 1) = pred;
482 | 	*(i8x16 *)P(0, 2) = pred;
483 | 	*(i8x16 *)P(0, 3) = pred;
484 | 	*(i8x16 *)P(0, 4) = pred;
485 | 	*(i8x16 *)P(0, 5) = pred;
486 | 	*(i8x16 *)P(0, 6) = pred;
487 | 	*(i8x16 *)P(0, 7) = pred;
488 | 	*(i8x16 *)P(0, 8) = pred;
489 | 	*(i8x16 *)P(0, 9) = pred;
490 | 	*(i8x16 *)P(0, 10) = pred;
491 | 	*(i8x16 *)P(0, 11) = pred;
492 | 	*(i8x16 *)P(0, 12) = pred;
493 | 	*(i8x16 *)P(0, 13) = pred;
494 | 	*(i8x16 *)P(0, 14) = pred;
495 | 	*(i8x16 *)P(0, 15) = pred;
496 | }
497 | 
498 | 
499 | 
500 | /**
501 |  * Intra Chroma
502 |  */
503 | static void decode_intraChroma(int mode, uint8_t * restrict p, size_t stride, i16x8 clip) {
504 | 	#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
505 | 		static const i8x16 shufDC = {0, 0, 0, 0, 4, 4, 4, 4, 8, 8, 8, 8, 12, 12, 12, 12};
506 | 		static const i8x16 shufDCA = {0, 0, 0, 0, 12, 12, 12, 12, 0, 0, 0, 0, 12, 12, 12, 12};
507 | 		static const i8x16 shufDCB = {0, 0, 0, 0, 0, 0, 0, 0, 12, 12, 12, 12, 12, 12, 12, 12};
508 | 	#else
509 | 		static const i8x16 shufDC = {1, 1, 1, 1, 5, 5, 5, 5, 9, 9, 9, 9, 13, 13, 13, 13};
510 | 		static const i8x16 shufDCA = {1, 1, 1, 1, 13, 13, 13, 13, 1, 1, 1, 1, 13, 13, 13, 13};
511 | 		static const i8x16 shufDCB = {1, 1, 1, 1, 1, 1, 1, 1, 13, 13, 13, 13, 13, 13, 13, 13};
512 | 	#endif
513 | 	INIT_P();
514 | 	i8x16 bt, rt, bl, rl, shuf;
515 | 	i64x2 bpred, rpred;
516 | 	switch (mode) {
517 | 	default: __builtin_unreachable();
518 | 	
519 | 	case IC8x8_DC_8:
520 | 		bt = load64(P(0, -2));
521 | 		rt = load64(P(0, -1));
522 | 		bl = ldleft8(0, 2, 4, 6, 8, 10, 12, 14);
523 | 		rl = ldleft8(1, 3, 5, 7, 9, 11, 13, 15);
524 | 		shuf = shufDC;
525 | 	chroma_dc_8x8_sum: {
526 | 		#if defined(__SSE2__)
527 | 			i8x16 b = ziplo64(bt, bl);
528 | 			i8x16 r = ziplo64(rt, rl);
529 | 			i16x8 b01 = sumh8(shuffle32(b, 0, 2, 1, 1));
530 | 			i16x8 r01 = sumh8(shuffle32(r, 0, 2, 1, 1));
531 | 			i16x8 b23 = sumh8(shuffle32(b, 3, 3, 1, 3));
532 | 			i16x8 r23 = sumh8(shuffle32(r, 3, 3, 1, 3));
533 | 			bpred = shuffle(shrru16(packs32(b01, b23), 3), shuf);
534 | 			rpred = shuffle(shrru16(packs32(r01, r23), 3), shuf);
535 | 		#elif defined(__ARM_NEON)
536 | 			i8x16 t = ziplo64(bt, rt);
537 | 			i8x16 l = ziplo64(bl, rl);
538 | 			i16x8 v0 = vpaddlq_u8(vtrn1q_s32(t, l)); // top-left sums
539 | 			i16x8 v1 = vpaddlq_u8(vtrn2q_s32(t, t)); // top-right sums
540 | 			i16x8 v2 = vpaddlq_u8(vtrn2q_s32(l, l)); // bottom-left sums
541 | 			i16x8 v3 = vpaddlq_u8(vtrn2q_s32(t, l)); // bottom-right sums
542 | 			i8x16 v4 = shrru16(vpaddq_u16(vpaddq_u16(v0, v1), vpaddq_u16(v2, v3)), 3);
543 | 			bpred = shuffle(v4, shuf);
544 | 			rpred = shuffle(shr128(v4, 2), shuf);
545 | 		#endif
546 | 		} break;
547 | 	case IC8x8_DCA_8:
548 | 		bt = bl = load64(P(0, -2));
549 | 		rt = rl = load64(P(0, -1));
550 | 		shuf = shufDCA;
551 | 		goto chroma_dc_8x8_sum;
552 | 	case IC8x8_DCB_8:
553 | 		bt = bl = ldleft8(0, 2, 4, 6, 8, 10, 12, 14);
554 | 		rt = rl = ldleft8(1, 3, 5, 7, 9, 11, 13, 15);
555 | 		shuf = shufDCB;
556 | 		goto chroma_dc_8x8_sum;
557 | 	case IC8x8_DCAB_8:
558 | 		bpred = rpred = set8(-128);
559 | 		break;
560 | 	
561 | 	case IC8x8_H_8:
562 | 		*(int64_t *)P(0, 0) = ((i64x2)set8(*P(-1, 0)))[0];
563 | 		*(int64_t *)P(0, 1) = ((i64x2)set8(*P(-1, 1)))[0];
564 | 		*(int64_t *)P(0, 2) = ((i64x2)set8(*P(-1, 2)))[0];
565 | 		*(int64_t *)P(0, 3) = ((i64x2)set8(*P(-1, 3)))[0];
566 | 		*(int64_t *)P(0, 4) = ((i64x2)set8(*P(-1, 4)))[0];
567 | 		*(int64_t *)P(0, 5) = ((i64x2)set8(*P(-1, 5)))[0];
568 | 		*(int64_t *)P(0, 6) = ((i64x2)set8(*P(-1, 6)))[0];
569 | 		*(int64_t *)P(0, 7) = ((i64x2)set8(*P(-1, 7)))[0];
570 | 		*(int64_t *)P(0, 8) = ((i64x2)set8(*P(-1, 8)))[0];
571 | 		*(int64_t *)P(0, 9) = ((i64x2)set8(*P(-1, 9)))[0];
572 | 		*(int64_t *)P(0, 10) = ((i64x2)set8(*P(-1, 10)))[0];
573 | 		*(int64_t *)P(0, 11) = ((i64x2)set8(*P(-1, 11)))[0];
574 | 		*(int64_t *)P(0, 12) = ((i64x2)set8(*P(-1, 12)))[0];
575 | 		*(int64_t *)P(0, 13) = ((i64x2)set8(*P(-1, 13)))[0];
576 | 		*(int64_t *)P(0, 14) = ((i64x2)set8(*P(-1, 14)))[0];
577 | 		*(int64_t *)P(0, 15) = ((i64x2)set8(*P(-1, 15)))[0];
578 | 		return;
579 | 	
580 | 	case IC8x8_V_8: {
581 | 		i64x2 t = {*(int64_t *)P(0, -2), *(int64_t *)P(0, -1)};
582 | 		bpred = ziplo64(t, t);
583 | 		rpred = ziphi64(t, t);
584 | 		} break;
585 | 	
586 | 	case IC8x8_P_8: {
587 | 		i8x16 btl = load128(P(-1, -2));
588 | 		i8x16 rtl = load128(P(-1, -1));
589 | 		i8x16 btr = shr128(btl, 5);
590 | 		i8x16 rtr = shr128(rtl, 5);
591 | 		i8x16 blt = ldleft3(btl, 0, 2, 4);
592 | 		i8x16 rlt = ldleft3(rtl, 1, 3, 5);
593 | 		i8x16 blb = ldleft4(8, 10, 12, 14);
594 | 		i8x16 rlb = ldleft4(9, 11, 13, 15);
595 | 		#if defined(__SSE2__)
596 | 			i8x16 n = {-4, -3, -2, -1, -4, -3, -2, -1, -4, -3, -2, -1, -4, -3, -2, -1};
597 | 			i8x16 m = {1, 2, 3, 4, 1, 2, 3, 4, 1, 2, 3, 4, 1, 2, 3, 4};
598 | 			i8x16 v0 = ziplo32(btr, rtr);
599 | 			i8x16 v1 = ziplo32(blb, rlb);
600 | 			i16x8 v2 = maddubs(ziplo64(ziplo32(btl, rtl), ziplo32(blt, rlt)), n);
601 | 			i16x8 v3 = maddubs(ziplo64(v0, v1), m);
602 | 			i16x8 v4 = v2 + v3;
603 | 			i16x8 v5 = (((u32x4)v0 >> 24) + ((u32x4)v1 >> 24) - -1) << 4;
604 | 			i16x8 HV = v4 + shufflelo(shufflehi(v4, 1, 0, 3, 2), 1, 0, 3, 2); // Hb,Hb,Hr,Hr,Vb,Vb,Vr,Vr
605 | 			i16x8 ba = broadcast16(v5, 0); // 16..8176
606 | 			i16x8 ra = broadcast16(v5, 2); // 16..8176
607 | 		#elif defined(__ARM_NEON)
608 | 			i16x8 v0 = sublou8(ziplo32(btr, rtr), vrev32q_s8(ziplo32(btl, rtl)));
609 | 			i16x8 v1 = sublou8(ziplo32(blb, rlb), vrev32q_s8(ziplo32(blt, rlt)));
610 | 			i16x8 m = {1, 2, 3, 4, 1, 2, 3, 4};
611 | 			i16x8 v4 = vpaddq_s16(v0 * m, v1 * m);
612 | 			i16x8 HV = (i16x8)vtrn1q_s16(v4, v4) + (i16x8)vtrn2q_s16(v4, v4);
613 | 			i16x8 ba = (broadcast16((i16x8)addlou8(btr, blb) - -1, 3)) << 4;
614 | 			i16x8 ra = (broadcast16((i16x8)addlou8(rtr, rlb) - -1, 3)) << 4;
615 | 		#endif
616 | 		i16x8 hv = shrrs16(HV + (HV >> 4), 1); // (17 * HV + 16) >> 5
617 | 		i16x8 bb = broadcast32(hv, 0);
618 | 		i16x8 rb = broadcast32(hv, 1);
619 | 		i16x8 bc = broadcast32(hv, 2);
620 | 		i16x8 rc = broadcast32(hv, 3);
621 | 		i16x8 bp = ba - bc - bc - bc + bb * (i16x8){-3, -2, -1, 0, 1, 2, 3, 4};
622 | 		i16x8 rp = ra - rc - rc - rc + rb * (i16x8){-3, -2, -1, 0, 1, 2, 3, 4};
623 | 		for (int i = 8; i--; bp += bc, rp += rc, p += stride * 2) {
624 | 			i64x2 v6 = shrpus16(bp, rp, 5);
625 | 			*(int64_t *)p = v6[0];
626 | 			*(int64_t *)(p + stride) = v6[1];
627 | 		}
628 | 		} return;
629 | 	}
630 | 	*(int64_t *)P(0, 0) = bpred[0];
631 | 	*(int64_t *)P(0, 1) = rpred[0];
632 | 	*(int64_t *)P(0, 2) = bpred[0];
633 | 	*(int64_t *)P(0, 3) = rpred[0];
634 | 	*(int64_t *)P(0, 4) = bpred[0];
635 | 	*(int64_t *)P(0, 5) = rpred[0];
636 | 	*(int64_t *)P(0, 6) = bpred[0];
637 | 	*(int64_t *)P(0, 7) = rpred[0];
638 | 	*(int64_t *)P(0, 8) = bpred[1];
639 | 	*(int64_t *)P(0, 9) = rpred[1];
640 | 	*(int64_t *)P(0, 10) = bpred[1];
641 | 	*(int64_t *)P(0, 11) = rpred[1];
642 | 	*(int64_t *)P(0, 12) = bpred[1];
643 | 	*(int64_t *)P(0, 13) = rpred[1];
644 | 	*(int64_t *)P(0, 14) = bpred[1];
645 | 	*(int64_t *)P(0, 15) = rpred[1];
646 | }
647 | 


--------------------------------------------------------------------------------
/edge264_mvpred.c:
--------------------------------------------------------------------------------
  1 | #include "edge264_internal.h"
  2 | 
  3 | #if defined(__SSE2__)
  4 | 	static always_inline i16x8 temporal_scale(i16x8 mvCol, int16_t DistScaleFactor) {
  5 | 		i32x4 neg = set32(-1);
  6 | 		i32x4 mul = set32(DistScaleFactor + 0xff800000u);
  7 | 		i32x4 lo = madd16(ziplo16(mvCol, neg), mul);
  8 | 		i32x4 hi = madd16(ziphi16(mvCol, neg), mul);
  9 | 		return packs32(lo >> 8, hi >> 8);
 10 | 	}
 11 | #elif defined(__ARM_NEON)
 12 | 	static always_inline i16x8 temporal_scale(i16x8 mvCol, int16_t DistScaleFactor) {
 13 | 		i32x4 lo = vmull_n_s16(vget_low_s16(mvCol), DistScaleFactor);
 14 | 		i32x4 hi = vmull_high_n_s16(mvCol, DistScaleFactor);
 15 | 		return vqrshrn_high_n_s32(vqrshrn_n_s32(lo, 8), hi, 8);
 16 | 	}
 17 | #endif
 18 | static always_inline i16x8 mvs_near_zero(i16x8 mvCol, i32x4 zero) {
 19 | 	return (u32x4)(abs16(mvCol) >> 1) == zero;
 20 | }
 21 | 
 22 | 
 23 | 
 24 | /**
 25 |  * Decoding of P_Skip is put in a function for reuse when recovering a P slice.
 26 |  */
 27 | static noinline void decode_P_skip(Edge264Context *ctx) {
 28 | 	mb->f.inter_eqs_s = little_endian32(0x1b5fbbff);
 29 | 	mb->refPic_s[0] = ((i32x4)set8(ctx->t.RefPicList_v[0][0]))[0];
 30 | 	mb->refPic_s[1] = -1;
 31 | 	int refIdxA = mbA->refIdx[1];
 32 | 	int refIdxB = mbB->refIdx[2];
 33 | 	int mvA = mbA->mvs_s[5];
 34 | 	int mvB = mbB->mvs_s[10];
 35 | 	i16x8 mv = {};
 36 | 	if ((refIdxA | mvA) && (refIdxB | mvB) && !(ctx->unavail4x4[0] & 3)) {
 37 | 		int refIdxC;
 38 | 		if (__builtin_expect(ctx->unavail4x4[5] & 4, 0)) {
 39 | 			refIdxC = mbD->refIdx[3];
 40 | 			mv = (i32x4){mbD->mvs_s[15]};
 41 | 		} else {
 42 | 			refIdxC = mbC->refIdx[2];
 43 | 			mv = (i32x4){mbC->mvs_s[10]};
 44 | 		}
 45 | 		// B/C unavailability (->A) was ruled out, thus not tested here
 46 | 		int eq = !refIdxA + !refIdxB * 2 + !refIdxC * 4;
 47 | 		if (__builtin_expect(0xe9 >> eq & 1, 1)) {
 48 | 			mv = median16((i32x4){mvA}, (i32x4){mvB}, mv);
 49 | 		} else if (eq != 4) {
 50 | 			mv = (i32x4){(eq == 1) ? mvA : mvB};
 51 | 		}
 52 | 	}
 53 | 	i16x8 mvs = broadcast32(mv, 0);
 54 | 	mb->mvs_v[0] = mb->mvs_v[1] = mb->mvs_v[2] = mb->mvs_v[3] = mvs;
 55 | 	mb->mvs_v[4] = mb->mvs_v[5] = mb->mvs_v[6] = mb->mvs_v[7] = (i16x8){};
 56 | 	decode_inter(ctx, 0, 16, 16);
 57 | }
 58 | 
 59 | 
 60 | 
 61 | /**
 62 |  * These functions are designed to optimize the parsing of motion vectors for
 63 |  * block sizes 16x16, 8x16 and 16x8. Each call computes a prediction from
 64 |  * neighbours, adds the mvd pair, then ends with a call to decode_inter.
 65 |  */
 66 | static inline void decode_inter_16x16(Edge264Context *ctx, i16x8 mvd, int lx)
 67 | {
 68 | 	// compare neighbouring indices and compute mvp
 69 | 	int refIdx = mb->refIdx[lx * 4];
 70 | 	int refIdxA = mbA->refIdx[lx * 4 + 1];
 71 | 	int refIdxB = mbB->refIdx[lx * 4 + 2];
 72 | 	int eqA = refIdx == refIdxA;
 73 | 	int refIdxC;
 74 | 	i16x8 mvp;
 75 | 	if (__builtin_expect(ctx->unavail4x4[5] & 4, 0)) {
 76 | 		refIdxC = mbD->refIdx[lx * 4 + 3];
 77 | 		mvp = (i32x4){mbD->mvs_s[lx * 16 + 15]};
 78 | 		eqA |= ctx->unavail4x4[0] == 14;
 79 | 	} else {
 80 | 		refIdxC = mbC->refIdx[lx * 4 + 2];
 81 | 		mvp = (i32x4){mbC->mvs_s[lx * 16 + 10]};
 82 | 	}
 83 | 	int eq = eqA + (refIdx==refIdxB) * 2 + (refIdx==refIdxC) * 4;
 84 | 	if (__builtin_expect(0xe9 >> eq & 1, 1)) {
 85 | 		i16x8 mvA = (i32x4){mbA->mvs_s[lx * 16 + 5]};
 86 | 		i16x8 mvB = (i32x4){mbB->mvs_s[lx * 16 + 10]};
 87 | 		mvp = median16(mvA, mvB, mvp);
 88 | 	} else if (eq == 1) {
 89 | 		mvp = (i32x4){mbA->mvs_s[lx * 16 + 5]};
 90 | 	} else if (eq == 2) {
 91 | 		mvp = (i32x4){mbB->mvs_s[lx * 16 + 10]};
 92 | 	}
 93 | 	
 94 | 	// sum mvp and mvd, broadcast everything to memory and tail-jump to decoding
 95 | 	i16x8 mv = mvp + mvd;
 96 | 	i16x8 mvs = broadcast32(mv, 0);
 97 | 	mb->absMvd_v[lx * 2] = mb->absMvd_v[lx * 2 + 1] = pack_absMvd(mvd);
 98 | 	mb->mvs_v[lx * 4] = mb->mvs_v[lx * 4 + 1] = mb->mvs_v[lx * 4 + 2] = mb->mvs_v[lx * 4 + 3] = mvs;
 99 | 	decode_inter(ctx, lx * 16, 16, 16);
100 | }
101 | 
102 | static inline void decode_inter_8x16_left(Edge264Context *ctx, i16x8 mvd, int lx)
103 | {
104 | 	// compare neighbouring indices and compute mvp
105 | 	i16x8 mvp, mvC;
106 | 	int refIdx = mb->refIdx[lx * 4];
107 | 	int refIdxA = mbA->refIdx[lx * 4 + 1];
108 | 	if (refIdx == refIdxA || ctx->unavail4x4[0] == 14) {
109 | 		mvp = (i32x4){mbA->mvs_s[lx * 16 + 5]};
110 | 	} else {
111 | 		int refIdxB = mbB->refIdx[lx * 4 + 2];
112 | 		int refIdxC;
113 | 		if (__builtin_expect(ctx->unavail4x4[0] & 2, 0)) {
114 | 			refIdxC = mbD->refIdx[lx * 4 + 3];
115 | 			mvC = (i32x4){mbD->mvs_s[lx * 16 + 15]};
116 | 		} else {
117 | 			refIdxC = mbB->refIdx[lx * 4 + 3];
118 | 			mvC = (i32x4){mbB->mvs_s[lx * 16 + 14]};
119 | 		}
120 | 		if (refIdx == refIdxB) {
121 | 			mvp = (i32x4){mbB->mvs_s[lx * 16 + 10]};
122 | 			if (refIdx == refIdxC) {
123 | 				i16x8 mvA = (i32x4){mbA->mvs_s[lx * 16 + 5]};
124 | 				mvp = median16(mvA, mvp, mvC);
125 | 			}
126 | 		} else { // refIdx != refIdxA/B
127 | 			mvp = mvC;
128 | 			if (refIdx != refIdxC) {
129 | 				i16x8 mvA = (i32x4){mbA->mvs_s[lx * 16 + 5]};
130 | 				i16x8 mvB = (i32x4){mbB->mvs_s[lx * 16 + 10]};
131 | 				mvp = median16(mvA, mvB, mvp);
132 | 			}
133 | 		}
134 | 	}
135 | 	
136 | 	// sum mvp and mvd, broadcast everything to memory and call decoding
137 | 	i16x8 mv = mvp + mvd;
138 | 	i16x8 mvs = broadcast32(mv, 0);
139 | 	mb->absMvd_l[lx * 4] = mb->absMvd_l[lx * 4 + 2] = ((i64x2)pack_absMvd(mvd))[0];
140 | 	mb->mvs_v[lx * 4] = mb->mvs_v[lx * 4 + 2] = mvs;
141 | 	decode_inter(ctx, lx * 16, 8, 16);
142 | }
143 | 
144 | static inline void decode_inter_8x16_right(Edge264Context *ctx, i16x8 mvd, int lx)
145 | {
146 | 	// compare neighbouring indices and compute mvp
147 | 	i16x8 mvp, mvC;
148 | 	int refIdx = mb->refIdx[lx * 4 + 1];
149 | 	int refIdxC;
150 | 	if (__builtin_expect(ctx->unavail4x4[5] & 4, 0)) {
151 | 		refIdxC = mbB->refIdx[lx * 4 + 2];
152 | 		mvC = (i32x4){mbB->mvs_s[lx * 16 + 11]};
153 | 	} else {
154 | 		refIdxC = mbC->refIdx[lx * 4 + 2];
155 | 		mvC = (i32x4){mbC->mvs_s[lx * 16 + 10]};
156 | 	}
157 | 	if (refIdx == refIdxC) {
158 | 		mvp = mvC;
159 | 	} else {
160 | 		int refIdxA = mb->refIdx[lx * 4];
161 | 		int refIdxB = mbB->refIdx[lx * 4 + 3];
162 | 		if (refIdx == refIdxB) {
163 | 			mvp = (i32x4){mbB->mvs_s[lx * 16 + 14]};
164 | 			if (refIdx == refIdxA) {
165 | 				i16x8 mvA = (i32x4){mb->mvs_s[lx * 16]};
166 | 				mvp = median16(mvA, mvp, mvC);
167 | 			}
168 | 		} else { // refIdx != B/C
169 | 			mvp = (i32x4){mb->mvs_s[lx * 16]};
170 | 			if (refIdx != refIdxA && ctx->unavail4x4[5] != 14) {
171 | 				i16x8 mvB = (i32x4){mbB->mvs_s[lx * 16 + 14]};
172 | 				mvp = median16(mvp, mvB, mvC);
173 | 			}
174 | 		}
175 | 	}
176 | 	
177 | 	// sum mvp and mvd, broadcast everything to memory and call decoding
178 | 	i16x8 mv = mvp + mvd;
179 | 	i16x8 mvs = broadcast32(mv, 0);
180 | 	mb->absMvd_l[lx * 4 + 1] = mb->absMvd_l[lx * 4 + 3] = ((i64x2)pack_absMvd(mvd))[0];
181 | 	mb->mvs_v[lx * 4 + 1] = mb->mvs_v[lx * 4 + 3] = mvs;
182 | 	decode_inter(ctx, lx * 16 + 4, 8, 16);
183 | }
184 | 
185 | static inline void decode_inter_16x8_top(Edge264Context *ctx, i16x8 mvd, int lx)
186 | {
187 | 	// compare neighbouring indices and compute mvp
188 | 	i16x8 mvp, mvC;
189 | 	int refIdx = mb->refIdx[lx * 4];
190 | 	int refIdxB = mbB->refIdx[lx * 4 + 2];
191 | 	if (refIdx == refIdxB) {
192 | 		mvp = (i32x4){mbB->mvs_s[lx * 16 + 10]};
193 | 	} else {
194 | 		int refIdxA = mbA->refIdx[lx * 4 + 1];
195 | 		int eqA = refIdx == refIdxA;
196 | 		int refIdxC;
197 | 		if (__builtin_expect(ctx->unavail4x4[5] & 4, 0)) {
198 | 			refIdxC = mbD->refIdx[lx * 4 + 3];
199 | 			mvC = (i32x4){mbD->mvs_s[lx * 16 + 15]};
200 | 			eqA |= ctx->unavail4x4[0] == 14;
201 | 		} else {
202 | 			refIdxC = mbC->refIdx[lx * 4 + 2];
203 | 			mvC = (i32x4){mbC->mvs_s[lx * 16 + 10]};
204 | 		}
205 | 		if (refIdx == refIdxC) {
206 | 			mvp = mvC;
207 | 			if (eqA) {
208 | 				i16x8 mvA = (i32x4){mbA->mvs_s[lx * 16 + 5]};
209 | 				i16x8 mvB = (i32x4){mbB->mvs_s[lx * 16 + 10]};
210 | 				mvp = median16(mvA, mvB, mvp);
211 | 			}
212 | 		} else { // refIdx != refIdxB/C
213 | 			mvp = (i32x4){mbA->mvs_s[lx * 16 + 5]};
214 | 			if (!eqA) {
215 | 				i16x8 mvB = (i32x4){mbB->mvs_s[lx * 16 + 10]};
216 | 				mvp = median16(mvp, mvB, mvC);
217 | 			}
218 | 		}
219 | 	}
220 | 	
221 | 	// sum mvp and mvd, broadcast everything to memory and tail-jump to decoding
222 | 	i16x8 mv = mvp + mvd;
223 | 	i16x8 mvs = broadcast32(mv, 0);
224 | 	mb->absMvd_v[lx * 2] = pack_absMvd(mvd);
225 | 	mb->mvs_v[lx * 4 + 0] = mb->mvs_v[lx * 4 + 1] = mvs;
226 | 	decode_inter(ctx, lx * 16, 16, 8);
227 | }
228 | 
229 | static inline void decode_inter_16x8_bottom(Edge264Context *ctx, i16x8 mvd, int lx)
230 | {
231 | 	// compare neighbouring indices and compute mvp
232 | 	i16x8 mvp;
233 | 	int refIdx = mb->refIdx[lx * 4 + 2];
234 | 	int refIdxA = mbA->refIdx[lx * 4 + 3];
235 | 	if (refIdx == refIdxA) {
236 | 		mvp = (i32x4){mbA->mvs_s[lx * 16 + 13]};
237 | 	} else {
238 | 		int refIdxB = mb->refIdx[lx * 4];
239 | 		int refIdxC = mbA->refIdx[lx * 4 + 1];
240 | 		if (refIdx == refIdxB) {
241 | 			mvp = (i32x4){mb->mvs_s[lx * 16]};
242 | 			if (refIdx == refIdxC) {
243 | 				i16x8 mvA = (i32x4){mbA->mvs_s[lx * 16 + 13]};
244 | 				i16x8 mvC = (i32x4){mbA->mvs_s[lx * 16 + 7]};
245 | 				mvp = median16(mvA, mvp, mvC);
246 | 			}
247 | 		} else {
248 | 			mvp = (i32x4){mbA->mvs_s[lx * 16 + 7]};
249 | 			if (refIdx != refIdxC) {
250 | 				i16x8 mvA = (i32x4){mbA->mvs_s[lx * 16 + 13]};
251 | 				i16x8 mvB = (i32x4){mb->mvs_s[lx * 16]};
252 | 				mvp = median16(mvA, mvB, mvp);
253 | 			}
254 | 		}
255 | 	}
256 | 	
257 | 	// sum mvp and mvd, broadcast everything to memory and tail-jump to decoding
258 | 	i16x8 mv = mvp + mvd;
259 | 	i16x8 mvs = broadcast32(mv, 0);
260 | 	mb->absMvd_v[lx * 2 + 1] = pack_absMvd(mvd);
261 | 	mb->mvs_v[lx * 4 + 2] = mb->mvs_v[lx * 4 + 3] = mvs;
262 | 	decode_inter(ctx, lx * 16 + 8, 16, 8);
263 | }
264 | 
265 | 
266 | 
267 | /**
268 |  * Initialise the reference indices and motion vectors of an entire macroblock
269 |  * with direct prediction (8.4.1.2).
270 |  */
271 | static always_inline void decode_direct_spatial_mv_pred(Edge264Context *ctx, unsigned direct_flags)
272 | {
273 | 	// load all refIdxN and mvN in vector registers
274 | 	i8x16 shuf = {0, 0, 0, 0, 4, 4, 4, 4, -1, -1, -1, -1, -1, -1, -1, -1};
275 | 	i16x8 mvA = (i32x4){mbA->mvs_s[5], mbA->mvs_s[21]};
276 | 	i16x8 mvB = (i32x4){mbB->mvs_s[10], mbB->mvs_s[26]};
277 | 	i16x8 mvC;
278 | 	i8x16 refIdxA = shuffle(shr128((i64x2){mbA->refIdx_l}, 1), shuf);
279 | 	i8x16 refIdxB = shuffle(shr128((i64x2){mbB->refIdx_l}, 2), shuf);
280 | 	i8x16 refIdxC;
281 | 	if (__builtin_expect(ctx->unavail4x4[5] & 4, 0)) {
282 | 		mvC = (i32x4){mbD->mvs_s[15], mbD->mvs_s[31]};
283 | 		refIdxC = shuffle(shr128((i64x2){mbD->refIdx_l}, 3), shuf);
284 | 	} else {
285 | 		mvC = (i32x4){mbC->mvs_s[10], mbC->mvs_s[26]};
286 | 		refIdxC = shuffle(shr128((i64x2){mbC->refIdx_l}, 2), shuf);
287 | 	}
288 | 	
289 | 	// initialize mv along refIdx since it will equal one of refIdxA/B/C
290 | 	i8x16 cmp_AB = (u8x16)refIdxA < (u8x16)refIdxB; // unsigned comparisons
291 | 	i8x16 refIdxm = ifelse_mask(cmp_AB, refIdxA, refIdxB); // umin(refIdxA, refIdxB)
292 | 	i8x16 refIdxM = ifelse_mask(cmp_AB, refIdxB, refIdxA); // umax(refIdxA, refIdxB)
293 | 	i16x8 mvm = ifelse_mask(cmp_AB, mvA, mvB);
294 | 	i8x16 cmp_mC = (u8x16)refIdxm < (u8x16)refIdxC;
295 | 	i8x16 refIdx = ifelse_mask(cmp_mC, refIdxm, refIdxC); // umin(refIdxm, refIdxC)
296 | 	i16x8 mvmm = ifelse_mask(cmp_mC, mvm, mvC);
297 | 	
298 | 	// select median if refIdx equals another of refIdxA/B/C
299 | 	i8x16 cmp_med = (refIdxm == refIdxC) | (refIdx == refIdxM); // 3 cases: A=B<C, A=C<B, B=C<A
300 | 	i16x8 mv01 = ifelse_mask(cmp_med, median16(mvA, mvB, mvC), mvmm);
301 | 	i16x8 mvs0 = broadcast32(mv01, 0);
302 | 	i16x8 mvs4 = broadcast32(mv01, 1);
303 | 	
304 | 	// direct zero prediction applies only to refIdx (mvLX are zero already)
305 | 	refIdx ^= (i8x16)((i64x2)refIdx == -1);
306 | 	mb->refPic_s[0] = ((i32x4)shufflen(ctx->t.RefPicList_v[0], refIdx))[0];
307 | 	mb->refPic_s[1] = ((i32x4)shufflen(ctx->t.RefPicList_v[2], refIdx))[1];
308 | 	//printf("<li>refIdxL0A/B/C=%d/%d/%d, refIdxL1A/B/C=%d/%d/%d, mvsL0A/B/C=[%d,%d]/[%d,%d]/[%d,%d], mvsL1A/B/C=[%d,%d]/[%d,%d]/[%d,%d] -> refIdxL0/1=%d/%d, mvsL0/1=[%d,%d]/[%d,%d]</li>\n", refIdxA[0], refIdxB[0], refIdxC[0], refIdxA[4], refIdxB[4], refIdxC[4], mvA[0], mvA[1], mvB[0], mvB[1], mvC[0], mvC[1], mvA[2], mvA[3], mvB[2], mvB[3], mvC[2], mvC[3], refIdx[0], refIdx[4], mv01[0], mv01[1], mv01[2], mv01[3]);
309 | 	
310 | 	// trick from ffmpeg: skip computations on refCol/mvCol if both mvs are zero
311 | 	if (((i64x2)mv01)[0] != 0 || direct_flags != 0xffffffff) {
312 | 		i16x8 colZeroMask0 = {}, colZeroMask1 = {}, colZeroMask2 = {}, colZeroMask3 = {};
313 | 		unsigned colZeroFlags = 0;
314 | 		if (ctx->col_short_term) {
315 | 			const Edge264Macroblock *mbCol = ctx->mbCol;
316 | 			i8x16 refColL0 = (i32x4){mbCol->refIdx_s[0]};
317 | 			i8x16 offsets = refColL0 & 32;
318 | 			i16x8 mvCol0 = *(i16x8*)(mbCol->mvs + offsets[0]);
319 | 			i16x8 mvCol1 = *(i16x8*)(mbCol->mvs + offsets[1] + 8);
320 | 			i16x8 mvCol2 = *(i16x8*)(mbCol->mvs + offsets[2] + 16);
321 | 			i16x8 mvCol3 = *(i16x8*)(mbCol->mvs + offsets[3] + 24);
322 | 			i8x16 refCol = ifelse_msb(refColL0, (i32x4){mbCol->refIdx_s[1]}, refColL0);
323 | 			if (ctx->t.direct_8x8_inference_flag) {
324 | 				mvCol0 = broadcast32(mvCol0, 0);
325 | 				mvCol1 = broadcast32(mvCol1, 1);
326 | 				mvCol2 = broadcast32(mvCol2, 2);
327 | 				mvCol3 = broadcast32(mvCol3, 3);
328 | 			}
329 | 			
330 | 			// initialize colZeroFlags and masks for motion vectors
331 | 			unsigned refColZero = ((i32x4)(refCol == 0))[0];
332 | 			i8x16 zero = {};
333 | 			if (refColZero & 1)
334 | 				colZeroMask0 = mvs_near_zero(mvCol0, zero);
335 | 			if (refColZero & 1 << 8)
336 | 				colZeroMask1 = mvs_near_zero(mvCol1, zero);
337 | 			if (refColZero & 1 << 16)
338 | 				colZeroMask2 = mvs_near_zero(mvCol2, zero);
339 | 			if (refColZero & 1 << 24)
340 | 				colZeroMask3 = mvs_near_zero(mvCol3, zero);
341 | 			colZeroFlags = movemask(packs16(packs32(colZeroMask0, colZeroMask1), packs32(colZeroMask2, colZeroMask3)));
342 | 		}
343 | 		
344 | 		// skip computations on colZeroFlags if none are set
345 | 		if (colZeroFlags != 0 || direct_flags != 0xffffffff) {
346 | 			unsigned mvd_flags = direct_flags;
347 | 			i16x8 mvs1 = mvs0, mvs2 = mvs0, mvs3 = mvs0, mvs5 = mvs4, mvs6 = mvs4, mvs7 = mvs4;
348 | 			if (refIdx[0] == 0) {
349 | 				colZeroFlags += colZeroFlags << 16;
350 | 				mvs0 &= ~colZeroMask0;
351 | 				mvs1 &= ~colZeroMask1;
352 | 				mvs2 &= ~colZeroMask2;
353 | 				mvs3 &= ~colZeroMask3;
354 | 			} else {
355 | 				colZeroFlags <<= 16;
356 | 				mvd_flags = (refIdx[0] < 0) ? mvd_flags & 0xffff0000 : mvd_flags;
357 | 			}
358 | 			if (refIdx[4] == 0) {
359 | 				mvs4 &= ~colZeroMask0;
360 | 				mvs5 &= ~colZeroMask1;
361 | 				mvs6 &= ~colZeroMask2;
362 | 				mvs7 &= ~colZeroMask3;
363 | 			} else {
364 | 				colZeroFlags &= 0x0000ffff;
365 | 				mvd_flags = (refIdx[4] < 0) ? mvd_flags & 0x0000ffff : mvd_flags;
366 | 			}
367 | 			
368 | 			// conditional memory storage
369 | 			if (direct_flags & 1) {
370 | 				mb->refIdx[0] = refIdx[0];
371 | 				mb->refIdx[4] = refIdx[4];
372 | 				mb->mvs_v[0] = mvs0;
373 | 				mb->mvs_v[4] = mvs4;
374 | 			}
375 | 			if (direct_flags & 1 << 4) {
376 | 				mb->refIdx[1] = refIdx[0];
377 | 				mb->refIdx[5] = refIdx[4];
378 | 				mb->mvs_v[1] = mvs1;
379 | 				mb->mvs_v[5] = mvs5;
380 | 			}
381 | 			if (direct_flags & 1 << 8) {
382 | 				mb->refIdx[2] = refIdx[0];
383 | 				mb->refIdx[6] = refIdx[4];
384 | 				mb->mvs_v[2] = mvs2;
385 | 				mb->mvs_v[6] = mvs6;
386 | 			}
387 | 			if (direct_flags & 1 << 12) {
388 | 				mb->refIdx[3] = refIdx[0];
389 | 				mb->refIdx[7] = refIdx[4];
390 | 				mb->mvs_v[3] = mvs3;
391 | 				mb->mvs_v[7] = mvs7;
392 | 			}
393 | 			
394 | 			// iteratively cut the area into blocks with uniform colZeroFlags values
395 | 			static const uint16_t scopes[16] = {0xffff, 0x505, 0x33, 0x1, 0xf0f, 0x505, 0x3, 0x1, 0xff, 0x5, 0x33, 0x1, 0xf, 0x5, 0x3, 0x1};
396 | 			static const u16x8 masks = {0xffff, 0xff, 0xf0f, 0xf, 0x33, 0x3, 0x5, 0x1};
397 | 			static const uint32_t eqs[8] = {0x1b5fbbff, 0x1b5f, 0x1b00bb, 0x1b, 0x0105, 0x1, 0x2, 0};
398 | 			static const int8_t widths[8] = {16, 16, 8, 8, 16, 8, 4, 4};
399 | 			static const int8_t heights[8] = {16, 8, 16, 8, 4, 4, 8, 4};
400 | 			uint64_t inter_eqs = 0;
401 | 			do {
402 | 				int i = __builtin_ctz(mvd_flags);
403 | 				unsigned t = mvd_flags >> i & scopes[i & 15];
404 | 				unsigned c = colZeroFlags >> i;
405 | 				i16x8 mt = (u16x8){t, t, t, t, t, t, t, t} & masks;
406 | 				i16x8 mc = (u16x8){c, c, c, c, c, c, c, c};
407 | 				int type = __builtin_ctz(movemask(((mt & mc) == masks) | ((mt & ~mc) == masks))) >> 1;
408 | 				mvd_flags ^= (unsigned)((uint16_t *)&masks)[type] << i;
409 | 				inter_eqs |= (uint64_t)eqs[type] << i * 2;
410 | 				decode_inter(ctx, i, widths[type], heights[type]);
411 | 			} while (mvd_flags);
412 | 			mb->f.inter_eqs_s |= little_endian32(inter_eqs & inter_eqs >> 32);
413 | 			return;
414 | 		}
415 | 	}
416 | 	
417 | 	// fallback if we did not need colZeroFlags
418 | 	mb->refIdx_l = ((i64x2)refIdx)[0];
419 | 	mb->mvs_v[0] = mb->mvs_v[1] = mb->mvs_v[2] = mb->mvs_v[3] = mvs0;
420 | 	mb->mvs_v[4] = mb->mvs_v[5] = mb->mvs_v[6] = mb->mvs_v[7] = mvs4;
421 | 	mb->f.inter_eqs_s = little_endian32(0x1b5fbbff);
422 | 	if (refIdx[0] >= 0)
423 | 		decode_inter(ctx, 0, 16, 16);
424 | 	if (refIdx[4] >= 0)
425 | 		decode_inter(ctx, 16, 16, 16);
426 | }
427 | 
428 | static always_inline void decode_direct_temporal_mv_pred(Edge264Context *ctx, unsigned direct_flags)
429 | {
430 | 	// load refPicCol and mvCol
431 | 	const Edge264Macroblock *mbCol = ctx->mbCol;
432 | 	i8x16 refPicColL0 = (i32x4){mbCol->refPic_s[0]};
433 | 	i8x16 offsets = refPicColL0 & 32;
434 | 	i16x8 mvCol0 = *(i16x8*)(mbCol->mvs + offsets[0]);
435 | 	i16x8 mvCol1 = *(i16x8*)(mbCol->mvs + offsets[1] + 8);
436 | 	i16x8 mvCol2 = *(i16x8*)(mbCol->mvs + offsets[2] + 16);
437 | 	i16x8 mvCol3 = *(i16x8*)(mbCol->mvs + offsets[3] + 24);
438 | 	i8x16 refPicCol = ifelse_msb(refPicColL0, (i32x4){mbCol->refPic_s[1]}, refPicColL0);
439 | 	unsigned inter_eqs = little_endian32(mbCol->f.inter_eqs_s);
440 | 	if (ctx->t.direct_8x8_inference_flag) {
441 | 		mvCol0 = broadcast32(mvCol0, 0);
442 | 		mvCol1 = broadcast32(mvCol1, 1);
443 | 		mvCol2 = broadcast32(mvCol2, 2);
444 | 		mvCol3 = broadcast32(mvCol3, 3);
445 | 		inter_eqs |= 0x1b1b1b1b;
446 | 	}
447 | 	
448 | 	// conditional memory storage
449 | 	i8x16 refIdx = shufflez2(ctx->MapPicToList0_v, refPicCol);
450 | 	mb->refPic_s[0] = ((i32x4)shufflen(ctx->t.RefPicList_v[0], refIdx))[0]; // overwritten by parse_ref_idx later if refIdx!=0
451 | 	mb->refPic_s[1] = ((i32x4)broadcast8(ctx->t.RefPicList_v[2], 0))[0]; // refIdxL1 is 0
452 | 	if (direct_flags & 1) {
453 | 		mb->refIdx[0] = refIdx[0];
454 | 		mb->refIdx[4] = 0;
455 | 		mb->mvs_v[0] = temporal_scale(mvCol0, ctx->DistScaleFactor[refIdx[0]]);
456 | 		mb->mvs_v[4] = mb->mvs_v[0] - mvCol0;
457 | 	} else {
458 | 		inter_eqs &= ~0x000000ff;
459 | 	}
460 | 	if (direct_flags & 1 << 4) {
461 | 		mb->refIdx[1] = refIdx[1];
462 | 		mb->refIdx[5] = 0;
463 | 		mb->mvs_v[1] = temporal_scale(mvCol1, ctx->DistScaleFactor[refIdx[1]]);
464 | 		mb->mvs_v[5] = mb->mvs_v[1] - mvCol1;
465 | 	} else {
466 | 		inter_eqs &= ~0x0000bb44;
467 | 	}
468 | 	if (direct_flags & 1 << 8) {
469 | 		mb->refIdx[2] = refIdx[2];
470 | 		mb->refIdx[6] = 0;
471 | 		mb->mvs_v[2] = temporal_scale(mvCol2, ctx->DistScaleFactor[refIdx[2]]);
472 | 		mb->mvs_v[6] = mb->mvs_v[2] - mvCol2;
473 | 	} else {
474 | 		inter_eqs &= ~0x005f00a0;
475 | 	}
476 | 	if (direct_flags & 1 << 12) {
477 | 		mb->refIdx[3] = refIdx[3];
478 | 		mb->refIdx[7] = 0;
479 | 		mb->mvs_v[3] = temporal_scale(mvCol3, ctx->DistScaleFactor[refIdx[3]]);
480 | 		mb->mvs_v[7] = mb->mvs_v[3] - mvCol3;
481 | 	} else { // edge case: 16x16 with a direct8x8 block on the bottom-right corner
482 | 		inter_eqs = (inter_eqs == 0x1b5fbbff) ? 0x001b1b5f : inter_eqs & ~0x1b44a000;
483 | 	}
484 | 	
485 | 	// execute decode_inter for the positions given in the mask
486 | 	mb->f.inter_eqs_s |= little_endian32(inter_eqs);
487 | 	direct_flags &= 0xffff;
488 | 	do {
489 | 		static uint16_t masks[16] = {0x1, 0x3, 0x5, 0xf, 0x1, 0x33, 0x5, 0xff, 0x1, 0x3, 0x0505, 0x0f0f, 0x1, 0x33, 0x0505, 0xffff};
490 | 		static int8_t widths[16] = {4, 8, 4, 8, 4, 16, 4, 16, 4, 8, 4, 8, 4, 16, 4, 16};
491 | 		static int8_t heights[16] = {4, 4, 8, 8, 4, 4, 8, 8, 4, 4, 16, 16, 4, 4, 16, 16};
492 | 		int i = __builtin_ctz(direct_flags);
493 | 		int type = extract_neighbours(inter_eqs >> i * 2) & ~i;
494 | 		direct_flags ^= masks[type] << i;
495 | 		decode_inter(ctx, i, widths[type], heights[type]);
496 | 		decode_inter(ctx, i + 16, widths[type], heights[type]);
497 | 	} while (direct_flags);
498 | }
499 | 
500 | static noinline void decode_direct_mv_pred(Edge264Context *ctx, unsigned direct_flags) {
501 | 	if (ctx->t.direct_spatial_mv_pred_flag) {
502 | 		decode_direct_spatial_mv_pred(ctx, direct_flags);
503 | 	} else {
504 | 		decode_direct_temporal_mv_pred(ctx, direct_flags);
505 | 	}
506 | }
507 | 


--------------------------------------------------------------------------------
/edge264_residual.c:
--------------------------------------------------------------------------------
  1 | #include "edge264_internal.h"
  2 | 
  3 | #if defined(__SSE2__)
  4 | 	#define addlou8s16(a, b) (cvtlo8u16(a) + (i16x8)b)
  5 | 	#define addhiu8s16(a, b) ((i16x8)ziphi8(a, (i8x16){}) + (i16x8)b)
  6 | 	#define mullou8(a, b) (cvtlo8u16(a) * cvtlo8u16(b))
  7 | 	#define mulhiu8(a, b) ((u16x8)ziphi8(a, (i8x16){}) * (u16x8)ziphi8(b, (i8x16){}))
  8 | 	#define shlrrs32(a, l, r, off) ((((i32x4)(a) << (l)) + (off)) >> (r))
  9 | 	#define shrrs32(a, i, off) (((i32x4)a + off) >> i)
 10 | 	#define shrps32(a, b, i) (i16x8)_mm_packs_epi32((i32x4)(a) >> i, (i32x4)(b) >> i)
 11 | 	#define unziplo32(a, b) shuffleps(a, b, 0, 2, 0, 2)
 12 | 	#define unziphi32(a, b) shuffleps(a, b, 1, 3, 1, 3)
 13 | 	static always_inline i16x8 scale32(i32x4 c0, i32x4 c1, u16x8 ls, int mul, i32x4 off, i32x4 sh) {return packs32(_mm_sra_epi32((i32x4)_mm_madd_epi16(cvtlo16u32(ls), c0) + off, sh), _mm_sra_epi32((i32x4)_mm_madd_epi16(ziphi16(ls, (i8x16){}), c1) + off, sh));}
 14 | #elif defined(__ARM_NEON)
 15 | 	#define addlou8s16(a, b) (i16x8)vaddw_u8(b, vget_low_u8(a))
 16 | 	#define addhiu8s16(a, b) (i16x8)vaddw_high_u8(b, a)
 17 | 	#define mullou8(a, b) (u16x8)vmull_u8(vget_low_u8(a), vget_low_u8(b))
 18 | 	#define mulhiu8(a, b) (u16x8)vmull_high_u8(a, b)
 19 | 	#define shlrrs32(a, l, r, off) (i32x4)vrshlq_s32(a, vdupq_n_s32((l) - (r)))
 20 | 	#define shrrs32(a, i, off) (i32x4)vrshrq_n_s32(a, i)
 21 | 	#define shrps32(a, b, i) (u8x16)vqshrn_high_n_s32(vqshrn_n_s32(a, i), b, i)
 22 | 	#define unziplo32(a, b) (i32x4)vuzp1q_s32(a, b)
 23 | 	#define unziphi32(a, b) (i32x4)vuzp2q_s32(a, b)
 24 | 	static always_inline i16x8 scale32(i32x4 c0, i32x4 c1, u16x8 ls, int mul, i32x4 off, i32x4 sh) {return vrshrn_high_n_s32(vrshrn_n_s32((i32x4)vmull_n_s16(vget_low_s16(ls), mul) * c0, 6), (i32x4)vmull_high_n_s16(ls, mul) * c1, 6);}
 25 | #endif
 26 | 
 27 | static const i8x4 normAdjust4x4[6] = {
 28 | 	{10, 16, 13},
 29 | 	{11, 18, 14},
 30 | 	{13, 20, 16},
 31 | 	{14, 23, 18},
 32 | 	{16, 25, 20},
 33 | 	{18, 29, 23}
 34 | };
 35 | static const i8x8 normAdjust8x8[6] = {
 36 | 	{20, 18, 32, 19, 25, 24},
 37 | 	{22, 19, 35, 21, 28, 26},
 38 | 	{26, 23, 42, 24, 33, 31},
 39 | 	{28, 25, 45, 26, 35, 33},
 40 | 	{32, 28, 51, 30, 40, 38},
 41 | 	{36, 32, 58, 34, 46, 43}
 42 | };
 43 | 
 44 | 
 45 | 
 46 | /**
 47 |  * Inverse 4x4 transform
 48 |  *
 49 |  * Here we try to stay close to the spec's pseudocode, avoiding minor
 50 |  * optimisations that would make the code hard to understand.
 51 |  */
 52 | static noinline void add_idct4x4(Edge264Context *ctx, int iYCbCr, int DCidx, uint8_t *p)
 53 | {
 54 | 	// loading and scaling
 55 | 	unsigned qP = ctx->t.QP[iYCbCr];
 56 | 	int sh = qP / 6;
 57 | 	i8x16 vm = load32(&normAdjust4x4[qP % 6]);
 58 | 	i8x16 nA = shuffle(vm, (i8x16){0, 2, 0, 2, 2, 1, 2, 1, 0, 2, 0, 2, 2, 1, 2, 1});
 59 | 	i8x16 wS = ctx->t.pps.weightScale4x4_v[iYCbCr + mb->mbIsInterFlag * 3];
 60 | 	i16x8 LS0 = mullou8(wS, nA);
 61 | 	i16x8 LS1 = mulhiu8(wS, nA);
 62 | 	i32x4 s8 = set32(8); // for SSE
 63 | 	i32x4 d0 = shlrrs32(ctx->c_v[0] * cvtlo16u32(LS0), sh, 4, s8);
 64 | 	i32x4 d1 = shlrrs32(ctx->c_v[1] * cvthi16u32(LS0), sh, 4, s8);
 65 | 	i32x4 d2 = shlrrs32(ctx->c_v[2] * cvtlo16u32(LS1), sh, 4, s8);
 66 | 	i32x4 d3 = shlrrs32(ctx->c_v[3] * cvthi16u32(LS1), sh, 4, s8);
 67 | 	if (DCidx >= 0)
 68 | 		d0[0] = ctx->c[16 + DCidx];
 69 | 	
 70 | 	// horizontal 1D transform
 71 | 	i32x4 e0 = d0 + d2;
 72 | 	i32x4 e1 = d0 - d2;
 73 | 	i32x4 e2 = (d1 >> 1) - d3;
 74 | 	i32x4 e3 = (d3 >> 1) + d1;
 75 | 	i32x4 f0 = e0 + e3;
 76 | 	i32x4 f1 = e1 + e2;
 77 | 	i32x4 f2 = e1 - e2;
 78 | 	i32x4 f3 = e0 - e3;
 79 | 	
 80 | 	// matrix transposition
 81 | 	i32x4 x0 = ziplo32(f0, f1);
 82 | 	i32x4 x1 = ziplo32(f2, f3);
 83 | 	i32x4 x2 = ziphi32(f0, f1);
 84 | 	i32x4 x3 = ziphi32(f2, f3);
 85 | 	f0 = (i32x4)ziplo64(x0, x1) + 32;
 86 | 	f1 = ziphi64(x0, x1);
 87 | 	f2 = ziplo64(x2, x3);
 88 | 	f3 = ziphi64(x2, x3);
 89 | 	
 90 | 	// vertical 1D transform
 91 | 	i32x4 g0 = f0 + f2;
 92 | 	i32x4 g1 = f0 - f2;
 93 | 	i32x4 g2 = (f1 >> 1) - f3;
 94 | 	i32x4 g3 = (f3 >> 1) + f1;
 95 | 	i32x4 h0 = g0 + g3;
 96 | 	i32x4 h1 = g1 + g2;
 97 | 	i32x4 h2 = g1 - g2;
 98 | 	i32x4 h3 = g0 - g3;
 99 | 	
100 | 	// final residual values
101 | 	i16x8 r0 = shrps32(h0, h1, 6);
102 | 	i16x8 r1 = shrps32(h2, h3, 6);
103 | 	
104 | 	// addition to values in place, clipping and storage
105 | 	size_t stride = ctx->t.stride[iYCbCr];
106 | 	INIT_P();
107 | 	if (ctx->t.samples_clip[iYCbCr][0] == 255) {
108 | 		i8x16 p0 = (i32x4){*(int32_t *)P(0, 0), *(int32_t *)P(0, 1)};
109 | 		i8x16 p1 = (i32x4){*(int32_t *)P(0, 2), *(int32_t *)P(0, 3)};
110 | 		i32x4 u = packus16(addlou8s16(p0, r0), addlou8s16(p1, r1));
111 | 		*(int32_t *)P(0, 0) = u[0];
112 | 		*(int32_t *)P(0, 1) = u[1];
113 | 		*(int32_t *)P(0, 2) = u[2];
114 | 		*(int32_t *)P(0, 3) = u[3];
115 | 	}
116 | }
117 | 
118 | static void add_dc4x4(Edge264Context *ctx, int iYCbCr, int DCidx, uint8_t *p) {
119 | 	i32x4 r = set16((ctx->c[16 + DCidx] + 32) >> 6);
120 | 	size_t stride = ctx->t.stride[iYCbCr];
121 | 	INIT_P();
122 | 	if (ctx->t.samples_clip[iYCbCr][0] == 255) {
123 | 		i8x16 p0 = (i32x4){*(int32_t *)P(0, 0), *(int32_t *)P(0, 1)};
124 | 		i8x16 p1 = (i32x4){*(int32_t *)P(0, 2), *(int32_t *)P(0, 3)};
125 | 		i32x4 u = packus16(addlou8s16(p0, r), addlou8s16(p1, r));
126 | 		*(int32_t *)P(0, 0) = u[0];
127 | 		*(int32_t *)P(0, 1) = u[1];
128 | 		*(int32_t *)P(0, 2) = u[2];
129 | 		*(int32_t *)P(0, 3) = u[3];
130 | 	}
131 | }
132 | 
133 | 
134 | 
135 | /**
136 |  * Inverse 8x8 transform
137 |  */
138 | static void add_idct8x8(Edge264Context *ctx, int iYCbCr, uint8_t *p)
139 | {
140 | 	// loading and scaling
141 | 	unsigned qP = ctx->t.QP[iYCbCr];
142 | 	if (ctx->t.samples_clip[iYCbCr][0] == 255) {
143 | 		int div = qP / 6;
144 | 		i8x16 vm = load64(&normAdjust8x8[qP % 6]);
145 | 		i8x16 *wS = ctx->t.pps.weightScale8x8_v + (iYCbCr * 2 + mb->mbIsInterFlag) * 4;
146 | 		i8x16 nA0 = shuffle(vm, (i8x16){0, 3, 4, 3, 0, 3, 4, 3, 3, 1, 5, 1, 3, 1, 5, 1});
147 | 		i8x16 nA1 = shuffle(vm, (i8x16){4, 5, 2, 5, 4, 5, 2, 5, 3, 1, 5, 1, 3, 1, 5, 1});
148 | 		i16x8 LS0 = mullou8(wS[0], nA0);
149 | 		i16x8 LS1 = mulhiu8(wS[0], nA0);
150 | 		i16x8 LS2 = mullou8(wS[1], nA1);
151 | 		i16x8 LS3 = mulhiu8(wS[1], nA1);
152 | 		i16x8 LS4 = mullou8(wS[2], nA0);
153 | 		i16x8 LS5 = mulhiu8(wS[2], nA0);
154 | 		i16x8 LS6 = mullou8(wS[3], nA1);
155 | 		i16x8 LS7 = mulhiu8(wS[3], nA1);
156 | 		i32x4 *c = ctx->c_v;
157 | 		i16x8 d0, d1, d2, d3, d4, d5, d6, d7;
158 | 		if (__builtin_expect(div < 6, 1)) {
159 | 			int mul = 1 << div; // for NEON
160 | 			i32x4 off = set32(1 << (5 - div)); // for SSE
161 | 			i32x4 sh = {6 - div}; // for SSE
162 | 			d0 = scale32(c[0], c[1], LS0, mul, off, sh);
163 | 			d1 = scale32(c[2], c[3], LS1, mul, off, sh);
164 | 			d2 = scale32(c[4], c[5], LS2, mul, off, sh);
165 | 			d3 = scale32(c[6], c[7], LS3, mul, off, sh);
166 | 			d4 = scale32(c[8], c[9], LS4, mul, off, sh);
167 | 			d5 = scale32(c[10], c[11], LS5, mul, off, sh);
168 | 			d6 = scale32(c[12], c[13], LS6, mul, off, sh);
169 | 			d7 = scale32(c[14], c[15], LS7, mul, off, sh);
170 | 		} else {
171 | 			int sh = div - 6;
172 | 			// FIXME use shift function for SSE
173 | 			d0 = packs32(c[0], c[1]) * (LS0 << sh);
174 | 			d1 = packs32(c[2], c[3]) * (LS1 << sh);
175 | 			d2 = packs32(c[4], c[5]) * (LS2 << sh);
176 | 			d3 = packs32(c[6], c[7]) * (LS3 << sh);
177 | 			d4 = packs32(c[8], c[9]) * (LS4 << sh);
178 | 			d5 = packs32(c[10], c[11]) * (LS5 << sh);
179 | 			d6 = packs32(c[12], c[13]) * (LS6 << sh);
180 | 			d7 = packs32(c[14], c[15]) * (LS7 << sh);
181 | 		}
182 | 		
183 | 		for (int i = 2;;) {
184 | 			// 1D transform
185 | 			i16x8 e0 = d0 + d4;
186 | 			i16x8 e1 = d5 - d3 - ((d7 >> 1) + d7);
187 | 			i16x8 e2 = d0 - d4;
188 | 			i16x8 e3 = d1 + d7 - ((d3 >> 1) + d3);
189 | 			i16x8 e4 = (d2 >> 1) - d6;
190 | 			i16x8 e5 = d7 - d1 + ((d5 >> 1) + d5);
191 | 			i16x8 e6 = (d6 >> 1) + d2;
192 | 			i16x8 e7 = d3 + d5 + ((d1 >> 1) + d1);
193 | 			i16x8 f0 = e0 + e6;
194 | 			i16x8 f1 = (e7 >> 2) + e1;
195 | 			i16x8 f2 = e2 + e4;
196 | 			i16x8 f3 = (e5 >> 2) + e3;
197 | 			i16x8 f4 = e2 - e4;
198 | 			i16x8 f5 = (e3 >> 2) - e5;
199 | 			i16x8 f6 = e0 - e6;
200 | 			i16x8 f7 = e7 - (e1 >> 2);
201 | 			
202 | 			// Compilers freak out whenever output uses other registers.
203 | 			d0 = f0 + f7;
204 | 			d1 = f2 + f5;
205 | 			d2 = f4 + f3;
206 | 			d3 = f6 + f1;
207 | 			d4 = f6 - f1;
208 | 			d5 = f4 - f3;
209 | 			d6 = f2 - f5;
210 | 			d7 = f0 - f7;
211 | 			if (--i == 0)
212 | 				break;
213 | 			
214 | 			// matrix transposition
215 | 			i16x8 x0 = ziplo16(d0, d1);
216 | 			i16x8 x1 = ziplo16(d2, d3);
217 | 			i16x8 x2 = ziplo16(d4, d5);
218 | 			i16x8 x3 = ziplo16(d6, d7);
219 | 			i16x8 x4 = ziphi16(d0, d1);
220 | 			i16x8 x5 = ziphi16(d2, d3);
221 | 			i16x8 x6 = ziphi16(d4, d5);
222 | 			i16x8 x7 = ziphi16(d6, d7);
223 | 			i16x8 x8 = ziplo32(x0, x1);
224 | 			i16x8 x9 = ziplo32(x2, x3);
225 | 			i16x8 xA = ziplo32(x4, x5);
226 | 			i16x8 xB = ziplo32(x6, x7);
227 | 			i16x8 xC = ziphi32(x0, x1);
228 | 			i16x8 xD = ziphi32(x2, x3);
229 | 			i16x8 xE = ziphi32(x4, x5);
230 | 			i16x8 xF = ziphi32(x6, x7);
231 | 			d0 = (i16x8)ziplo64(x8, x9) + 32;
232 | 			d1 = ziphi64(x8, x9);
233 | 			d2 = ziplo64(xC, xD);
234 | 			d3 = ziphi64(xC, xD);
235 | 			d4 = ziplo64(xA, xB);
236 | 			d5 = ziphi64(xA, xB);
237 | 			d6 = ziplo64(xE, xF);
238 | 			d7 = ziphi64(xE, xF);
239 | 		}
240 | 		
241 | 		// final residual values
242 | 		i16x8 r0 = d0 >> 6;
243 | 		i16x8 r1 = d1 >> 6;
244 | 		i16x8 r2 = d2 >> 6;
245 | 		i16x8 r3 = d3 >> 6;
246 | 		i16x8 r4 = d4 >> 6;
247 | 		i16x8 r5 = d5 >> 6;
248 | 		i16x8 r6 = d6 >> 6;
249 | 		i16x8 r7 = d7 >> 6;
250 | 		
251 | 		// addition to values in place, clipping and storage
252 | 		size_t stride = ctx->t.stride[iYCbCr];
253 | 		INIT_P();
254 | 		i8x16 p0 = addlou8s16(load64(P(0, 0)), r0);
255 | 		i8x16 p1 = addlou8s16(load64(P(0, 1)), r1);
256 | 		i8x16 p2 = addlou8s16(load64(P(0, 2)), r2);
257 | 		i8x16 p3 = addlou8s16(load64(P(0, 3)), r3);
258 | 		i8x16 p4 = addlou8s16(load64(P(0, 4)), r4);
259 | 		i8x16 p5 = addlou8s16(load64(P(0, 5)), r5);
260 | 		i8x16 p6 = addlou8s16(load64(P(0, 6)), r6);
261 | 		i8x16 p7 = addlou8s16(load64(P(0, 7)), r7);
262 | 		i64x2 u0 = packus16(p0, p1);
263 | 		i64x2 u1 = packus16(p2, p3);
264 | 		i64x2 u2 = packus16(p4, p5);
265 | 		i64x2 u3 = packus16(p6, p7);
266 | 		*(int64_t *)P(0, 0) = u0[0];
267 | 		*(int64_t *)P(0, 1) = u0[1];
268 | 		*(int64_t *)P(0, 2) = u1[0];
269 | 		*(int64_t *)P(0, 3) = u1[1];
270 | 		*(int64_t *)P(0, 4) = u2[0];
271 | 		*(int64_t *)P(0, 5) = u2[1];
272 | 		*(int64_t *)P(0, 6) = u3[0];
273 | 		*(int64_t *)P(0, 7) = u3[1];
274 | 	} // FIXME else 16bit
275 | }
276 | 
277 | 
278 | 
279 | /**
280 |  * DC transforms
281 |  * 
282 |  * These functions do not gain enough from 8bit to justify distinct versions.
283 |  */
284 | static void transform_dc4x4(Edge264Context *ctx, int iYCbCr)
285 | {
286 | 	// load matrix in column order and multiply right
287 | 	i32x4 x0 = ctx->c_v[0] + ctx->c_v[1];
288 | 	i32x4 x1 = ctx->c_v[2] + ctx->c_v[3];
289 | 	i32x4 x2 = ctx->c_v[0] - ctx->c_v[1];
290 | 	i32x4 x3 = ctx->c_v[2] - ctx->c_v[3];
291 | 	i32x4 x4 = x0 + x1;
292 | 	i32x4 x5 = x0 - x1;
293 | 	i32x4 x6 = x2 - x3;
294 | 	i32x4 x7 = x2 + x3;
295 | 	
296 | 	// transpose
297 | 	i32x4 x8 = ziplo32(x4, x5);
298 | 	i32x4 x9 = ziplo32(x6, x7);
299 | 	i32x4 xA = ziphi32(x4, x5);
300 | 	i32x4 xB = ziphi32(x6, x7);
301 | 	i32x4 xC = ziplo64(x8, x9);
302 | 	i32x4 xD = ziphi64(x8, x9);
303 | 	i32x4 xE = ziplo64(xA, xB);
304 | 	i32x4 xF = ziphi64(xA, xB);
305 | 	
306 | 	// multiply left
307 | 	i32x4 xG = xC + xD;
308 | 	i32x4 xH = xE + xF;
309 | 	i32x4 xI = xC - xD;
310 | 	i32x4 xJ = xE - xF;
311 | 	i32x4 f0 = xG + xH;
312 | 	i32x4 f1 = xG - xH;
313 | 	i32x4 f2 = xI - xJ;
314 | 	i32x4 f3 = xI + xJ;
315 | 	
316 | 	// scale
317 | 	unsigned qP = ctx->t.QP[0]; // FIXME 4:4:4
318 | 	i32x4 s32 = set32(32);
319 | 	i32x4 LS = set32((ctx->t.pps.weightScale4x4[iYCbCr][0] * normAdjust4x4[qP % 6][0]) << (qP / 6));
320 | 	i32x4 dc0 = shrrs32(f0 * LS, 6, s32);
321 | 	i32x4 dc1 = shrrs32(f1 * LS, 6, s32);
322 | 	i32x4 dc2 = shrrs32(f2 * LS, 6, s32);
323 | 	i32x4 dc3 = shrrs32(f3 * LS, 6, s32);
324 | 	
325 | 	// store in zigzag order if needed later ...
326 | 	if (mb->bits[0] & 1 << 5) {
327 | 		ctx->c_v[4] = ziplo64(dc0, dc1);
328 | 		ctx->c_v[5] = ziphi64(dc0, dc1);
329 | 		ctx->c_v[6] = ziplo64(dc2, dc3);
330 | 		ctx->c_v[7] = ziphi64(dc2, dc3);
331 | 		
332 | 	// ... or prepare for storage in place
333 | 	} else {
334 | 		i32x4 r0 = (dc0 + s32) >> 6;
335 | 		i32x4 r1 = (dc1 + s32) >> 6;
336 | 		i32x4 r2 = (dc2 + s32) >> 6;
337 | 		i32x4 r3 = (dc3 + s32) >> 6;
338 | 		#if defined(__SSE2__)
339 | 			i16x8 lo0 = shuffle32(shufflelo(r0, 0, 0, 2, 2), 0, 0, 1, 1);
340 | 			i16x8 lo1 = shuffle32(shufflelo(r1, 0, 0, 2, 2), 0, 0, 1, 1);
341 | 			i16x8 lo2 = shuffle32(shufflelo(r2, 0, 0, 2, 2), 0, 0, 1, 1);
342 | 			i16x8 lo3 = shuffle32(shufflelo(r3, 0, 0, 2, 2), 0, 0, 1, 1);
343 | 			i16x8 hi0 = shuffle32(shufflehi(r0, 0, 0, 2, 2), 2, 2, 3, 3);
344 | 			i16x8 hi1 = shuffle32(shufflehi(r1, 0, 0, 2, 2), 2, 2, 3, 3);
345 | 			i16x8 hi2 = shuffle32(shufflehi(r2, 0, 0, 2, 2), 2, 2, 3, 3);
346 | 			i16x8 hi3 = shuffle32(shufflehi(r3, 0, 0, 2, 2), 2, 2, 3, 3);
347 | 		#elif defined(__ARM_NEON)
348 | 			i16x8 v0 = vtrn1q_s16(r0, r0);
349 | 			i16x8 v1 = vtrn1q_s16(r1, r1);
350 | 			i16x8 v2 = vtrn1q_s16(r2, r2);
351 | 			i16x8 v3 = vtrn1q_s16(r3, r3);
352 | 			i16x8 lo0 = ziplo32(v0, v0);
353 | 			i16x8 lo1 = ziplo32(v1, v1);
354 | 			i16x8 lo2 = ziplo32(v2, v2);
355 | 			i16x8 lo3 = ziplo32(v3, v3);
356 | 			i16x8 hi0 = ziphi32(v0, v0);
357 | 			i16x8 hi1 = ziphi32(v1, v1);
358 | 			i16x8 hi2 = ziphi32(v2, v2);
359 | 			i16x8 hi3 = ziphi32(v3, v3);
360 | 		#endif
361 | 		
362 | 		// add to predicted samples
363 | 		size_t stride = ctx->t.stride[iYCbCr];
364 | 		uint8_t *p = ctx->samples_mb[iYCbCr];
365 | 		if (ctx->t.samples_clip[iYCbCr][0] == 255) {
366 | 			i8x16 p0 = *(i8x16 *)(p             );
367 | 			i8x16 p1 = *(i8x16 *)(p + stride    );
368 | 			i8x16 p2 = *(i8x16 *)(p + stride * 2);
369 | 			i8x16 p3 = *(i8x16 *)(p + stride * 3);
370 | 			*(i8x16 *)(p             ) = packus16(addlou8s16(p0, lo0), addhiu8s16(p0, hi0));
371 | 			*(i8x16 *)(p + stride    ) = packus16(addlou8s16(p1, lo0), addhiu8s16(p1, hi0));
372 | 			*(i8x16 *)(p + stride * 2) = packus16(addlou8s16(p2, lo0), addhiu8s16(p2, hi0));
373 | 			*(i8x16 *)(p + stride * 3) = packus16(addlou8s16(p3, lo0), addhiu8s16(p3, hi0));
374 | 			i8x16 p4 = *(i8x16 *)(p += stride * 4);
375 | 			i8x16 p5 = *(i8x16 *)(p + stride    );
376 | 			i8x16 p6 = *(i8x16 *)(p + stride * 2);
377 | 			i8x16 p7 = *(i8x16 *)(p + stride * 3);
378 | 			*(i8x16 *)(p             ) = packus16(addlou8s16(p4, lo1), addhiu8s16(p4, hi1));
379 | 			*(i8x16 *)(p + stride    ) = packus16(addlou8s16(p5, lo1), addhiu8s16(p5, hi1));
380 | 			*(i8x16 *)(p + stride * 2) = packus16(addlou8s16(p6, lo1), addhiu8s16(p6, hi1));
381 | 			*(i8x16 *)(p + stride * 3) = packus16(addlou8s16(p7, lo1), addhiu8s16(p7, hi1));
382 | 			i8x16 p8 = *(i8x16 *)(p += stride * 4);
383 | 			i8x16 p9 = *(i8x16 *)(p + stride    );
384 | 			i8x16 pA = *(i8x16 *)(p + stride * 2);
385 | 			i8x16 pB = *(i8x16 *)(p + stride * 3);
386 | 			*(i8x16 *)(p             ) = packus16(addlou8s16(p8, lo2), addhiu8s16(p8, hi2));
387 | 			*(i8x16 *)(p + stride    ) = packus16(addlou8s16(p9, lo2), addhiu8s16(p9, hi2));
388 | 			*(i8x16 *)(p + stride * 2) = packus16(addlou8s16(pA, lo2), addhiu8s16(pA, hi2));
389 | 			*(i8x16 *)(p + stride * 3) = packus16(addlou8s16(pB, lo2), addhiu8s16(pB, hi2));
390 | 			i8x16 pC = *(i8x16 *)(p += stride * 4);
391 | 			i8x16 pD = *(i8x16 *)(p + stride    );
392 | 			i8x16 pE = *(i8x16 *)(p + stride * 2);
393 | 			i8x16 pF = *(i8x16 *)(p + stride * 3);
394 | 			*(i8x16 *)(p             ) = packus16(addlou8s16(pC, lo3), addhiu8s16(pC, hi3));
395 | 			*(i8x16 *)(p + stride    ) = packus16(addlou8s16(pD, lo3), addhiu8s16(pD, hi3));
396 | 			*(i8x16 *)(p + stride * 2) = packus16(addlou8s16(pE, lo3), addhiu8s16(pE, hi3));
397 | 			*(i8x16 *)(p + stride * 3) = packus16(addlou8s16(pF, lo3), addhiu8s16(pF, hi3));
398 | 		}
399 | 	}
400 | }
401 | 
402 | static void transform_dc2x2(Edge264Context *ctx)
403 | {
404 | 	// load both matrices interlaced+transposed and multiply right
405 | 	i32x4 d0 = ctx->c_v[0] + ctx->c_v[1];
406 | 	i32x4 d1 = ctx->c_v[0] - ctx->c_v[1];
407 | 	
408 | 	// transpose and multiply left
409 | 	i32x4 e0 = ziplo64(d0, d1);
410 | 	i32x4 e1 = ziphi64(d0, d1);
411 | 	i32x4 f0 = e0 + e1;
412 | 	i32x4 f1 = e0 - e1;
413 | 	
414 | 	// deinterlace and scale
415 | 	unsigned qPb = ctx->t.QP[1];
416 | 	unsigned qPr = ctx->t.QP[2];
417 | 	i32x4 LSb = set32((ctx->t.pps.weightScale4x4[1 + mb->mbIsInterFlag * 3][0] * normAdjust4x4[qPb % 6][0]) << (qPb / 6));
418 | 	i32x4 LSr = set32((ctx->t.pps.weightScale4x4[2 + mb->mbIsInterFlag * 3][0] * normAdjust4x4[qPr % 6][0]) << (qPr / 6));
419 | 	i32x4 dcCb = ((i32x4)unziplo32(f0, f1) * LSb) >> 5;
420 | 	i32x4 dcCr = ((i32x4)unziphi32(f0, f1) * LSr) >> 5;
421 | 	
422 | 	// store if needed later ...
423 | 	if (mb->f.CodedBlockPatternChromaAC) {
424 | 		ctx->c_v[4] = dcCb;
425 | 		ctx->c_v[5] = dcCr;
426 | 		
427 | 	// ... or prepare for storage in place
428 | 	} else {
429 | 		i32x4 s32 = set32(32); // for SSE
430 | 		i32x4 rb = shrrs32(dcCb, 6, s32);
431 | 		i32x4 rr = shrrs32(dcCr, 6, s32);
432 | 		#if defined(__SSE2__)
433 | 			i16x8 lob = shuffle32(shufflelo(rb, 0, 0, 2, 2), 0, 0, 1, 1);
434 | 			i16x8 lor = shuffle32(shufflelo(rr, 0, 0, 2, 2), 0, 0, 1, 1);
435 | 			i16x8 hib = shuffle32(shufflehi(rb, 0, 0, 2, 2), 2, 2, 3, 3);
436 | 			i16x8 hir = shuffle32(shufflehi(rr, 0, 0, 2, 2), 2, 2, 3, 3);
437 | 		#elif defined(__ARM_NEON)
438 | 			i16x8 vb = vtrn1q_s16(rb, rb);
439 | 			i16x8 vr = vtrn1q_s16(rr, rr);
440 | 			i16x8 lob = ziplo32(vb, vb);
441 | 			i16x8 lor = ziplo32(vr, vr);
442 | 			i16x8 hib = ziphi32(vb, vb);
443 | 			i16x8 hir = ziphi32(vr, vr);
444 | 		#endif
445 | 		
446 | 		// add to predicted samples
447 | 		uint8_t *p = ctx->samples_mb[1];
448 | 		size_t stride = ctx->t.stride[1] >> 1;
449 | 		if (ctx->t.samples_clip[1][0] == 255) {
450 | 			i16x8 b0 = addlou8s16(load64(p             ), lob);
451 | 			i16x8 r0 = addlou8s16(load64(p + stride    ), lor);
452 | 			i16x8 b1 = addlou8s16(load64(p + stride * 2), lob);
453 | 			i16x8 r1 = addlou8s16(load64(p + stride * 3), lor);
454 | 			i64x2 b8 = packus16(b0, b1);
455 | 			i64x2 r8 = packus16(r0, r1);
456 | 			*(int64_t *)(p             ) = b8[0];
457 | 			*(int64_t *)(p + stride    ) = r8[0];
458 | 			*(int64_t *)(p + stride * 2) = b8[1];
459 | 			*(int64_t *)(p + stride * 3) = r8[1];
460 | 			i16x8 b2 = addlou8s16(load64(p += stride * 4), lob);
461 | 			i16x8 r2 = addlou8s16(load64(p + stride    ), lor);
462 | 			i16x8 b3 = addlou8s16(load64(p + stride * 2), lob);
463 | 			i16x8 r3 = addlou8s16(load64(p + stride * 3), lor);
464 | 			i64x2 b9 = packus16(b2, b3);
465 | 			i64x2 r9 = packus16(r2, r3);
466 | 			*(int64_t *)(p             ) = b9[0];
467 | 			*(int64_t *)(p + stride    ) = r9[0];
468 | 			*(int64_t *)(p + stride * 2) = b9[1];
469 | 			*(int64_t *)(p + stride * 3) = r9[1];
470 | 			i16x8 b4 = addlou8s16(load64(p += stride * 4), hib);
471 | 			i16x8 r4 = addlou8s16(load64(p + stride    ), hir);
472 | 			i16x8 b5 = addlou8s16(load64(p + stride * 2), hib);
473 | 			i16x8 r5 = addlou8s16(load64(p + stride * 3), hir);
474 | 			i64x2 bA = packus16(b4, b5);
475 | 			i64x2 rA = packus16(r4, r5);
476 | 			*(int64_t *)(p             ) = bA[0];
477 | 			*(int64_t *)(p + stride    ) = rA[0];
478 | 			*(int64_t *)(p + stride * 2) = bA[1];
479 | 			*(int64_t *)(p + stride * 3) = rA[1];
480 | 			i16x8 b6 = addlou8s16(load64(p += stride * 4), hib);
481 | 			i16x8 r6 = addlou8s16(load64(p + stride    ), hir);
482 | 			i16x8 b7 = addlou8s16(load64(p + stride * 2), hib);
483 | 			i16x8 r7 = addlou8s16(load64(p + stride * 3), hir);
484 | 			i64x2 bB = packus16(b6, b7);
485 | 			i64x2 rB = packus16(r6, r7);
486 | 			*(int64_t *)(p             ) = bB[0];
487 | 			*(int64_t *)(p + stride    ) = rB[0];
488 | 			*(int64_t *)(p + stride * 2) = bB[1];
489 | 			*(int64_t *)(p + stride * 3) = rB[1];
490 | 		}
491 | 	}
492 | }
493 | 
494 | 
495 | 
496 | // legacy code for 16-bit 8x8 transform
497 | #if 0
498 | void FUNC_CTX(transform_dc2x4)
499 | {
500 | 	int iYCbCr = (0/*BlkIdx*/ - 8) >> 3; // BlkIdx is 16 or 24
501 | 	unsigned qP_DC = 0; //mb->QP[iYCbCr] + 3;
502 | 	int w = ctx->t.pps.weightScale4x4[iYCbCr + mb->mbIsInterFlag * 3][0];
503 | 	int nA = normAdjust4x4[qP_DC % 6][0];
504 | 	__m128i x0 = (__m128i)ctx->c_v[0]; // {c00, c01, c10, c11} as per 8.5.11.1
505 | 	__m128i x1 = (__m128i)ctx->c_v[1]; // {c20, c21, c30, c31}
506 | 	__m128i x2 = _mm_add_epi32(x0, x1); // {c00+c20, c01+c21, c10+c30, c11+c31}
507 | 	__m128i x3 = _mm_sub_epi32(x0, x1); // {c00-c20, c01-c21, c10-c30, c11-c31}
508 | 	__m128i x4 = ziplo64(x2, x3); // {c00+c20, c01+c21, c00-c20, c01-c21}
509 | 	__m128i x5 = ziphi64(x2, x3); // {c10+c30, c11+c31, c10-c30, c11-c31}
510 | 	__m128i x6 = _mm_add_epi32(x4, x5); // {d00, d01, d10, d11}
511 | 	__m128i x7 = _mm_sub_epi32(x4, x5); // {d30, d31, d20, d21}
512 | 	__m128i x8 = _mm_hadd_epi32(x6, x7); // {f00, f10, f30, f20}
513 | 	__m128i x9 = _mm_hsub_epi32(x6, x7); // {f01, f11, f31, f21}
514 | 	__m128i s = set32((w * nA) << (qP_DC / 6 + ctx->ps.BitDepth_C - 8));
515 | 	__m128i s32 = set32(32);
516 | 	__m128i dc0 = _mm_srai_epi32(_mm_add_epi32(_mm_mullo_epi32(x8, s), s32), 6);
517 | 	__m128i dc1 = _mm_srai_epi32(_mm_add_epi32(_mm_mullo_epi32(x9, s), s32), 6);
518 | 	__m128i *c = (__m128i *)ctx->c_v + 2 + iYCbCr * 2;
519 | 	c[0] = ziplo32(dc0, dc1);
520 | 	c[1] = _mm_shuffle_epi32(ziphi64(dc0, dc1), _MM_SHUFFLE(2, 0, 3, 1));
521 | }
522 | 
523 | #ifdef __AVX2__
524 | 		// loading
525 | 		// FIXME scaling
526 | 		__m256i d0 = (__m256i)ctx->c_V[0];
527 | 		__m256i d1 = (__m256i)ctx->c_V[1];
528 | 		__m256i d2 = (__m256i)ctx->c_V[2];
529 | 		__m256i d3 = (__m256i)ctx->c_V[3];
530 | 		__m256i d4 = (__m256i)ctx->c_V[4];
531 | 		__m256i d5 = (__m256i)ctx->c_V[5];
532 | 		__m256i d6 = (__m256i)ctx->c_V[6];
533 | 		__m256i d7 = (__m256i)ctx->c_V[7];
534 | 		
535 | 		for (int i = 2;;) {
536 | 			// 1D transform
537 | 			__m256i e0 = _mm256_add_epi32(d0, d4);
538 | 			__m256i e1 = _mm256_sub_epi32(_mm256_sub_epi32(d5, d3), _mm256_add_epi32(_mm256_srai_epi32(d7, 1), d7));
539 | 			__m256i e2 = _mm256_sub_epi32(d0, d4);
540 | 			__m256i e3 = _mm256_sub_epi32(_mm256_add_epi32(d1, d7), _mm256_add_epi32(_mm256_srai_epi32(d3, 1), d3));
541 | 			__m256i e4 = _mm256_sub_epi32(_mm256_srai_epi32(d2, 1), d6);
542 | 			__m256i e5 = _mm256_add_epi32(_mm256_sub_epi32(d7, d1), _mm256_add_epi32(_mm256_srai_epi32(d5, 1), d5));
543 | 			__m256i e6 = _mm256_add_epi32(_mm256_srai_epi32(d6, 1), d2);
544 | 			__m256i e7 = _mm256_add_epi32(_mm256_add_epi32(d3, d5), _mm256_add_epi32(_mm256_srai_epi32(d1, 1), d1));
545 | 			__m256i f0 = _mm256_add_epi32(e0, e6);
546 | 			__m256i f1 = _mm256_add_epi32(_mm256_srai_epi32(e7, 2), e1);
547 | 			__m256i f2 = _mm256_add_epi32(e2, e4);
548 | 			__m256i f3 = _mm256_add_epi32(_mm256_srai_epi32(e5, 2), e3);
549 | 			__m256i f4 = _mm256_sub_epi32(e2, e4);
550 | 			__m256i f5 = _mm256_sub_epi32(_mm256_srai_epi32(e3, 2), e5);
551 | 			__m256i f6 = _mm256_sub_epi32(e0, e6);
552 | 			__m256i f7 = _mm256_sub_epi32(e7, _mm256_srai_epi32(e1, 2));
553 | 			
554 | 			// Compilers freak out whenever output uses other registers.
555 | 			d0 = _mm256_add_epi32(f0, f7);
556 | 			d1 = _mm256_add_epi32(f2, f5);
557 | 			d2 = _mm256_add_epi32(f4, f3);
558 | 			d3 = _mm256_add_epi32(f6, f1);
559 | 			d4 = _mm256_sub_epi32(f6, f1);
560 | 			d5 = _mm256_sub_epi32(f4, f3);
561 | 			d6 = _mm256_sub_epi32(f2, f5);
562 | 			d7 = _mm256_sub_epi32(f0, f7);
563 | 			if (--i == 0)
564 | 				break;
565 | 			
566 | 			// matrix transposition
567 | 			__m256i y0 = _mm256_ziplo_epi32(d0, d1);
568 | 			__m256i y1 = _mm256_ziplo_epi32(d2, d3);
569 | 			__m256i y2 = _mm256_ziplo_epi32(d4, d5);
570 | 			__m256i y3 = _mm256_ziplo_epi32(d6, d7);
571 | 			__m256i y4 = _mm256_ziphi_epi32(d0, d1);
572 | 			__m256i y5 = _mm256_ziphi_epi32(d2, d3);
573 | 			__m256i y6 = _mm256_ziphi_epi32(d4, d5);
574 | 			__m256i y7 = _mm256_ziphi_epi32(d6, d7);
575 | 			__m256i y8 = _mm256_ziplo_epi64(y0, y1);
576 | 			__m256i y9 = _mm256_ziplo_epi64(y2, y3);
577 | 			__m256i yA = _mm256_ziplo_epi64(y4, y5);
578 | 			__m256i yB = _mm256_ziplo_epi64(y6, y7);
579 | 			__m256i yC = _mm256_ziphi_epi64(y0, y1);
580 | 			__m256i yD = _mm256_ziphi_epi64(y2, y3);
581 | 			__m256i yE = _mm256_ziphi_epi64(y4, y5);
582 | 			__m256i yF = _mm256_ziphi_epi64(y6, y7);
583 | 			d0 = _mm256_add_epi32(_mm256_permute2x128_si256(y8, y9, _MM_SHUFFLE(0, 2, 0, 0)), _mm256_set1_epi32(32));
584 | 			d1 = _mm256_permute2x128_si256(yC, yD, _MM_SHUFFLE(0, 2, 0, 0));
585 | 			d2 = _mm256_permute2x128_si256(yA, yB, _MM_SHUFFLE(0, 2, 0, 0));
586 | 			d3 = _mm256_permute2x128_si256(yE, yF, _MM_SHUFFLE(0, 2, 0, 0));
587 | 			d4 = _mm256_permute2x128_si256(y8, y9, _MM_SHUFFLE(0, 3, 0, 1));
588 | 			d5 = _mm256_permute2x128_si256(yC, yD, _MM_SHUFFLE(0, 3, 0, 1));
589 | 			d6 = _mm256_permute2x128_si256(yA, yB, _MM_SHUFFLE(0, 3, 0, 1));
590 | 			d7 = _mm256_permute2x128_si256(yE, yF, _MM_SHUFFLE(0, 3, 0, 1));
591 | 		}
592 | 		
593 | 		// final residual values
594 | 		__m256i yG = _mm256_packs_epi32(_mm256_srai_epi32(d0, 6), _mm256_srai_epi32(d1, 6));
595 | 		__m256i yH = _mm256_packs_epi32(_mm256_srai_epi32(d2, 6), _mm256_srai_epi32(d3, 6));
596 | 		__m256i yI = _mm256_packs_epi32(_mm256_srai_epi32(d4, 6), _mm256_srai_epi32(d5, 6));
597 | 		__m256i yJ = _mm256_packs_epi32(_mm256_srai_epi32(d6, 6), _mm256_srai_epi32(d7, 6));
598 | 		__m256i r0 = _mm256_permute4x64_epi64(yG, _MM_SHUFFLE(3, 1, 2, 0));
599 | 		__m256i r1 = _mm256_permute4x64_epi64(yH, _MM_SHUFFLE(3, 1, 2, 0));
600 | 		__m256i r2 = _mm256_permute4x64_epi64(yI, _MM_SHUFFLE(3, 1, 2, 0));
601 | 		__m256i r3 = _mm256_permute4x64_epi64(yJ, _MM_SHUFFLE(3, 1, 2, 0));
602 | 		
603 | 		// addition to values in place, clipping and storage
604 | 		size_t stride = ctx->t.stride[0]; // FIXME 4:4:4
605 | 		size_t stride3 = stride * 3;
606 | 		uint8_t *p = NULL; //ctx->frame + ctx->frame_offsets_x[0/*BlkIdx2i4x4[BlkIdx]*/] + ctx->frame_offsets_y[0/*BlkIdx2i4x4[BlkIdx]*/];
607 | 		uint8_t *q = p + stride * 4;
608 | 		__m256i zero = _mm256_setzero_si256();
609 | 		__m256i clip = _mm256_set1_epi16(ctx->t.samples_clip[0]); // FIXME 4:4:4
610 | 		__m256i p0 = _mm256_setr_m128i(*(__m128i *)(p             ), *(__m128i *)(p + stride ));
611 | 		__m256i p1 = _mm256_setr_m128i(*(__m128i *)(p + stride * 2), *(__m128i *)(p + stride3));
612 | 		__m256i p2 = _mm256_setr_m128i(*(__m128i *)(q             ), *(__m128i *)(q + stride ));
613 | 		__m256i p3 = _mm256_setr_m128i(*(__m128i *)(q + stride * 2), *(__m128i *)(q + stride3));
614 | 		__m256i u0 = _mm256_min_epi16(_mm256_max_epi16(_mm256_adds_epi16(p0, r0), zero), clip);
615 | 		__m256i u1 = _mm256_min_epi16(_mm256_max_epi16(_mm256_adds_epi16(p1, r1), zero), clip);
616 | 		__m256i u2 = _mm256_min_epi16(_mm256_max_epi16(_mm256_adds_epi16(p2, r2), zero), clip);
617 | 		__m256i u3 = _mm256_min_epi16(_mm256_max_epi16(_mm256_adds_epi16(p3, r3), zero), clip);
618 | 		*(__m128i *)(p             ) = _mm256_extracti128_si256(u0, 0);
619 | 		*(__m128i *)(p + stride    ) = _mm256_extracti128_si256(u0, 1);
620 | 		*(__m128i *)(p + stride * 2) = _mm256_extracti128_si256(u1, 0);
621 | 		*(__m128i *)(p + stride3   ) = _mm256_extracti128_si256(u1, 1);
622 | 		*(__m128i *)(q             ) = _mm256_extracti128_si256(u2, 0);
623 | 		*(__m128i *)(q + stride    ) = _mm256_extracti128_si256(u2, 1);
624 | 		*(__m128i *)(q + stride * 2) = _mm256_extracti128_si256(u3, 0);
625 | 		*(__m128i *)(q + stride3   ) = _mm256_extracti128_si256(u3, 1);
626 | 
627 | #else // !defined(__AVX2__)
628 | 		// load half of samples
629 | 		// FIXME scaline
630 | 		__m128i d0 = (__m128i)ctx->c_v[0];
631 | 		__m128i d1 = (__m128i)ctx->c_v[2];
632 | 		__m128i d2 = (__m128i)ctx->c_v[4];
633 | 		__m128i d3 = (__m128i)ctx->c_v[6];
634 | 		__m128i d4 = (__m128i)ctx->c_v[8];
635 | 		__m128i d5 = (__m128i)ctx->c_v[10];
636 | 		__m128i d6 = (__m128i)ctx->c_v[12];
637 | 		__m128i d7 = (__m128i)ctx->c_v[14];
638 | 		
639 | 		// This (crappy) version uses a nested loop trick to reduce code size
640 | 		for (int i = 2;; ) {
641 | 			for (int j = 2;; ) {
642 | 				// 1D transform
643 | 				__m128i e0 = _mm_add_epi32(d0, d4);
644 | 				__m128i e1 = _mm_sub_epi32(_mm_sub_epi32(d5, d3), _mm_add_epi32(_mm_srai_epi32(d7, 1), d7));
645 | 				__m128i e2 = _mm_sub_epi32(d0, d4);
646 | 				__m128i e3 = _mm_sub_epi32(_mm_add_epi32(d1, d7), _mm_add_epi32(_mm_srai_epi32(d3, 1), d3));
647 | 				__m128i e4 = _mm_sub_epi32(_mm_srai_epi32(d2, 1), d6);
648 | 				__m128i e5 = _mm_add_epi32(_mm_sub_epi32(d7, d1), _mm_add_epi32(_mm_srai_epi32(d5, 1), d5));
649 | 				__m128i e6 = _mm_add_epi32(_mm_srai_epi32(d6, 1), d2);
650 | 				__m128i e7 = _mm_add_epi32(_mm_add_epi32(d3, d5), _mm_add_epi32(_mm_srai_epi32(d1, 1), d1));
651 | 				__m128i f0 = _mm_add_epi32(e0, e6);
652 | 				__m128i f1 = _mm_add_epi32(_mm_srai_epi32(e7, 2), e1);
653 | 				__m128i f2 = _mm_add_epi32(e2, e4);
654 | 				__m128i f3 = _mm_add_epi32(_mm_srai_epi32(e5, 2), e3);
655 | 				__m128i f4 = _mm_sub_epi32(e2, e4);
656 | 				__m128i f5 = _mm_sub_epi32(_mm_srai_epi32(e3, 2), e5);
657 | 				__m128i f6 = _mm_sub_epi32(e0, e6);
658 | 				__m128i f7 = _mm_sub_epi32(e7, _mm_srai_epi32(e1, 2));
659 | 				d0 = _mm_add_epi32(f0, f7);
660 | 				d1 = _mm_add_epi32(f2, f5);
661 | 				d2 = _mm_add_epi32(f4, f3);
662 | 				d3 = _mm_add_epi32(f6, f1);
663 | 				d4 = _mm_sub_epi32(f6, f1);
664 | 				d5 = _mm_sub_epi32(f4, f3);
665 | 				d6 = _mm_sub_epi32(f2, f5);
666 | 				d7 = _mm_sub_epi32(f0, f7);
667 | 				if (--j == 0)
668 | 					break;
669 | 				
670 | 				// load other half of samples
671 | 				ctx->c_v[0] = (v4si)d0;
672 | 				d0 = (__m128i)ctx->c_v[1];
673 | 				ctx->c_v[2] = (v4si)d1;
674 | 				d1 = (__m128i)ctx->c_v[3];
675 | 				ctx->c_v[4] = (v4si)d2;
676 | 				d2 = (__m128i)ctx->c_v[5];
677 | 				ctx->c_v[6] = (v4si)d3;
678 | 				d3 = (__m128i)ctx->c_v[7];
679 | 				ctx->c_v[8] = (v4si)d4;
680 | 				d4 = (__m128i)ctx->c_v[9];
681 | 				ctx->c_v[10] = (v4si)d5;
682 | 				d5 = (__m128i)ctx->c_v[11];
683 | 				ctx->c_v[12] = (v4si)d6;
684 | 				d6 = (__m128i)ctx->c_v[13];
685 | 				ctx->c_v[14] = (v4si)d7;
686 | 				d7 = (__m128i)ctx->c_v[15];
687 | 			}
688 | 			if (--i == 0)
689 | 				break;
690 | 			
691 | 			// transpose the half matrix going to memory
692 | 			__m128i x0 = ziplo32((__m128i)ctx->c_v[8], (__m128i)ctx->c_v[10]);
693 | 			__m128i x1 = ziplo32((__m128i)ctx->c_v[12], (__m128i)ctx->c_v[14]);
694 | 			__m128i x2 = ziphi32((__m128i)ctx->c_v[8], (__m128i)ctx->c_v[10]);
695 | 			__m128i x3 = ziphi32((__m128i)ctx->c_v[12], (__m128i)ctx->c_v[14]);
696 | 			__m128i x4 = ziplo32(d4, d5);
697 | 			__m128i x5 = ziplo32(d6, d7);
698 | 			__m128i x6 = ziphi32(d4, d5);
699 | 			__m128i x7 = ziphi32(d6, d7);
700 | 			ctx->c_v[1] = (v4si)_mm_add_epi32(ziplo64(x0, x1), set32(32));
701 | 			ctx->c_v[3] = (v4si)ziphi64(x0, x1);
702 | 			ctx->c_v[5] = (v4si)ziplo64(x2, x3);
703 | 			ctx->c_v[7] = (v4si)ziphi64(x2, x3);
704 | 			ctx->c_v[9] = (v4si)ziplo64(x4, x5);
705 | 			ctx->c_v[11] = (v4si)ziphi64(x4, x5);
706 | 			ctx->c_v[13] = (v4si)ziplo64(x6, x7);
707 | 			ctx->c_v[15] = (v4si)ziphi64(x6, x7);
708 | 			
709 | 			// transpose the half matrix staying in registers
710 | 			__m128i x8 = ziplo32((__m128i)ctx->c_v[0], (__m128i)ctx->c_v[2]);
711 | 			__m128i x9 = ziplo32((__m128i)ctx->c_v[4], (__m128i)ctx->c_v[6]);
712 | 			__m128i xA = ziphi32((__m128i)ctx->c_v[0], (__m128i)ctx->c_v[2]);
713 | 			__m128i xB = ziphi32((__m128i)ctx->c_v[4], (__m128i)ctx->c_v[6]);
714 | 			__m128i xC = ziplo32(d0, d1);
715 | 			__m128i xD = ziplo32(d2, d3);
716 | 			__m128i xE = ziphi32(d0, d1);
717 | 			__m128i xF = ziphi32(d2, d3);
718 | 			d0 = _mm_add_epi32(ziplo64(x8, x9), set32(32));
719 | 			d1 = ziphi64(x8, x9);
720 | 			d2 = ziplo64(xA, xB);
721 | 			d3 = ziphi64(xA, xB);
722 | 			d4 = ziplo64(xC, xD);
723 | 			d5 = ziphi64(xC, xD);
724 | 			d6 = ziplo64(xE, xF);
725 | 			d7 = ziphi64(xE, xF);
726 | 		}
727 | 		
728 | 		// final residual values
729 | 		__m128i r0 = packs32(_mm_srai_epi32((__m128i)ctx->c_v[0], 6), _mm_srai_epi32(d0, 6));
730 | 		__m128i r1 = packs32(_mm_srai_epi32((__m128i)ctx->c_v[2], 6), _mm_srai_epi32(d1, 6));
731 | 		__m128i r2 = packs32(_mm_srai_epi32((__m128i)ctx->c_v[4], 6), _mm_srai_epi32(d2, 6));
732 | 		__m128i r3 = packs32(_mm_srai_epi32((__m128i)ctx->c_v[6], 6), _mm_srai_epi32(d3, 6));
733 | 		__m128i r4 = packs32(_mm_srai_epi32((__m128i)ctx->c_v[8], 6), _mm_srai_epi32(d4, 6));
734 | 		__m128i r5 = packs32(_mm_srai_epi32((__m128i)ctx->c_v[10], 6), _mm_srai_epi32(d5, 6));
735 | 		__m128i r6 = packs32(_mm_srai_epi32((__m128i)ctx->c_v[12], 6), _mm_srai_epi32(d6, 6));
736 | 		__m128i r7 = packs32(_mm_srai_epi32((__m128i)ctx->c_v[14], 6), _mm_srai_epi32(d7, 6));
737 | 		
738 | 		// addition to values in place, clipping and storage
739 | 		size_t stride = ctx->t.stride[0]; // FIXME 4:4:4
740 | 		size_t stride3 = stride * 3;
741 | 		uint8_t *p = NULL; //ctx->frame + ctx->frame_offsets_x[0/*BlkIdx2i4x4[BlkIdx]*/] + ctx->frame_offsets_y[0/*BlkIdx2i4x4[BlkIdx]*/];
742 | 		uint8_t *q = p + stride * 4;
743 | 		__m128i zero = _mm_setzero_si128();
744 | 		__m128i clip = set16(ctx->t.samples_clip[0]); // FIXME 4:4:4
745 | 		__m128i p0 = *(__m128i *)(p             );
746 | 		__m128i p1 = *(__m128i *)(p + stride    );
747 | 		__m128i p2 = *(__m128i *)(p + stride * 2);
748 | 		__m128i p3 = *(__m128i *)(p + stride3   );
749 | 		__m128i p4 = *(__m128i *)(q             );
750 | 		__m128i p5 = *(__m128i *)(q + stride    );
751 | 		__m128i p6 = *(__m128i *)(q + stride * 2);
752 | 		__m128i p7 = *(__m128i *)(q + stride3   );
753 | 		__m128i u0 = min16(max16(adds16(p0, r0), zero), clip);
754 | 		__m128i u1 = min16(max16(adds16(p1, r1), zero), clip);
755 | 		__m128i u2 = min16(max16(adds16(p2, r2), zero), clip);
756 | 		__m128i u3 = min16(max16(adds16(p3, r3), zero), clip);
757 | 		__m128i u4 = min16(max16(adds16(p4, r4), zero), clip);
758 | 		__m128i u5 = min16(max16(adds16(p5, r5), zero), clip);
759 | 		__m128i u6 = min16(max16(adds16(p6, r6), zero), clip);
760 | 		__m128i u7 = min16(max16(adds16(p7, r7), zero), clip);
761 | 		*(__m128i *)(p             ) = u0;
762 | 		*(__m128i *)(p + stride    ) = u1;
763 | 		*(__m128i *)(p + stride * 2) = u2;
764 | 		*(__m128i *)(p + stride3   ) = u3;
765 | 		*(__m128i *)(q             ) = u4;
766 | 		*(__m128i *)(q + stride    ) = u5;
767 | 		*(__m128i *)(q + stride * 2) = u6;
768 | 		*(__m128i *)(q + stride3   ) = u7;
769 | #endif // __AVX2__
770 | #endif
771 | 


--------------------------------------------------------------------------------
/edge264_sei.c:
--------------------------------------------------------------------------------
  1 | static int parse_buffering_period(Edge264Decoder *dec) {
  2 | 	if (get_ue16(&dec->_gb, 31))
  3 | 		return ENOTSUP;
  4 | 	log_dec(dec, "    nal_hrd_parameters:\n");
  5 | 	for (int i = dec->sps.nal_hrd_cpb_cnt; i--; ) {
  6 | 		int initial_cpb_removal_delay = get_uv(&dec->_gb, dec->sps.initial_cpb_removal_delay_length);
  7 | 		int initial_cpb_removal_delay_offset = get_uv(&dec->_gb, dec->sps.initial_cpb_removal_delay_length);
  8 | 		log_dec(dec, "      - initial_cpb_removal_delay: %u\n"
  9 | 			"        initial_cpb_removal_delay_offset: %u\n",
 10 | 			initial_cpb_removal_delay, initial_cpb_removal_delay_offset);
 11 | 	}
 12 | 	log_dec(dec, "    vcl_hrd_parameters:\n");
 13 | 	for (int i = dec->sps.vcl_hrd_cpb_cnt; i--; ) {
 14 | 		int initial_cpb_removal_delay = get_uv(&dec->_gb, dec->sps.initial_cpb_removal_delay_length);
 15 | 		int initial_cpb_removal_delay_offset = get_uv(&dec->_gb, dec->sps.initial_cpb_removal_delay_length);
 16 | 		log_dec(dec, "      - initial_cpb_removal_delay: %u\n"
 17 | 			"        initial_cpb_removal_delay_offset: %u\n",
 18 | 			initial_cpb_removal_delay, initial_cpb_removal_delay_offset);
 19 | 	}
 20 | 	return 0;
 21 | }
 22 | 
 23 | 
 24 | 
 25 | static int parse_pic_timing(Edge264Decoder *dec) {
 26 | 	static const char * const pic_struct_names[16] = {
 27 | 		"progressive frame", "top field", "bottom field", "top then bottom",
 28 | 		"bottom then top", "top then bottom then top",
 29 | 		"bottom then top then bottom", "frame doubling", "frame tripling",
 30 | 		[9 ... 15] = "Unknown"};
 31 | 	static const char * const ct_type_names[4] = {
 32 | 		"progressive", "interlaced", [2 ... 3] = "unknown"};
 33 | 	
 34 | 	if (dec->sps.nal_hrd_cpb_cnt | dec->sps.vcl_hrd_cpb_cnt) {
 35 | 		int cpb_removal_delay = get_uv(&dec->_gb, dec->sps.cpb_removal_delay_length);
 36 | 		int dpb_output_delay = get_uv(&dec->_gb, dec->sps.dpb_output_delay_length);
 37 | 		log_dec(dec, "    cpb_removal_delay: %u\n"
 38 | 			"    dpb_output_delay: %u\n",
 39 | 			cpb_removal_delay, dpb_output_delay);
 40 | 	}
 41 | 	if (dec->sps.pic_struct_present_flag) {
 42 | 		int pic_struct = get_uv(&dec->_gb, 4);
 43 | 		int NumClockTS = 0x3be95 >> (pic_struct * 2) & 3;
 44 | 		log_dec(dec, "    pic_struct: %s (%u)\n",
 45 | 			pic_struct_names[pic_struct], pic_struct);
 46 | 		log_dec(dec, "    clock_timestamps:\n");
 47 | 		while (NumClockTS--) {
 48 | 			if (!get_u1(&dec->_gb)) // clock_timestamp_flag
 49 | 				continue;
 50 | 			unsigned u = get_uv(&dec->_gb, 19);
 51 | 			if (u & 1 << 10) {
 52 | 				unsigned v = get_uv(&dec->_gb, 17);
 53 | 				dec->sS = v >> 11;
 54 | 				dec->mM = v >> 5 & 0x3f;
 55 | 				dec->hH = v & 0x1f;
 56 | 			} else if (get_u1(&dec->_gb)) { // seconds_flag
 57 | 				unsigned w = get_uv(&dec->_gb, 7);
 58 | 				dec->sS = w >> 1;
 59 | 				if (w & 1) {
 60 | 					unsigned x = get_uv(&dec->_gb, 7);
 61 | 					dec->mM = x >> 1;
 62 | 					if (x & 1)
 63 | 						dec->hH = get_uv(&dec->_gb, 5);
 64 | 				}
 65 | 			}
 66 | 			int tOffset = 0;
 67 | 			if (dec->sps.time_offset_length)
 68 | 				tOffset = get_uv(&dec->_gb, dec->sps.time_offset_length);
 69 | 			log_dec(dec, "      - scan_type: %s (%u)\n"
 70 | 				"        discontinuity_flag: %u\n"
 71 | 				"        clockTimestamp: \"%02u:%02u:%02u+%u/%u\"\n",
 72 | 				ct_type_names[u >> 17], u >> 17,
 73 | 				u >> 9 & 1,
 74 | 				dec->hH, dec->mM, dec->sS, (u & 0xff) * (dec->sps.num_units_in_tick * (1 + (u >> 16 & 1))) + tOffset, dec->sps.time_scale);
 75 | 		}
 76 | 	}
 77 | 	return 0;
 78 | }
 79 | 
 80 | 
 81 | 
 82 | static int parse_pan_scan_rect(Edge264Decoder *dec) {
 83 | 	int pan_scan_rect_id = get_ue32(&dec->_gb, 4294967294);
 84 | 	int pan_scan_rect_cancel_flag = get_u1(&dec->_gb);
 85 | 	log_dec(dec, "    pan_scan_rect_id: %u\n"
 86 | 		"    pan_scan_rect_cancel_flag: %u\n",
 87 | 		pan_scan_rect_id, pan_scan_rect_cancel_flag);
 88 | 	if (!pan_scan_rect_cancel_flag) {
 89 | 		log_dec(dec, "    fields:\n");
 90 | 		for (int i = get_ue16(&dec->_gb, 2) + 1; i--; ) {
 91 | 			int pan_scan_rect_left_offset = get_se32(&dec->_gb, -2147483647, 2147483647);
 92 | 			int pan_scan_rect_right_offset = get_se32(&dec->_gb, -2147483647, 2147483647);
 93 | 			int pan_scan_rect_top_offset = get_se32(&dec->_gb, -2147483647, 2147483647);
 94 | 			int pan_scan_rect_bottom_offset = get_se32(&dec->_gb, -2147483647, 2147483647);
 95 | 			log_dec(dec, "      - {left: %d, right: %d, top: %d, bottom: %d}\n",
 96 | 				pan_scan_rect_left_offset, pan_scan_rect_right_offset, pan_scan_rect_top_offset, pan_scan_rect_bottom_offset);
 97 | 		}
 98 | 		int pan_scan_rect_repetition_period = get_ue16(&dec->_gb, 16384);
 99 | 		log_dec(dec, "    pan_scan_rect_repetition_period: %u\n",
100 | 			pan_scan_rect_repetition_period);
101 | 	}
102 | 	return 0;
103 | }
104 | 
105 | 
106 | 
107 | typedef int (*SEI_Parser)(Edge264Decoder *dec);
108 | int ADD_VARIANT(parse_sei)(Edge264Decoder *dec, int non_blocking, void(*free_cb)(void*,int), void *free_arg)
109 | {
110 | 	static const char * const payloadType_names[206] = {
111 | 		[0 ... 205] = "Unknown",
112 | 		[0] = "Buffering period",
113 | 		[1] = "Picture timing",
114 | 		[2] = "Pan-scan rectangle",
115 | 	};
116 | 	static const SEI_Parser parse_sei_message[206] = {
117 | 		[0] = parse_buffering_period,
118 | 		[1] = parse_pic_timing,
119 | 		[2] = parse_pan_scan_rect,
120 | 	};
121 | 	
122 | 	refill(&dec->_gb, 0);
123 | 	log_dec(dec, "  sei_messages:\n");
124 | 	if (print_dec(dec, dec->log_header_arg))
125 | 		return ENOTSUP;
126 | 	int ret = 0;
127 | 	do {
128 | 		int byte, payloadType = 0, payloadSize = 0;
129 | 		do {
130 | 			byte = get_uv(&dec->_gb, 8);
131 | 			payloadType += byte;
132 | 		} while (byte == 255);
133 | 		do {
134 | 			byte = get_uv(&dec->_gb, 8);
135 | 			payloadSize += byte;
136 | 		} while (byte == 255);
137 | 		log_dec(dec, "  - payloadType: %s (%u)\n",
138 | 			payloadType_names[payloadType], payloadType);
139 | 		Edge264GetBits start = dec->_gb;
140 | 		int res = ENOTSUP;
141 | 		if (payloadType <= 205 && parse_sei_message[payloadType])
142 | 			res = parse_sei_message[payloadType](dec);
143 | 		if (res) {
144 | 			//ret = ENOTSUP;
145 | 			dec->_gb = start;
146 | 			while (payloadSize-- > 0)
147 | 				get_uv(&dec->_gb, 8);
148 | 		} else {
149 | 			int skip = (SIZE_BIT - 1 - ctz(dec->_gb.lsb_cache)) & 7;
150 | 			if (skip)
151 | 				get_uv(&dec->_gb, skip);
152 | 		}
153 | 		if (print_dec(dec, dec->log_sei_arg))
154 | 			return ENOTSUP;
155 | 	} while (dec->_gb.msb_cache << 1 || (dec->_gb.lsb_cache & (dec->_gb.lsb_cache - 1)) || (intptr_t)(dec->_gb.end - dec->_gb.CPB) > 0);
156 | 	return (dec->_gb.msb_cache & (size_t)1 << (SIZE_BIT - 1)) ? ret : EBADMSG;
157 | }
158 | 


--------------------------------------------------------------------------------
/edge264_test.c:
--------------------------------------------------------------------------------
  1 | #include <dirent.h>
  2 | #include <stdio.h>
  3 | #include <stdlib.h>
  4 | #include <string.h>
  5 | #include <time.h>
  6 | #ifdef _WIN32
  7 | 	#include <windows.h>
  8 | 	#include <psapi.h>
  9 | 	#define dlsym (void*)GetProcAddress
 10 | 	#define dlclose FreeLibrary
 11 | #else
 12 | 	#include <dlfcn.h>
 13 | 	#include <fcntl.h>
 14 | 	#include <sys/mman.h>
 15 | 	#include <sys/resource.h>
 16 | 	#include <sys/stat.h>
 17 | 	#include <sys/types.h>
 18 | 	#include <unistd.h>
 19 | #endif
 20 | #include "edge264.h"
 21 | 
 22 | 
 23 | 
 24 | /**
 25 |  * Using SDL2 without dependency on header file and development library
 26 |  */
 27 | #define SDL_INIT_VIDEO 0x20
 28 | #define SDL_INIT_EVENTS 0x4000
 29 | #define SDL_WINDOWPOS_CENTERED 0x2FFF0000
 30 | #define SDL_WINDOW_SHOWN 0x4
 31 | #define SDL_WINDOW_BORDERLESS 0x10
 32 | #define SDL_RENDERER_ACCELERATED 0x2
 33 | #define SDL_RENDERER_PRESENTVSYNC 0x4
 34 | #define SDL_PIXELFORMAT_IYUV 0x56555949
 35 | #define SDL_TEXTUREACCESS_STREAMING 1
 36 | #define SDL_BLENDMODE_BLEND 0x1
 37 | #define SDL_QUIT 0x100
 38 | #define SDL_KEYDOWN 0x300
 39 | #define SDLK_ESCAPE 27
 40 | typedef struct SDL_Window SDL_Window;
 41 | typedef struct SDL_Renderer SDL_Renderer;
 42 | typedef struct SDL_Texture SDL_Texture;
 43 | typedef union SDL_Event {
 44 | 	uint32_t type;
 45 | 	struct {
 46 | 		uint32_t _[4];
 47 | 		struct {
 48 | 			int _;
 49 | 			int32_t sym;
 50 | 		} keysym;
 51 | 	} key;
 52 | 	uint8_t padding[sizeof(void *) <= 8 ? 56 : sizeof(void *) == 16 ? 64 : 3 * sizeof(void *)];
 53 | } SDL_Event;
 54 | int (*SDL_Init)(uint32_t);
 55 | SDL_Window *(*SDL_CreateWindow)(const char *, int, int, int, int, uint32_t);
 56 | SDL_Renderer *(*SDL_CreateRenderer)(SDL_Window *, int, uint32_t);
 57 | void (*SDL_SetWindowSize)(SDL_Window *, int, int);
 58 | void (*SDL_SetWindowPosition)(SDL_Window *, int, int);
 59 | void (*SDL_DestroyTexture)(SDL_Texture *);
 60 | SDL_Texture *(*SDL_CreateTexture)(SDL_Renderer *, uint32_t, int, int, int);
 61 | int (*SDL_SetTextureBlendMode)(SDL_Texture *, int);
 62 | int (*SDL_SetTextureAlphaMod)(SDL_Texture *, uint8_t);
 63 | int (*SDL_UpdateYUVTexture)(SDL_Texture *, void *, const uint8_t *, int, const uint8_t *, int, const uint8_t *, int);
 64 | int (*SDL_RenderClear)(SDL_Renderer *);
 65 | int (*SDL_RenderCopy)(SDL_Renderer *, SDL_Texture *, void *, void *);
 66 | void (*SDL_RenderPresent)(SDL_Renderer *);
 67 | int (*SDL_PollEvent)(SDL_Event *);
 68 | void (*SDL_DestroyTexture)(SDL_Texture *);
 69 | void (*SDL_DestroyRenderer)(SDL_Renderer *);
 70 | void (*SDL_DestroyWindow)(SDL_Window *);
 71 | void (*SDL_Quit)(void);
 72 | static void *sdl2;
 73 | static int load_SDL2(void) {
 74 | 	const char *lib, *func;
 75 | 	#ifdef _WIN32
 76 | 		sdl2 = LoadLibraryA(lib = "SDL2.dll");
 77 | 	#else
 78 | 		sdl2 = dlopen(lib = "/Library/Frameworks/SDL2.framework/SDL2", RTLD_NOW | RTLD_GLOBAL) ?:
 79 | 			dlopen(lib = "SDL2.so", RTLD_NOW | RTLD_GLOBAL) ?:
 80 | 			dlopen(lib = "libSDL2.so", RTLD_NOW | RTLD_GLOBAL);
 81 | 	#endif
 82 | 	if (!sdl2) {
 83 | 		printf("SDL2 is needed for display but not found, please download the library file at https://www.libsdl.org/ and place it in the current folder\n");
 84 | 		return 1;
 85 | 	}
 86 | 	if (!(SDL_Init = dlsym(sdl2, func = "SDL_Init")) ||
 87 | 	    !(SDL_CreateWindow = dlsym(sdl2, func = "SDL_CreateWindow")) ||
 88 | 	    !(SDL_CreateRenderer = dlsym(sdl2, func = "SDL_CreateRenderer")) ||
 89 | 	    !(SDL_SetWindowSize = dlsym(sdl2, func = "SDL_SetWindowSize")) ||
 90 | 	    !(SDL_SetWindowPosition = dlsym(sdl2, func = "SDL_SetWindowPosition")) ||
 91 | 	    !(SDL_DestroyTexture = dlsym(sdl2, func = "SDL_DestroyTexture")) ||
 92 | 	    !(SDL_CreateTexture = dlsym(sdl2, func = "SDL_CreateTexture")) ||
 93 | 	    !(SDL_SetTextureBlendMode = dlsym(sdl2, func = "SDL_SetTextureBlendMode")) ||
 94 | 	    !(SDL_SetTextureAlphaMod = dlsym(sdl2, func = "SDL_SetTextureAlphaMod")) ||
 95 | 	    !(SDL_UpdateYUVTexture = dlsym(sdl2, func = "SDL_UpdateYUVTexture")) ||
 96 | 	    !(SDL_RenderClear = dlsym(sdl2, func = "SDL_RenderClear")) ||
 97 | 	    !(SDL_RenderCopy = dlsym(sdl2, func = "SDL_RenderCopy")) ||
 98 | 	    !(SDL_RenderPresent = dlsym(sdl2, func = "SDL_RenderPresent")) ||
 99 | 	    !(SDL_PollEvent = dlsym(sdl2, func = "SDL_PollEvent")) ||
100 | 	    !(SDL_DestroyTexture = dlsym(sdl2, func = "SDL_DestroyTexture")) ||
101 | 	    !(SDL_DestroyRenderer = dlsym(sdl2, func = "SDL_DestroyRenderer")) ||
102 | 	    !(SDL_DestroyWindow = dlsym(sdl2, func = "SDL_DestroyWindow")) ||
103 | 	    !(SDL_Quit = dlsym(sdl2, func = "SDL_Quit"))) {
104 | 		printf("Missing function %s in %s\n", func, lib);
105 | 		return 1;
106 | 	}
107 | 	return 0;
108 | }
109 | 
110 | 
111 | 
112 | /**
113 |  * Static variables and helper functions
114 |  */
115 | #define RESET  "\e[0m"
116 | #define BOLD   "\e[1m"
117 | #define RED    "\e[31m"
118 | #define GREEN  "\e[32m"
119 | #define YELLOW "\e[33m"
120 | #define BLUE   "\e[34m"
121 | 
122 | static int display = 0;
123 | static int print_failed = 0;
124 | static int print_passed = 0;
125 | static int print_unsupported = 0;
126 | static int enable_yuv = 1;
127 | static const char *moveup = "";
128 | FILE *trace_file = NULL;
129 | static Edge264Decoder *d;
130 | static Edge264Frame out;
131 | static const uint8_t *conf[2];
132 | static SDL_Window *window;
133 | static SDL_Renderer *renderer;
134 | static SDL_Texture *texture0, *texture1;
135 | static int width, height, mvc_display;
136 | static int count_pass, count_unsup, count_fail, count_flag;
137 | 
138 | #ifndef min
139 | 	static inline int min(int a, int b) { return (a < b) ? a : b; }
140 | 	static inline int max(int a, int b) { return (a > b) ? a : b; }
141 | #endif
142 | 
143 | static int flt(const struct dirent *a) {
144 | 	char *ext = strrchr(a->d_name, '.');
145 | 	return ext != NULL && strcmp(ext, ".264") == 0;
146 | }
147 | static int cmp(const struct dirent **a, const struct dirent **b) {
148 | 	return -strcasecmp((*a)->d_name, (*b)->d_name);
149 | }
150 | 
151 | 
152 | 
153 | static int draw_frame()
154 | {
155 | 	// create or resize the window if necessary
156 | 	int has_second_view = out.samples_mvc[0] != NULL;
157 | 	if (width != out.width_Y || height != out.height_Y || mvc_display != has_second_view) {
158 | 		width = out.width_Y;
159 | 		height = out.height_Y;
160 | 		mvc_display = has_second_view;
161 | 		if (window == NULL) {
162 | 			SDL_Init(SDL_INIT_VIDEO | SDL_INIT_EVENTS);
163 | 			window = SDL_CreateWindow("edge264_test", SDL_WINDOWPOS_CENTERED, SDL_WINDOWPOS_CENTERED, width, height, SDL_WINDOW_SHOWN | SDL_WINDOW_BORDERLESS);
164 | 			renderer = SDL_CreateRenderer(window, -1, SDL_RENDERER_ACCELERATED | SDL_RENDERER_PRESENTVSYNC);
165 | 		} else {
166 | 			SDL_SetWindowSize(window, width, height);
167 | 			SDL_SetWindowPosition(window, SDL_WINDOWPOS_CENTERED, SDL_WINDOWPOS_CENTERED);
168 | 			SDL_DestroyTexture(texture0);
169 | 			SDL_DestroyTexture(texture1);
170 | 		}
171 | 		texture0 = SDL_CreateTexture(renderer, SDL_PIXELFORMAT_IYUV, SDL_TEXTUREACCESS_STREAMING, width, height);
172 | 		texture1 = SDL_CreateTexture(renderer, SDL_PIXELFORMAT_IYUV, SDL_TEXTUREACCESS_STREAMING, width, height);
173 | 		SDL_SetTextureBlendMode(texture1, SDL_BLENDMODE_BLEND);
174 | 		SDL_SetTextureAlphaMod(texture1, 128);
175 | 	}
176 | 	
177 | 	// upload the image to a texture and render!
178 | 	SDL_UpdateYUVTexture(texture0, NULL, out.samples[0], out.stride_Y, out.samples[1], out.stride_C, out.samples[2], out.stride_C);
179 | 	SDL_RenderClear(renderer);
180 | 	SDL_RenderCopy(renderer, texture0, NULL, NULL);
181 | 	if (mvc_display) {
182 | 		SDL_UpdateYUVTexture(texture1, NULL, out.samples_mvc[0], out.stride_Y, out.samples_mvc[1], out.stride_C, out.samples_mvc[2], out.stride_C);
183 | 		SDL_RenderCopy(renderer, texture1, NULL, NULL);
184 | 	}
185 | 	SDL_RenderPresent(renderer);
186 | 	
187 | 	// Is user closing the window?
188 | 	SDL_Event event;
189 | 	while (SDL_PollEvent(&event)) {
190 | 		if (event.type == SDL_QUIT ||
191 | 			(event.type == SDL_KEYDOWN && event.key.keysym.sym == SDLK_ESCAPE)) {
192 | 			return 1;
193 | 		}
194 | 	}
195 | 	return 0;
196 | }
197 | 
198 | 
199 | 
200 | static int check_frame()
201 | {
202 | 	// check that the number of returned views is as expected
203 | 	if ((out.samples_mvc[0] != NULL) != (conf[1] != NULL)) {
204 | 		printf("Number of returned views (%d) does not match number of YUV files found (%d)\n", (out.samples_mvc[0] != NULL) + 1, (conf[1] != NULL) + 1);
205 | 		moveup = "";
206 | 		return -2;
207 | 	}
208 | 	
209 | 	// check that each macroblock matches the conformance buffer
210 | 	int poc = out.TopFieldOrderCnt;
211 | 	int cropt = out.frame_crop_offsets[0];
212 | 	int cropr = out.frame_crop_offsets[1];
213 | 	int cropb = out.frame_crop_offsets[2];
214 | 	int cropl = out.frame_crop_offsets[3];
215 | 	int pic_width_in_mbs = (cropl + out.width_Y + cropr) >> 4;
216 | 	int pic_height_in_mbs = (cropt + out.height_Y + cropb) >> 4;
217 | 	for (int view = 0; view < 2 && conf[view] != NULL; view += 1) {
218 | 		for (int row = 0; row < pic_width_in_mbs; row += 1) {
219 | 			for (int col = 0; col < pic_height_in_mbs; col += 1) {
220 | 				for (int iYCbCr = 0; iYCbCr < 3; iYCbCr++) {
221 | 					int stride = (iYCbCr == 0) ? out.stride_Y : out.stride_C;
222 | 					int depth = (iYCbCr == 0) ? out.pixel_depth_Y : out.pixel_depth_C;
223 | 					int sh_width = (iYCbCr > 0 && out.width_C < out.width_Y);
224 | 					int sh_height = (iYCbCr > 0 && out.height_C < out.height_Y);
225 | 					const uint8_t *p = (view ? out.samples_mvc : out.samples)[iYCbCr];
226 | 					const uint8_t *q = conf[view] +
227 | 						(iYCbCr > 0) * (out.width_Y << out.pixel_depth_Y) * out.height_Y +
228 | 						(iYCbCr > 1) * (out.width_C << out.pixel_depth_C) * out.height_C;
229 | 					int xl = max(col * 16 - cropl, 0) >> sh_width;
230 | 					int xr = min(col * 16 - cropl + 16, out.width_Y) >> sh_width;
231 | 					int invalid = 0;
232 | 					for (int y = max(row * 16 - cropt, 0) >> sh_height; xl < xr && y < min(row * 16 - cropt + 16, out.height_Y) >> sh_height; y++)
233 | 						invalid |= memcmp(p + y * stride + (xl << depth), q + y * (out.width_Y >> sh_width << depth) + (xl << depth), (xr - xl) << depth);
234 | 					if (invalid) {
235 | 						printf("Erroneous macroblock (poc %d, view %d, row %d, column %d, %s plane):\n",
236 | 							poc, view, row, col, (iYCbCr == 0) ? "Luma" : (iYCbCr == 1) ? "Cb" : "Cr");
237 | 						for (int y = (row * 16 - cropt) >> sh_height; y < (row * 16 - cropt + 16) >> sh_height; y++) {
238 | 							for (int x = (col * 16 - cropl) >> sh_width; x < (col * 16 - cropl + 16) >> sh_width; x++) {
239 | 								// FIXME 16 bit
240 | 								printf(y < 0 || y >= out.height_Y >> sh_height || x < 0 || x >= out.width_Y >> sh_width ? "    " :
241 | 									p[y * stride + x] == q[y * (out.width_Y >> sh_width) + x] ? " %3d" :
242 | 									RED " %3d" RESET, p[y * stride + x]);
243 | 							}
244 | 							printf("\n");
245 | 						}
246 | 						printf("Expected macroblock:\n");
247 | 						for (int y = (row * 16 - cropt) >> sh_height; y < (row * 16 - cropt + 16) >> sh_height; y++) {
248 | 							for (int x = (col * 16 - cropl) >> sh_width; x < (col * 16 - cropl + 16) >> sh_width; x++) {
249 | 								printf(y < 0 || y >= out.height_Y >> sh_height || x < 0 || x >= out.width_Y >> sh_width ? "    " :
250 | 									" %3d", q[y * (out.width_Y >> sh_width) + x]);
251 | 							}
252 | 							printf("\n");
253 | 						}
254 | 						moveup = "";
255 | 						return -2;
256 | 					}
257 | 				}
258 | 			}
259 | 		}
260 | 		conf[view] += (out.width_Y << out.pixel_depth_Y) * out.height_Y + (out.width_C << out.pixel_depth_C) * out.height_C * 2;
261 | 	}
262 | 	return 0;
263 | }
264 | 
265 | 
266 | 
267 | static int decode_file(const char *name0)
268 | {
269 | 	// process file names
270 | 	if (trace_file)
271 | 		fprintf(trace_file, "\n--- # %s\n", name0);
272 | 	int len = strrchr(name0, '.') - name0;
273 | 	if (strcmp(name0 + len + 1, "264") != 0)
274 | 		return 0;
275 | 	char name1[len + 5], name2[len + 7];
276 | 	snprintf(name1, sizeof(name1), "%.*s.yuv", len, name0);
277 | 	snprintf(name2, sizeof(name2), "%.*s.1.yuv", len, name0);
278 | 	
279 | 	// open and memory map all input files
280 | 	int quit = 0;
281 | 	conf[0] = conf[1] = NULL;
282 | 	#ifdef _WIN32
283 | 		HANDLE f0 = NULL, f1 = NULL, f2 = NULL, m0 = NULL, m1 = NULL, m2 = NULL;
284 | 		void *v0 = NULL, *v1 = NULL, *v2 = NULL;
285 | 		if ((f0 = CreateFileA(name0, GENERIC_READ, FILE_SHARE_READ, NULL, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL)) == INVALID_HANDLE_VALUE ||
286 | 			(m0 = CreateFileMappingA(f0, NULL, PAGE_READONLY, 0, 0, NULL)) == NULL ||
287 | 			(v0 = MapViewOfFile(m0, FILE_MAP_READ, 0, 0, 0)) == NULL) {
288 | 			printf("Error opening file %s: %lu\n", name0, GetLastError());
289 | 			quit = 1;
290 | 		}
291 | 		if (enable_yuv) {
292 | 			if ((f1 = CreateFileA(name1, GENERIC_READ, FILE_SHARE_READ, NULL, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL)) == INVALID_HANDLE_VALUE) {
293 | 				printf("%s%s not found\n", moveup, name1);
294 | 				moveup = "";
295 | 			} else if ((m1 = CreateFileMappingA(f1, NULL, PAGE_READONLY, 0, 0, NULL)) == NULL ||
296 | 				(conf[0] = v1 = MapViewOfFile(m1, FILE_MAP_READ, 0, 0, 0)) == NULL) {
297 | 				printf("Error opening file %s: %lu\n", name1, GetLastError());
298 | 				quit = 1;
299 | 			}
300 | 			f2 = CreateFileA(name2, GENERIC_READ, FILE_SHARE_READ, NULL, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL);
301 | 			if (f2 != INVALID_HANDLE_VALUE &&
302 | 				((m2 = CreateFileMappingA(f2, NULL, PAGE_READONLY, 0, 0, NULL)) == NULL ||
303 | 				(conf[1] = v2 = MapViewOfFile(m2, FILE_MAP_READ, 0, 0, 0)) == NULL)) {
304 | 				printf("Error opening file %s: %lu\n", name2, GetLastError());
305 | 				quit = 1;
306 | 			}
307 | 		}
308 | 		const uint8_t *nal = v0;
309 | 		const uint8_t *end0 = v0 + GetFileSize(f0, NULL);
310 | 		const uint8_t *end1 = v1 + GetFileSize(f1, NULL);
311 | 	#else
312 | 		int fd0 = -1, fd1 = -1, fd2 = -1;
313 | 		struct stat st0, st1, st2;
314 | 		uint8_t *mm0 = MAP_FAILED, *mm1 = MAP_FAILED, *mm2 = MAP_FAILED;
315 | 		if ((fd0 = open(name0, O_RDONLY)) < 0 ||
316 | 			fstat(fd0, &st0) < 0 ||
317 | 			(mm0 = mmap(NULL, st0.st_size, PROT_READ, MAP_SHARED, fd0, 0)) == MAP_FAILED) {
318 | 			perror(name0);
319 | 			quit = 1;
320 | 		}
321 | 		if (enable_yuv) {
322 | 			if ((fd1 = open(name1, O_RDONLY)) < 0) {
323 | 				printf("%s%s not found\n", moveup, name1);
324 | 				moveup = "";
325 | 			} else if (fstat(fd1, &st1) < 0 ||
326 | 				(conf[0] = mm1 = mmap(NULL, st1.st_size, PROT_READ, MAP_SHARED, fd1, 0)) == MAP_FAILED) {
327 | 				perror(name1);
328 | 				quit = 1;
329 | 			}
330 | 			fd2 = open(name2, O_RDONLY);
331 | 			if (fd2 >= 0 && (fstat(fd2, &st2) < 0 ||
332 | 				(conf[1] = mm2 = mmap(NULL, st2.st_size, PROT_READ, MAP_SHARED, fd2, 0)) == MAP_FAILED)) {
333 | 				perror(name2);
334 | 				quit = 1;
335 | 			}
336 | 		}
337 | 		const uint8_t *nal = mm0;
338 | 		const uint8_t *end0 = mm0 + st0.st_size;
339 | 		const uint8_t *end1 = mm1 + st1.st_size;
340 | 	#endif
341 | 	
342 | 	// print the success counts
343 | 	if (!quit) {
344 | 		if (count_flag > 0)
345 | 			printf("%s%d " GREEN "PASS" RESET ", %d " YELLOW "UNSUPPORTED" RESET ", %d " RED "FAIL" RESET ", %d " BLUE "FLAGGED" RESET " (%s)\n", moveup, count_pass, count_unsup, count_fail, count_flag, name0);
346 | 		else
347 | 			printf("%s%d " GREEN "PASS" RESET ", %d " YELLOW "UNSUPPORTED" RESET ", %d " RED "FAIL" RESET " (%s)\n", moveup, count_pass, count_unsup, count_fail, name0);
348 | 		moveup = "\e[A\e[K";
349 | 		
350 | 		// decode the entire file and FAIL on any error
351 | 		nal += 3 + (nal[2] == 0); // skip the [0]001 delimiter
352 | 		int res;
353 | 		do {
354 | 			res = edge264_decode_NAL(d, nal, end0, 0, NULL, NULL, &nal);
355 | 			while (!edge264_get_frame(d, &out, 0)) {
356 | 				if (conf[0] != NULL && check_frame()) {
357 | 					res = EBADMSG;
358 | 					break;
359 | 				} else if (display && draw_frame()) {
360 | 					res = ENODATA;
361 | 					quit = 1;
362 | 					break;
363 | 				}
364 | 			}
365 | 		} while (res == 0 || res == ENOBUFS);
366 | 		edge264_flush(d);
367 | 		if (res == ENOBUFS || (res == ENODATA && conf[0] != NULL && conf[0] != end1))
368 | 			res = EBADMSG;
369 | 		// FIXME interrupt all threads before closing the files!
370 | 		
371 | 		// print the file that was decoded
372 | 		count_pass += res == ENODATA;
373 | 		count_unsup += res == ENOTSUP;
374 | 		count_fail += res == EBADMSG;
375 | 		count_flag += res == ESRCH;
376 | 		if (res == ENODATA && print_passed) {
377 | 			printf("%s%s: " GREEN "PASS" RESET "\n", moveup, name0);
378 | 			moveup = "";
379 | 		} else if (res == ENOTSUP && print_unsupported) {
380 | 			printf("%s%s: " YELLOW "UNSUPPORTED" RESET "\n", moveup, name0);
381 | 			moveup = "";
382 | 		} else if (res == EBADMSG && print_failed) {
383 | 			printf("%s%s: " RED "FAIL" RESET "\n", moveup, name0);
384 | 			moveup = "";
385 | 		} else if (res == ESRCH) {
386 | 			printf("%s%s: " BLUE "FLAGGED" RESET "\n", moveup, name0);
387 | 			moveup = "";
388 | 		}
389 | 	}
390 | 	
391 | 	// close everything
392 | 	#ifdef _WIN32
393 | 		UnmapViewOfFile(v0);
394 | 		UnmapViewOfFile(v1);
395 | 		UnmapViewOfFile(v2);
396 | 		CloseHandle(m0);
397 | 		CloseHandle(m1);
398 | 		CloseHandle(m2);
399 | 		CloseHandle(f0);
400 | 		CloseHandle(f1);
401 | 		CloseHandle(f2);
402 | 	#else
403 | 		munmap(mm0, st0.st_size);
404 | 		munmap(mm1, st1.st_size);
405 | 		munmap(mm2, st2.st_size);
406 | 		close(fd0);
407 | 		close(fd1);
408 | 		close(fd2);
409 | 	#endif
410 | 	return quit;
411 | }
412 | 
413 | 
414 | 
415 | int main(int argc, char *argv[])
416 | {
417 | 	// read command-line options
418 | 	const char *file_name = "conformance";
419 | 	int benchmark = 0;
420 | 	int help = 0;
421 | 	int n_threads = -1;
422 | 	int trace = 0;
423 | 	for (int i = 1; i < argc; i++) {
424 | 		if (argv[i][0] != '-') {
425 | 			file_name = argv[i];
426 | 		} else for (int j = 1; argv[i][j]; j++) {
427 | 			switch (argv[i][j]) {
428 | 				case 'b': benchmark = 1; break;
429 | 				case 'd': display = 1; break;
430 | 				case 'f': print_failed = 1; break;
431 | 				case 'p': print_passed = 1; break;
432 | 				case 's': n_threads = 0; break;
433 | 				case 'u': print_unsupported = 1; break;
434 | 				case 'v': trace = 1; break;
435 | 				case 'V': trace = 2; n_threads = 0; break;
436 | 				case 'y': enable_yuv = 0; break;
437 | 				default: help = 1; break;
438 | 			}
439 | 		}
440 | 	}
441 | 	if (trace) {
442 | 		trace_file = fopen("trace.yaml", "w");
443 | 		setvbuf(trace_file, NULL, _IONBF, BUFSIZ);
444 | 	}
445 | 	
446 | 	// load SDL2 if requested
447 | 	if (display && load_SDL2())
448 | 		return 1;
449 | 	
450 | 	// print help if any argument was unknown
451 | 	if (help) {
452 | 		printf("Usage: " BOLD "%s [video.264|directory] [-hbdfpsuvVy]" RESET "\n"
453 | 			"Decodes a video or all videos inside a directory (./conformance by default),\n"
454 | 			"comparing their outputs with inferred YUV pairs (.yuv and .1.yuv extensions).\n"
455 | 			"-h\tprint this help and exit\n"
456 | 			"-b\tbenchmark decoding time and memory usage\n"
457 | 			"-d\tenable display of the videos (requires SDL2)\n"
458 | 			"-f\tprint names of failed files in directory\n"
459 | 			"-p\tprint names of passed files in directory\n"
460 | 			"-s\tsingle-threaded operation\n"
461 | 			"-u\tprint names of unsupported files in directory\n"
462 | 			"-v\tenable output of headers to file trace.yaml (large)\n"
463 | 			"-V\tadd output of macroblocks to trace.yaml (very large, implies -vs)\n"
464 | 			"-y\tdisable comparison against YUV pairs\n"
465 | 			, argv[0]);
466 | 		return 0;
467 | 	}
468 | 	
469 | 	struct timespec t0, t1;
470 | 	clock_gettime(CLOCK_MONOTONIC, &t0);
471 | 	d = edge264_alloc(n_threads, (void(*)(const char*, void*))fputs, trace_file, trace_file, (trace > 1) ? trace_file : NULL);
472 | 	
473 | 	// check if input is a directory by trying to move into it
474 | 	if (chdir(file_name) < 0) {
475 | 		decode_file(file_name);
476 | 	} else {
477 | 		#ifdef _WIN32
478 | 			DIR *dp;
479 | 			struct dirent *ep;
480 | 			dp = opendir(".");
481 | 			while ((ep = readdir(dp)) && !decode_file(ep->d_name));
482 | 			closedir(dp);
483 | 		#else
484 | 			struct dirent **entries;
485 | 			int n = scandir(".", &entries, flt, cmp);
486 | 			while (--n >= 0 && !decode_file(entries[n]->d_name))
487 | 				free(entries[n]);
488 | 			free(entries);
489 | 		#endif
490 | 	}
491 | 	
492 | 	if (count_flag > 0)
493 | 		printf("%s%d " GREEN "PASS" RESET ", %d " YELLOW "UNSUPPORTED" RESET ", %d " RED "FAIL" RESET ", %d " BLUE "FLAGGED" RESET "\n", moveup, count_pass, count_unsup, count_fail, count_flag);
494 | 	else
495 | 		printf("%s%d " GREEN "PASS" RESET ", %d " YELLOW "UNSUPPORTED" RESET ", %d " RED "FAIL" RESET "\n", moveup, count_pass, count_unsup, count_fail);
496 | 	edge264_free(&d);
497 | 	
498 | 	// close SDL if enabled
499 | 	if (display) {
500 | 		SDL_DestroyTexture(texture0);
501 | 		SDL_DestroyTexture(texture1);
502 | 		SDL_DestroyRenderer(renderer);
503 | 		SDL_DestroyWindow(window);
504 | 		SDL_Quit();
505 | 		dlclose(sdl2);
506 | 	}
507 | 	
508 | 	// closing information
509 | 	if (benchmark) {
510 | 		clock_gettime(CLOCK_MONOTONIC, &t1);
511 | 		int64_t time_msec = (int64_t)(t1.tv_sec - t0.tv_sec) * 1000 + (t1.tv_nsec - t0.tv_nsec) / 1000000;
512 | 		#ifdef _WIN32
513 | 			HANDLE p = GetCurrentProcess();
514 | 			FILETIME c, e, k, u;
515 | 			GetProcessTimes(p, &c, &e, &k, &u);
516 | 			int64_t cpu_msec = ((int64_t)u.dwHighDateTime << 32 | u.dwLowDateTime) / 10000;
517 | 			PROCESS_MEMORY_COUNTERS m;
518 | 			GetProcessMemoryInfo(p, &m, sizeof(m));
519 | 			int mem_kb = m.PeakPagefileUsage / 1000;
520 | 		#else
521 | 			struct rusage rusage;
522 | 			getrusage(RUSAGE_SELF, &rusage);
523 | 			int64_t cpu_msec = (int64_t)rusage.ru_utime.tv_sec * 1000 + rusage.ru_utime.tv_usec / 1000;
524 | 			long mem_kb = rusage.ru_maxrss / 1000;
525 | 		#endif
526 | 		printf("time: %.3lfs\nCPU: %.3lfs\nmemory: %.3lfMB\n", (double)time_msec / 1000, (double)cpu_msec / 1000, (double)mem_kb / 1000);
527 | 	}
528 | 	if (trace_file)
529 | 		fclose(trace_file);
530 | 	return 0;
531 | }
532 | 


--------------------------------------------------------------------------------
/tools/gen_avc.py:
--------------------------------------------------------------------------------
  1 | #!/usr/bin/env python3
  2 | from functools import reduce
  3 | import re
  4 | import sys
  5 | import types
  6 | import yaml
  7 | 
  8 | # make yaml use SimpleNamespace instead of dict
  9 | class SafeConstructor(yaml.loader.SafeConstructor):
 10 | 	def construct_yaml_map(self, node):
 11 | 		data = types.SimpleNamespace()
 12 | 		yield data
 13 | 		value = self.construct_mapping(node)
 14 | 		vars(data).update(value)
 15 | SafeConstructor.add_constructor(u'tag:yaml.org,2002:map', SafeConstructor.construct_yaml_map)
 16 | class SafeLoader(yaml.loader.Reader, yaml.loader.Scanner, yaml.loader.Parser, yaml.loader.Composer, SafeConstructor, yaml.loader.Resolver):
 17 | 	def __init__(self, stream):
 18 | 		yaml.loader.Reader.__init__(self, stream)
 19 | 		yaml.loader.Scanner.__init__(self)
 20 | 		yaml.loader.Parser.__init__(self)
 21 | 		yaml.loader.Composer.__init__(self)
 22 | 		SafeConstructor.__init__(self)
 23 | 		yaml.loader.Resolver.__init__(self)
 24 | 
 25 | 
 26 | 
 27 | def ctz(v):
 28 | 	return (v & -v).bit_length() - 1
 29 | 
 30 | def gen_ue(bits, v):
 31 | 	v += 1
 32 | 	return bits << (v.bit_length() * 2 - 1) | v
 33 | 
 34 | def gen_se(bits, v):
 35 | 	v = v * 2 if v > 0 else -v * 2 + 1
 36 | 	return bits << (v.bit_length() * 2 - 1) | v
 37 | 
 38 | 
 39 | 
 40 | def gen_slice_data_cavlc(bits, f, slice_type, BitDepth_Y, BitDepth_C, macroblocks):
 41 | 	for mb in macroblocks:
 42 | 		# flush the bits buffer to file
 43 | 		num = bits.bit_length() - 1
 44 | 		bits ^= 1 << num
 45 | 		f.write(bits.to_bytes(num // 8, byteorder="big"))
 46 | 		bits = bits & ((1 << (num % 8)) - 1) | 1 << (num % 8)
 47 | 		if "mb_skip_run" in vars(mb):
 48 | 			bits = gen_ue(bits, mb.mb_skip_run)
 49 | 		if "mb_field_decoding_flag" in vars(mb):
 50 | 			bits = bits << 1 | mb.mb_field_decoding_flag
 51 | 		# macroblock_layer()
 52 | 		bits = gen_ue(bits, mb.mb_type)
 53 | 		if mb.mb_type == [30, 48, 25][slice_type]: # I_PCM
 54 | 			num = bits.bit_length() - 1
 55 | 			bits <<= -num % 8 # pcm_alignment_zero_bit
 56 | 			for sample in mb.pcm_samples_Y:
 57 | 				bits = bits << BitDepth_Y | sample
 58 | 			for sample in mb.pcm_samples_Cb + mb.pcm_samples_Cr:
 59 | 				bits = bits << BitDepth_C | sample
 60 | 		
 61 | 	return bits
 62 | 	
 63 | def gen_slice_layer_without_partitioning(bits, f, slice):
 64 | 	bits = gen_ue(bits, slice.first_mb_in_slice)
 65 | 	bits = gen_ue(bits, slice.slice_type)
 66 | 	slice_type = slice.slice_type % 5
 67 | 	bits = gen_ue(bits, slice.pic_parameter_set_id)
 68 | 	if "colour_plane_id" in vars(slice):
 69 | 		bits = bits << 2 | slice.colour_plane_id
 70 | 	bits = bits << slice.frame_num.bits | (slice.frame_num.absolute & (1 << slice.frame_num.bits) - 1)
 71 | 	if "field_pic_flag" in vars(slice):
 72 | 		bits = bits << 1 | slice.field_pic_flag
 73 | 		if slice.field_pic_flag:
 74 | 			bits = bits << 1 | slice.bottom_field_flag
 75 | 	IdrPicFlag = slice.nal_unit_type == 5 or "non_idr_flag" in vars(slice) and not slice.non_idr_flag
 76 | 	if IdrPicFlag:
 77 | 		bits = gen_ue(bits, slice.idr_pic_id)
 78 | 	if slice.pic_order_cnt.type == 0:
 79 | 		bits = bits << slice.pic_order_cnt.bits | (slice.pic_order_cnt.absolute & (1 << slice.pic_order_cnt) - 1)
 80 | 		if "bottom" in vars(slice.pic_order_cnt):
 81 | 			bits = gen_se(bits, slice.pic_order_cnt.bottom - slice.pic_order_cnt.absolute)
 82 | 	if slice.pic_order_cnt.type == 1 and "delta0" in vars(slice.pic_order_cnt):
 83 | 		bits = gen_se(bits, slice.pic_order_cnt.delta0)
 84 | 		if "delta1" in vars(slice.pic_order_cnt):
 85 | 			bits = gen_se(bits, slice.pic_order_cnt.delta1)
 86 | 	if slice_type == 1:
 87 | 		bits = bits << 1 | slice.direct_spatial_mv_pred_flag
 88 | 	if slice_type <= 1:
 89 | 		bits = bits << 1 | int("num_ref_idx_active" in vars(slice))
 90 | 		if "num_ref_idx_active" in vars(slice):
 91 | 			bits = gen_ue(bits, slice.num_ref_idx_active.l0)
 92 | 			if slice_type == 1:
 93 | 				bits = gen_ue(bits, slice.num_ref_idx_active.l1)
 94 | 	field_to_idc = {"sref": 1, "lref": 2, "view": 5}
 95 | 	for i in range(slice_type + 1):
 96 | 		bits = bits << 1 | int(f"ref_pic_list_modification_l{i}" in vars(slice))
 97 | 		if f"ref_pic_list_modification_l{i}" in vars(slice):
 98 | 			for field, diff in vars(slice)[f"ref_pic_list_modification_l{i}"]:
 99 | 				bits = gen_ue(bits, field_to_idc[field] - (diff < 0))
100 | 				bits = gen_ue(bits, diff if field == "lref" else abs(diff) - 1)
101 | 			bits = gen_ue(bits, 3)
102 | 	if "explicit_weights_l0" in vars(slice):
103 | 		bits = gen_ue(bits, int(re.findall(r"\d+", slice.explicit_weights_l0[0].Y)[1]))
104 | 		bits = gen_ue(bits, int(re.findall(r"\d+", slice.explicit_weights_l0[0].Cb)[1]))
105 | 		for i in range(slice_type + 1):
106 | 			for ref in vars(slice)[f"explicit_weights_l{i}"]:
107 | 				for plane in ("Y", "Cb", "Cr"):
108 | 					weight, denom, offset = map(int, re.findall(r"\-?\d+", vars(ref)[plane]))
109 | 					bits = bits << 1 | int(weight != 2 ** denom or offset != 0)
110 | 					if weight != 2 ** denom or offset != 0:
111 | 						bits = gen_se(bits, weight)
112 | 						bits = gen_se(bits, offset)
113 | 	if slice.nal_ref_idc:
114 | 		if IdrPicFlag:
115 | 			bits = bits << 1 | slice.no_output_of_prior_pics_flag
116 | 			bits = bits << 1 | slice.long_term_reference_flag
117 | 		else:
118 | 			bits = bits << 1 | int("memory_management_control_operations" in vars(slice))
119 | 			if "memory_management_control_operations" in vars(slice):
120 | 				for mmco in slice.memory_management_control_operations:
121 | 					bits = gen_ue(bits, mmco.idc)
122 | 					if "sref" in mmco:
123 | 						bits = gen_ue(bits, -mmco.sref)
124 | 					if "lref" in mmco:
125 | 						bits = gen_ue(bits, mmco.lref)
126 | 				bits = bits << 1 | 1 # memory_management_control_operation == 0
127 | 	if "cabac_init_idc" in vars(slice):
128 | 		bits = gen_ue(bits, slice.cabac_init_idc)
129 | 	bits = gen_se(bits, slice.slice_qp_delta)
130 | 	if "disable_deblocking_filter_idc" in vars(slice):
131 | 		bits = gen_ue(bits, slice.disable_deblocking_filter_idc)
132 | 		if slice.disable_deblocking_filter_idc != 1:
133 | 			bits = gen_se(bits, slice.slice_alpha_c0_offset >> 1)
134 | 			bits = gen_se(bits, slice.slice_beta_offset >> 1)
135 | 	if "CAVLC_macroblocks" in vars(slice):
136 | 		bits = gen_slice_data_cavlc(bits, f, slice_type, slice.FrameFormat.BitDepth_Y, slice.FrameFormat.BitDepth_C, slice.CAVLC_macroblocks)
137 | 	else:
138 | 		bits = gen_slice_data_cabac(bits, f, slice_type, slice.FrameFormat.BitDepth_Y, slice.FrameFormat.BitDepth_C, slice.CABAC_macroblocks)
139 | 	return bits
140 | 
141 | 
142 | 
143 | def gen_hrd_parameters(bits, hrd):
144 | 	bits = gen_ue(bits, len(hrd.CPBs) - 1)
145 | 	bit_rate_scale = min(ctz(reduce(lambda a, b: a | b, (cpb.bit_rate for cpb in hrd.CPBs))) - 6, 15)
146 | 	cpb_size_scale = min(ctz(reduce(lambda a, b: a | b, (cpb.size for cpb in hrd.CPBs))) - 4, 15)
147 | 	bits = bits << 4 | bit_rate_scale
148 | 	bits = bits << 4 | cpb_size_scale
149 | 	for cpb in hrd.CPBs:
150 | 		bits = gen_ue(bits, (cpb.bit_rate >> 6 >> bit_rate_scale) - 1)
151 | 		bits = gen_ue(bits, (cpb.size >> 4 >> cpb_size_scale) - 1)
152 | 		bits = bits << 1 | cpb.cbr_flag
153 | 	bits = bits << 5 | hrd.initial_cpb_removal_delay_length - 1
154 | 	bits = bits << 5 | hrd.cpb_removal_delay_length - 1
155 | 	bits = bits << 5 | hrd.dpb_output_delay_length - 1
156 | 	bits = bits << 5 | hrd.time_offset_length
157 | 	return bits
158 | 
159 | def gen_vui_parameters(bits, sps, vui):
160 | 	bits = bits << 1 | int("aspect_ratio" in vars(vui))
161 | 	if "aspect_ratio" in vars(vui):
162 | 		bits = bits << 8 | vui.aspect_ratio.idc
163 | 		if vui.aspect_ratio.idc == 255:
164 | 			bits = bits << 16 | vui.aspect_ratio.width
165 | 			bits = bits << 16 | vui.aspect_ratio.height
166 | 	bits = bits << 1 | min(vui.overscan_appropriate_flag + 1, 1)
167 | 	if vui.overscan_appropriate_flag >= 0:
168 | 		bits = bits << 1 | vui.overscan_appropriate_flag
169 | 	bits = bits << 1 | int("video_format" in vars(vui))
170 | 	if "video_format" in vars(vui):
171 | 		bits = bits << 3 | vui.video_format
172 | 		bits = bits << 1 | vui.video_full_range_flag
173 | 		bits = bits << 1 | int("colour_primaries" in vars(vui))
174 | 		if "colour_primaries" in vars(vui):
175 | 			bits = bits << 8 | vui.colour_primaries
176 | 			bits = bits << 8 | vui.transfer_characteristics
177 | 			bits = bits << 8 | vui.matrix_coefficients
178 | 	bits = bits << 1 | int("chroma_sample_loc" in vars(vui))
179 | 	if "chroma_sample_loc" in vars(vui):
180 | 		bits = gen_ue(bits, vui.chroma_sample_loc.top)
181 | 		bits = gen_ue(bits, vui.chroma_sample_loc.bottom)
182 | 	bits = bits << 1 | int("num_units_in_tick" in vars(vui))
183 | 	if "num_units_in_tick" in vars(vui):
184 | 		bits = bits << 32 | vui.num_units_in_tick
185 | 		bits = bits << 32 | vui.time_scale
186 | 		bits = bits << 1 | vui.fixed_frame_rate_flag
187 | 	bits = bits << 1 | int("nal_hrd_parameters" in vars(vui))
188 | 	if "nal_hrd_parameters" in vars(vui):
189 | 		bits = gen_hrd_parameters(bits, vui.nal_hrd_parameters)
190 | 	bits = bits << 1 | int("vcl_hrd_parameters" in vars(vui))
191 | 	if "vcl_hrd_parameters" in vars(vui):
192 | 		bits = gen_hrd_parameters(bits, vui.vcl_hrd_parameters)
193 | 	if "nal_hrd_parameters" in vars(vui) or "vcl_hrd_parameters" in vars(vui):
194 | 		bits = bits << 1 | vui.low_delay_hrd_flag
195 | 	bits = bits << 1 | vui.pic_struct_present_flag
196 | 	bits = bits << 1 | int("motion_vectors_over_pic_boundaries_flag" in vars(vui))
197 | 	if "motion_vectors_over_pic_boundaries_flag" in vars(vui):
198 | 		bits = bits << 1 | vui.motion_vectors_over_pic_boundaries_flag
199 | 		PicSizeInMbs = sps.pic_size_in_mbs.width * sps.pic_size_in_mbs.height
200 | 		RawMbBits = 256 * sps.bit_depth.luma + (64 << sps.chroma_format_idc & ~64) * sps.bit_depth.chroma
201 | 		bits = gen_ue(bits, (PicSizeInMbs * RawMbBits) // vui.max_bytes_per_pic // 8 if vui.max_bytes_per_pic else 0)
202 | 		bits = gen_ue(bits, (128 + RawMbBits) // vui.max_bits_per_mb if vui.max_bits_per_mb else 0)
203 | 		bits = gen_ue(bits, vui.log2_max_mv_length_horizontal)
204 | 		bits = gen_ue(bits, vui.log2_max_mv_length_vertical)
205 | 		bits = gen_ue(bits, vui.max_num_reorder_frames)
206 | 		bits = gen_ue(bits, vui.max_dec_frame_buffering)
207 | 	return bits
208 | 
209 | def gen_seq_parameter_set(bits, f, sps):
210 | 	bits = bits << 8 | sps.profile_idc
211 | 	print(sps.constraint_set_flags)
212 | 	bits = bits << 8 | sum(f << (7 - i) for i, f in enumerate(sps.constraint_set_flags))
213 | 	bits = bits << 8 | int(sps.level_idc * 10)
214 | 	bits = bits << 1 | 1 # seq_parameter_set_id
215 | 	if sps.profile_idc in (100, 110, 122, 244, 44, 83, 86, 118, 128, 138, 139, 134, 135):
216 | 		bits = gen_ue(bits, sps.chroma_format_idc)
217 | 		if sps.chroma_format_idc == 3:
218 | 			bits = bits << 1 | sps.separate_colour_plane_flag
219 | 		bits = gen_ue(bits, sps.bit_depth.luma - 8)
220 | 		bits = gen_ue(bits, sps.bit_depth.chroma - 8)
221 | 		bits = bits << 1 | sps.qpprime_y_zero_transform_bypass_flag
222 | 		bits = bits << 1 | int("seq_scaling_matrix" in vars(sps))
223 | 		if "seq_scaling_matrix" in vars(sps):
224 | 			for scaling_list in sps.seq_scaling_matrix:
225 | 				bits = bits << 1 | min(len(scaling_list), 1) # seq_scaling_list_present_flag
226 | 				for lastScale, nextScale in zip([8] + scaling_list, scaling_list):
227 | 					bits = gen_se(bits, (nextScale - lastScale + 128) % 256 - 128) # delta_scale
228 | 	bits = gen_ue(bits, sps.log2_max_frame_num - 4)
229 | 	bits = gen_ue(bits, sps.pic_order_cnt_type)
230 | 	if sps.pic_order_cnt_type == 0:
231 | 		bits = gen_ue(bits, sps.log2_max_pic_order_cnt_lsb - 4)
232 | 	elif sps.pic_order_cnt_type == 1:
233 | 		bits = bits << 1 | sps.delta_pic_order_always_zero_flag
234 | 		bits = gen_se(bits, sps.offset_for_non_ref_pic)
235 | 		bits = gen_se(bits, sps.offset_for_top_to_bottom_field)
236 | 		bits = gen_ue(bits, len(sps.offsets_for_ref_frames))
237 | 		for offset in sps.offsets_for_ref_frames:
238 | 			bits = gen_se(bits, offset)
239 | 	bits = gen_ue(bits, sps.max_num_ref_frames)
240 | 	bits = bits << 1 | sps.gaps_in_frame_num_value_allowed_flag
241 | 	bits = gen_ue(bits, sps.pic_size_in_mbs.width - 1)
242 | 	bits = gen_ue(bits, sps.pic_size_in_mbs.height - 1)
243 | 	bits = bits << 1 | sps.frame_mbs_only_flag
244 | 	if not sps.frame_mbs_only_flag:
245 | 		bits = bits << 1 | sps.mb_adaptive_frame_field_flag
246 | 	bits = bits << 1 | sps.direct_8x8_inference_flag
247 | 	bits = bits << 1 | int("frame_crop_offsets" in vars(sps))
248 | 	if "frame_crop_offsets" in vars(sps):
249 | 		for side in ("left", "right", "top", "bottom"):
250 | 			bits = gen_ue(bits, vars(sps.frame_crop_offsets)[side])
251 | 	bits = bits << 1 | int("vui_parameters" in vars(sps))
252 | 	if "vui_parameters" in vars(sps):
253 | 		bits = gen_vui_parameters(bits, sps, sps.vui_parameters)
254 | 	return bits
255 | 
256 | 
257 | 
258 | def gen_pic_parameter_set(bits, f, pps):
259 | 	bits = gen_ue(bits, pps.pic_parameter_set_id)
260 | 	bits = bits << 1 | 1 # seq_parameter_set_id
261 | 	bits = bits << 1 | pps.entropy_coding_mode_flag
262 | 	bits = bits << 1 | pps.bottom_field_pic_order_in_frame_present_flag
263 | 	bits = bits << 1 | 1 # num_slice_groups
264 | 	bits = gen_ue(bits, pps.num_ref_idx_default_active.l0 - 1)
265 | 	bits = gen_ue(bits, pps.num_ref_idx_default_active.l1 - 1)
266 | 	bits = bits << 1 | pps.weighted_pred_flag
267 | 	bits = bits << 2 | pps.weighted_bipred_idc
268 | 	bits = gen_se(bits, pps.pic_init_qp - 26)
269 | 	bits = bits << 1 | 1 # pic_init_qs
270 | 	bits = gen_se(bits, pps.chroma_qp_index_offset)
271 | 	bits = bits << 1 | pps.deblocking_filter_control_present_flag
272 | 	bits = bits << 1 | pps.constrained_intra_pred_flag
273 | 	bits = bits << 1 # redundant_pic_cnt_present_flag
274 | 	if "transform_8x8_mode_flag" in vars(pps):
275 | 		bits = bits << 1 | pps.transform_8x8_mode_flag
276 | 		bits = bits << 1 | int("pic_scaling_matrix" in vars(pps))
277 | 		if "pic_scaling_matrix" in vars(pps):
278 | 			for scaling_list in pps.pic_scaling_matrix:
279 | 				bits = bits << 1 | min(len(scaling_list), 1) # seq_scaling_list_present_flag
280 | 				for lastScale, nextScale in zip([8] + scaling_list, scaling_list):
281 | 					bits = gen_se(bits, (nextScale - lastScale + 128) % 256 - 128) # delta_scale
282 | 		bits = gen_se(bits, pps.second_chroma_qp_index_offset)
283 | 	return bits
284 | 
285 | 
286 | 
287 | def gen_access_unit_delimiter(bits, f, aud):
288 | 	bits = bits << 3 | aud.primary_pic_type
289 | 	return bits
290 | 
291 | 
292 | 
293 | def gen_seq_parameter_set_extension(bits, f, spse):
294 | 	bits = bits << 1 | 1 # seq_parameter_set_id
295 | 	bits = gen_ue(bits, spse.aux_format_idc)
296 | 	if spse.aux_format_idc:
297 | 		bits = gen_ue(bits, spse.bit_depth_aux - 8)
298 | 		bits = bits << 1 | spse.alpha_incr_flag
299 | 		bits = bits << spse.bit_depth_aux << 1 | spse.alpha_opaque_value
300 | 		bits = bits << spse.bit_depth_aux << 1 | spse.alpha_transparent_value
301 | 	bits = bits << 1 # additional_extension_flag
302 | 	return bits
303 | 
304 | 
305 | 
306 | def gen_mvc_vui_parameters_extension(bits, ssps, vui):
307 | 	bits = gen_ue(bits, len(vui.vui_mvc_operation_points) - 1)
308 | 	for op in vui.vui_mvc_operation_points:
309 | 		bits = bits << 3 | op.temporal_id
310 | 		bits = gen_ue(bits, len(op.target_views) - 1)
311 | 		for view_id in op.target_views:
312 | 			bits = gen_ue(bits, view_id)
313 | 		bits = bits << 1 | int("num_units_in_tick" in vars(vui))
314 | 		if "num_units_in_tick" in vars(vui):
315 | 			bits = bits << 32 | vui.num_units_in_tick
316 | 			bits = bits << 32 | vui.time_scale
317 | 			bits = bits << 1 | vui.fixed_frame_rate_flag
318 | 		bits = bits << 1 | int("nal_hrd_parameters" in vars(vui))
319 | 		if "nal_hrd_parameters" in vars(vui):
320 | 			bits = gen_hrd_parameters(bits, vui.nal_hrd_parameters)
321 | 		bits = bits << 1 | int("vcl_hrd_parameters" in vars(vui))
322 | 		if "vcl_hrd_parameters" in vars(vui):
323 | 			bits = gen_hrd_parameters(bits, vui.vcl_hrd_parameters)
324 | 		if "nal_hrd_parameters" in vars(vui) or "vcl_hrd_parameters" in vars(vui):
325 | 			bits = bits << 1 | vui.low_delay_hrd_flag
326 | 		bits = bits << 1 | vui.pic_struct_present_flag
327 | 	return bits
328 | 
329 | def gen_subset_seq_parameter_set(bits, f, ssps):
330 | 	bits = gen_seq_parameter_set(bits, f, ssps)
331 | 	if ssps.profile_idc in (118, 128, 134):
332 | 		bits = bits << 1 | 1 # bit_equal_to_one
333 | 		bits = gen_ue(bits, 1) # num_views_minus1
334 | 		bits = gen_ue(bits, ssps.view_ids[0])
335 | 		bits = gen_ue(bits, ssps.view_ids[1])
336 | 		bits = gen_ue(bits, ssps.num_anchor_refs.l0)
337 | 		if ssps.num_anchor_refs.l0:
338 | 			bits = gen_ue(bits, ssps.view_ids[0])
339 | 		bits = gen_ue(bits, ssps.num_anchor_refs.l1)
340 | 		if ssps.num_anchor_refs.l1:
341 | 			bits = gen_ue(bits, ssps.view_ids[0])
342 | 		bits = gen_ue(bits, ssps.num_non_anchor_refs.l0)
343 | 		if ssps.num_non_anchor_refs.l0:
344 | 			bits = gen_ue(bits, ssps.view_ids[0])
345 | 		bits = gen_ue(bits, ssps.num_non_anchor_refs.l1)
346 | 		if ssps.num_non_anchor_refs.l1:
347 | 			bits = gen_ue(bits, ssps.view_ids[0])
348 | 		bits = gen_ue(bits, len(ssps.level_values_signalled) - 1)
349 | 		for level in ssps.level_values_signalled:
350 | 			bits = bits << 8 | level.idc
351 | 			bits = gen_ue(bits, len(level.operation_points) - 1)
352 | 			for op in level.operation_points:
353 | 				bits = bits << 3 | op.temporal_id
354 | 				bits = gen_ue(bits, len(op.target_views) - 1)
355 | 				for view_id in op.target_views:
356 | 					bits = gen_ue(bits, view_id)
357 | 				bits = gen_ue(bits, op.num_views - 1)
358 | 		bits = bits << 1 | int("mvc_vui_parameters" in vars(ssps))
359 | 		if "mvc_vui_parameters" in vars(ssps):
360 | 			bits = gen_mvc_vui_parameters_extension(bits, ssps, ssps.mvc_vui_parameters)
361 | 	bits = bits << 1 # additional_extension2_flag
362 | 	return bits
363 | 
364 | 
365 | 
366 | gen_bits = {
367 | 	7: gen_seq_parameter_set,
368 | 	8: gen_pic_parameter_set,
369 | 	9: gen_access_unit_delimiter,
370 | 	13: gen_seq_parameter_set_extension,
371 | 	15: gen_subset_seq_parameter_set
372 | }
373 | 
374 | def main():
375 | 	if len(sys.argv) != 3:
376 | 		print(f"Usage: {sys.argv[0]} input.yaml output.264")
377 | 		exit()
378 | 	with open(sys.argv[1], "r") as f:
379 | 		nals = yaml.load(f, Loader=SafeLoader)
380 | 	with open(sys.argv[2], "wb") as f:
381 | 		for nal in nals:
382 | 			bits = 1 # leading set bit
383 | 			bits = bits << 32 | 1 # start code
384 | 			bits <<= 1 # forbidden_zero_bit
385 | 			bits = bits << 2 | nal.nal_ref_idc
386 | 			bits = bits << 5 | nal.nal_unit_type
387 | 			bits = gen_bits[nal.nal_unit_type](bits, f, nal)
388 | 			bits = bits << 1 | 1 # rbsp_stop_one_bit
389 | 			num = bits.bit_length() - 1
390 | 			bits ^= 1 << num
391 | 			bits <<= -num % 8
392 | 			f.write(bits.to_bytes((num + 7) // 8, byteorder="big"))
393 | 
394 | if __name__ == "__main__":
395 | 	main()


--------------------------------------------------------------------------------
/tools/split_gops.c:
--------------------------------------------------------------------------------
  1 | #include <stdio.h>
  2 | #include <string.h>
  3 | #ifdef _WIN32
  4 | 	#include <windows.h>
  5 | #else
  6 | 	#include <fcntl.h>
  7 | 	#include <sys/mman.h>
  8 | 	#include <sys/stat.h>
  9 | 	#include <unistd.h>
 10 | #endif
 11 | 
 12 | #include "./edge264.h"
 13 | 
 14 | #define RESET  "\e[0m"
 15 | #define BOLD   "\e[1m"
 16 | 
 17 | int main(int argc, char *argv[]) {
 18 | 	// parse command-line options
 19 | 	const char *input_name = NULL;
 20 | 	int help = 0;
 21 | 	int mock = 0;
 22 | 	int print_nal = 0;
 23 | 	int scp_len = 4;
 24 | 	for (int i = 1; i < argc; i++) {
 25 | 		if (argv[i][0] != '-') {
 26 | 			input_name = argv[i];
 27 | 		} else for (int j = 1; argv[i][j]; j++) {
 28 | 			switch (argv[i][j]) {
 29 | 				case 'm': mock = 1; break;
 30 | 				case 'p': print_nal = 1; break;
 31 | 				case 't': scp_len = 3; break;
 32 | 				default: help = 1; break;
 33 | 			}
 34 | 		}
 35 | 	}
 36 | 	if (help || input_name == NULL) {
 37 | 		printf("Usage: " BOLD "%s video.264 [-hm]" RESET "\n"
 38 | 			"Splits a video in GOPs in files [video]-N.264 (N=0,1,2,..).\n"
 39 | 			"-h\tprint this help and exit\n"
 40 | 			"-m\tmock intended splits instead of actually writing to files\n"
 41 | 			"-p\tprint all NAL unit types while seeking\n"
 42 | 			"-t\tseek three-byte start codes instead of four-byte\n",
 43 | 			argv[0]);
 44 | 		return 0;
 45 | 	}
 46 | 	
 47 | 	// open and memory map input file
 48 | 	#ifdef _WIN32
 49 | 		HANDLE input_f = NULL;
 50 | 		HANDLE input_fm = NULL;
 51 | 		void *input_vf = NULL;
 52 | 		if ((input_f = CreateFileA(input_name, GENERIC_READ, FILE_SHARE_READ, NULL, OPEN_EXISTING, FILE_ATTRIBUTE_NORMAL, NULL)) == INVALID_HANDLE_VALUE ||
 53 | 			(input_fm = CreateFileMappingA(input_f, NULL, PAGE_READONLY, 0, 0, NULL)) == NULL ||
 54 | 			(input_vf = MapViewOfFile(input_fm, FILE_MAP_READ, 0, 0, 0)) == NULL) {
 55 | 			printf("Error opening file %s: %lu\n", input_name, GetLastError());
 56 | 			return 1;
 57 | 		}
 58 | 		const uint8_t *nal = input_vf;
 59 | 		const uint8_t *end = nal + GetFileSize(input_f, NULL);
 60 | 	#else
 61 | 		int input_fd = -1;
 62 | 		struct stat input_st;
 63 | 		uint8_t *input_mm = MAP_FAILED;
 64 | 		if ((input_fd = open(input_name, O_RDONLY)) < 0 ||
 65 | 			fstat(input_fd, &input_st) < 0 ||
 66 | 			(input_mm = mmap(NULL, input_st.st_size, PROT_READ, MAP_SHARED, input_fd, 0)) == MAP_FAILED) {
 67 | 			perror(input_name);
 68 | 			return 1;
 69 | 		}
 70 | 		const uint8_t *nal = input_mm;
 71 | 		const uint8_t *end = nal + input_st.st_size;
 72 | 	#endif
 73 | 	
 74 | 	// loop on NAL units and split on last AUD for each new SPS
 75 | 	int input_len = strrchr(input_name, '.') - input_name;
 76 | 	const uint8_t *from = nal;
 77 | 	const uint8_t *to = NULL;
 78 | 	int count = 0;
 79 | 	while (nal < end) {
 80 | 		const uint8_t *scp = edge264_find_start_code(nal, end, scp_len - 3);
 81 | 		nal = scp + scp_len;
 82 | 		int nal_unit_type = nal < end ? *nal & 31 : 0;
 83 | 		int is_delimiter = !nal_unit_type || nal_unit_type == 9;
 84 | 		int is_flusher = !nal_unit_type || nal_unit_type == 5;
 85 | 		if (print_nal && nal_unit_type)
 86 | 			printf("NAL unit type %d\n", nal_unit_type);
 87 | 		if (is_delimiter)
 88 | 			to = scp;
 89 | 		if (is_flusher && from < to) {
 90 | 			char output_name[input_len + 16];
 91 | 			snprintf(output_name, sizeof(output_name), "%.*s-%d.264", input_len, input_name, count++);
 92 | 			if (mock) {
 93 | 				printf("Write %zd bytes in %s\n", to - from, output_name);
 94 | 			} else {
 95 | 				FILE *output_file = fopen(output_name, "w");
 96 | 				fwrite(from, to - from, 1, output_file);
 97 | 				fclose(output_file);
 98 | 			}
 99 | 			from = to;
100 | 		}
101 | 	}
102 | 	
103 | 	// close everything
104 | 	#ifdef _WIN32
105 | 		UnmapViewOfFile(input_vf);
106 | 		CloseHandle(input_fm);
107 | 		CloseHandle(input_f);
108 | 	#else
109 | 		munmap(input_mm, input_st.st_size);
110 | 		close(input_fd);
111 | 	#endif
112 | 	return 0;
113 | }
114 | 


--------------------------------------------------------------------------------