├── .gitmodules ├── LICENSE ├── README.cartwheel ├── README.md ├── SECURITY.md └── patches ├── 0021-lavc-decode-Add-get_hw_config-function.patch ├── 0022-lavc-decode-Add-internal-surface-re-allocate-method-.patch ├── 0023-lavc-vaapi_-vp9-vvc-add-surface-internal-re-allocati.patch ├── 0033-lavc-vaapi_hevc-add-skip_frame-invalid-to-skip-inval.patch ├── 0034-FFmpeg-vaapi-HEVC-SCC-encode.patch ├── 0050-libavcodec-qsvdec.c-extract-frame-packing-arrangemen.patch ├── 0055-libavcodec-qsvenc_hevc-add-main10sp-support-to-hevc_.patch ├── 0060-avutils-hwcontext-When-deriving-a-hwdevice-search-fo.patch ├── 0062-lavu-add-sub-frame-side-data.patch ├── 0063-lavc-add-sub-frame-options-and-flag.patch ├── 0064-lavc-hevc_vaapi-enable-sub-frame-support.patch ├── 0065-examples-seperate-vaapi_decode-from-hw_decode.patch ├── 0066-lavfi-Add-exportsubframe-filter.patch ├── 0067-avfilter-vf_xcam-add-xcam-video-filter.patch ├── 0075-lavf-dnn-enable-libtorch-backend-support-FRVSR-model.patch ├── 0092-avutil-hwcontext-add-a-function-to-get-the-AVHWDevic.patch ├── 0093-avfilter-vf_hwmap-get-the-AVHWDeviceType-from-outlin.patch ├── 0094-lavfi-avfiltergraph-move-convert-codes-into-function.patch ├── 0095-lavfi-format-wrap-auto-filters-into-structures.patch ├── 0096-lavfi-format-add-a-hwmap-auto-conversion-filter.patch ├── 0116-lavfi-add-LibTorch-backend-multi-frame-inference-sup.patch ├── 0117-libavfilter-dnn-dnn_backend_torch-Add-async-to-torch.patch ├── 0118-libavfilter-dnn-dnn_backend_torch-add-multi-device-s.patch ├── 0132-ffmpeg-raisr-filter.patch ├── 0133-libavfilter-raisr_opencl-Add-raisr_opencl-filter.patch ├── 0135-avutil-hwcontext_d3d11va-enable-D3D11_RESOURCE_MISC_.patch ├── 0151-libavfilter-dnn_backend_openvino-Factor-out-preproce.patch ├── 0152-libavfilter-dnn_backend_openvino-Add-remote_context-.patch ├── 0153-libavfilter-vf_dnn_detect-Add-vaapi-into-detect.patch ├── 0154-libavfilter-dnn_backend_openvino-Add-vaSurface-crop-.patch ├── 0155-libavfilter-vf_dnn_classify-Add-vaapi-into-classify.patch ├── 0165-libavcodec-qsvenc-enable-Alpha-Encode-for-HEVC.patch ├── 0166-libavcodec-qsvenc-enable-Screen-Content-Tool-Encode-.patch ├── 0167-lavc-qsvenc-Support-calculate-encoded-frame-quality-.patch ├── 0168-lavc-qsvenc-Make-ffmpeg-compilable-with-experimental.patch └── 0169-lavc-qsvenc-Add-sliding-window-bitrate-control-for-C.patch /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "ffmpeg"] 2 | path = ffmpeg 3 | url = https://github.com/ffmpeg/ffmpeg 4 | branch = master 5 | -------------------------------------------------------------------------------- /README.cartwheel: -------------------------------------------------------------------------------- 1 | # cartwheel : ffmpeg 2 | 3 | Intel developer staging area for upstream patch contributions to [FFmpeg](https://github.com/ffmpeg/ffmpeg). 4 | 5 | The upstream project is tracked as a submodule in this repo. 6 | 7 | ## upstream process 8 | To improve upstream patch quality and reduce patchset rebase efforts, we use the following process 9 | 1. Send PR1 to https://github.com/intel-media-ci/ffmpeg, add module maintainers to Reviewers 10 | 1. Got approval from one maintainer 11 | 1. Wait for 1 more workday to see any objections. If it's an urgent issue, please ask another reviewer + 1 for you. 12 | 1. Close PR1, and send it to the community, ping the community with some frequency if it's no response. 13 | 1. If the community has no response in 4 weeks or it’s an urgent issue, send PR2 to https://github.com/intel/cartwheel-ffmpeg 14 | 1. Provide upstream patch link, ping maintainers for merging 15 | 1. Ping community until the patch merged in upstream 16 | 17 | 18 | ## maintainers 19 | * @xhaihao @feiwan1 @xuguangxin for Linux and OneVPL 20 | * @galinart for Windows 21 | * @guoyejun @Semmer2 for DNN 22 | * @uartie @FocusLuo @Bin-CI for CI system 23 | 24 | ## clone 25 | ```shell 26 | $ git clone https://github.com/intel/cartwheel-ffmpeg --recursive 27 | ``` 28 | 29 | ## apply patches 30 | ```shell 31 | # at top-level directory 32 | $ git submodule update --init --recursive 33 | # at submodule directory 34 | $ cd ffmpeg 35 | # It is recommended to create a branch before applying the patches 36 | $ git checkout -b 37 | $ git am ../patches/*.patch 38 | ``` 39 | 40 | ## build 41 | [how to]( https://github.com/intel/cartwheel-ffmpeg/wiki/How-to-build-cartwheel-ffmpeg) 42 | 43 | ## rebase 44 | ```shell 45 | # at top-level directory 46 | $ git pull --rebase --recurse-submodule 47 | ``` 48 | 49 | ## administrators 50 | 51 | To update the submodule reference commit id to the latest upstream: 52 | 53 | ```shell 54 | # at top-level directory 55 | $ git submodule update --remote --recursive 56 | $ git commit -sam "$(git diff --submodule | head -1 | sed 's/:$//')" 57 | ``` 58 | 59 | ...verify the patches still apply successfully. If they don't apply, fix them and include in new commit(s). 60 | 61 | ## additional information 62 | 63 | For more information and examples about Git Submodules, see https://git-scm.com/book/en/v2/Git-Tools-Submodules 64 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | README.cartwheel -------------------------------------------------------------------------------- /SECURITY.md: -------------------------------------------------------------------------------- 1 | # Security Policy 2 | Intel is committed to rapidly addressing security vulnerabilities affecting our customers and providing clear guidance on the solution, impact, severity and mitigation. 3 | 4 | ## Reporting a Vulnerability 5 | Please report any security vulnerabilities in this project utilizing the guidelines [here](https://www.intel.com/content/www/us/en/security-center/vulnerability-handling-guidelines.html). -------------------------------------------------------------------------------- /patches/0021-lavc-decode-Add-get_hw_config-function.patch: -------------------------------------------------------------------------------- 1 | From 92be7fa47898d46ac23aab21d6c0f04519c4b482 Mon Sep 17 00:00:00 2001 2 | From: Linjie Fu 3 | Date: Thu, 2 Jan 2020 11:06:45 +0800 4 | Subject: [PATCH 21/77] lavc/decode: Add get_hw_config function 5 | 6 | Wrap the procedure of getting the hardware config from a pixel format 7 | into a function. 8 | 9 | Signed-off-by: Linjie Fu 10 | --- 11 | libavcodec/decode.c | 33 +++++++++++++++++++++------------ 12 | 1 file changed, 21 insertions(+), 12 deletions(-) 13 | 14 | diff --git a/libavcodec/decode.c b/libavcodec/decode.c 15 | index 1893caa6a6..bfa82199ae 100644 16 | --- a/libavcodec/decode.c 17 | +++ b/libavcodec/decode.c 18 | @@ -1097,6 +1097,26 @@ static void hwaccel_uninit(AVCodecContext *avctx) 19 | av_buffer_unref(&avctx->hw_frames_ctx); 20 | } 21 | 22 | +static const AVCodecHWConfigInternal *get_hw_config(AVCodecContext *avctx, enum AVPixelFormat fmt) 23 | +{ 24 | + const AVCodecHWConfigInternal *hw_config; 25 | + int i; 26 | + 27 | + if (ffcodec(avctx->codec)->hw_configs) { 28 | + for (i = 0;; i++) { 29 | + hw_config = ffcodec(avctx->codec)->hw_configs[i]; 30 | + if (!hw_config) 31 | + break; 32 | + if (hw_config->public.pix_fmt == fmt) 33 | + break; 34 | + } 35 | + } else { 36 | + hw_config = NULL; 37 | + } 38 | + 39 | + return hw_config; 40 | +} 41 | + 42 | int ff_get_format(AVCodecContext *avctx, const enum AVPixelFormat *fmt) 43 | { 44 | const AVPixFmtDescriptor *desc; 45 | @@ -1154,18 +1174,7 @@ int ff_get_format(AVCodecContext *avctx, const enum AVPixelFormat *fmt) 46 | break; 47 | } 48 | 49 | - if (ffcodec(avctx->codec)->hw_configs) { 50 | - for (i = 0;; i++) { 51 | - hw_config = ffcodec(avctx->codec)->hw_configs[i]; 52 | - if (!hw_config) 53 | - break; 54 | - if (hw_config->public.pix_fmt == user_choice) 55 | - break; 56 | - } 57 | - } else { 58 | - hw_config = NULL; 59 | - } 60 | - 61 | + hw_config = get_hw_config(avctx, user_choice); 62 | if (!hw_config) { 63 | // No config available, so no extra setup required. 64 | ret = user_choice; 65 | -- 66 | 2.17.1 67 | 68 | -------------------------------------------------------------------------------- /patches/0022-lavc-decode-Add-internal-surface-re-allocate-method-.patch: -------------------------------------------------------------------------------- 1 | From 2ca290ab08101689f9a99845c8ffdcb59034bd18 Mon Sep 17 00:00:00 2001 2 | From: Linjie Fu 3 | Date: Thu, 2 Jan 2020 11:12:15 +0800 4 | Subject: [PATCH 02/44] lavc/decode: Add internal surface re-allocate method 5 | for hwaccel 6 | 7 | Add HWACCEL_CAP_INTERNAL_ALLOC flag to indicate hwaccels are able to 8 | re-allocate surface internally through ff_decode_get_hw_frames_ctx. 9 | 10 | Signed-off-by: Linjie Fu 11 | --- 12 | libavcodec/decode.c | 37 +++++++++++++++++++++++++++++++++++ 13 | libavcodec/hwaccel_internal.h | 1 + 14 | 2 files changed, 38 insertions(+) 15 | 16 | diff --git a/libavcodec/decode.c b/libavcodec/decode.c 17 | index 0aa238a3eb..de4574d4ff 100644 18 | --- a/libavcodec/decode.c 19 | +++ b/libavcodec/decode.c 20 | @@ -1278,6 +1278,33 @@ static const AVCodecHWConfigInternal *get_hw_config(AVCodecContext *avctx, enum 21 | return hw_config; 22 | } 23 | 24 | +static int hwaccel_realloc_surface(AVCodecContext *avctx) 25 | +{ 26 | + const AVCodecHWConfigInternal *hw_config; 27 | + int ret; 28 | + 29 | + if (avctx->hw_frames_ctx) 30 | + av_buffer_unref(&avctx->hw_frames_ctx); 31 | + 32 | + hw_config = get_hw_config(avctx, avctx->pix_fmt); 33 | + if (!hw_config) 34 | + return AV_PIX_FMT_NONE; 35 | + 36 | + if (avctx->hw_device_ctx && 37 | + hw_config->public.methods & AV_CODEC_HW_CONFIG_METHOD_HW_DEVICE_CTX) { 38 | + const AVHWDeviceContext *device_ctx = 39 | + (AVHWDeviceContext*)avctx->hw_device_ctx->data; 40 | + ret = ff_decode_get_hw_frames_ctx(avctx, device_ctx->type); 41 | + if (ret < 0) { 42 | + av_log(avctx, AV_LOG_WARNING, "Failed to re-allocate hwaccel surface internally.\n"); 43 | + return AV_PIX_FMT_NONE; 44 | + } 45 | + } else 46 | + return AV_PIX_FMT_NONE; 47 | + 48 | + return hw_config->public.pix_fmt; 49 | +} 50 | + 51 | int ff_get_format(AVCodecContext *avctx, const enum AVPixelFormat *fmt) 52 | { 53 | const AVPixFmtDescriptor *desc; 54 | @@ -1304,6 +1331,16 @@ int ff_get_format(AVCodecContext *avctx, const enum AVPixelFormat *fmt) 55 | return AV_PIX_FMT_NONE; 56 | 57 | for (;;) { 58 | + if (avctx->internal->hwaccel_priv_data && 59 | + avctx->hwaccel && 60 | + (ffhwaccel(avctx->hwaccel)->caps_internal & HWACCEL_CAP_INTERNAL_ALLOC)) { 61 | + err = hwaccel_realloc_surface(avctx); 62 | + if (err < 0) 63 | + av_log(avctx, AV_LOG_WARNING, "Try to re-initialize all.\n"); 64 | + else 65 | + return err; 66 | + } 67 | + 68 | // Remove the previous hwaccel, if there was one. 69 | ff_hwaccel_uninit(avctx); 70 | 71 | diff --git a/libavcodec/hwaccel_internal.h b/libavcodec/hwaccel_internal.h 72 | index edfe283150..11c5727598 100644 73 | --- a/libavcodec/hwaccel_internal.h 74 | +++ b/libavcodec/hwaccel_internal.h 75 | @@ -29,6 +29,7 @@ 76 | 77 | #define HWACCEL_CAP_ASYNC_SAFE (1 << 0) 78 | #define HWACCEL_CAP_THREAD_SAFE (1 << 1) 79 | +#define HWACCEL_CAP_INTERNAL_ALLOC (1 << 2) 80 | 81 | typedef struct FFHWAccel { 82 | /** 83 | -- 84 | 2.34.1 85 | 86 | -------------------------------------------------------------------------------- /patches/0023-lavc-vaapi_-vp9-vvc-add-surface-internal-re-allocati.patch: -------------------------------------------------------------------------------- 1 | From 8a910a6c44ef32b5ebce563faf551d8e32c78b2a Mon Sep 17 00:00:00 2001 2 | From: Linjie Fu 3 | Date: Thu, 2 Jan 2020 11:16:31 +0800 4 | Subject: [PATCH] lavc/vaapi_{vp9, vvc}: add surface internal re-allocation 5 | capability 6 | 7 | Signed-off-by: Linjie Fu 8 | --- 9 | libavcodec/vaapi_vp9.c | 2 +- 10 | libavcodec/vaapi_vvc.c | 2 +- 11 | 2 files changed, 2 insertions(+), 2 deletions(-) 12 | 13 | diff --git a/libavcodec/vaapi_vp9.c b/libavcodec/vaapi_vp9.c 14 | index a28fc75a59..60416d19ca 100644 15 | --- a/libavcodec/vaapi_vp9.c 16 | +++ b/libavcodec/vaapi_vp9.c 17 | @@ -181,5 +181,5 @@ const FFHWAccel ff_vp9_vaapi_hwaccel = { 18 | .uninit = ff_vaapi_decode_uninit, 19 | .frame_params = ff_vaapi_common_frame_params, 20 | .priv_data_size = sizeof(VAAPIDecodeContext), 21 | - .caps_internal = HWACCEL_CAP_ASYNC_SAFE, 22 | + .caps_internal = HWACCEL_CAP_ASYNC_SAFE | HWACCEL_CAP_INTERNAL_ALLOC, 23 | }; 24 | diff --git a/libavcodec/vaapi_vvc.c b/libavcodec/vaapi_vvc.c 25 | index 310b56e5a4..5f040a73ad 100644 26 | --- a/libavcodec/vaapi_vvc.c 27 | +++ b/libavcodec/vaapi_vvc.c 28 | @@ -653,5 +653,5 @@ const FFHWAccel ff_vvc_vaapi_hwaccel = { 29 | .uninit = &ff_vaapi_decode_uninit, 30 | .frame_params = &ff_vaapi_common_frame_params, 31 | .priv_data_size = sizeof(VAAPIDecodeContext), 32 | - .caps_internal = HWACCEL_CAP_ASYNC_SAFE, 33 | + .caps_internal = HWACCEL_CAP_ASYNC_SAFE | HWACCEL_CAP_INTERNAL_ALLOC, 34 | }; 35 | -- 36 | 2.34.1 37 | 38 | -------------------------------------------------------------------------------- /patches/0033-lavc-vaapi_hevc-add-skip_frame-invalid-to-skip-inval.patch: -------------------------------------------------------------------------------- 1 | From 015c913a10daa24046ac26b86c393a9cf14c17fb Mon Sep 17 00:00:00 2001 2 | From: Linjie Fu 3 | Date: Tue, 14 Jul 2020 14:56:25 +0800 4 | Subject: [PATCH] lavc/vaapi_hevc: add -skip_frame invalid to skip invalid 5 | nalus 6 | 7 | Works in single thread mode: 8 | $ ffmpeg -threads 1 -hwaccel vaapi -i input-100frames.h265 -f null - 9 | 10 | For multi thread, add '-skip_frame invalid' option to skip the invalid 11 | nalus before an IRAP: 12 | 13 | $ ffmpeg -skip_frame invalid -hwaccel vaapi -i input-100frames.h265 -f null - 14 | 15 | Signed-off-by: Linjie Fu 16 | --- 17 | libavcodec/defs.h | 1 + 18 | libavcodec/hevc/hevcdec.c | 6 +++++- 19 | libavcodec/options_table.h | 1 + 20 | libavcodec/pthread_frame.c | 1 + 21 | 4 files changed, 8 insertions(+), 1 deletion(-) 22 | 23 | diff --git a/libavcodec/defs.h b/libavcodec/defs.h 24 | index 7ddfdcad0b..e46f2a5227 100644 25 | --- a/libavcodec/defs.h 26 | +++ b/libavcodec/defs.h 27 | @@ -213,6 +213,7 @@ enum AVDiscard{ 28 | * keyframes for intra-only or drop just some bidir frames). */ 29 | AVDISCARD_NONE =-16, ///< discard nothing 30 | AVDISCARD_DEFAULT = 0, ///< discard useless packets like 0 size packets in avi 31 | + AVDISCARD_INVALID = 1, ///< discard invalid packets like NALs before IRAP 32 | AVDISCARD_NONREF = 8, ///< discard all non reference 33 | AVDISCARD_BIDIR = 16, ///< discard all bidirectional frames 34 | AVDISCARD_NONINTRA= 24, ///< discard all non intra frames 35 | diff --git a/libavcodec/hevc/hevcdec.c b/libavcodec/hevc/hevcdec.c 36 | index 88f2bcecad..f6018af4fe 100644 37 | --- a/libavcodec/hevc/hevcdec.c 38 | +++ b/libavcodec/hevc/hevcdec.c 39 | @@ -600,8 +600,11 @@ static int hls_slice_header(SliceHeader *sh, const HEVCContext *s, GetBitContext 40 | sh->first_slice_in_pic_flag = get_bits1(gb); 41 | 42 | sh->no_output_of_prior_pics_flag = 0; 43 | - if (IS_IRAP(s)) 44 | + if (IS_IRAP(s)) { 45 | sh->no_output_of_prior_pics_flag = get_bits1(gb); 46 | + if (s->avctx->skip_frame == AVDISCARD_INVALID) 47 | + s->avctx->skip_frame = AVDISCARD_DEFAULT; 48 | + } 49 | 50 | pps_id = get_ue_golomb_long(gb); 51 | if (pps_id >= HEVC_MAX_PPS_COUNT || !s->ps.pps_list[pps_id]) { 52 | @@ -3162,6 +3165,7 @@ static int decode_slice(HEVCContext *s, const H2645NAL *nal, GetBitContext *gb) 53 | 54 | if ((s->avctx->skip_frame >= AVDISCARD_BIDIR && s->sh.slice_type == HEVC_SLICE_B) || 55 | (s->avctx->skip_frame >= AVDISCARD_NONINTRA && s->sh.slice_type != HEVC_SLICE_I) || 56 | + (s->avctx->skip_frame >= AVDISCARD_INVALID && !IS_IRAP(s)) || 57 | (s->avctx->skip_frame >= AVDISCARD_NONKEY && !IS_IRAP(s)) || 58 | ((s->nal_unit_type == HEVC_NAL_RASL_R || s->nal_unit_type == HEVC_NAL_RASL_N) && 59 | s->no_rasl_output_flag)) { 60 | diff --git a/libavcodec/options_table.h b/libavcodec/options_table.h 61 | index 33f1bce887..4225747b39 100644 62 | --- a/libavcodec/options_table.h 63 | +++ b/libavcodec/options_table.h 64 | @@ -256,6 +256,7 @@ static const AVOption avcodec_options[] = { 65 | {"skip_frame" , "skip decoding for the selected frames", OFFSET(skip_frame), AV_OPT_TYPE_INT, {.i64 = AVDISCARD_DEFAULT }, INT_MIN, INT_MAX, V|D, .unit = "avdiscard"}, 66 | {"none" , "discard no frame", 0, AV_OPT_TYPE_CONST, {.i64 = AVDISCARD_NONE }, INT_MIN, INT_MAX, V|D, .unit = "avdiscard"}, 67 | {"default" , "discard useless frames", 0, AV_OPT_TYPE_CONST, {.i64 = AVDISCARD_DEFAULT }, INT_MIN, INT_MAX, V|D, .unit = "avdiscard"}, 68 | +{"invalid" , "discard NALUs before IRAP", 0, AV_OPT_TYPE_CONST, {.i64 = AVDISCARD_INVALID }, INT_MIN, INT_MAX, V|D, .unit = "avdiscard"}, 69 | {"noref" , "discard all non-reference frames", 0, AV_OPT_TYPE_CONST, {.i64 = AVDISCARD_NONREF }, INT_MIN, INT_MAX, V|D, .unit = "avdiscard"}, 70 | {"bidir" , "discard all bidirectional frames", 0, AV_OPT_TYPE_CONST, {.i64 = AVDISCARD_BIDIR }, INT_MIN, INT_MAX, V|D, .unit = "avdiscard"}, 71 | {"nointra" , "discard all frames except I frames", 0, AV_OPT_TYPE_CONST, {.i64 = AVDISCARD_NONINTRA}, INT_MIN, INT_MAX, V|D, .unit = "avdiscard"}, 72 | diff --git a/libavcodec/pthread_frame.c b/libavcodec/pthread_frame.c 73 | index 85a3dce929..537c8b7ae6 100644 74 | --- a/libavcodec/pthread_frame.c 75 | +++ b/libavcodec/pthread_frame.c 76 | @@ -293,6 +293,7 @@ static int update_context_from_thread(AVCodecContext *dst, const AVCodecContext 77 | dst->has_b_frames = src->has_b_frames; 78 | dst->idct_algo = src->idct_algo; 79 | dst->properties = src->properties; 80 | + dst->skip_frame = src->skip_frame; 81 | 82 | dst->bits_per_coded_sample = src->bits_per_coded_sample; 83 | dst->sample_aspect_ratio = src->sample_aspect_ratio; 84 | -- 85 | 2.25.1 86 | 87 | -------------------------------------------------------------------------------- /patches/0034-FFmpeg-vaapi-HEVC-SCC-encode.patch: -------------------------------------------------------------------------------- 1 | From 33f729d784c84d2aa9c173b04db6f3d280b22737 Mon Sep 17 00:00:00 2001 2 | From: Fei Wang 3 | Date: Wed, 18 Sep 2024 10:26:51 +0800 4 | Subject: [PATCH] FFmpeg vaapi HEVC SCC encode. 5 | 6 | Depend on 0008 hevc low power enable patch. 7 | 8 | Main: 9 | ffmpeg -hwaccel vaapi -vaapi_device /dev/dri/renderD128 -v verbose -f rawvideo -pix_fmt yuv420p -s:v 320x240 -r:v 30 -i 320x240_100.i420 -vf 'format=nv12,hwupload' -c:v hevc_vaapi -profile:v main -rc_mode 10 | CQP -g 30 -slices 1 -bf 3 -low_power 1 -vframes 150 -y out.h265 11 | 12 | Main10: 13 | ffmpeg -hwaccel vaapi -vaapi_device /dev/dri/renderD128 -v verbose -f rawvideo -pix_fmt p010 -s:v 320x240 -r:v 30 -i 320x240_100.p010 -vf 'format=p010,hwupload' -c:v hevc_vaapi -profile:v scc -rc_mode CQP 14 | -g 30 -slices 1 -bf 3 -low_power 1 -vframes 150 -y out.h265 15 | 16 | Main 444: 17 | ffmpeg -hwaccel vaapi -vaapi_device /dev/dri/renderD128 -v verbose -f rawvideo -pix_fmt vuyx -s:v 320x240 -r:v 30 -i 320x240_100.ayuv -vf 'format=vuyx,hwupload' -c:v hevc_vaapi -profile:v scc -rc_mode CQP 18 | -g 30 -slices 1 -bf 3 -low_power 1 -vframes 150 -y out.h265 19 | 20 | Main 444-10: 21 | ffmpeg -hwaccel vaapi -vaapi_device /dev/dri/renderD128 -v verbose -f rawvideo -pix_fmt y410 -s:v 320x240 -r:v 30 -i 320x240_100.y410 -vf 'format=y410,hwupload' -c:v hevc_vaapi -profile:v scc -rc_mode CQP 22 | -g 30 -slices 1 -bf 3 -low_power 1 -vframes 150 -y out.h265 23 | 24 | Signed-off-by: Fei Wang 25 | --- 26 | libavcodec/hw_base_encode_h265.c | 18 ++++++++++++++++-- 27 | libavcodec/vaapi_encode_h265.c | 23 ++++++++++++++++++++++- 28 | 2 files changed, 38 insertions(+), 3 deletions(-) 29 | 30 | diff --git a/libavcodec/hw_base_encode_h265.c b/libavcodec/hw_base_encode_h265.c 31 | index dceec8aba9..52ad8f69be 100644 32 | --- a/libavcodec/hw_base_encode_h265.c 33 | +++ b/libavcodec/hw_base_encode_h265.c 34 | @@ -104,9 +104,9 @@ int ff_hw_base_encode_init_params_h265(FFHWBaseEncodeContext *base_ctx, 35 | ptl->general_max_422chroma_constraint_flag = chroma_format <= 2; 36 | ptl->general_max_420chroma_constraint_flag = chroma_format <= 1; 37 | ptl->general_max_monochrome_constraint_flag = chroma_format == 0; 38 | - 39 | - ptl->general_intra_constraint_flag = base_ctx->gop_size == 1; 40 | ptl->general_one_picture_only_constraint_flag = 0; 41 | + ptl->general_intra_constraint_flag = 42 | + (avctx->profile == AV_PROFILE_HEVC_SCC) ? 0 : base_ctx->gop_size == 1; 43 | 44 | ptl->general_lower_bit_rate_constraint_flag = 1; 45 | 46 | @@ -291,6 +291,14 @@ int ff_hw_base_encode_init_params_h265(FFHWBaseEncodeContext *base_ctx, 47 | vui->log2_max_mv_length_horizontal = 15; 48 | vui->log2_max_mv_length_vertical = 15; 49 | 50 | + if (avctx->profile == AV_PROFILE_HEVC_SCC) { 51 | + sps->sps_extension_present_flag = 1; 52 | + sps->sps_scc_extension_flag = 1; 53 | + sps->sps_curr_pic_ref_enabled_flag = 1; 54 | + sps->palette_mode_enabled_flag = 1; 55 | + sps->palette_max_size = 64; 56 | + sps->delta_palette_max_predictor_size = 32; 57 | + } 58 | 59 | // PPS 60 | 61 | @@ -347,5 +355,11 @@ int ff_hw_base_encode_init_params_h265(FFHWBaseEncodeContext *base_ctx, 62 | 63 | pps->pps_loop_filter_across_slices_enabled_flag = 1; 64 | 65 | + if (avctx->profile == AV_PROFILE_HEVC_SCC) { 66 | + pps->pps_extension_present_flag = 1; 67 | + pps->pps_scc_extension_flag =1; 68 | + pps->pps_curr_pic_ref_enabled_flag = 1; 69 | + } 70 | + 71 | return 0; 72 | } 73 | diff --git a/libavcodec/vaapi_encode_h265.c b/libavcodec/vaapi_encode_h265.c 74 | index f4a9003c82..7d11123903 100644 75 | --- a/libavcodec/vaapi_encode_h265.c 76 | +++ b/libavcodec/vaapi_encode_h265.c 77 | @@ -392,6 +392,10 @@ static int vaapi_encode_h265_init_sequence_params(AVCodecContext *avctx) 78 | sps->log2_diff_max_min_pcm_luma_coding_block_size, 79 | 80 | .vui_parameters_present_flag = 0, 81 | + 82 | + .scc_fields.bits = { 83 | + .palette_mode_enabled_flag = sps->palette_mode_enabled_flag, 84 | + } 85 | }; 86 | 87 | *vpic = (VAEncPictureParameterBufferHEVC) { 88 | @@ -444,6 +448,9 @@ static int vaapi_encode_h265_init_sequence_params(AVCodecContext *avctx) 89 | .enable_gpu_weighted_prediction = 0, 90 | .no_output_of_prior_pics_flag = 0, 91 | }, 92 | + .scc_fields.bits = { 93 | + .pps_curr_pic_ref_enabled_flag = pps->pps_curr_pic_ref_enabled_flag, 94 | + } 95 | }; 96 | 97 | if (pps->tiles_enabled_flag) { 98 | @@ -805,8 +812,13 @@ static int vaapi_encode_h265_init_slice_params(AVCodecContext *avctx, 99 | sh->collocated_ref_idx = 0; 100 | } 101 | 102 | + if (avctx->profile == AV_PROFILE_HEVC_SCC) { 103 | + sh->num_ref_idx_active_override_flag = 1; 104 | + sh->num_ref_idx_l0_active_minus1 = pps->num_ref_idx_l0_default_active_minus1 + 1; 105 | + } else { 106 | sh->num_ref_idx_active_override_flag = 0; 107 | sh->num_ref_idx_l0_active_minus1 = pps->num_ref_idx_l0_default_active_minus1; 108 | + } 109 | sh->num_ref_idx_l1_active_minus1 = pps->num_ref_idx_l1_default_active_minus1; 110 | } 111 | 112 | @@ -828,7 +840,9 @@ static int vaapi_encode_h265_init_slice_params(AVCodecContext *avctx, 113 | .slice_type = sh->slice_type, 114 | .slice_pic_parameter_set_id = sh->slice_pic_parameter_set_id, 115 | 116 | - .num_ref_idx_l0_active_minus1 = sh->num_ref_idx_l0_active_minus1, 117 | + .num_ref_idx_l0_active_minus1 = sh->num_ref_idx_active_override_flag ? 118 | + sh->num_ref_idx_l0_active_minus1 - 1 : 119 | + sh->num_ref_idx_l0_active_minus1, 120 | .num_ref_idx_l1_active_minus1 = sh->num_ref_idx_l1_active_minus1, 121 | 122 | .luma_log2_weight_denom = sh->luma_log2_weight_denom, 123 | @@ -1019,6 +1033,12 @@ static const VAAPIEncodeProfile vaapi_encode_h265_profiles[] = { 124 | { AV_PROFILE_HEVC_REXT, 8, 3, 0, 0, VAProfileHEVCMain444 }, 125 | { AV_PROFILE_HEVC_REXT, 10, 3, 0, 0, VAProfileHEVCMain444_10 }, 126 | { AV_PROFILE_HEVC_REXT, 12, 3, 0, 0, VAProfileHEVCMain444_12 }, 127 | + { AV_PROFILE_HEVC_SCC, 8, 3, 1, 1, VAProfileHEVCSccMain }, 128 | + { AV_PROFILE_HEVC_SCC, 10, 3, 1, 1, VAProfileHEVCSccMain10 }, 129 | + { AV_PROFILE_HEVC_SCC, 8, 3, 0, 0, VAProfileHEVCSccMain444 }, 130 | +#endif 131 | +#if VA_CHECK_VERSION(1, 9, 0) 132 | + { AV_PROFILE_HEVC_SCC, 10, 3, 0, 0, VAProfileHEVCSccMain444_10 }, 133 | #endif 134 | { AV_PROFILE_UNKNOWN } 135 | }; 136 | @@ -1118,6 +1138,7 @@ static const AVOption vaapi_encode_h265_options[] = { 137 | { PROFILE("main", AV_PROFILE_HEVC_MAIN) }, 138 | { PROFILE("main10", AV_PROFILE_HEVC_MAIN_10) }, 139 | { PROFILE("rext", AV_PROFILE_HEVC_REXT) }, 140 | + { PROFILE("scc", AV_PROFILE_HEVC_SCC) }, 141 | #undef PROFILE 142 | 143 | { "tier", "Set tier (general_tier_flag)", 144 | -- 145 | 2.34.1 146 | 147 | -------------------------------------------------------------------------------- /patches/0050-libavcodec-qsvdec.c-extract-frame-packing-arrangemen.patch: -------------------------------------------------------------------------------- 1 | From 73cec1372565a981264309802900aadc23973c23 Mon Sep 17 00:00:00 2001 2 | From: Wenbinc-Bin 3 | Date: Mon, 18 Jan 2021 16:22:29 +0800 4 | Subject: [PATCH] libavcodec/qsvdec.c: extract frame packing arrangement data 5 | 6 | Use h264_sei to parse SEI data got from MediaSDK. Extract frame 7 | packing arrangement information from SEI data and config AVStereo3D 8 | side data for decoded frame. 9 | 10 | Sigend-off-by: Wenbin Chen 11 | Sigend-off-by: Tong Wu 12 | --- 13 | libavcodec/qsv_internal.h | 2 + 14 | libavcodec/qsvdec.c | 155 ++++++++++++++++++++++++++++++++++++++ 15 | 2 files changed, 157 insertions(+) 16 | 17 | diff --git a/libavcodec/qsv_internal.h b/libavcodec/qsv_internal.h 18 | index d970cd20f0..9f8f6ac87e 100644 19 | --- a/libavcodec/qsv_internal.h 20 | +++ b/libavcodec/qsv_internal.h 21 | @@ -56,6 +56,8 @@ 22 | 23 | #define QSV_MAX_FRAME_EXT_PARAMS 4 24 | 25 | +#define QSV_PAYLOAD_SIZE 1024 26 | + 27 | #define QSV_VERSION_ATLEAST(MAJOR, MINOR) \ 28 | (MFX_VERSION_MAJOR > (MAJOR) || \ 29 | MFX_VERSION_MAJOR == (MAJOR) && MFX_VERSION_MINOR >= (MINOR)) 30 | 31 | diff --git a/libavcodec/qsvdec.c b/libavcodec/qsvdec.c 32 | index df0d49bc10..02ebd6808b 100644 33 | --- a/libavcodec/qsvdec.c 34 | +++ b/libavcodec/qsvdec.c 35 | @@ -43,12 +43,15 @@ 36 | #include "libavutil/film_grain_params.h" 37 | #include "libavutil/mastering_display_metadata.h" 38 | #include "libavutil/avassert.h" 39 | +#include "libavutil/stereo3d.h" 40 | 41 | #include "avcodec.h" 42 | #include "codec_internal.h" 43 | #include "internal.h" 44 | #include "decode.h" 45 | #include "hwconfig.h" 46 | +#include "get_bits.h" 47 | +#include "h264_sei.h" 48 | #include "qsv.h" 49 | #include "qsv_internal.h" 50 | #include "libavutil/refstruct.h" 51 | @@ -111,8 +114,12 @@ typedef struct QSVContext { 52 | 53 | char *load_plugins; 54 | 55 | + mfxPayload payload; 56 | + 57 | mfxExtBuffer **ext_buffers; 58 | int nb_ext_buffers; 59 | + H264SEIContext sei; 60 | + H264ParamSets ps; 61 | } QSVContext; 62 | 63 | static const AVCodecHWConfigInternal *const qsv_hw_configs[] = { 64 | @@ -784,6 +784,147 @@ static int qsv_export_hdr_side_data_av1(AVCodecContext *avctx, mfxExtMasteringDi 65 | 66 | #endif 67 | 68 | +static int h264_decode_fpa(H2645SEIFramePacking *fpa, AVFrame *frame) 69 | +{ 70 | + if (!fpa || !frame) { 71 | + return AVERROR(EINVAL); 72 | + } 73 | + 74 | + if (!fpa->arrangement_cancel_flag && 75 | + fpa->arrangement_type <= 6 && 76 | + fpa->content_interpretation_type > 0 && 77 | + fpa->content_interpretation_type < 3) { 78 | + AVStereo3D *stereo = av_stereo3d_create_side_data(frame); 79 | + if (stereo) { 80 | + switch (fpa->arrangement_type) { 81 | + case 0: 82 | + stereo->type = AV_STEREO3D_CHECKERBOARD; 83 | + break; 84 | + case 1: 85 | + stereo->type = AV_STEREO3D_COLUMNS; 86 | + break; 87 | + case 2: 88 | + stereo->type = AV_STEREO3D_LINES; 89 | + break; 90 | + case 3: 91 | + if (fpa->quincunx_sampling_flag) 92 | + stereo->type = AV_STEREO3D_SIDEBYSIDE_QUINCUNX; 93 | + else 94 | + stereo->type = AV_STEREO3D_SIDEBYSIDE; 95 | + break; 96 | + case 4: 97 | + stereo->type = AV_STEREO3D_TOPBOTTOM; 98 | + break; 99 | + case 5: 100 | + stereo->type = AV_STEREO3D_FRAMESEQUENCE; 101 | + if (fpa->current_frame_is_frame0_flag) 102 | + stereo->view = AV_STEREO3D_VIEW_LEFT; 103 | + else 104 | + stereo->view = AV_STEREO3D_VIEW_RIGHT; 105 | + break; 106 | + case 6: 107 | + stereo->type = AV_STEREO3D_2D; 108 | + break; 109 | + } 110 | + 111 | + if (fpa->content_interpretation_type == 2) 112 | + stereo->flags = AV_STEREO3D_FLAG_INVERT; 113 | + } 114 | + } 115 | + return 0; 116 | +} 117 | + 118 | +static int h264_parse_side_data(AVCodecContext *avctx, QSVContext *q, AVFrame *frame) 119 | +{ 120 | + GetBitContext gb_payload; 121 | + uint8_t *sei_buffer; 122 | + int sei_buffer_index; 123 | + int ret; 124 | + 125 | + /* remove emulation prevention bytes */ 126 | + sei_buffer = (uint8_t *)av_mallocz(q->payload.NumBit / 8); 127 | + if (!sei_buffer) { 128 | + av_freep(&sei_buffer); 129 | + return AVERROR(ENOMEM); 130 | + } 131 | + sei_buffer_index = 0; 132 | + for (int i = 0; i < q->payload.NumBit / 8; i++) { 133 | + if (q->payload.Data[i] == 3) 134 | + i++; 135 | + sei_buffer[sei_buffer_index] = q->payload.Data[i]; 136 | + sei_buffer_index += 1; 137 | + } 138 | + 139 | + ret = init_get_bits8(&gb_payload, sei_buffer, sei_buffer_index+1); 140 | + if (ret < 0) { 141 | + av_freep(&sei_buffer); 142 | + return ret; 143 | + } 144 | + 145 | + ret = ff_h264_sei_decode(&q->sei, &gb_payload, &q->ps, avctx); 146 | + if (ret < 0) { 147 | + av_freep(&sei_buffer); 148 | + return ret; 149 | + } 150 | + 151 | + switch (q->payload.Type) { 152 | + case SEI_TYPE_FRAME_PACKING_ARRANGEMENT: 153 | + ret = h264_decode_fpa(&q->sei.common.frame_packing, frame); 154 | + break; 155 | + default: 156 | + break; 157 | + } 158 | + 159 | + av_freep(&sei_buffer); 160 | + return ret; 161 | +} 162 | + 163 | +static int extract_frame_side_data(AVCodecContext *avctx, QSVContext *q, AVFrame *frame) 164 | +{ 165 | + mfxU64 ts; 166 | + mfxStatus sts; 167 | + int ret = 0; 168 | + 169 | + if (q->payload.BufSize == 0) { 170 | + q->payload.Data = av_mallocz(QSV_PAYLOAD_SIZE); 171 | + if (!q->payload.Data) { 172 | + av_freep(&q->payload.Data); 173 | + return AVERROR(ENOMEM); 174 | + } 175 | + q->payload.BufSize = QSV_PAYLOAD_SIZE; 176 | + } 177 | + 178 | + sts = MFX_ERR_NONE; 179 | + while (sts == MFX_ERR_NONE) { 180 | + 181 | + sts = MFXVideoDECODE_GetPayload(q->session, &ts, &q->payload); 182 | + 183 | + if (sts == MFX_ERR_NOT_ENOUGH_BUFFER) { 184 | + av_log(avctx, AV_LOG_WARNING, "Space for SEI is not enough. One SEI will be skipped\n"); 185 | + continue; 186 | + } else if (sts != MFX_ERR_NONE || q->payload.NumBit == 0) { 187 | + break; 188 | + } 189 | + 190 | + if (q->payload.Type != SEI_TYPE_FRAME_PACKING_ARRANGEMENT) 191 | + continue; 192 | + 193 | + switch (avctx->codec_id) { 194 | + case AV_CODEC_ID_H264: 195 | + ret = h264_parse_side_data(avctx, q, frame); 196 | + break; 197 | + default: 198 | + break; 199 | + } 200 | + 201 | + if (ret < 0) { 202 | + av_log(avctx, AV_LOG_WARNING, "parse side data failed\n"); 203 | + break; 204 | + } 205 | + } 206 | + return ret; 207 | +} 208 | + 209 | static int qsv_decode(AVCodecContext *avctx, QSVContext *q, 210 | AVFrame *frame, int *got_frame, 211 | const AVPacket *avpkt) 212 | @@ -895,6 +1043,10 @@ static int qsv_decode(AVCodecContext *avctx, QSVContext *q, 213 | 214 | outsurf = &aframe.frame->surface; 215 | 216 | + ret = extract_frame_side_data(avctx, q, frame); 217 | + if (ret < 0) 218 | + av_log(avctx, AV_LOG_WARNING, "Extracting side from packet failed\n"); 219 | + 220 | frame->pts = MFX_PTS_TO_PTS(outsurf->Data.TimeStamp, avctx->pkt_timebase); 221 | #if QSV_VERSION_ATLEAST(1, 34) 222 | if ((avctx->export_side_data & AV_CODEC_EXPORT_DATA_FILM_GRAIN) && 223 | @@ -986,6 +1138,8 @@ static void qsv_decode_close_qsvcontext(QSVContext *q) 224 | av_buffer_unref(&q->frames_ctx.hw_frames_ctx); 225 | av_refstruct_unref(&q->frames_ctx.mids); 226 | av_buffer_pool_uninit(&q->pool); 227 | + 228 | + av_freep(&q->payload.Data); 229 | } 230 | 231 | static int qsv_process_data(AVCodecContext *avctx, QSVContext *q, 232 | -- 233 | 2.34.1 234 | 235 | -------------------------------------------------------------------------------- /patches/0055-libavcodec-qsvenc_hevc-add-main10sp-support-to-hevc_.patch: -------------------------------------------------------------------------------- 1 | From dc39e386767dabce7374a2c3f5a5438471d1955c Mon Sep 17 00:00:00 2001 2 | From: "Chen,Wenbin" 3 | Date: Thu, 26 Aug 2021 16:52:50 +0800 4 | Subject: [PATCH 07/47] libavcodec/qsvenc_hevc: add main10sp support to 5 | hevc_qsv 6 | 7 | Main10sp is a combination of Main10 and one_pic_only flag. 8 | This profile encode 10bit single still picture. 9 | A option "main10sp" is added to ffmpeg-qsv. This option 10 | set MFX_PROFILE_HEVC_MAIN10 profile and 11 | MFX_HEVC_CONSTR_REXT_ONE_PICTURE_ONLY flag to enable main10sp 12 | in ffmpeg-qsv. 13 | 14 | Signed-off-by: Wenbin Chen 15 | --- 16 | libavcodec/qsvenc.c | 31 ++++++++++++++++++++++++++++++- 17 | libavcodec/qsvenc.h | 6 +++++- 18 | libavcodec/qsvenc_hevc.c | 3 +++ 19 | 3 files changed, 38 insertions(+), 2 deletions(-) 20 | 21 | diff --git a/libavcodec/qsvenc.c b/libavcodec/qsvenc.c 22 | index d881c11160..325e4f3571 100644 23 | --- a/libavcodec/qsvenc.c 24 | +++ b/libavcodec/qsvenc.c 25 | @@ -205,6 +205,7 @@ static void dump_video_param(AVCodecContext *avctx, QSVEncContext *q, 26 | #endif 27 | 28 | const char *tmp_str = NULL; 29 | + mfxExtHEVCParam *exthevcparam = NULL; 30 | 31 | if (q->co2_idx > 0) 32 | co2 = (mfxExtCodingOption2*)coding_opts[q->co2_idx]; 33 | @@ -220,6 +221,8 @@ static void dump_video_param(AVCodecContext *avctx, QSVEncContext *q, 34 | exthypermodeparam = (mfxExtHyperModeParam *)coding_opts[q->exthypermodeparam_idx]; 35 | #endif 36 | 37 | + if (q->exthevcparam_idx > 0) 38 | + exthevcparam = (mfxExtHEVCParam *)coding_opts[q->exthevcparam_idx]; 39 | av_log(avctx, AV_LOG_VERBOSE, "profile: %s; level: %"PRIu16"\n", 40 | print_profile(avctx->codec_id, info->CodecProfile), info->CodecLevel); 41 | 42 | @@ -400,6 +403,11 @@ static void dump_video_param(AVCodecContext *avctx, QSVEncContext *q, 43 | av_log(avctx, AV_LOG_VERBOSE, "\n"); 44 | } 45 | #endif 46 | + if (exthevcparam && 47 | + exthevcparam->GeneralConstraintFlags == MFX_HEVC_CONSTR_REXT_ONE_PICTURE_ONLY && 48 | + avctx->codec_id == AV_CODEC_ID_HEVC && 49 | + info->CodecProfile == MFX_PROFILE_HEVC_MAIN10) 50 | + av_log(avctx, AV_LOG_VERBOSE, "Main10sp (Main10 profile and one_pic_only flag): enable\n"); 51 | } 52 | 53 | static void dump_video_vp9_param(AVCodecContext *avctx, QSVEncContext *q, 54 | @@ -1209,6 +1217,18 @@ static int init_video_param(AVCodecContext *avctx, QSVEncContext *q) 55 | q->extparam_internal[q->nb_extparam_internal++] = (mfxExtBuffer *)&q->exthevctiles; 56 | } 57 | 58 | + if (avctx->codec_id == AV_CODEC_ID_HEVC && q->main10sp) { 59 | + if (QSV_RUNTIME_VERSION_ATLEAST(q->ver, 2, 0)) { 60 | + q->param.mfx.CodecProfile = MFX_PROFILE_HEVC_MAIN10; 61 | + q->exthevcparam.Header.BufferId = MFX_EXTBUFF_HEVC_PARAM; 62 | + q->exthevcparam.Header.BufferSz = sizeof(q->exthevcparam); 63 | + q->exthevcparam.GeneralConstraintFlags = MFX_HEVC_CONSTR_REXT_ONE_PICTURE_ONLY; 64 | + q->extparam_internal[q->nb_extparam_internal++] = (mfxExtBuffer *)&q->exthevcparam; 65 | + } else 66 | + av_log(avctx, AV_LOG_WARNING, 67 | + "This version of runtime doesn't support 10bit single still picture\n"); 68 | + } 69 | + 70 | q->extvsi.VideoFullRange = (avctx->color_range == AVCOL_RANGE_JPEG); 71 | q->extvsi.ColourDescriptionPresent = 0; 72 | 73 | @@ -1461,12 +1481,17 @@ static int qsv_retrieve_enc_params(AVCodecContext *avctx, QSVEncContext *q) 74 | }; 75 | #endif 76 | 77 | - mfxExtBuffer *ext_buffers[6 + QSV_HAVE_HE]; 78 | + mfxExtHEVCParam hevc_param_buf = { 79 | + .Header.BufferId = MFX_EXTBUFF_HEVC_PARAM, 80 | + .Header.BufferSz = sizeof(hevc_param_buf), 81 | + }; 82 | 83 | + mfxExtBuffer *ext_buffers[7 + QSV_HAVE_HE]; 84 | int need_pps = avctx->codec_id != AV_CODEC_ID_MPEG2VIDEO; 85 | int ret, ext_buf_num = 0, extradata_offset = 0; 86 | 87 | q->co2_idx = q->co3_idx = q->exthevctiles_idx = q->exthypermodeparam_idx = -1; 88 | + q->exthevcparam_idx = -1; 89 | ext_buffers[ext_buf_num++] = (mfxExtBuffer*)&extradata; 90 | ext_buffers[ext_buf_num++] = (mfxExtBuffer*)&co; 91 | 92 | @@ -1494,6 +1519,10 @@ static int qsv_retrieve_enc_params(AVCodecContext *avctx, QSVEncContext *q) 93 | ext_buffers[ext_buf_num++] = (mfxExtBuffer*)&hyper_mode_param_buf; 94 | } 95 | #endif 96 | + if (avctx->codec_id == AV_CODEC_ID_HEVC && QSV_RUNTIME_VERSION_ATLEAST(q->ver, 2, 0)) { 97 | + q->exthevcparam_idx = ext_buf_num; 98 | + ext_buffers[ext_buf_num++] = (mfxExtBuffer*)&hevc_param_buf; 99 | + } 100 | 101 | q->param.ExtParam = ext_buffers; 102 | q->param.NumExtParam = ext_buf_num; 103 | diff --git a/libavcodec/qsvenc.h b/libavcodec/qsvenc.h 104 | index e3eb083746..522f3b5b55 100644 105 | --- a/libavcodec/qsvenc.h 106 | +++ b/libavcodec/qsvenc.h 107 | @@ -177,6 +177,7 @@ typedef struct QSVEncContext { 108 | mfxExtMultiFrameControl extmfc; 109 | #endif 110 | mfxExtHEVCTiles exthevctiles; 111 | + mfxExtHEVCParam exthevcparam; 112 | mfxExtVP9Param extvp9param; 113 | #if QSV_HAVE_EXT_AV1_PARAM 114 | mfxExtAV1TileParam extav1tileparam; 115 | @@ -193,7 +194,7 @@ typedef struct QSVEncContext { 116 | 117 | mfxExtVideoSignalInfo extvsi; 118 | 119 | - mfxExtBuffer *extparam_internal[5 + (QSV_HAVE_MF * 2) + (QSV_HAVE_EXT_AV1_PARAM * 2) + QSV_HAVE_HE]; 120 | + mfxExtBuffer *extparam_internal[6 + (QSV_HAVE_MF * 2) + (QSV_HAVE_EXT_AV1_PARAM * 2) + QSV_HAVE_HE]; 121 | int nb_extparam_internal; 122 | 123 | mfxExtBuffer **extparam_str; 124 | @@ -321,6 +322,9 @@ typedef struct QSVEncContext { 125 | int dual_gfx; 126 | 127 | AVDictionary *qsv_params; 128 | + int exthevcparam_idx; 129 | + int main10sp; 130 | + 131 | } QSVEncContext; 132 | 133 | int ff_qsv_enc_init(AVCodecContext *avctx, QSVEncContext *q); 134 | diff --git a/libavcodec/qsvenc_hevc.c b/libavcodec/qsvenc_hevc.c 135 | index 2ccbe0464b..ef590a5fa3 100644 136 | --- a/libavcodec/qsvenc_hevc.c 137 | +++ b/libavcodec/qsvenc_hevc.c 138 | @@ -364,6 +364,9 @@ static const AVOption options[] = { 139 | { "int_ref_qp_delta", "QP difference for the refresh MBs", OFFSET(qsv.int_ref_qp_delta), AV_OPT_TYPE_INT, { .i64 = INT16_MIN }, INT16_MIN, INT16_MAX, VE }, 140 | { "int_ref_cycle_dist", "Distance between the beginnings of the intra-refresh cycles in frames", OFFSET(qsv.int_ref_cycle_dist), AV_OPT_TYPE_INT, { .i64 = -1 }, -1, INT16_MAX, VE }, 141 | 142 | +#if QSV_ONEVPL 143 | + { "main10sp", "This profile allow to encode 10 bit single still picture", OFFSET(qsv.main10sp), AV_OPT_TYPE_BOOL, { .i64 = 0 }, 0, 1, VE}, 144 | +#endif 145 | { NULL }, 146 | }; 147 | 148 | -- 149 | 2.34.1 150 | 151 | -------------------------------------------------------------------------------- /patches/0060-avutils-hwcontext-When-deriving-a-hwdevice-search-fo.patch: -------------------------------------------------------------------------------- 1 | From 85b7306c95d044c090bbfb2636639687be7fd070 Mon Sep 17 00:00:00 2001 2 | From: Soft Works 3 | Date: Thu, 25 Nov 2021 02:41:32 +0000 4 | Subject: [PATCH 09/46] avutils/hwcontext: When deriving a hwdevice, search for 5 | existing device in both directions 6 | 7 | The test /libavutil/tests/hwdevice checks that when deriving a device 8 | from a source device and then deriving back to the type of the source 9 | device, the result is matching the original source device, i.e. the 10 | derivation mechanism doesn't create a new device in this case. 11 | 12 | Previously, this test was usually passed, but only due to two different 13 | kind of flaws: 14 | 15 | 1. The test covers only a single level of derivation (and back) 16 | 17 | It derives device Y from device X and then Y back to the type of X and 18 | checks whether the result matches X. 19 | 20 | What it doesn't check for, are longer chains of derivation like: 21 | 22 | CUDA1 > OpenCL2 > CUDA3 and then back to OpenCL4 23 | 24 | In that case, the second derivation returns the first device (CUDA3 == 25 | CUDA1), but when deriving OpenCL4, hwcontext.c was creating a new 26 | OpenCL4 context instead of returning OpenCL2, because there was no link 27 | from CUDA1 to OpenCL2 (only backwards from OpenCL2 to CUDA1) 28 | 29 | If the test would check for two levels of derivation, it would have 30 | failed. 31 | 32 | This patch fixes those (yet untested) cases by introducing forward 33 | references (derived_device) in addition to the existing back references 34 | (source_device). 35 | 36 | 2. hwcontext_qsv didn't properly set the source_device 37 | 38 | In case of QSV, hwcontext_qsv creates a source context internally 39 | (vaapi, dxva2 or d3d11va) without calling av_hwdevice_ctx_create_derived 40 | and without setting source_device. 41 | 42 | This way, the hwcontext test ran successful, but what practically 43 | happened, was that - for example - deriving vaapi from qsv didn't return 44 | the original underlying vaapi device and a new one was created instead: 45 | Exactly what the test is intended to detect and prevent. It just 46 | couldn't do so, because the original device was hidden (= not set as the 47 | source_device of the QSV device). 48 | 49 | This patch properly makes these setting and fixes all derivation 50 | scenarios. 51 | 52 | (at a later stage, /libavutil/tests/hwdevice should be extended to check 53 | longer derivation chains as well) 54 | 55 | Reviewed-by: Lynne 56 | Reviewed-by: Anton Khirnov 57 | Tested-by: Wenbin Chen 58 | Signed-off-by: softworkz 59 | Signed-off-by: Haihao Xiang 60 | --- 61 | libavutil/hwcontext.c | 55 ++++++++++++++++++++++++---------- 62 | libavutil/hwcontext.h | 1 + 63 | libavutil/hwcontext_internal.h | 21 +++++++++++++ 64 | libavutil/hwcontext_qsv.c | 15 ++++++++-- 65 | 4 files changed, 74 insertions(+), 18 deletions(-) 66 | 67 | diff --git a/libavutil/hwcontext.c b/libavutil/hwcontext.c 68 | index fa99a0d8a4..b5e280844c 100644 69 | --- a/libavutil/hwcontext.c 70 | +++ b/libavutil/hwcontext.c 71 | @@ -84,21 +84,6 @@ static const char *const hw_type_names[] = { 72 | [AV_HWDEVICE_TYPE_VULKAN] = "vulkan", 73 | }; 74 | 75 | -typedef struct FFHWDeviceContext { 76 | - /** 77 | - * The public AVHWDeviceContext. See hwcontext.h for it. 78 | - */ 79 | - AVHWDeviceContext p; 80 | - 81 | - const HWContextType *hw_type; 82 | - 83 | - /** 84 | - * For a derived device, a reference to the original device 85 | - * context it was derived from. 86 | - */ 87 | - AVBufferRef *source_device; 88 | -} FFHWDeviceContext; 89 | - 90 | enum AVHWDeviceType av_hwdevice_find_type_by_name(const char *name) 91 | { 92 | int type; 93 | @@ -143,6 +128,7 @@ static void hwdevice_ctx_free(void *opaque, uint8_t *data) 94 | { 95 | FFHWDeviceContext *ctxi = (FFHWDeviceContext*)data; 96 | AVHWDeviceContext *ctx = &ctxi->p; 97 | + int i; 98 | 99 | /* uninit might still want access the hw context and the user 100 | * free() callback might destroy it, so uninit has to be called first */ 101 | @@ -153,6 +139,8 @@ static void hwdevice_ctx_free(void *opaque, uint8_t *data) 102 | ctx->free(ctx); 103 | 104 | av_buffer_unref(&ctxi->source_device); 105 | + for (i = 0; i < AV_HWDEVICE_TYPE_NB; i++) 106 | + av_buffer_unref(&ctxi->derived_devices[i]); 107 | 108 | av_freep(&ctx->hwctx); 109 | av_freep(&ctx); 110 | @@ -633,6 +621,28 @@ fail: 111 | return ret; 112 | } 113 | 114 | +static AVBufferRef* find_derived_hwdevice_ctx(AVBufferRef *src_ref, enum AVHWDeviceType type) 115 | +{ 116 | + AVBufferRef *tmp_ref; 117 | + FFHWDeviceContext *src_ctxi; 118 | + AVHWDeviceContext *src_ctx; 119 | + int i; 120 | + 121 | + src_ctxi = (FFHWDeviceContext *)src_ref->data; 122 | + src_ctx = &src_ctxi->p; 123 | + if (src_ctx->type == type) 124 | + return src_ref; 125 | + 126 | + for (i = 0; i < AV_HWDEVICE_TYPE_NB; i++) 127 | + if (src_ctxi->derived_devices[i]) { 128 | + tmp_ref = find_derived_hwdevice_ctx(src_ctxi->derived_devices[i], type); 129 | + if (tmp_ref) 130 | + return tmp_ref; 131 | + } 132 | + 133 | + return NULL; 134 | +} 135 | + 136 | int av_hwdevice_ctx_create_derived_opts(AVBufferRef **dst_ref_ptr, 137 | enum AVHWDeviceType type, 138 | AVBufferRef *src_ref, 139 | @@ -656,6 +666,16 @@ int av_hwdevice_ctx_create_derived_opts(AVBufferRef **dst_ref_ptr, 140 | tmp_ref = tmp_ctx->source_device; 141 | } 142 | 143 | + tmp_ref = find_derived_hwdevice_ctx(src_ref, type); 144 | + if (tmp_ref) { 145 | + dst_ref = av_buffer_ref(tmp_ref); 146 | + if (!dst_ref) { 147 | + ret = AVERROR(ENOMEM); 148 | + goto fail; 149 | + } 150 | + goto done; 151 | + } 152 | + 153 | dst_ref = av_hwdevice_ctx_alloc(type); 154 | if (!dst_ref) { 155 | ret = AVERROR(ENOMEM); 156 | @@ -676,6 +696,11 @@ int av_hwdevice_ctx_create_derived_opts(AVBufferRef **dst_ref_ptr, 157 | ret = AVERROR(ENOMEM); 158 | goto fail; 159 | } 160 | + tmp_ctx->derived_devices[type] = av_buffer_ref(dst_ref); 161 | + if (!tmp_ctx->derived_devices[type]) { 162 | + ret = AVERROR(ENOMEM); 163 | + goto fail; 164 | + } 165 | ret = av_hwdevice_ctx_init(dst_ref); 166 | if (ret < 0) 167 | goto fail; 168 | diff --git a/libavutil/hwcontext.h b/libavutil/hwcontext.h 169 | index bac30debae..ab05297c59 100644 170 | --- a/libavutil/hwcontext.h 171 | +++ b/libavutil/hwcontext.h 172 | @@ -38,6 +38,7 @@ enum AVHWDeviceType { 173 | AV_HWDEVICE_TYPE_MEDIACODEC, 174 | AV_HWDEVICE_TYPE_VULKAN, 175 | AV_HWDEVICE_TYPE_D3D12VA, 176 | + AV_HWDEVICE_TYPE_NB, ///< number of hw device types, not part of API/ABI. 177 | }; 178 | 179 | /** 180 | diff --git a/libavutil/hwcontext_internal.h b/libavutil/hwcontext_internal.h 181 | index e32b786238..55a6f82d4a 100644 182 | --- a/libavutil/hwcontext_internal.h 183 | +++ b/libavutil/hwcontext_internal.h 184 | @@ -164,4 +164,25 @@ extern const HWContextType ff_hwcontext_type_videotoolbox; 185 | extern const HWContextType ff_hwcontext_type_mediacodec; 186 | extern const HWContextType ff_hwcontext_type_vulkan; 187 | 188 | +typedef struct FFHWDeviceContext { 189 | + /** 190 | + * The public AVHWDeviceContext. See hwcontext.h for it. 191 | + */ 192 | + AVHWDeviceContext p; 193 | + 194 | + const HWContextType *hw_type; 195 | + 196 | + /** 197 | + * For a derived device, a reference to the original device 198 | + * context it was derived from. 199 | + */ 200 | + AVBufferRef *source_device; 201 | + 202 | + /** 203 | + * An array of reference to device contexts which 204 | + * were derived from this device. 205 | + */ 206 | + AVBufferRef *derived_devices[AV_HWDEVICE_TYPE_NB]; 207 | +} FFHWDeviceContext; 208 | + 209 | #endif /* AVUTIL_HWCONTEXT_INTERNAL_H */ 210 | diff --git a/libavutil/hwcontext_qsv.c b/libavutil/hwcontext_qsv.c 211 | index 378cd5e826..c17eff89da 100644 212 | --- a/libavutil/hwcontext_qsv.c 213 | +++ b/libavutil/hwcontext_qsv.c 214 | @@ -365,7 +365,7 @@ static void qsv_frames_uninit(AVHWFramesContext *ctx) 215 | av_buffer_unref(&s->child_frames_ref); 216 | } 217 | 218 | -static void qsv_pool_release_dummy(void *opaque, uint8_t *data) 219 | +static void qsv_release_dummy(void *opaque, uint8_t *data) 220 | { 221 | } 222 | 223 | @@ -378,7 +378,7 @@ static AVBufferRef *qsv_pool_alloc(void *opaque, size_t size) 224 | if (s->nb_surfaces_used < hwctx->nb_surfaces) { 225 | s->nb_surfaces_used++; 226 | return av_buffer_create((uint8_t*)(s->surfaces_internal + s->nb_surfaces_used - 1), 227 | - sizeof(*hwctx->surfaces), qsv_pool_release_dummy, NULL, 0); 228 | + sizeof(*hwctx->surfaces), qsv_release_dummy, NULL, 0); 229 | } 230 | 231 | return NULL; 232 | @@ -2248,8 +2248,17 @@ static int qsv_device_create(AVHWDeviceContext *ctx, const char *device, 233 | child_device = (AVHWDeviceContext*)priv->child_device_ctx->data; 234 | 235 | impl = choose_implementation(device, child_device_type); 236 | + ret = qsv_device_derive_from_child(ctx, impl, child_device, 0); 237 | + if (ret >= 0) { 238 | + FFHWDeviceContext *fctx = (FFHWDeviceContext*)ctx; 239 | + FFHWDeviceContext *fchild_device = (FFHWDeviceContext*)child_device; 240 | + fctx->source_device = av_buffer_ref(priv->child_device_ctx); 241 | + fchild_device->derived_devices[ctx->type] = av_buffer_create((uint8_t*)fctx, sizeof(*fctx), qsv_release_dummy, fctx, 0); 242 | + if (!fchild_device->derived_devices[ctx->type]) 243 | + return AVERROR(ENOMEM); 244 | + } 245 | 246 | - return qsv_device_derive_from_child(ctx, impl, child_device, 0); 247 | + return ret; 248 | } 249 | 250 | const HWContextType ff_hwcontext_type_qsv = { 251 | -- 252 | 2.34.1 253 | 254 | -------------------------------------------------------------------------------- /patches/0062-lavu-add-sub-frame-side-data.patch: -------------------------------------------------------------------------------- 1 | From 3c40560035d4574fca45b6febbd7ea9195ece1e8 Mon Sep 17 00:00:00 2001 2 | From: Fei Wang 3 | Date: Fri, 29 Apr 2022 15:59:38 +0800 4 | Subject: [PATCH] lavu: add sub frame side data 5 | 6 | Sub frame side data allows attach another AVFrame as side data into 7 | the target AVFrame. 8 | 9 | Note: remove version bump in libavutil/version.h compare to version 10 | submitted to community: 11 | 12 | https://patchwork.ffmpeg.org/project/ffmpeg/list/?series=6504 13 | 14 | Signed-off-by: Fei Wang 15 | --- 16 | libavutil/Makefile | 4 ++- 17 | libavutil/frame.c | 5 ++- 18 | libavutil/frame.h | 5 +++ 19 | libavutil/sub_frame_metadata.c | 61 ++++++++++++++++++++++++++++++++++ 20 | libavutil/sub_frame_metadata.h | 35 +++++++++++++++++++ 21 | 5 files changed, 108 insertions(+), 2 deletions(-) 22 | create mode 100644 libavutil/sub_frame_metadata.c 23 | create mode 100644 libavutil/sub_frame_metadata.h 24 | 25 | diff --git a/libavutil/Makefile b/libavutil/Makefile 26 | index 847878d7a4..dda2e90e94 100644 27 | --- a/libavutil/Makefile 28 | +++ b/libavutil/Makefile 29 | @@ -94,7 +94,8 @@ HEADERS = adler32.h \ 30 | xtea.h \ 31 | tea.h \ 32 | tx.h \ 33 | - video_hint.h 34 | + video_hint.h \ 35 | + sub_frame_metadata.h 36 | 37 | ARCH_HEADERS = bswap.h \ 38 | intmath.h \ 39 | @@ -190,6 +191,7 @@ OBJS = adler32.o \ 40 | version.o \ 41 | video_enc_params.o \ 42 | video_hint.o \ 43 | + sub_frame_metadata.o \ 44 | 45 | 46 | OBJS-$(CONFIG_CUDA) += hwcontext_cuda.o 47 | diff --git a/libavutil/frame.c b/libavutil/frame.c 48 | index ab0d8f53..08a04c87 100644 49 | --- a/libavutil/frame.c 50 | +++ b/libavutil/frame.c 51 | @@ -58,6 +58,7 @@ static const AVSideDataDescriptor sd_props[] = { 52 | [AV_FRAME_DATA_SPHERICAL] = { "Spherical Mapping", AV_SIDE_DATA_PROP_GLOBAL | AV_SIDE_DATA_PROP_SIZE_DEPENDENT }, 53 | [AV_FRAME_DATA_ICC_PROFILE] = { "ICC profile", AV_SIDE_DATA_PROP_GLOBAL | AV_SIDE_DATA_PROP_COLOR_DEPENDENT }, 54 | [AV_FRAME_DATA_SEI_UNREGISTERED] = { "H.26[45] User Data Unregistered SEI message", AV_SIDE_DATA_PROP_MULTI }, 55 | + [AV_FRAME_DATA_SUB_FRAME] = { "Sub frame Metadata" }, 56 | [AV_FRAME_DATA_VIDEO_HINT] = { "Encoding video hint", AV_SIDE_DATA_PROP_SIZE_DEPENDENT }, 57 | }; 58 | 59 | @@ -373,7 +374,9 @@ FF_ENABLE_DEPRECATION_WARNINGS 60 | if ( sd_src->type == AV_FRAME_DATA_PANSCAN 61 | && (src->width != dst->width || src->height != dst->height)) 62 | continue; 63 | - if (force_copy) { 64 | + /* Don't copy sub frame side data, otherwise sub frame's pointers in 65 | + * dst may be invalid. */ 66 | + if (force_copy && sd_src->type != AV_FRAME_DATA_SUB_FRAME) { 67 | sd_dst = av_frame_new_side_data(dst, sd_src->type, 68 | sd_src->size); 69 | if (!sd_dst) { 70 | diff --git a/libavutil/frame.h b/libavutil/frame.h 71 | index f7806566d5..7ec9c8b65b 100644 72 | --- a/libavutil/frame.h 73 | +++ b/libavutil/frame.h 74 | @@ -243,6 +243,11 @@ enum AVFrameSideDataType { 75 | * The data is an int storing the view ID. 76 | */ 77 | AV_FRAME_DATA_VIEW_ID, 78 | + 79 | + /** 80 | + * Sub frame of a target frame, as described by AVFrame. 81 | + */ 82 | + AV_FRAME_DATA_SUB_FRAME, 83 | }; 84 | 85 | enum AVActiveFormatDescription { 86 | diff --git a/libavutil/sub_frame_metadata.c b/libavutil/sub_frame_metadata.c 87 | new file mode 100644 88 | index 0000000000..82ea32383f 89 | --- /dev/null 90 | +++ b/libavutil/sub_frame_metadata.c 91 | @@ -0,0 +1,61 @@ 92 | +/* 93 | + * This file is part of FFmpeg. 94 | + * 95 | + * FFmpeg is free software; you can redistribute it and/or 96 | + * modify it under the terms of the GNU Lesser General Public 97 | + * License as published by the Free Software Foundation; either 98 | + * version 2.1 of the License, or (at your option) any later version. 99 | + * 100 | + * FFmpeg is distributed in the hope that it will be useful, 101 | + * but WITHOUT ANY WARRANTY; without even the implied warranty of 102 | + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 103 | + * Lesser General Public License for more details. 104 | + * 105 | + * You should have received a copy of the GNU Lesser General Public 106 | + * License along with FFmpeg; if not, write to the Free Software 107 | + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA 108 | + */ 109 | + 110 | +#include "sub_frame_metadata.h" 111 | + 112 | +static void sub_frame_free(void *opaque, uint8_t *data) 113 | +{ 114 | + AVFrame *frame = (AVFrame*)data; 115 | + 116 | + av_frame_free(&frame); 117 | +} 118 | + 119 | +static AVFrame *sub_frame_alloc(size_t *out_size) 120 | +{ 121 | + AVFrame *sub_frame = av_frame_alloc(); 122 | + if (!sub_frame) 123 | + return NULL; 124 | + 125 | + *out_size = sizeof(*sub_frame); 126 | + 127 | + return sub_frame; 128 | +} 129 | + 130 | +AVFrame *av_sub_frame_create_side_data(AVFrame *frame) 131 | +{ 132 | + AVBufferRef *buf; 133 | + AVFrame *sub_frame; 134 | + size_t size; 135 | + 136 | + sub_frame = sub_frame_alloc(&size); 137 | + if (!sub_frame) 138 | + return NULL; 139 | + 140 | + buf = av_buffer_create((uint8_t *)sub_frame, size, &sub_frame_free, NULL, 0); 141 | + if (!buf) { 142 | + av_frame_free(&sub_frame); 143 | + return NULL; 144 | + } 145 | + 146 | + if (!av_frame_new_side_data_from_buf(frame, AV_FRAME_DATA_SUB_FRAME, buf)) { 147 | + av_buffer_unref(&buf); 148 | + return NULL; 149 | + } 150 | + 151 | + return sub_frame; 152 | +} 153 | diff --git a/libavutil/sub_frame_metadata.h b/libavutil/sub_frame_metadata.h 154 | new file mode 100644 155 | index 0000000000..621fb31e42 156 | --- /dev/null 157 | +++ b/libavutil/sub_frame_metadata.h 158 | @@ -0,0 +1,35 @@ 159 | +/* 160 | + * This file is part of FFmpeg. 161 | + * 162 | + * FFmpeg is free software; you can redistribute it and/or 163 | + * modify it under the terms of the GNU Lesser General Public 164 | + * License as published by the Free Software Foundation; either 165 | + * version 2.1 of the License, or (at your option) any later version. 166 | + * 167 | + * FFmpeg is distributed in the hope that it will be useful, 168 | + * but WITHOUT ANY WARRANTY; without even the implied warranty of 169 | + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 170 | + * Lesser General Public License for more details. 171 | + * 172 | + * You should have received a copy of the GNU Lesser General Public 173 | + * License along with FFmpeg; if not, write to the Free Software 174 | + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA 175 | + */ 176 | + 177 | +#ifndef AVUTIL_SUB_FRAME_METADATA_H 178 | +#define AVUTIL_SUB_FRAME_METADATA_H 179 | + 180 | +#include "frame.h" 181 | + 182 | +/** 183 | + * Allocate a AVFrame structure and add it to the input frame as 184 | + * the side data. The allocated AVFrame will be freed automatically 185 | + * once the buf of created side data reference count decrease to zero. 186 | + * 187 | + * @param frame The frame which side data is added to. 188 | + * 189 | + * @return The AVFrame structure to be filled by caller. 190 | + */ 191 | +AVFrame *av_sub_frame_create_side_data(AVFrame *frame); 192 | + 193 | +#endif /* AVUTIL_SUB_FRAME_METADATA_H */ 194 | -- 195 | 2.34.1 196 | 197 | -------------------------------------------------------------------------------- /patches/0063-lavc-add-sub-frame-options-and-flag.patch: -------------------------------------------------------------------------------- 1 | From f2c449a62dee3804c56fc5b5356e19e08897706f Mon Sep 17 00:00:00 2001 2 | From: Fei Wang 3 | Date: Fri, 29 Apr 2022 15:59:39 +0800 4 | Subject: [PATCH] lavc: add sub frame options and flag 5 | 6 | Note: remove version bump in libavcodec/version.h compare to version 7 | submitted to community: 8 | 9 | https://patchwork.ffmpeg.org/project/ffmpeg/list/?series=6504 10 | 11 | Signed-off-by: Fei Wang 12 | --- 13 | doc/codecs.texi | 9 +++++++++ 14 | libavcodec/avcodec.h | 13 +++++++++++++ 15 | libavcodec/options_table.h | 2 ++ 16 | 3 files changed, 24 insertions(+) 17 | 18 | diff --git a/doc/codecs.texi b/doc/codecs.texi 19 | index 6bdeb664e7..fdcfdcaea3 100644 20 | --- a/doc/codecs.texi 21 | +++ b/doc/codecs.texi 22 | @@ -664,6 +664,9 @@ for codecs that support it. At present, those are H.264 and VP9. 23 | @item film_grain 24 | Export film grain parameters through frame side data (see @code{AV_FRAME_DATA_FILM_GRAIN_PARAMS}). 25 | Supported at present by AV1 decoders. 26 | +@item sub_frame 27 | +Export sub frame through frame side data (see @code{AV_FRAME_DATA_SUB_FRAME}). 28 | +Supported at present by hevc VAAPI decoder. 29 | @item enhancements 30 | Export picture enhancement metadata through frame side data, e.g. LCEVC (see @code{AV_FRAME_DATA_LCEVC}). 31 | @end table 32 | @@ -1020,6 +1023,12 @@ Note: The required alignment depends on if @code{AV_CODEC_FLAG_UNALIGNED} is set 33 | CPU. @code{AV_CODEC_FLAG_UNALIGNED} cannot be changed from the command line. Also hardware 34 | decoders will not apply left/top Cropping. 35 | 36 | +@item sub_frame_opts @var{dict} (@emph{decoding,video}) 37 | +Sub frames parameters, like width/height/format etc. 38 | +@example 39 | +-sub_frame_opts "width=640:height=480:format=nv12" 40 | +@end example 41 | + 42 | 43 | @end table 44 | 45 | diff --git a/libavcodec/avcodec.h b/libavcodec/avcodec.h 46 | index 77ca8dee1f..7fdec356fd 100644 47 | --- a/libavcodec/avcodec.h 48 | +++ b/libavcodec/avcodec.h 49 | @@ -425,6 +425,12 @@ typedef struct RcOverride{ 50 | */ 51 | #define AV_CODEC_EXPORT_DATA_ENHANCEMENTS (1 << 4) 52 | 53 | +/** 54 | + * Decoding only. 55 | + * export sub frame through frame side data. 56 | + */ 57 | +#define AV_CODEC_EXPORT_DATA_SUB_FRAME (1 << 5) 58 | + 59 | /** 60 | * The decoder will keep a reference to the frame and may reuse it later. 61 | */ 62 | @@ -2086,6 +2092,13 @@ typedef struct AVCodecContext { 63 | */ 64 | AVFrameSideData **decoded_side_data; 65 | int nb_decoded_side_data; 66 | + /** 67 | + * Set sub frame's parameters like: width/height/format etc. 68 | + * 69 | + * - decoding: set by user 70 | + * - encoding: unused 71 | + */ 72 | + AVDictionary *sub_frame_opts; 73 | } AVCodecContext; 74 | 75 | /** 76 | diff --git a/libavcodec/options_table.h b/libavcodec/options_table.h 77 | index c8fc0bd532..3300b72ca2 100644 78 | --- a/libavcodec/options_table.h 79 | +++ b/libavcodec/options_table.h 80 | @@ -94,6 +94,7 @@ static const AVOption avcodec_options[] = { 81 | {"venc_params", "export video encoding parameters through frame side data", 0, AV_OPT_TYPE_CONST, {.i64 = AV_CODEC_EXPORT_DATA_VIDEO_ENC_PARAMS}, INT_MIN, INT_MAX, V|D, .unit = "export_side_data"}, 82 | {"film_grain", "export film grain parameters through frame side data", 0, AV_OPT_TYPE_CONST, {.i64 = AV_CODEC_EXPORT_DATA_FILM_GRAIN}, INT_MIN, INT_MAX, V|D, .unit = "export_side_data"}, 83 | {"enhancements", "export picture enhancement metadata through frame side data", 0, AV_OPT_TYPE_CONST, {.i64 = AV_CODEC_EXPORT_DATA_ENHANCEMENTS}, INT_MIN, INT_MAX, V|D, .unit = "export_side_data"}, 84 | +{"sub_frame", "export sub frame through frame side data", 0, AV_OPT_TYPE_CONST, {.i64 = AV_CODEC_EXPORT_DATA_SUB_FRAME}, INT_MIN, INT_MAX, V|D, .unit = "export_side_data"}, 85 | {"time_base", NULL, OFFSET(time_base), AV_OPT_TYPE_RATIONAL, {.dbl = 0}, 0, INT_MAX}, 86 | {"g", "set the group of picture (GOP) size", OFFSET(gop_size), AV_OPT_TYPE_INT, {.i64 = 12 }, INT_MIN, INT_MAX, V|E}, 87 | {"ar", "set audio sampling rate (in Hz)", OFFSET(sample_rate), AV_OPT_TYPE_INT, {.i64 = DEFAULT }, 0, INT_MAX, A|D|E}, 88 | @@ -413,6 +414,7 @@ static const AVOption avcodec_options[] = { 89 | {"mastering_display_metadata", .default_val.i64 = AV_PKT_DATA_MASTERING_DISPLAY_METADATA, .type = AV_OPT_TYPE_CONST, .flags = A|D, .unit = "side_data_pkt" }, 90 | {"content_light_level", .default_val.i64 = AV_PKT_DATA_CONTENT_LIGHT_LEVEL, .type = AV_OPT_TYPE_CONST, .flags = A|D, .unit = "side_data_pkt" }, 91 | {"icc_profile", .default_val.i64 = AV_PKT_DATA_ICC_PROFILE, .type = AV_OPT_TYPE_CONST, .flags = A|D, .unit = "side_data_pkt" }, 92 | +{"sub_frame_opts", "set sub frame opts", OFFSET(sub_frame_opts), AV_OPT_TYPE_DICT, {.str = NULL}, -1, INT_MAX, V|D}, 93 | {NULL}, 94 | }; 95 | 96 | -- 97 | 2.34.1 98 | 99 | -------------------------------------------------------------------------------- /patches/0064-lavc-hevc_vaapi-enable-sub-frame-support.patch: -------------------------------------------------------------------------------- 1 | From 74687432b13e613cd83d0b5e524ee02063d5c273 Mon Sep 17 00:00:00 2001 2 | From: Fei Wang 3 | Date: Fri, 29 Apr 2022 15:59:40 +0800 4 | Subject: [PATCH] lavc/hevc_vaapi: enable sub frame support 5 | 6 | Intel HW provide a feature that allows decoder output another scaled 7 | frame beside original frame. And the scaled frame will attach to main 8 | frame as sub frame side data. 9 | 10 | The use case is mainly for video analysis. For example, scaled down 11 | frame can be used for analysis, and the result can be applied back 12 | to main frame. 13 | 14 | Normally, we use scale_vaapi for scaling in vaapi transcode pipeline 15 | if want to get a smaller resolution frame. While now sub frame can 16 | be used instead. For some platforms, the sub frame scaling is much 17 | more faster than scale_vaapi. For example, the decode + sub frame 18 | cmd will improve ~50% performance than decode + scaling on my DG2 19 | i3-11100B@3.6GHz. 20 | 21 | decode + sub frame cmd: 22 | ffmpeg -hwaccel vaapi -hwaccel_device /dev/dri/renderD128 \ 23 | -hwaccel_output_format vaapi -export_side_data sub_frame \ 24 | -sub_frame_opts "width=300:height=300:format=nv12" \ 25 | -i 1920x1080.h265 -f null - & 26 | 27 | decode + scaling cmd: 28 | ffmpeg -hwaccel vaapi -hwaccel_device /dev/dri/renderD128 \ 29 | -hwaccel_output_format vaapi -i 1920x1080.h265 \ 30 | -vf 'scale_vaapi=w=300:h=300:format=nv12' -f null - & 31 | 32 | Signed-off-by: Fei Wang 33 | --- 34 | libavcodec/vaapi_decode.c | 46 ++++++++++++++++++++++++++- 35 | libavcodec/vaapi_decode.h | 4 +++ 36 | libavcodec/vaapi_hevc.c | 31 +++++++++++++++++++ 37 | libavutil/hwcontext_vaapi.c | 62 +++++++++++++++++++++++++++++++++++-- 38 | libavutil/hwcontext_vaapi.h | 15 ++++++++- 39 | 5 files changed, 153 insertions(+), 5 deletions(-) 40 | 41 | diff --git a/libavcodec/vaapi_decode.c b/libavcodec/vaapi_decode.c 42 | index 4ac2df5b43..ed981fa88c 100644 43 | --- a/libavcodec/vaapi_decode.c 44 | +++ b/libavcodec/vaapi_decode.c 45 | @@ -178,6 +178,10 @@ int ff_vaapi_decode_issue(AVCodecContext *avctx, 46 | av_log(avctx, AV_LOG_DEBUG, "Decode to surface %#x.\n", 47 | pic->output_surface); 48 | 49 | + if (ctx->hwfc->enable_sub_frame) 50 | + av_log(avctx, AV_LOG_DEBUG, "Decode sub frame to surface %#x.\n", 51 | + pic->sub_frame_surface); 52 | + 53 | vas = vaBeginPicture(ctx->hwctx->display, ctx->va_context, 54 | pic->output_surface); 55 | if (vas != VA_STATUS_SUCCESS) { 56 | @@ -491,6 +495,9 @@ static int vaapi_decode_make_config(AVCodecContext *avctx, 57 | AVHWDeviceContext *device = (AVHWDeviceContext*)device_ref->data; 58 | AVVAAPIDeviceContext *hwctx = device->hwctx; 59 | 60 | + VAConfigAttrib attr; 61 | + int attr_num = 0, support_dec_processing = 0; 62 | + 63 | codec_desc = avcodec_descriptor_get(avctx->codec_id); 64 | if (!codec_desc) { 65 | err = AVERROR(EINVAL); 66 | @@ -569,8 +576,23 @@ static int vaapi_decode_make_config(AVCodecContext *avctx, 67 | } 68 | } 69 | 70 | + if (avctx->export_side_data & AV_CODEC_EXPORT_DATA_SUB_FRAME) { 71 | + attr.type = VAConfigAttribDecProcessing; 72 | + vas = vaGetConfigAttributes(hwctx->display, matched_va_profile, 73 | + VAEntrypointVLD, &attr, 1); 74 | + if (vas != VA_STATUS_SUCCESS) { 75 | + av_log(avctx, AV_LOG_ERROR, "Failed to query decode process " 76 | + "attributes: %d (%s).\n", vas, vaErrorStr(vas)); 77 | + return AVERROR_EXTERNAL; 78 | + } else if (attr.value & VA_DEC_PROCESSING) { 79 | + support_dec_processing = 1; 80 | + attr_num++; 81 | + } else 82 | + av_log(avctx, AV_LOG_WARNING, "Hardware doesn't support decode processing.\n"); 83 | + } 84 | + 85 | vas = vaCreateConfig(hwctx->display, matched_va_profile, 86 | - VAEntrypointVLD, NULL, 0, 87 | + VAEntrypointVLD, &attr, attr_num, 88 | va_config); 89 | if (vas != VA_STATUS_SUCCESS) { 90 | av_log(avctx, AV_LOG_ERROR, "Failed to create decode " 91 | @@ -615,10 +637,32 @@ static int vaapi_decode_make_config(AVCodecContext *avctx, 92 | 93 | if (frames_ref) { 94 | AVHWFramesContext *frames = (AVHWFramesContext *)frames_ref->data; 95 | + AVVAAPIFramesContext *avfc = frames->hwctx; 96 | 97 | frames->format = AV_PIX_FMT_VAAPI; 98 | frames->width = avctx->coded_width; 99 | frames->height = avctx->coded_height; 100 | + avfc->enable_sub_frame = support_dec_processing; 101 | + 102 | + if (avfc->enable_sub_frame) { 103 | + avfc->sub_frame_width = avctx->coded_width; 104 | + avfc->sub_frame_height = avctx->coded_height; 105 | + avfc->sub_frame_sw_format = AV_PIX_FMT_NV12; 106 | + if (avctx->sub_frame_opts) { 107 | + AVDictionaryEntry *e = NULL; 108 | + while ((e = av_dict_get(avctx->sub_frame_opts, "", e, AV_DICT_IGNORE_SUFFIX))) { 109 | + if (!strcmp(e->key, "width")) 110 | + avfc->sub_frame_width= atoi(e->value); 111 | + else if (!strcmp(e->key, "height")) 112 | + avfc->sub_frame_height = atoi(e->value); 113 | + else if (!strcmp(e->key, "format")) 114 | + avfc->sub_frame_sw_format = av_get_pix_fmt(e->value); 115 | + } 116 | + } 117 | + av_log(avctx, AV_LOG_DEBUG, "Sub frame set with width:%d, height:%d, " 118 | + "format:%s.\n", avfc->sub_frame_width, avfc->sub_frame_height, 119 | + av_get_pix_fmt_name(avfc->sub_frame_sw_format)); 120 | + } 121 | 122 | err = vaapi_decode_find_best_format(avctx, device, 123 | *va_config, frames); 124 | diff --git a/libavcodec/vaapi_decode.h b/libavcodec/vaapi_decode.h 125 | index 7813473c98..6504f1a101 100644 126 | --- a/libavcodec/vaapi_decode.h 127 | +++ b/libavcodec/vaapi_decode.h 128 | @@ -42,6 +42,10 @@ typedef struct VAAPIDecodePicture { 129 | int nb_slices; 130 | VABufferID *slice_buffers; 131 | int nb_slice_buffers_allocated; 132 | + 133 | + VASurfaceID sub_frame_surface; 134 | + VARectangle sub_frame_src; 135 | + VARectangle sub_frame_dst; 136 | } VAAPIDecodePicture; 137 | 138 | typedef struct VAAPIDecodeContext { 139 | diff --git a/libavcodec/vaapi_hevc.c b/libavcodec/vaapi_hevc.c 140 | index 0c5a829220..a140acbe1e 100644 141 | --- a/libavcodec/vaapi_hevc.c 142 | +++ b/libavcodec/vaapi_hevc.c 143 | @@ -39,6 +39,7 @@ typedef struct VAAPIDecodePictureHEVC { 144 | VAPictureParameterBufferHEVC pic_param; 145 | VASliceParameterBufferHEVC last_slice_param; 146 | #endif 147 | + VAProcPipelineParameterBuffer proc_param; 148 | const uint8_t *last_buffer; 149 | size_t last_size; 150 | 151 | @@ -133,6 +134,7 @@ static int vaapi_hevc_start_frame(AVCodecContext *avctx, 152 | 153 | const ScalingList *scaling_list = NULL; 154 | int pic_param_size, err, i; 155 | + AVFrameSideData *sd; 156 | 157 | #if VA_CHECK_VERSION(1, 2, 0) 158 | int num_comps, pre_palette_size; 159 | @@ -336,6 +338,35 @@ static int vaapi_hevc_start_frame(AVCodecContext *avctx, 160 | goto fail; 161 | } 162 | 163 | + sd = av_frame_get_side_data(h->cur_frame->f, AV_FRAME_DATA_SUB_FRAME); 164 | + if (sd) { 165 | + VAProcPipelineParameterBuffer *proc_param = &pic->proc_param; 166 | + AVFrame *sub_frame = (AVFrame *)sd->data; 167 | + 168 | + memset(proc_param, 0, sizeof(VAProcPipelineParameterBuffer)); 169 | + 170 | + pic->pic.sub_frame_src.x = pic->pic.sub_frame_src.y = 0; 171 | + pic->pic.sub_frame_src.width = sps->width; 172 | + pic->pic.sub_frame_src.height = sps->height; 173 | + 174 | + pic->pic.sub_frame_dst.x = pic->pic.sub_frame_dst.y = 0; 175 | + pic->pic.sub_frame_dst.width = sub_frame->width; 176 | + pic->pic.sub_frame_dst.height = sub_frame->height; 177 | + 178 | + pic->pic.sub_frame_surface = ff_vaapi_get_surface_id(sub_frame); 179 | + proc_param->surface = pic->pic.output_surface; 180 | + proc_param->surface_region = &pic->pic.sub_frame_src; 181 | + proc_param->output_region = &pic->pic.sub_frame_dst; 182 | + proc_param->additional_outputs = &pic->pic.sub_frame_surface; 183 | + proc_param->num_additional_outputs = 1; 184 | + 185 | + err = ff_vaapi_decode_make_param_buffer(avctx, &pic->pic, 186 | + VAProcPipelineParameterBufferType, 187 | + &pic->proc_param, sizeof(VAProcPipelineParameterBuffer)); 188 | + if (err < 0) 189 | + goto fail; 190 | + } 191 | + 192 | return 0; 193 | 194 | fail: 195 | diff --git a/libavutil/hwcontext_vaapi.c b/libavutil/hwcontext_vaapi.c 196 | index 95aa38d9d2..e9f46948a6 100644 197 | --- a/libavutil/hwcontext_vaapi.c 198 | +++ b/libavutil/hwcontext_vaapi.c 199 | @@ -58,8 +58,7 @@ typedef HRESULT (WINAPI *PFN_CREATE_DXGI_FACTORY)(REFIID riid, void **ppFactory) 200 | #include "hwcontext_vaapi.h" 201 | #include "mem.h" 202 | #include "pixdesc.h" 203 | -#include "pixfmt.h" 204 | - 205 | +#include "sub_frame_metadata.h" 206 | 207 | typedef struct VAAPIDevicePriv { 208 | #if HAVE_VAAPI_X11 209 | @@ -101,6 +100,8 @@ typedef struct VAAPIFramesContext { 210 | // Caches whether VA_SURFACE_ATTRIB_MEM_TYPE_DRM_PRIME_2 is unsupported for 211 | // surface imports. 212 | int prime_2_import_unsupported; 213 | + 214 | + AVBufferRef *sub_frames_ref; 215 | } VAAPIFramesContext; 216 | 217 | typedef struct VAAPIMapping { 218 | @@ -549,7 +550,7 @@ static AVBufferRef *vaapi_pool_alloc(void *opaque, size_t size) 219 | return ref; 220 | } 221 | 222 | -static int vaapi_frames_init(AVHWFramesContext *hwfc) 223 | +static int vaapi_hw_frames_init(AVHWFramesContext *hwfc) 224 | { 225 | VAAPIFramesContext *ctx = hwfc->hwctx; 226 | AVVAAPIFramesContext *avfc = &ctx->p; 227 | @@ -701,17 +702,57 @@ fail: 228 | return err; 229 | } 230 | 231 | +static int vaapi_frames_init(AVHWFramesContext *hwfc) 232 | +{ 233 | + VAAPIFramesContext *ctx = hwfc->hwctx; 234 | + AVVAAPIFramesContext *avfc = &ctx->p; 235 | + AVHWFramesContext *sub_frames_ctx; 236 | + int ret; 237 | + 238 | + ret = vaapi_hw_frames_init(hwfc); 239 | + if (ret < 0) 240 | + return ret; 241 | + 242 | + if (avfc->enable_sub_frame){ 243 | + ctx->sub_frames_ref = av_hwframe_ctx_alloc(hwfc->device_ref); 244 | + if (!ctx->sub_frames_ref) { 245 | + return AVERROR(ENOMEM); 246 | + } 247 | + sub_frames_ctx = (AVHWFramesContext*)ctx->sub_frames_ref->data; 248 | + 249 | + sub_frames_ctx->width = avfc->sub_frame_width; 250 | + sub_frames_ctx->height = avfc->sub_frame_height; 251 | + sub_frames_ctx->format = AV_PIX_FMT_VAAPI; 252 | + sub_frames_ctx->sw_format = avfc->sub_frame_sw_format; 253 | + 254 | + ret = av_hwframe_ctx_init(ctx->sub_frames_ref); 255 | + if (ret < 0) { 256 | + av_buffer_unref(&ctx->sub_frames_ref); 257 | + av_log(hwfc, AV_LOG_ERROR, "Error to init sub frame hw context.\n"); 258 | + return ret; 259 | + } 260 | + } 261 | + 262 | + return 0; 263 | +} 264 | + 265 | static void vaapi_frames_uninit(AVHWFramesContext *hwfc) 266 | { 267 | VAAPIFramesContext *ctx = hwfc->hwctx; 268 | AVVAAPIFramesContext *avfc = &ctx->p; 269 | 270 | + av_buffer_unref(&ctx->sub_frames_ref); 271 | av_freep(&avfc->surface_ids); 272 | av_freep(&ctx->attributes); 273 | } 274 | 275 | static int vaapi_get_buffer(AVHWFramesContext *hwfc, AVFrame *frame) 276 | { 277 | + VAAPIFramesContext *ctx = hwfc->hwctx; 278 | + AVVAAPIFramesContext *avfc = &ctx->p; 279 | + AVFrame *sub_frame; 280 | + int ret; 281 | + 282 | frame->buf[0] = av_buffer_pool_get(hwfc->pool); 283 | if (!frame->buf[0]) 284 | return AVERROR(ENOMEM); 285 | @@ -721,6 +762,21 @@ static int vaapi_get_buffer(AVHWFramesContext *hwfc, AVFrame *frame) 286 | frame->width = hwfc->width; 287 | frame->height = hwfc->height; 288 | 289 | + if (avfc->enable_sub_frame) { 290 | + if (!ctx->sub_frames_ref) 291 | + return AVERROR(ENOSYS); 292 | + 293 | + sub_frame = av_sub_frame_create_side_data(frame); 294 | + if (!sub_frame) 295 | + return AVERROR(ENOMEM); 296 | + 297 | + ret = av_hwframe_get_buffer(ctx->sub_frames_ref, sub_frame, 0); 298 | + if (ret < 0) { 299 | + av_log(ctx, AV_LOG_ERROR, "Can't get sub frame.\n"); 300 | + return ret; 301 | + } 302 | + } 303 | + 304 | return 0; 305 | } 306 | 307 | diff --git a/libavutil/hwcontext_vaapi.h b/libavutil/hwcontext_vaapi.h 308 | index 0b2e071cb3..aea0ec9263 100644 309 | --- a/libavutil/hwcontext_vaapi.h 310 | +++ b/libavutil/hwcontext_vaapi.h 311 | @@ -19,6 +19,7 @@ 312 | #ifndef AVUTIL_HWCONTEXT_VAAPI_H 313 | #define AVUTIL_HWCONTEXT_VAAPI_H 314 | 315 | +#include "pixfmt.h" 316 | #include 317 | 318 | /** 319 | @@ -81,7 +82,7 @@ typedef struct AVVAAPIDeviceContext { 320 | } AVVAAPIDeviceContext; 321 | 322 | /** 323 | - * VAAPI-specific data associated with a frame pool. 324 | + * VAAPI-specific data associated with a frame pool and sub frame. 325 | * 326 | * Allocated as AVHWFramesContext.hwctx. 327 | */ 328 | @@ -100,6 +101,18 @@ typedef struct AVVAAPIFramesContext { 329 | */ 330 | VASurfaceID *surface_ids; 331 | int nb_surfaces; 332 | + 333 | + /** 334 | + * Set by the user to indicate if need to enable sub frame support. 335 | + */ 336 | + int enable_sub_frame; 337 | + 338 | + /** 339 | + * Sub frame width/height/format. Only avaliable if enable_sub_frame 340 | + * is true. 341 | + */ 342 | + int sub_frame_width, sub_frame_height; 343 | + enum AVPixelFormat sub_frame_sw_format; 344 | } AVVAAPIFramesContext; 345 | 346 | /** 347 | -- 348 | 2.34.1 349 | 350 | -------------------------------------------------------------------------------- /patches/0065-examples-seperate-vaapi_decode-from-hw_decode.patch: -------------------------------------------------------------------------------- 1 | From 72e6ac27bf10fd30273130c2e171e81c34cbe37c Mon Sep 17 00:00:00 2001 2 | From: Fei Wang 3 | Date: Fri, 29 Apr 2022 15:59:41 +0800 4 | Subject: [PATCH] examples: seperate vaapi_decode from hw_decode 5 | 6 | Now vaapi_decode can be used to test vaapi decode and vaapi decode 7 | with sub frame. 8 | 9 | decode example: 10 | $ vaapi_decode 1920x1080.h265 out.yuv 11 | 12 | decode with sub frame example: 13 | $ vaapi_decode 1920x1080.h265 out.yuv width=640:height=480:format=argb sub_out.argb 14 | 15 | Signed-off-by: Fei Wang 16 | --- 17 | configure | 2 + 18 | doc/examples/Makefile | 1 + 19 | doc/examples/vaapi_decode.c | 296 ++++++++++++++++++++++++++++++++++++ 20 | 3 files changed, 299 insertions(+) 21 | create mode 100644 doc/examples/vaapi_decode.c 22 | 23 | diff --git a/configure b/configure 24 | index 932805ccf3c1..11d74cb60fdf 100755 25 | --- a/configure 26 | +++ b/configure 27 | @@ -1731,6 +1731,7 @@ EXAMPLE_LIST=" 28 | show_metadata_example 29 | transcode_aac_example 30 | transcode_example 31 | + vaapi_decode_example 32 | vaapi_encode_example 33 | vaapi_transcode_example 34 | qsv_transcode_example 35 | @@ -3795,6 +3796,7 @@ scale_video_example_deps="avutil swscale" 36 | show_metadata_example_deps="avformat avutil" 37 | transcode_aac_example_deps="avcodec avformat swresample" 38 | transcode_example_deps="avfilter avcodec avformat avutil" 39 | +vaapi_decode_example_deps="avcodec avformat avutil" 40 | vaapi_encode_example_deps="avcodec avutil h264_vaapi_encoder" 41 | vaapi_transcode_example_deps="avcodec avformat avutil h264_vaapi_encoder" 42 | qsv_transcode_example_deps="avcodec avformat avutil h264_qsv_encoder" 43 | diff --git a/doc/examples/Makefile b/doc/examples/Makefile 44 | index 4efed6b11d8e..583c98e54e6e 100644 45 | --- a/doc/examples/Makefile 46 | +++ b/doc/examples/Makefile 47 | @@ -19,6 +19,7 @@ EXAMPLES-$(CONFIG_SCALE_VIDEO_EXAMPLE) += scale_video 48 | EXAMPLES-$(CONFIG_SHOW_METADATA_EXAMPLE) += show_metadata 49 | EXAMPLES-$(CONFIG_TRANSCODE_AAC_EXAMPLE) += transcode_aac 50 | EXAMPLES-$(CONFIG_TRANSCODE_EXAMPLE) += transcode 51 | +EXAMPLES-$(CONFIG_VAAPI_DECODE_EXAMPLE) += vaapi_decode 52 | EXAMPLES-$(CONFIG_VAAPI_ENCODE_EXAMPLE) += vaapi_encode 53 | EXAMPLES-$(CONFIG_VAAPI_TRANSCODE_EXAMPLE) += vaapi_transcode 54 | EXAMPLES-$(CONFIG_QSV_TRANSCODE_EXAMPLE) += qsv_transcode 55 | diff --git a/doc/examples/vaapi_decode.c b/doc/examples/vaapi_decode.c 56 | new file mode 100644 57 | index 000000000000..a9c8d48240db 58 | --- /dev/null 59 | +++ b/doc/examples/vaapi_decode.c 60 | @@ -0,0 +1,296 @@ 61 | +/* 62 | + * Video Acceleration API (video decoding) decode sample 63 | + * 64 | + * Permission is hereby granted, free of charge, to any person obtaining a copy 65 | + * of this software and associated documentation files (the "Software"), to deal 66 | + * in the Software without restriction, including without limitation the rights 67 | + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 68 | + * copies of the Software, and to permit persons to whom the Software is 69 | + * furnished to do so, subject to the following conditions: 70 | + * 71 | + * The above copyright notice and this permission notice shall be included in 72 | + * all copies or substantial portions of the Software. 73 | + * 74 | + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 75 | + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 76 | + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 77 | + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 78 | + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 79 | + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN 80 | + * THE SOFTWARE. 81 | + */ 82 | + 83 | +/** 84 | + * @file 85 | + * VAAPI-accelerated decoding example. 86 | + * 87 | + * @example vaapi_decode.c 88 | + * This example shows how to do VAAPI-accelerated decoding with output 89 | + * frames from the HW video surfaces. Also support decoding with sub frame. 90 | + */ 91 | + 92 | +#include 93 | + 94 | +#include 95 | +#include 96 | +#include 97 | +#include 98 | +#include 99 | +#include 100 | +#include 101 | + 102 | +static AVBufferRef *hw_device_ctx = NULL; 103 | +static enum AVPixelFormat hw_pix_fmt; 104 | +static FILE *output_file = NULL; 105 | +static FILE *sub_frame_output = NULL; 106 | + 107 | +static int hw_decoder_init(AVCodecContext *ctx, const enum AVHWDeviceType type) 108 | +{ 109 | + int err = 0; 110 | + 111 | + if ((err = av_hwdevice_ctx_create(&hw_device_ctx, type, 112 | + NULL, NULL, 0)) < 0) { 113 | + fprintf(stderr, "Failed to create specified HW device.\n"); 114 | + return err; 115 | + } 116 | + ctx->hw_device_ctx = av_buffer_ref(hw_device_ctx); 117 | + 118 | + return err; 119 | +} 120 | + 121 | +static enum AVPixelFormat get_hw_format(AVCodecContext *ctx, 122 | + const enum AVPixelFormat *pix_fmts) 123 | +{ 124 | + const enum AVPixelFormat *p; 125 | + 126 | + for (p = pix_fmts; *p != -1; p++) { 127 | + if (*p == hw_pix_fmt) 128 | + return *p; 129 | + } 130 | + 131 | + fprintf(stderr, "Failed to get HW surface format.\n"); 132 | + return AV_PIX_FMT_NONE; 133 | +} 134 | + 135 | +static int retrieve_write(AVFrame *frame, AVFrame *sw_frame, FILE *file) 136 | +{ 137 | + AVFrame *tmp_frame = NULL; 138 | + uint8_t *buffer = NULL; 139 | + int size; 140 | + int ret = 0; 141 | + 142 | + if (frame->format == hw_pix_fmt) { 143 | + /* retrieve data from GPU to CPU */ 144 | + if ((ret = av_hwframe_transfer_data(sw_frame, frame, 0)) < 0) { 145 | + fprintf(stderr, "Error transferring the data to system memory\n"); 146 | + goto fail; 147 | + } 148 | + tmp_frame = sw_frame; 149 | + } else 150 | + tmp_frame = frame; 151 | + 152 | + size = av_image_get_buffer_size(tmp_frame->format, tmp_frame->width, 153 | + tmp_frame->height, 1); 154 | + buffer = av_malloc(size); 155 | + if (!buffer) { 156 | + fprintf(stderr, "Can not alloc buffer\n"); 157 | + ret = AVERROR(ENOMEM); 158 | + goto fail; 159 | + } 160 | + ret = av_image_copy_to_buffer(buffer, size, 161 | + (const uint8_t * const *)tmp_frame->data, 162 | + (const int *)tmp_frame->linesize, tmp_frame->format, 163 | + tmp_frame->width, tmp_frame->height, 1); 164 | + if (ret < 0) { 165 | + fprintf(stderr, "Can not copy image to buffer\n"); 166 | + goto fail; 167 | + } 168 | + 169 | + if ((ret = fwrite(buffer, 1, size, file)) < 0) { 170 | + fprintf(stderr, "Failed to dump raw data.\n"); 171 | + goto fail; 172 | + } 173 | + 174 | +fail: 175 | + av_freep(&buffer); 176 | + 177 | + return ret; 178 | +} 179 | + 180 | +static int decode_write(AVCodecContext *avctx, AVPacket *packet) 181 | +{ 182 | + AVFrame *frame = NULL, *sw_frame = NULL; 183 | + AVFrame *sub_frame = NULL, *sw_sub_frame = NULL; 184 | + AVFrameSideData *sd = NULL; 185 | + int ret = 0; 186 | + 187 | + ret = avcodec_send_packet(avctx, packet); 188 | + if (ret < 0) { 189 | + fprintf(stderr, "Error during decoding\n"); 190 | + return ret; 191 | + } 192 | + 193 | + while (1) { 194 | + if (!(frame = av_frame_alloc()) || !(sw_frame = av_frame_alloc())) { 195 | + fprintf(stderr, "Can not alloc frame\n"); 196 | + ret = AVERROR(ENOMEM); 197 | + goto fail; 198 | + } 199 | + 200 | + ret = avcodec_receive_frame(avctx, frame); 201 | + if (ret == AVERROR(EAGAIN) || ret == AVERROR_EOF) { 202 | + av_frame_free(&frame); 203 | + av_frame_free(&sw_frame); 204 | + return 0; 205 | + } else if (ret < 0) { 206 | + fprintf(stderr, "Error while decoding\n"); 207 | + goto fail; 208 | + } 209 | + 210 | + ret = retrieve_write(frame, sw_frame, output_file); 211 | + if (ret < 0) { 212 | + fprintf(stderr, "Error while retrieve and write data\n"); 213 | + goto fail; 214 | + } 215 | + 216 | + sd = av_frame_get_side_data(frame, AV_FRAME_DATA_SUB_FRAME); 217 | + if (sd) { 218 | + if (!(sw_sub_frame = av_frame_alloc())) { 219 | + fprintf(stderr, "Can not alloc sub frame\n"); 220 | + ret = AVERROR(ENOMEM); 221 | + goto fail; 222 | + } 223 | + 224 | + sub_frame = (AVFrame *)sd->data; 225 | + 226 | + ret = retrieve_write(sub_frame, sw_sub_frame, sub_frame_output); 227 | + if (ret < 0) { 228 | + fprintf(stderr, "Error while retrieve and write sub frame data\n"); 229 | + goto fail; 230 | + } 231 | + 232 | + av_frame_remove_side_data(frame, AV_FRAME_DATA_SUB_FRAME); 233 | + sd = NULL; 234 | + } 235 | + 236 | + fail: 237 | + av_frame_free(&frame); 238 | + av_frame_free(&sw_frame); 239 | + av_frame_free(&sw_sub_frame); 240 | + if (ret < 0) 241 | + return ret; 242 | + } 243 | +} 244 | + 245 | +int main(int argc, char *argv[]) 246 | +{ 247 | + AVFormatContext *input_ctx = NULL; 248 | + AVStream *video = NULL; 249 | + AVCodecContext *decoder_ctx = NULL; 250 | + const AVCodec *decoder = NULL; 251 | + AVPacket *packet = NULL; 252 | + int video_stream, ret, i; 253 | + 254 | + if (argc !=3 && argc != 5) { 255 | + fprintf(stderr, "Decode only Usage: %s \n", argv[0]); 256 | + fprintf(stderr, "Decode with sub frame Usage: %s \n", argv[0]); 257 | + return -1; 258 | + } 259 | + 260 | + // av_log_set_level(AV_LOG_DEBUG); 261 | + 262 | + packet = av_packet_alloc(); 263 | + if (!packet) { 264 | + fprintf(stderr, "Failed to allocate AVPacket\n"); 265 | + return -1; 266 | + } 267 | + 268 | + /* open the input file */ 269 | + if (avformat_open_input(&input_ctx, argv[1], NULL, NULL) != 0) { 270 | + fprintf(stderr, "Cannot open input file '%s'\n", argv[1]); 271 | + return -1; 272 | + } 273 | + 274 | + if (avformat_find_stream_info(input_ctx, NULL) < 0) { 275 | + fprintf(stderr, "Cannot find input stream information.\n"); 276 | + return -1; 277 | + } 278 | + 279 | + /* find the video stream information */ 280 | + ret = av_find_best_stream(input_ctx, AVMEDIA_TYPE_VIDEO, -1, -1, &decoder, 0); 281 | + if (ret < 0) { 282 | + fprintf(stderr, "Cannot find a video stream in the input file\n"); 283 | + return -1; 284 | + } 285 | + video_stream = ret; 286 | + 287 | + for (i = 0;; i++) { 288 | + const AVCodecHWConfig *config = avcodec_get_hw_config(decoder, i); 289 | + if (!config) { 290 | + fprintf(stderr, "Decoder %s does not support device type %s.\n", 291 | + decoder->name, av_hwdevice_get_type_name(AV_HWDEVICE_TYPE_VAAPI)); 292 | + return -1; 293 | + } 294 | + if (config->methods & AV_CODEC_HW_CONFIG_METHOD_HW_DEVICE_CTX && 295 | + config->device_type == AV_HWDEVICE_TYPE_VAAPI) { 296 | + hw_pix_fmt = config->pix_fmt; 297 | + break; 298 | + } 299 | + } 300 | + 301 | + if (!(decoder_ctx = avcodec_alloc_context3(decoder))) 302 | + return AVERROR(ENOMEM); 303 | + 304 | + video = input_ctx->streams[video_stream]; 305 | + if (avcodec_parameters_to_context(decoder_ctx, video->codecpar) < 0) 306 | + return -1; 307 | + 308 | + decoder_ctx->get_format = get_hw_format; 309 | + 310 | + if (argc == 5) { 311 | + sub_frame_output = fopen(argv[4], "w+b"); 312 | + decoder_ctx->export_side_data = decoder_ctx->export_side_data | AV_CODEC_EXPORT_DATA_SUB_FRAME; 313 | + if ((ret = av_dict_parse_string(&decoder_ctx->sub_frame_opts, argv[3], "=", ":", 0)) < 0) { 314 | + av_log(decoder_ctx, AV_LOG_ERROR, "Failed to parse option string '%s'.\n", argv[3]); 315 | + av_dict_free(&decoder_ctx->sub_frame_opts); 316 | + return -1; 317 | + } 318 | + } 319 | + 320 | + if (hw_decoder_init(decoder_ctx, AV_HWDEVICE_TYPE_VAAPI) < 0) 321 | + return -1; 322 | + 323 | + if ((ret = avcodec_open2(decoder_ctx, decoder, NULL)) < 0) { 324 | + fprintf(stderr, "Failed to open codec for stream #%u\n", video_stream); 325 | + return -1; 326 | + } 327 | + 328 | + /* open the file to dump raw data */ 329 | + output_file = fopen(argv[2], "w+b"); 330 | + 331 | + /* actual decoding and dump the raw data */ 332 | + while (ret >= 0) { 333 | + if ((ret = av_read_frame(input_ctx, packet)) < 0) 334 | + break; 335 | + 336 | + if (video_stream == packet->stream_index) 337 | + ret = decode_write(decoder_ctx, packet); 338 | + 339 | + av_packet_unref(packet); 340 | + } 341 | + 342 | + /* flush the decoder */ 343 | + ret = decode_write(decoder_ctx, NULL); 344 | + 345 | + if (output_file) 346 | + fclose(output_file); 347 | + if (sub_frame_output) 348 | + fclose(sub_frame_output); 349 | + av_packet_free(&packet); 350 | + av_dict_free(&decoder_ctx->sub_frame_opts); 351 | + avcodec_free_context(&decoder_ctx); 352 | + avformat_close_input(&input_ctx); 353 | + av_buffer_unref(&hw_device_ctx); 354 | + 355 | + return 0; 356 | +} 357 | -- 358 | 2.38.1 359 | 360 | -------------------------------------------------------------------------------- /patches/0066-lavfi-Add-exportsubframe-filter.patch: -------------------------------------------------------------------------------- 1 | From 48899dbabe34f8be0d36bf7f622aec5fa18056f1 Mon Sep 17 00:00:00 2001 2 | From: Fei Wang 3 | Date: Mon, 27 Feb 2023 13:45:58 +0800 4 | Subject: [PATCH] lavfi: Add exportsubframe filter 5 | 6 | Example cmdline: 7 | ffmpeg -hwaccel vaapi -hwaccel_device /dev/dri/renderD128 -hwaccel_output_format vaapi \ 8 | -export_side_data sub_frame -sub_frame_opts "width=640:height=480:format=nv12" \ 9 | -i 1080p_10.h265 -vf exportsubframe=w=640:h=480 -c:v h264_vaapi -y out.h264 10 | 11 | Note, please ensure resolution set in sub_frame_opts and exportsubframe 12 | are same. Otherwise, exportsubframe filter will output error like: 13 | "Invalid sub frame, expect 64x48, actual 640x480." 14 | 15 | Signed-off-by: Fei Wang 16 | --- 17 | libavfilter/Makefile | 1 + 18 | libavfilter/allfilters.c | 1 + 19 | libavfilter/vf_exportsubframe.c | 139 ++++++++++++++++++++++++++++++++ 20 | 3 files changed, 141 insertions(+) 21 | create mode 100644 libavfilter/vf_exportsubframe.c 22 | 23 | diff --git a/libavfilter/Makefile b/libavfilter/Makefile 24 | index 71e198bbf9..0657e42123 100644 25 | --- a/libavfilter/Makefile 26 | +++ b/libavfilter/Makefile 27 | @@ -291,6 +291,7 @@ OBJS-$(CONFIG_EROSION_FILTER) += vf_neighbor.o 28 | OBJS-$(CONFIG_EROSION_OPENCL_FILTER) += vf_neighbor_opencl.o opencl.o \ 29 | opencl/neighbor.o 30 | OBJS-$(CONFIG_ESTDIF_FILTER) += vf_estdif.o 31 | +OBJS-$(CONFIG_EXPORTSUBFRAME_FILTER) += vf_exportsubframe.o 32 | OBJS-$(CONFIG_EXPOSURE_FILTER) += vf_exposure.o 33 | OBJS-$(CONFIG_EXTRACTPLANES_FILTER) += vf_extractplanes.o 34 | OBJS-$(CONFIG_FADE_FILTER) += vf_fade.o 35 | diff --git a/libavfilter/allfilters.c b/libavfilter/allfilters.c 36 | index d7db46c2af..737ebae919 100644 37 | --- a/libavfilter/allfilters.c 38 | +++ b/libavfilter/allfilters.c 39 | @@ -268,6 +268,7 @@ extern const AVFilter ff_vf_eq; 40 | extern const AVFilter ff_vf_erosion; 41 | extern const AVFilter ff_vf_erosion_opencl; 42 | extern const AVFilter ff_vf_estdif; 43 | +extern const AVFilter ff_vf_exportsubframe; 44 | extern const AVFilter ff_vf_exposure; 45 | extern const AVFilter ff_vf_extractplanes; 46 | extern const AVFilter ff_vf_fade; 47 | diff --git a/libavfilter/vf_exportsubframe.c b/libavfilter/vf_exportsubframe.c 48 | new file mode 100644 49 | index 0000000000..4d16d2ff93 50 | --- /dev/null 51 | +++ b/libavfilter/vf_exportsubframe.c 52 | @@ -0,0 +1,139 @@ 53 | +/* 54 | + * This file is part of FFmpeg. 55 | + * 56 | + * FFmpeg is free software; you can redistribute it and/or 57 | + * modify it under the terms of the GNU Lesser General Public 58 | + * License as published by the Free Software Foundation; either 59 | + * version 2.1 of the License, or (at your option) any later version. 60 | + * 61 | + * FFmpeg is distributed in the hope that it will be useful, 62 | + * but WITHOUT ANY WARRANTY; without even the implied warranty of 63 | + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 64 | + * Lesser General Public License for more details. 65 | + * 66 | + * You should have received a copy of the GNU Lesser General Public 67 | + * License along with FFmpeg; if not, write to the Free Software 68 | + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA 69 | + */ 70 | + 71 | +/** 72 | + * @file 73 | + * Export subframe filter 74 | + */ 75 | + 76 | +#include 77 | + 78 | +#include "libavutil/opt.h" 79 | +#include "libavutil/pixdesc.h" 80 | + 81 | +#include "filters.h" 82 | + 83 | +typedef struct ExportSubframeContext { 84 | + const AVClass *class; 85 | + int w, h; 86 | +} ExportSubframeContext; 87 | + 88 | +static int exportsubframe_filter_frame(AVFilterLink *inlink, AVFrame *input_frame) 89 | +{ 90 | + AVFilterContext *avctx = inlink->dst; 91 | + AVFilterLink *outlink = avctx->outputs[0]; 92 | + ExportSubframeContext *ctx = avctx->priv; 93 | + AVFrame *output_frame = NULL, *sub_frame = NULL; 94 | + AVFrameSideData *sd = NULL; 95 | + int ret; 96 | + 97 | + av_log(avctx, AV_LOG_DEBUG, "Filter input: %s, %ux%u (%"PRId64").\n", 98 | + av_get_pix_fmt_name(input_frame->format), 99 | + input_frame->width, input_frame->height, input_frame->pts); 100 | + 101 | + sd = av_frame_get_side_data(input_frame, AV_FRAME_DATA_SUB_FRAME); 102 | + if (sd) { 103 | + sub_frame = (AVFrame *)sd->data; 104 | + output_frame = av_frame_alloc(); 105 | + if (!output_frame) { 106 | + ret = AVERROR(ENOMEM); 107 | + goto fail; 108 | + } 109 | + 110 | + if (sub_frame->width != ctx->w || sub_frame->height != ctx->h) { 111 | + av_log(ctx, AV_LOG_ERROR, "Invalid sub frame, expect %ux%u, actual %ux%u.\n", 112 | + ctx->w, ctx->h, sub_frame->width, sub_frame->height); 113 | + ret = AVERROR_INVALIDDATA; 114 | + goto fail; 115 | + } 116 | + 117 | + ret = av_frame_ref(output_frame, sub_frame); 118 | + if (ret < 0) 119 | + goto fail; 120 | + 121 | + av_frame_remove_side_data(input_frame, AV_FRAME_DATA_SUB_FRAME); 122 | + 123 | + ret = av_frame_copy_props(output_frame, input_frame); 124 | + if (ret < 0) 125 | + goto fail; 126 | + 127 | + } else { 128 | + av_log(ctx, AV_LOG_ERROR, "No sub frame found.\n"); 129 | + return AVERROR_INVALIDDATA; 130 | + } 131 | + 132 | + av_frame_free(&input_frame); 133 | + 134 | + av_log(avctx, AV_LOG_DEBUG, "Filter output: %s, %ux%u (%"PRId64").\n", 135 | + av_get_pix_fmt_name(output_frame->format), 136 | + output_frame->width, output_frame->height, output_frame->pts); 137 | + 138 | + return ff_filter_frame(outlink, output_frame); 139 | + 140 | +fail: 141 | + av_frame_free(&input_frame); 142 | + av_frame_free(&output_frame); 143 | + return ret; 144 | +} 145 | + 146 | +static int config_output(AVFilterLink *outlink) 147 | +{ 148 | + AVFilterContext *avctx = outlink->src; 149 | + AVFilterLink *inlink = outlink->src->inputs[0]; 150 | + ExportSubframeContext *ctx = avctx->priv; 151 | + 152 | + outlink->w = ctx->w; 153 | + outlink->h = ctx->h; 154 | + outlink->sample_aspect_ratio = inlink->sample_aspect_ratio; 155 | + 156 | + return 0; 157 | +} 158 | +#define OFFSET(x) offsetof(ExportSubframeContext, x) 159 | +#define FLAGS (AV_OPT_FLAG_AUDIO_PARAM | AV_OPT_FLAG_VIDEO_PARAM | AV_OPT_FLAG_FILTERING_PARAM) 160 | +static const AVOption exportsubframe_options[] = { 161 | + { "w", "set subframe width", OFFSET(w), AV_OPT_TYPE_INT, { .i64 = 0 }, 0, INT_MAX, FLAGS }, 162 | + { "h", "set subframe height outputs", OFFSET(h), AV_OPT_TYPE_INT, { .i64 = 0 }, 0, INT_MAX, FLAGS }, 163 | + { NULL } 164 | +}; 165 | + 166 | +AVFILTER_DEFINE_CLASS(exportsubframe); 167 | + 168 | +static const AVFilterPad inputs[] = { 169 | + { 170 | + .name = "default", 171 | + .type = AVMEDIA_TYPE_VIDEO, 172 | + .filter_frame = &exportsubframe_filter_frame, 173 | + }, 174 | +}; 175 | + 176 | +static const AVFilterPad outputs[] = { 177 | + { 178 | + .name = "default", 179 | + .type = AVMEDIA_TYPE_VIDEO, 180 | + .config_props = config_output, 181 | + }, 182 | +}; 183 | + 184 | +const AVFilter ff_vf_exportsubframe = { 185 | + .name = "exportsubframe", 186 | + .description = NULL_IF_CONFIG_SMALL("Export and output subframe."), 187 | + .priv_size = sizeof(ExportSubframeContext), 188 | + .priv_class = &exportsubframe_class, 189 | + FILTER_INPUTS(inputs), 190 | + FILTER_OUTPUTS(outputs), 191 | +}; 192 | -- 193 | 2.25.1 194 | 195 | -------------------------------------------------------------------------------- /patches/0075-lavf-dnn-enable-libtorch-backend-support-FRVSR-model.patch: -------------------------------------------------------------------------------- 1 | From 555795fca429c639a74bd5c6f1abd173c3b1db7a Mon Sep 17 00:00:00 2001 2 | From: Ting Fu 3 | Date: Wed, 29 Jun 2022 07:46:44 +0000 4 | Subject: [PATCH] lavf/dnn: enable libtorch backend support FRVSR model 5 | 6 | Signed-off-by: Ting Fu 7 | --- 8 | libavfilter/dnn/dnn_backend_torch.cpp | 44 ++++++++++++++++++++++++--- 9 | 1 file changed, 40 insertions(+), 4 deletions(-) 10 | 11 | diff --git a/libavfilter/dnn/dnn_backend_torch.cpp b/libavfilter/dnn/dnn_backend_torch.cpp 12 | index ea493f5873..33921471a6 100644 13 | --- a/libavfilter/dnn/dnn_backend_torch.cpp 14 | +++ b/libavfilter/dnn/dnn_backend_torch.cpp 15 | @@ -36,6 +36,8 @@ extern "C" { 16 | #include "safe_queue.h" 17 | } 18 | 19 | +typedef enum {UNKNOWN_MODEL = -1, BASICVSR, FRVSR} ModelType; 20 | + 21 | typedef struct THModel { 22 | DNNModel model; 23 | DnnContext *ctx; 24 | @@ -43,6 +45,7 @@ typedef struct THModel { 25 | SafeQueue *request_queue; 26 | Queue *task_queue; 27 | Queue *lltask_queue; 28 | + ModelType model_type; 29 | } THModel; 30 | 31 | typedef struct THInferRequest { 32 | @@ -210,9 +213,15 @@ static int fill_model_input_th(THModel *th_model, THRequestItem *request) 33 | avpriv_report_missing_feature(NULL, "model function type %d", th_model->model.func_type); 34 | break; 35 | } 36 | - *infer_request->input_tensor = torch::from_blob(input.data, 37 | - {1, input.dims[channel_idx], input.dims[height_idx], input.dims[width_idx]}, 38 | - deleter, torch::kFloat32); 39 | + if (th_model->model_type == FRVSR) { 40 | + *infer_request->input_tensor = torch::from_blob(input.data, 41 | + {1, 3, input.dims[height_idx], input.dims[width_idx]}, 42 | + deleter, torch::kFloat32); 43 | + } else { 44 | + *infer_request->input_tensor = torch::from_blob(input.data, 45 | + {1, input.dims[channel_idx], input.dims[height_idx], input.dims[width_idx]}, 46 | + deleter, torch::kFloat32); 47 | + } 48 | return 0; 49 | 50 | err: 51 | @@ -256,7 +265,24 @@ static int th_start_inference(void *args) 52 | *infer_request->input_tensor = infer_request->input_tensor->to(device); 53 | inputs.push_back(*infer_request->input_tensor); 54 | 55 | - *infer_request->output = th_model->jit_model->forward(inputs).toTensor(); 56 | + if (th_model->model_type == FRVSR) { 57 | + auto size = infer_request->input_tensor->sizes(); 58 | + int height = size[2]; 59 | + int width = size[3]; 60 | + torch::Tensor lr_prev = torch::zeros({1, 3, height, width}, torch::TensorOptions().dtype(torch::kFloat32) 61 | + .device(device)); 62 | + torch::Tensor hr_prev = torch::zeros({1, 3, height * 4, width * 4}, torch::TensorOptions().dtype(torch::kFloat32) 63 | + .device(device)); 64 | + inputs.push_back(lr_prev); 65 | + inputs.push_back(hr_prev); 66 | + } 67 | + 68 | + auto outputs = th_model->jit_model->forward(inputs); 69 | + if (th_model->model_type == FRVSR) { 70 | + *infer_request->output = outputs.toTuple()->elements()[0].toTensor(); 71 | + } else { 72 | + *infer_request->output = outputs.toTensor(); 73 | + } 74 | 75 | return 0; 76 | } 77 | @@ -422,6 +448,7 @@ static DNNModel *dnn_load_model_th(DnnContext *ctx, DNNFunctionType func_type, A 78 | DNNModel *model = NULL; 79 | THModel *th_model = NULL; 80 | THRequestItem *item = NULL; 81 | + torch::jit::NameTensor first_param; 82 | const char *device_name = ctx->device ? ctx->device : "cpu"; 83 | 84 | th_model = (THModel *)av_mallocz(sizeof(THModel)); 85 | @@ -450,6 +477,7 @@ static DNNModel *dnn_load_model_th(DnnContext *ctx, DNNFunctionType func_type, A 86 | av_log(ctx, AV_LOG_ERROR, "Failed to load torch model\n"); 87 | goto fail; 88 | } 89 | + first_param = *th_model->jit_model->named_parameters().begin(); 90 | 91 | th_model->request_queue = ff_safe_queue_create(); 92 | if (!th_model->request_queue) { 93 | @@ -480,6 +508,14 @@ static DNNModel *dnn_load_model_th(DnnContext *ctx, DNNFunctionType func_type, A 94 | goto fail; 95 | } 96 | 97 | + if (!first_param.name.find("fnet")) { 98 | + th_model->model_type = FRVSR; 99 | + } else if (!first_param.name.find("spynet")) { 100 | + th_model->model_type = BASICVSR; 101 | + } else { 102 | + th_model->model_type = UNKNOWN_MODEL; 103 | + } 104 | + 105 | th_model->lltask_queue = ff_queue_create(); 106 | if (!th_model->lltask_queue) { 107 | goto fail; 108 | -- 109 | 2.34.1 110 | 111 | -------------------------------------------------------------------------------- /patches/0092-avutil-hwcontext-add-a-function-to-get-the-AVHWDevic.patch: -------------------------------------------------------------------------------- 1 | From b61f88ad16f0a6e558c9e1d157831af46367304e Mon Sep 17 00:00:00 2001 2 | From: Tong Wu 3 | Date: Mon, 4 Jul 2022 16:09:53 +0800 4 | Subject: [PATCH 1/5] avutil/hwcontext: add a function to get the 5 | AVHWDeviceType 6 | 7 | Add a function to get the corresponding AVHWDeviceType from a given 8 | hardware pixel format. 9 | 10 | Signed-off-by: Tong Wu 11 | --- 12 | libavutil/hwcontext.c | 11 +++++++++++ 13 | libavutil/hwcontext.h | 12 ++++++++++++ 14 | 2 files changed, 23 insertions(+) 15 | 16 | diff --git a/libavutil/hwcontext.c b/libavutil/hwcontext.c 17 | index cf14707182..9a19e53704 100644 18 | --- a/libavutil/hwcontext.c 19 | +++ b/libavutil/hwcontext.c 20 | @@ -80,6 +80,17 @@ static const char *const hw_type_names[] = { 21 | [AV_HWDEVICE_TYPE_VULKAN] = "vulkan", 22 | }; 23 | 24 | +enum AVHWDeviceType av_hwdevice_get_type_by_pix_fmt(enum AVPixelFormat fmt) 25 | +{ 26 | + for (int i = 0; hw_table[i]; i++) { 27 | + for (int j = 0; hw_table[i]->pix_fmts[j] != AV_PIX_FMT_NONE; j++) { 28 | + if (hw_table[i]->pix_fmts[j] == fmt) 29 | + return hw_table[i]->type; 30 | + } 31 | + } 32 | + return AV_HWDEVICE_TYPE_NONE; 33 | +} 34 | + 35 | enum AVHWDeviceType av_hwdevice_find_type_by_name(const char *name) 36 | { 37 | int type; 38 | diff --git a/libavutil/hwcontext.h b/libavutil/hwcontext.h 39 | index 7cea44f9d4..b32b1fd9ca 100644 40 | --- a/libavutil/hwcontext.h 41 | +++ b/libavutil/hwcontext.h 42 | @@ -230,6 +230,18 @@ typedef struct AVHWFramesContext { 43 | int width, height; 44 | } AVHWFramesContext; 45 | 46 | +/** 47 | + * Get the device type by a given pixel format. 48 | + * 49 | + * This function only returns a preferred device type which supports the given 50 | + * pixel format. There is no guarantee that the device type is unique. 51 | + * 52 | + * @param fmt Pixel format from enum AVPixelFormat. 53 | + * @return The type from enum AVHWDeviceType, or AV_HWDEVICE_TYPE_NONE if 54 | + * not found. 55 | + */ 56 | +enum AVHWDeviceType av_hwdevice_get_type_by_pix_fmt(enum AVPixelFormat fmt); 57 | + 58 | /** 59 | * Look up an AVHWDeviceType by name. 60 | * 61 | -- 62 | 2.41.0.windows.1 63 | 64 | -------------------------------------------------------------------------------- /patches/0093-avfilter-vf_hwmap-get-the-AVHWDeviceType-from-outlin.patch: -------------------------------------------------------------------------------- 1 | From 9b4342ee2830990850dce5365f1bb133d4d6e37a Mon Sep 17 00:00:00 2001 2 | From: Tong Wu 3 | Date: Mon, 4 Jul 2022 16:09:54 +0800 4 | Subject: [PATCH] avfilter/vf_hwmap: get the AVHWDeviceType from outlink format 5 | 6 | When both derive_device_type and device context are not 7 | presented during hw->hw map, the hwmap filter should be 8 | able to retrieve AVHWDeviceType from outlink->format and create 9 | corresponding hwdevice context. 10 | 11 | Signed-off-by: Tong Wu 12 | --- 13 | libavfilter/vf_hwmap.c | 35 ++++++++++++++++++++++------------- 14 | 1 file changed, 22 insertions(+), 13 deletions(-) 15 | 16 | diff --git a/libavfilter/vf_hwmap.c b/libavfilter/vf_hwmap.c 17 | index d3e84e39d7..14b806136c 100644 18 | --- a/libavfilter/vf_hwmap.c 19 | +++ b/libavfilter/vf_hwmap.c 20 | @@ -73,16 +73,31 @@ static int hwmap_config_output(AVFilterLink *outlink) 21 | device_is_derived = 0; 22 | 23 | if (inl->hw_frames_ctx) { 24 | + enum AVHWDeviceType type; 25 | hwfc = (AVHWFramesContext*)inl->hw_frames_ctx->data; 26 | 27 | - if (ctx->derive_device_type) { 28 | - enum AVHWDeviceType type; 29 | + desc = av_pix_fmt_desc_get(outlink->format); 30 | + if (!desc) { 31 | + err = AVERROR(EINVAL); 32 | + goto fail; 33 | + } 34 | 35 | - type = av_hwdevice_find_type_by_name(ctx->derive_device_type); 36 | - if (type == AV_HWDEVICE_TYPE_NONE) { 37 | - av_log(avctx, AV_LOG_ERROR, "Invalid device type.\n"); 38 | - err = AVERROR(EINVAL); 39 | - goto fail; 40 | + if (ctx->derive_device_type || (!device && (desc->flags & AV_PIX_FMT_FLAG_HWACCEL))) { 41 | + if (ctx->derive_device_type) { 42 | + type = av_hwdevice_find_type_by_name(ctx->derive_device_type); 43 | + if (type == AV_HWDEVICE_TYPE_NONE) { 44 | + av_log(avctx, AV_LOG_ERROR, "Invalid device type.\n"); 45 | + err = AVERROR(EINVAL); 46 | + goto fail; 47 | + } 48 | + } else { 49 | + type = av_hwdevice_get_type_by_pix_fmt(outlink->format); 50 | + if (type == AV_HWDEVICE_TYPE_NONE) { 51 | + av_log(avctx, AV_LOG_ERROR, "Could not get device type from " 52 | + "format %s.\n", av_get_pix_fmt_name(outlink->format)); 53 | + err = AVERROR(EINVAL); 54 | + goto fail; 55 | + } 56 | } 57 | 58 | err = av_hwdevice_ctx_create_derived(&device, type, 59 | @@ -95,12 +110,6 @@ static int hwmap_config_output(AVFilterLink *outlink) 60 | device_is_derived = 1; 61 | } 62 | 63 | - desc = av_pix_fmt_desc_get(outlink->format); 64 | - if (!desc) { 65 | - err = AVERROR(EINVAL); 66 | - goto fail; 67 | - } 68 | - 69 | if (inlink->format == hwfc->format && 70 | (desc->flags & AV_PIX_FMT_FLAG_HWACCEL) && 71 | !ctx->reverse) { 72 | -- 73 | 2.34.1 74 | 75 | -------------------------------------------------------------------------------- /patches/0094-lavfi-avfiltergraph-move-convert-codes-into-function.patch: -------------------------------------------------------------------------------- 1 | From f0de0922100c1398570f7bf929ded0c2e7c5a4ee Mon Sep 17 00:00:00 2001 2 | From: Tong Wu 3 | Date: Mon, 4 Jul 2022 16:09:55 +0800 4 | Subject: [PATCH 1/3] lavfi/avfiltergraph: move convert codes into functions 5 | 6 | This patch moves the auto-insert filter codes into two functions. 7 | 8 | Signed-off-by: Tong Wu 9 | --- 10 | libavfilter/avfiltergraph.c | 146 ++++++++++++++++++++++-------------- 11 | 1 file changed, 88 insertions(+), 58 deletions(-) 12 | 13 | diff --git a/libavfilter/avfiltergraph.c b/libavfilter/avfiltergraph.c 14 | index 674711ec35..b80bba3571 100644 15 | --- a/libavfilter/avfiltergraph.c 16 | +++ b/libavfilter/avfiltergraph.c 17 | @@ -391,6 +391,83 @@ static int formats_declared(AVFilterContext *f) 18 | return 1; 19 | } 20 | 21 | +static int insert_auto_filter(AVFilterContext **convert, AVFilterGraph *graph, 22 | + AVFilterLink *link, const AVFilterNegotiation *neg, 23 | + int *converter_count, void *log_ctx) 24 | +{ 25 | + int ret; 26 | + const AVFilter *filter; 27 | + AVFilterContext *ctx; 28 | + AVFilterLink *inlink, *outlink; 29 | + char inst_name[30]; 30 | + const char *opts; 31 | + 32 | + if (!(filter = avfilter_get_by_name(neg->conversion_filter))) { 33 | + av_log(log_ctx, AV_LOG_ERROR, 34 | + "'%s' filter not present, cannot convert formats.\n", 35 | + neg->conversion_filter); 36 | + return AVERROR(EINVAL); 37 | + } 38 | + snprintf(inst_name, sizeof(inst_name), "auto_%s_%d", 39 | + neg->conversion_filter, (*converter_count)++); 40 | + opts = FF_FIELD_AT(char *, neg->conversion_opts_offset, *graph); 41 | + ret = avfilter_graph_create_filter(&ctx, filter, inst_name, opts, NULL, graph); 42 | + if (ret < 0) 43 | + return ret; 44 | + 45 | + if ((ret = avfilter_insert_filter(link, ctx, 0, 0)) < 0) 46 | + return ret; 47 | + 48 | + if ((ret = filter_query_formats(ctx)) < 0) 49 | + return ret; 50 | + 51 | + inlink = ctx->inputs[0]; 52 | + outlink = ctx->outputs[0]; 53 | + av_assert0( inlink->incfg.formats->refcount > 0); 54 | + av_assert0( inlink->outcfg.formats->refcount > 0); 55 | + av_assert0(outlink->incfg.formats->refcount > 0); 56 | + av_assert0(outlink->outcfg.formats->refcount > 0); 57 | + if (outlink->type == AVMEDIA_TYPE_VIDEO) { 58 | + av_assert0( inlink-> incfg.color_spaces->refcount > 0); 59 | + av_assert0( inlink->outcfg.color_spaces->refcount > 0); 60 | + av_assert0(outlink-> incfg.color_spaces->refcount > 0); 61 | + av_assert0(outlink->outcfg.color_spaces->refcount > 0); 62 | + av_assert0( inlink-> incfg.color_ranges->refcount > 0); 63 | + av_assert0( inlink->outcfg.color_ranges->refcount > 0); 64 | + av_assert0(outlink-> incfg.color_ranges->refcount > 0); 65 | + av_assert0(outlink->outcfg.color_ranges->refcount > 0); 66 | + } else if (outlink->type == AVMEDIA_TYPE_AUDIO) { 67 | + av_assert0( inlink-> incfg.samplerates->refcount > 0); 68 | + av_assert0( inlink->outcfg.samplerates->refcount > 0); 69 | + av_assert0(outlink-> incfg.samplerates->refcount > 0); 70 | + av_assert0(outlink->outcfg.samplerates->refcount > 0); 71 | + av_assert0( inlink-> incfg.channel_layouts->refcount > 0); 72 | + av_assert0( inlink->outcfg.channel_layouts->refcount > 0); 73 | + av_assert0(outlink-> incfg.channel_layouts->refcount > 0); 74 | + av_assert0(outlink->outcfg.channel_layouts->refcount > 0); 75 | + } 76 | + 77 | + *convert = ctx; 78 | + return 0; 79 | +} 80 | + 81 | +static int merge_auto_filter(AVFilterContext *convert, const AVFilterNegotiation *neg) 82 | +{ 83 | + int ret; 84 | + AVFilterLink *inlink = convert->inputs[0]; 85 | + AVFilterLink *outlink = convert->outputs[0]; 86 | +#define MERGE(merger, link) \ 87 | + ((merger)->merge(FF_FIELD_AT(void *, (merger)->offset, (link)->incfg), \ 88 | + FF_FIELD_AT(void *, (merger)->offset, (link)->outcfg))) 89 | + for (unsigned neg_step = 0; neg_step < neg->nb_mergers; neg_step++) { 90 | + const AVFilterFormatsMerger *m = &neg->mergers[neg_step]; 91 | + if ((ret = MERGE(m, inlink)) <= 0 || 92 | + (ret = MERGE(m, outlink)) <= 0) 93 | + break; 94 | + } 95 | + return ret; 96 | +} 97 | + 98 | /** 99 | * Perform one round of query_formats() and merging formats lists on the 100 | * filter graph. 101 | @@ -465,10 +542,6 @@ static int query_formats(AVFilterGraph *graph, void *log_ctx) 102 | 103 | if (convert_needed) { 104 | AVFilterContext *convert; 105 | - const AVFilter *filter; 106 | - AVFilterLink *inlink, *outlink; 107 | - char inst_name[30]; 108 | - const char *opts; 109 | 110 | if (fffiltergraph(graph)->disable_auto_convert) { 111 | av_log(log_ctx, AV_LOG_ERROR, 112 | @@ -479,63 +552,20 @@ static int query_formats(AVFilterGraph *graph, void *log_ctx) 113 | } 114 | 115 | /* couldn't merge format lists. auto-insert conversion filter */ 116 | - if (!(filter = avfilter_get_by_name(neg->conversion_filter))) { 117 | - av_log(log_ctx, AV_LOG_ERROR, 118 | - "'%s' filter not present, cannot convert formats.\n", 119 | - neg->conversion_filter); 120 | - return AVERROR(EINVAL); 121 | - } 122 | - snprintf(inst_name, sizeof(inst_name), "auto_%s_%d", 123 | - neg->conversion_filter, converter_count++); 124 | - opts = FF_FIELD_AT(char *, neg->conversion_opts_offset, *graph); 125 | - ret = avfilter_graph_create_filter(&convert, filter, inst_name, opts, NULL, graph); 126 | - if (ret < 0) 127 | - return ret; 128 | - if ((ret = avfilter_insert_filter(link, convert, 0, 0)) < 0) 129 | + ret = insert_auto_filter(&convert, graph, link, neg, &converter_count, log_ctx); 130 | + if (ret < 0) { 131 | + av_log(log_ctx, AV_LOG_ERROR, "Failed to insert an auto filter.\n"); 132 | return ret; 133 | + } 134 | 135 | - if ((ret = filter_query_formats(convert)) < 0) 136 | + ret = merge_auto_filter(convert, neg); 137 | + if (ret < 0) 138 | return ret; 139 | - 140 | - inlink = convert->inputs[0]; 141 | - outlink = convert->outputs[0]; 142 | - av_assert0( inlink->incfg.formats->refcount > 0); 143 | - av_assert0( inlink->outcfg.formats->refcount > 0); 144 | - av_assert0(outlink->incfg.formats->refcount > 0); 145 | - av_assert0(outlink->outcfg.formats->refcount > 0); 146 | - if (outlink->type == AVMEDIA_TYPE_VIDEO) { 147 | - av_assert0( inlink-> incfg.color_spaces->refcount > 0); 148 | - av_assert0( inlink->outcfg.color_spaces->refcount > 0); 149 | - av_assert0(outlink-> incfg.color_spaces->refcount > 0); 150 | - av_assert0(outlink->outcfg.color_spaces->refcount > 0); 151 | - av_assert0( inlink-> incfg.color_ranges->refcount > 0); 152 | - av_assert0( inlink->outcfg.color_ranges->refcount > 0); 153 | - av_assert0(outlink-> incfg.color_ranges->refcount > 0); 154 | - av_assert0(outlink->outcfg.color_ranges->refcount > 0); 155 | - } else if (outlink->type == AVMEDIA_TYPE_AUDIO) { 156 | - av_assert0( inlink-> incfg.samplerates->refcount > 0); 157 | - av_assert0( inlink->outcfg.samplerates->refcount > 0); 158 | - av_assert0(outlink-> incfg.samplerates->refcount > 0); 159 | - av_assert0(outlink->outcfg.samplerates->refcount > 0); 160 | - av_assert0( inlink-> incfg.channel_layouts->refcount > 0); 161 | - av_assert0( inlink->outcfg.channel_layouts->refcount > 0); 162 | - av_assert0(outlink-> incfg.channel_layouts->refcount > 0); 163 | - av_assert0(outlink->outcfg.channel_layouts->refcount > 0); 164 | - } 165 | -#define MERGE(merger, link) \ 166 | - ((merger)->merge(FF_FIELD_AT(void *, (merger)->offset, (link)->incfg), \ 167 | - FF_FIELD_AT(void *, (merger)->offset, (link)->outcfg))) 168 | - for (neg_step = 0; neg_step < neg->nb_mergers; neg_step++) { 169 | - const AVFilterFormatsMerger *m = &neg->mergers[neg_step]; 170 | - if ((ret = MERGE(m, inlink)) <= 0 || 171 | - (ret = MERGE(m, outlink)) <= 0) { 172 | - if (ret < 0) 173 | - return ret; 174 | - av_log(log_ctx, AV_LOG_ERROR, 175 | - "Impossible to convert between the formats supported by the filter " 176 | - "'%s' and the filter '%s'\n", link->src->name, link->dst->name); 177 | - return AVERROR(ENOSYS); 178 | - } 179 | + else if (ret == 0) { 180 | + av_log(log_ctx, AV_LOG_ERROR, 181 | + "Impossible to convert between the formats supported by the filter " 182 | + "'%s' and the filter '%s'\n", link->src->name, link->dst->name); 183 | + return AVERROR(ENOSYS); 184 | } 185 | } 186 | } 187 | -- 188 | 2.41.0.windows.1 189 | 190 | -------------------------------------------------------------------------------- /patches/0095-lavfi-format-wrap-auto-filters-into-structures.patch: -------------------------------------------------------------------------------- 1 | From c084786efd4eee9a99f7fc0976628899617c1f63 Mon Sep 17 00:00:00 2001 2 | From: Tong Wu 3 | Date: Mon, 4 Jul 2022 16:09:56 +0800 4 | Subject: [PATCH 2/3] lavfi/format: wrap auto filters into structures 5 | 6 | This patch wraps auto conversion filters into new structures, making it 7 | easier to add more auto filters. And it adds a loop to automatically insert 8 | every possible conversion filter until merge succeeds. 9 | 10 | Signed-off-by: Tong Wu 11 | --- 12 | libavfilter/avfiltergraph.c | 76 +++++++++++++++++++++++++------------ 13 | libavfilter/formats.c | 22 +++++++++-- 14 | libavfilter/formats.h | 10 ++++- 15 | 3 files changed, 78 insertions(+), 30 deletions(-) 16 | 17 | diff --git a/libavfilter/avfiltergraph.c b/libavfilter/avfiltergraph.c 18 | index b80bba3571..d3ecdda070 100644 19 | --- a/libavfilter/avfiltergraph.c 20 | +++ b/libavfilter/avfiltergraph.c 21 | @@ -393,24 +393,22 @@ static int formats_declared(AVFilterContext *f) 22 | 23 | static int insert_auto_filter(AVFilterContext **convert, AVFilterGraph *graph, 24 | AVFilterLink *link, const AVFilterNegotiation *neg, 25 | - int *converter_count, void *log_ctx) 26 | + unsigned conv_step, int *converter_count, void *log_ctx) 27 | { 28 | int ret; 29 | const AVFilter *filter; 30 | AVFilterContext *ctx; 31 | AVFilterLink *inlink, *outlink; 32 | char inst_name[30]; 33 | - const char *opts; 34 | + const char *opts = FF_FIELD_AT(char *, neg->conversion_filters[conv_step].conversion_opts_offset, *graph); 35 | + const char *name = neg->conversion_filters[conv_step].conversion_filter; 36 | 37 | - if (!(filter = avfilter_get_by_name(neg->conversion_filter))) { 38 | + if (!(filter = avfilter_get_by_name(name))) { 39 | av_log(log_ctx, AV_LOG_ERROR, 40 | - "'%s' filter not present, cannot convert formats.\n", 41 | - neg->conversion_filter); 42 | + "'%s' filter not present, cannot convert formats.\n", name); 43 | return AVERROR(EINVAL); 44 | } 45 | - snprintf(inst_name, sizeof(inst_name), "auto_%s_%d", 46 | - neg->conversion_filter, (*converter_count)++); 47 | - opts = FF_FIELD_AT(char *, neg->conversion_opts_offset, *graph); 48 | + snprintf(inst_name, sizeof(inst_name), "auto_%s_%d", name, (*converter_count)++); 49 | ret = avfilter_graph_create_filter(&ctx, filter, inst_name, opts, NULL, graph); 50 | if (ret < 0) 51 | return ret; 52 | @@ -505,7 +503,7 @@ static int query_formats(AVFilterGraph *graph, void *log_ctx) 53 | for (j = 0; j < filter->nb_inputs; j++) { 54 | AVFilterLink *link = filter->inputs[j]; 55 | const AVFilterNegotiation *neg; 56 | - unsigned neg_step; 57 | + unsigned neg_step, conv_step; 58 | int convert_needed = 0; 59 | 60 | if (!link) 61 | @@ -541,8 +539,6 @@ static int query_formats(AVFilterGraph *graph, void *log_ctx) 62 | } 63 | 64 | if (convert_needed) { 65 | - AVFilterContext *convert; 66 | - 67 | if (fffiltergraph(graph)->disable_auto_convert) { 68 | av_log(log_ctx, AV_LOG_ERROR, 69 | "The filters '%s' and '%s' do not have a common format " 70 | @@ -552,20 +548,52 @@ static int query_formats(AVFilterGraph *graph, void *log_ctx) 71 | } 72 | 73 | /* couldn't merge format lists. auto-insert conversion filter */ 74 | - ret = insert_auto_filter(&convert, graph, link, neg, &converter_count, log_ctx); 75 | - if (ret < 0) { 76 | - av_log(log_ctx, AV_LOG_ERROR, "Failed to insert an auto filter.\n"); 77 | - return ret; 78 | - } 79 | + for (conv_step = 0; conv_step < neg->nb_conversion_filters; conv_step++) { 80 | + AVFilterContext *convert; 81 | + ret = insert_auto_filter(&convert, graph, link, neg, 82 | + conv_step, &converter_count, log_ctx); 83 | + if (ret < 0) { 84 | + av_log(log_ctx, AV_LOG_ERROR, "Failed to insert an auto filter.\n"); 85 | + return ret; 86 | + } 87 | 88 | - ret = merge_auto_filter(convert, neg); 89 | - if (ret < 0) 90 | - return ret; 91 | - else if (ret == 0) { 92 | - av_log(log_ctx, AV_LOG_ERROR, 93 | - "Impossible to convert between the formats supported by the filter " 94 | - "'%s' and the filter '%s'\n", link->src->name, link->dst->name); 95 | - return AVERROR(ENOSYS); 96 | + ret = merge_auto_filter(convert, neg); 97 | + if (ret < 0) 98 | + return ret; 99 | + else if (ret > 0) 100 | + break; 101 | + else if (conv_step < neg->nb_conversion_filters - 1) { 102 | + AVFilterLink *inlink = convert->inputs[0]; 103 | + AVFilterLink *outlink = convert->outputs[0]; 104 | + av_log(log_ctx, AV_LOG_VERBOSE, 105 | + "Impossible to convert between the formats supported by the filter " 106 | + "'%s' and the filter '%s', try another conversion filter.\n", 107 | + link->src->name, link->dst->name); 108 | + unsigned dstpad_idx = outlink->dstpad - outlink->dst->input_pads; 109 | + converter_count--; 110 | + /* re-hookup the link */ 111 | + inlink->dst = outlink->dst; 112 | + inlink->dstpad = &outlink->dst->input_pads[dstpad_idx]; 113 | + outlink->dst->inputs[dstpad_idx] = inlink; 114 | + if (outlink->outcfg.formats) 115 | + ff_formats_changeref(&outlink->outcfg.formats, 116 | + &inlink->outcfg.formats); 117 | + if (outlink->outcfg.samplerates) 118 | + ff_formats_changeref(&outlink->outcfg.samplerates, 119 | + &inlink->outcfg.samplerates); 120 | + if (outlink->outcfg.channel_layouts) 121 | + ff_channel_layouts_changeref(&outlink->outcfg.channel_layouts, 122 | + &inlink->outcfg.channel_layouts); 123 | + /* remove the previous auto filter */ 124 | + convert->inputs[0] = NULL; 125 | + convert->outputs[0]->dst = NULL; 126 | + avfilter_free(convert); 127 | + } else { 128 | + av_log(log_ctx, AV_LOG_ERROR, 129 | + "Impossible to convert between the formats supported by the filter " 130 | + "'%s' and the filter '%s'\n", link->src->name, link->dst->name); 131 | + return AVERROR(ENOSYS); 132 | + } 133 | } 134 | } 135 | } 136 | diff --git a/libavfilter/formats.c b/libavfilter/formats.c 137 | index 681f0b1203..63adf2d485 100644 138 | --- a/libavfilter/formats.c 139 | +++ b/libavfilter/formats.c 140 | @@ -381,18 +381,32 @@ static const AVFilterFormatsMerger mergers_audio[] = { 141 | }, 142 | }; 143 | 144 | +static const AVFilterFormatsFilter filters_video[] = { 145 | + { 146 | + .conversion_filter = "scale", 147 | + .conversion_opts_offset = offsetof(AVFilterGraph, scale_sws_opts), 148 | + }, 149 | +}; 150 | + 151 | +static const AVFilterFormatsFilter filters_audio[] = { 152 | + { 153 | + .conversion_filter = "aresample", 154 | + .conversion_opts_offset = offsetof(AVFilterGraph, aresample_swr_opts), 155 | + } 156 | +}; 157 | + 158 | static const AVFilterNegotiation negotiate_video = { 159 | .nb_mergers = FF_ARRAY_ELEMS(mergers_video), 160 | .mergers = mergers_video, 161 | - .conversion_filter = "scale", 162 | - .conversion_opts_offset = offsetof(AVFilterGraph, scale_sws_opts), 163 | + .nb_conversion_filters = FF_ARRAY_ELEMS(filters_video), 164 | + .conversion_filters = filters_video, 165 | }; 166 | 167 | static const AVFilterNegotiation negotiate_audio = { 168 | .nb_mergers = FF_ARRAY_ELEMS(mergers_audio), 169 | .mergers = mergers_audio, 170 | - .conversion_filter = "aresample", 171 | - .conversion_opts_offset = offsetof(AVFilterGraph, aresample_swr_opts), 172 | + .nb_conversion_filters = FF_ARRAY_ELEMS(filters_audio), 173 | + .conversion_filters = filters_audio, 174 | }; 175 | 176 | const AVFilterNegotiation *ff_filter_get_negotiation(AVFilterLink *link) 177 | diff --git a/libavfilter/formats.h b/libavfilter/formats.h 178 | index 82b3af4be1..acdd5b4691 100644 179 | --- a/libavfilter/formats.h 180 | +++ b/libavfilter/formats.h 181 | @@ -388,6 +388,12 @@ typedef struct AVFilterFormatMerger { 182 | int (*can_merge)(const void *a, const void *b); 183 | } AVFilterFormatsMerger; 184 | 185 | +typedef struct AVFilterFormatFilter { 186 | + const char *conversion_filter; 187 | + unsigned conversion_opts_offset; 188 | +} AVFilterFormatsFilter; 189 | + 190 | + 191 | /** 192 | * Callbacks and properties to describe the steps of a format negotiation. 193 | * 194 | @@ -476,8 +482,8 @@ typedef struct AVFilterFormatMerger { 195 | typedef struct AVFilterNegotiation { 196 | unsigned nb_mergers; 197 | const AVFilterFormatsMerger *mergers; 198 | - const char *conversion_filter; 199 | - unsigned conversion_opts_offset; 200 | + unsigned nb_conversion_filters; 201 | + const AVFilterFormatsFilter *conversion_filters; 202 | } AVFilterNegotiation; 203 | 204 | const AVFilterNegotiation *ff_filter_get_negotiation(AVFilterLink *link); 205 | -- 206 | 2.41.0.windows.1 207 | 208 | -------------------------------------------------------------------------------- /patches/0096-lavfi-format-add-a-hwmap-auto-conversion-filter.patch: -------------------------------------------------------------------------------- 1 | From 25823b80e7ecd155d190c84f78885e70c009f934 Mon Sep 17 00:00:00 2001 2 | From: Tong Wu 3 | Date: Mon, 4 Jul 2022 16:09:57 +0800 4 | Subject: [PATCH 3/3] lavfi/format: add a hwmap auto conversion filter 5 | 6 | When two formats lists cannot be merged, a scale filter is 7 | auto-inserted. However, when it comes to hardware map, we have to 8 | manually add a hwmap filter to do the conversion. This patch introduces 9 | an auto hwmap filter to do the hwmap conversion automatically. 10 | 11 | Signed-off-by: Tong Wu 12 | --- 13 | libavfilter/avfiltergraph.c | 3 ++- 14 | libavfilter/formats.c | 4 ++++ 15 | 2 files changed, 6 insertions(+), 1 deletion(-) 16 | 17 | diff --git a/libavfilter/avfiltergraph.c b/libavfilter/avfiltergraph.c 18 | index d3ecdda070..db79ef0f5f 100644 19 | --- a/libavfilter/avfiltergraph.c 20 | +++ b/libavfilter/avfiltergraph.c 21 | @@ -400,7 +400,8 @@ static int insert_auto_filter(AVFilterContext **convert, AVFilterGraph *graph, 22 | AVFilterContext *ctx; 23 | AVFilterLink *inlink, *outlink; 24 | char inst_name[30]; 25 | - const char *opts = FF_FIELD_AT(char *, neg->conversion_filters[conv_step].conversion_opts_offset, *graph); 26 | + const char *opts = neg->conversion_filters[conv_step].conversion_opts_offset == 0 ? NULL : 27 | + FF_FIELD_AT(char *, neg->conversion_filters[conv_step].conversion_opts_offset, *graph); 28 | const char *name = neg->conversion_filters[conv_step].conversion_filter; 29 | 30 | if (!(filter = avfilter_get_by_name(name))) { 31 | diff --git a/libavfilter/formats.c b/libavfilter/formats.c 32 | index 63adf2d485..54fd17477f 100644 33 | --- a/libavfilter/formats.c 34 | +++ b/libavfilter/formats.c 35 | @@ -386,6 +386,10 @@ static const AVFilterFormatsFilter filters_video[] = { 36 | .conversion_filter = "scale", 37 | .conversion_opts_offset = offsetof(AVFilterGraph, scale_sws_opts), 38 | }, 39 | + { 40 | + .conversion_filter = "hwmap", 41 | + .conversion_opts_offset = 0, 42 | + } 43 | }; 44 | 45 | static const AVFilterFormatsFilter filters_audio[] = { 46 | -- 47 | 2.41.0.windows.1 48 | 49 | -------------------------------------------------------------------------------- /patches/0117-libavfilter-dnn-dnn_backend_torch-Add-async-to-torch.patch: -------------------------------------------------------------------------------- 1 | From bf6812ea9f780401c02c0d4015ad898576d6f4e3 Mon Sep 17 00:00:00 2001 2 | From: Wenbin Chen 3 | Date: Tue, 14 Nov 2023 16:38:52 +0800 4 | Subject: [PATCH] libavfilter/dnn/dnn_backend_torch: Add async to torch backend 5 | 6 | Signed-off-by: Wenbin Chen 7 | --- 8 | libavfilter/dnn/dnn_backend_torch.cpp | 55 ++++++++++++++++++--------- 9 | 1 file changed, 37 insertions(+), 18 deletions(-) 10 | 11 | diff --git a/libavfilter/dnn/dnn_backend_torch.cpp b/libavfilter/dnn/dnn_backend_torch.cpp 12 | index 47ec6d4ea6..da1d07409b 100644 13 | --- a/libavfilter/dnn/dnn_backend_torch.cpp 14 | +++ b/libavfilter/dnn/dnn_backend_torch.cpp 15 | @@ -69,6 +69,8 @@ static const AVOption dnn_th_options[] = { 16 | { NULL } 17 | }; 18 | 19 | +static void dnn_free_model_th(DNNModel **model); 20 | + 21 | static int extract_lltask_from_task(TaskItem *task, Queue *lltask_queue) 22 | { 23 | THModel *th_model = (THModel *)task->model; 24 | @@ -380,6 +382,7 @@ static int execute_model_th(THRequestItem *request, Queue *lltask_queue) 25 | LastLevelTaskItem *lltask; 26 | TaskItem *task = NULL; 27 | int ret = 0; 28 | + DnnContext *ctx; 29 | 30 | if (ff_queue_size(lltask_queue) == 0) { 31 | destroy_request_item(&request); 32 | @@ -394,13 +397,17 @@ static int execute_model_th(THRequestItem *request, Queue *lltask_queue) 33 | } 34 | task = lltask->task; 35 | th_model = (THModel *)task->model; 36 | + ctx = th_model->ctx; 37 | 38 | ret = fill_model_input_th(th_model, request); 39 | if ( ret != 0) { 40 | goto err; 41 | } 42 | if (task->async) { 43 | - avpriv_report_missing_feature(th_model->ctx, "LibTorch async"); 44 | + if (ff_dnn_start_inference_async(ctx, &request->exec_module) != 0) { 45 | + goto err; 46 | + } 47 | + return 0; 48 | } else { 49 | ret = th_start_inference((void *)(request)); 50 | if (ret != 0) { 51 | @@ -512,29 +519,41 @@ static DNNModel *dnn_load_model_th(DnnContext *ctx, DNNFunctionType func_type, A 52 | } 53 | first_param = *th_model->jit_model->named_parameters().begin(); 54 | 55 | +#if !HAVE_PTHREAD_CANCEL 56 | + if (ctx->options.async) { 57 | + ctx->options.async = 0; 58 | + av_log(filter_ctx, AV_LOG_WARNING, "pthread is not supported, roll back to sync.\n"); 59 | + } 60 | +#endif 61 | + 62 | th_model->request_queue = ff_safe_queue_create(); 63 | if (!th_model->request_queue) { 64 | goto fail; 65 | } 66 | 67 | - item = (THRequestItem *)av_mallocz(sizeof(THRequestItem)); 68 | - if (!item) { 69 | - goto fail; 70 | - } 71 | - item->lltask = NULL; 72 | - item->infer_request = th_create_inference_request(); 73 | - if (!item->infer_request) { 74 | - av_log(NULL, AV_LOG_ERROR, "Failed to allocate memory for Torch inference request\n"); 75 | - goto fail; 76 | - } 77 | - item->exec_module.start_inference = &th_start_inference; 78 | - item->exec_module.callback = &infer_completion_callback; 79 | - item->exec_module.args = item; 80 | + if (ctx->nireq <= 0) 81 | + ctx->nireq = 1; 82 | 83 | - if (ff_safe_queue_push_back(th_model->request_queue, item) < 0) { 84 | - goto fail; 85 | + for (int i = 0; i < ctx->nireq; i++) { 86 | + item = (THRequestItem *)av_mallocz(sizeof(THRequestItem)); 87 | + if (!item) { 88 | + goto fail; 89 | + } 90 | + item->lltask = NULL; 91 | + item->infer_request = th_create_inference_request(); 92 | + if (!item->infer_request) { 93 | + av_log(ctx, AV_LOG_ERROR, "Failed to allocate memory for Torch inference request\n"); 94 | + goto fail; 95 | + } 96 | + item->exec_module.start_inference = &th_start_inference; 97 | + item->exec_module.callback = &infer_completion_callback; 98 | + item->exec_module.args = item; 99 | + 100 | + if (ff_safe_queue_push_back(th_model->request_queue, item) < 0) { 101 | + goto fail; 102 | + } 103 | + item = NULL; 104 | } 105 | - item = NULL; 106 | 107 | th_model->task_queue = ff_queue_create(); 108 | if (!th_model->task_queue) { 109 | @@ -589,7 +608,7 @@ static int dnn_execute_model_th(const DNNModel *model, DNNExecBaseParams *exec_p 110 | return AVERROR(ENOMEM); 111 | } 112 | 113 | - ret = ff_dnn_fill_task(task, exec_params, th_model, 0, 1); 114 | + ret = ff_dnn_fill_task(task, exec_params, th_model, ctx->async, 1); 115 | if (ret != 0) { 116 | av_freep(&task); 117 | av_log(ctx, AV_LOG_ERROR, "unable to fill task.\n"); 118 | -- 119 | 2.34.1 120 | 121 | -------------------------------------------------------------------------------- /patches/0118-libavfilter-dnn-dnn_backend_torch-add-multi-device-s.patch: -------------------------------------------------------------------------------- 1 | From ee97bfc4d8a2aa400369808f9d166263ff8730fa Mon Sep 17 00:00:00 2001 2 | From: Wenbin Chen 3 | Date: Mon, 20 Nov 2023 14:32:26 +0800 4 | Subject: [PATCH] libavfilter/dnn/dnn_backend_torch: add multi-device support 5 | 6 | Now ffmpeg-libtoch can be run on multi-device at same time. When set device 7 | user need to input all devices splited by '&' and also set async=1. The 8 | inference tasks will be distributed to idle device dynamically. 9 | 10 | example command: 11 | ffmpeg -i input.mp4 -vf dnn_processing=dnn_backend=torch:model=model.pt:backend_configs="device='xpu\\\&cpu'" output.mp4 12 | 13 | Signed-off-by: Wenbin Chen 14 | --- 15 | libavfilter/dnn/dnn_backend_common.c | 1 - 16 | libavfilter/dnn/dnn_backend_common.h | 1 + 17 | libavfilter/dnn/dnn_backend_torch.cpp | 104 ++++++++++++++++++++------ 18 | 3 files changed, 83 insertions(+), 23 deletions(-) 19 | 20 | diff --git a/libavfilter/dnn/dnn_backend_common.c b/libavfilter/dnn/dnn_backend_common.c 21 | index 6eb90ba963..4dbe96517c 100644 22 | --- a/libavfilter/dnn/dnn_backend_common.c 23 | +++ b/libavfilter/dnn/dnn_backend_common.c 24 | @@ -139,7 +139,6 @@ int ff_dnn_start_inference_async(void *ctx, DNNAsyncExecModule *async_module) 25 | DNNAsyncStatusType ff_dnn_get_result_common(Queue *task_queue, AVFrame **in, AVFrame **out) 26 | { 27 | TaskItem *task = ff_queue_peek_front(task_queue); 28 | - 29 | if (!task) { 30 | return DAST_EMPTY_QUEUE; 31 | } 32 | diff --git a/libavfilter/dnn/dnn_backend_common.h b/libavfilter/dnn/dnn_backend_common.h 33 | index 2844804b36..f694657fce 100644 34 | --- a/libavfilter/dnn/dnn_backend_common.h 35 | +++ b/libavfilter/dnn/dnn_backend_common.h 36 | @@ -54,6 +54,7 @@ typedef struct TaskItem { 37 | uint32_t nb_output; 38 | uint32_t inference_todo; 39 | uint32_t inference_done; 40 | + uint32_t device_idx; 41 | } TaskItem; 42 | 43 | // one task might have multiple inferences 44 | diff --git a/libavfilter/dnn/dnn_backend_torch.cpp b/libavfilter/dnn/dnn_backend_torch.cpp 45 | index c3d210f238..3bdbd21b84 100644 46 | --- a/libavfilter/dnn/dnn_backend_torch.cpp 47 | +++ b/libavfilter/dnn/dnn_backend_torch.cpp 48 | @@ -29,6 +29,7 @@ 49 | extern "C" { 50 | #include "dnn_io_proc.h" 51 | #include "dnn_backend_common.h" 52 | +#include "libavutil/avstring.h" 53 | #include "libavutil/fifo.h" 54 | #include "libavutil/opt.h" 55 | #include "libavutil/mem.h" 56 | @@ -46,11 +47,15 @@ typedef struct THModel { 57 | Queue *task_queue; 58 | Queue *lltask_queue; 59 | ModelType model_type; 60 | + char **device_names; 61 | + int nb_models; 62 | + SafeQueue *jit_model_queue; 63 | } THModel; 64 | 65 | typedef struct THInferRequest { 66 | torch::Tensor *output; 67 | torch::Tensor *input_tensor; 68 | + torch::jit::Module *jit_model; 69 | } THInferRequest; 70 | 71 | typedef struct THRequestItem { 72 | @@ -69,6 +74,29 @@ static const AVOption dnn_th_options[] = { 73 | 74 | static void dnn_free_model_th(DNNModel **model); 75 | 76 | +static char **th_separate_device_name(char *device_str, int *nb_devices) 77 | +{ 78 | + char *saveptr = NULL; 79 | + char **device_names; 80 | + int i = 0; 81 | + *nb_devices = 0; 82 | + while(device_str[i] != '\0') { 83 | + if(device_str[i] == '&') 84 | + *nb_devices += 1; 85 | + i++; 86 | + } 87 | + *nb_devices += 1; 88 | + device_names = (char **)av_mallocz(*nb_devices * sizeof(*device_names)); 89 | + if (!device_names) 90 | + return NULL; 91 | + 92 | + for (int i = 0; i < *nb_devices; i++) { 93 | + device_names[i] = av_strtok(device_str, "&", &saveptr); 94 | + device_str = NULL; 95 | + } 96 | + return device_names; 97 | +} 98 | + 99 | static int extract_lltask_from_task(TaskItem *task, Queue *lltask_queue) 100 | { 101 | THModel *th_model = (THModel *)task->model; 102 | @@ -144,7 +172,12 @@ static void dnn_free_model_th(DNNModel **model) 103 | av_freep(&item); 104 | } 105 | ff_queue_destroy(th_model->task_queue); 106 | - delete th_model->jit_model; 107 | + while (ff_safe_queue_size(th_model->jit_model_queue) != 0) { 108 | + torch::jit::Module *jit_model = (torch::jit::Module *)ff_safe_queue_pop_front(th_model->jit_model_queue); 109 | + delete jit_model; 110 | + } 111 | + ff_safe_queue_destroy(th_model->jit_model_queue); 112 | + av_freep(&th_model->device_names); 113 | av_freep(&th_model); 114 | *model = NULL; 115 | } 116 | @@ -191,6 +224,13 @@ static int fill_model_input_th(THModel *th_model, THRequestItem *request) 117 | if ( ret != 0) { 118 | goto err; 119 | } 120 | + 121 | + infer_request->jit_model = (torch::jit::Module *)ff_safe_queue_pop_front(th_model->jit_model_queue); 122 | + if (!infer_request->jit_model) { 123 | + av_log(ctx, AV_LOG_ERROR, "unable to get jit_model.\n"); 124 | + return AVERROR(EINVAL); 125 | + } 126 | + 127 | width_idx = dnn_get_width_idx_by_layout(input.layout); 128 | height_idx = dnn_get_height_idx_by_layout(input.layout); 129 | channel_idx = dnn_get_channel_idx_by_layout(input.layout); 130 | @@ -274,7 +314,7 @@ static int th_start_inference(void *args) 131 | return DNN_GENERIC_ERROR; 132 | } 133 | // Transfer tensor to the same device as model 134 | - c10::Device device = (*th_model->jit_model->parameters().begin()).device(); 135 | + c10::Device device = (*infer_request->jit_model->parameters().begin()).device(); 136 | if (infer_request->input_tensor->device() != device) 137 | *infer_request->input_tensor = infer_request->input_tensor->to(device); 138 | inputs.push_back(*infer_request->input_tensor); 139 | @@ -291,7 +331,7 @@ static int th_start_inference(void *args) 140 | inputs.push_back(hr_prev); 141 | } 142 | 143 | - auto outputs = th_model->jit_model->forward(inputs); 144 | + auto outputs = infer_request->jit_model->forward(inputs); 145 | if (th_model->model_type == FRVSR) { 146 | *infer_request->output = outputs.toTuple()->elements()[0].toTensor(); 147 | } else { 148 | @@ -367,7 +407,11 @@ static void infer_completion_callback(void *args) { 149 | av_freep(&request->lltask); 150 | err: 151 | th_free_request(infer_request); 152 | - 153 | + if (ff_safe_queue_push_back(th_model->jit_model_queue, infer_request->jit_model) < 0) { 154 | + delete infer_request->jit_model; 155 | + av_log(&th_model->ctx, AV_LOG_ERROR, "Unable to push back jit_model when failed to start inference.\n"); 156 | + } 157 | + infer_request->jit_model = NULL; 158 | if (ff_safe_queue_push_back(th_model->request_queue, request) < 0) { 159 | destroy_request_item(&request); 160 | av_log(th_model->ctx, AV_LOG_ERROR, "Unable to push back request_queue when failed to start inference.\n"); 161 | @@ -484,6 +528,7 @@ static DNNModel *dnn_load_model_th(DnnContext *ctx, DNNFunctionType func_type, A 162 | THModel *th_model = NULL; 163 | THRequestItem *item = NULL; 164 | torch::jit::NameTensor first_param; 165 | + torch::jit::Module *jit_model; 166 | const char *device_name = ctx->device ? ctx->device : "cpu"; 167 | 168 | th_model = (THModel *)av_mallocz(sizeof(THModel)); 169 | @@ -492,27 +537,42 @@ static DNNModel *dnn_load_model_th(DnnContext *ctx, DNNFunctionType func_type, A 170 | model = &th_model->model; 171 | th_model->ctx = ctx; 172 | 173 | - c10::Device device = c10::Device(device_name); 174 | - if (device.is_xpu()) { 175 | - if (!at::hasXPU()) { 176 | - av_log(ctx, AV_LOG_ERROR, "No XPU device found\n"); 177 | - goto fail; 178 | - } 179 | - at::detail::getXPUHooks().initXPU(); 180 | - } else if (!device.is_cpu()) { 181 | - av_log(ctx, AV_LOG_ERROR, "Not supported device:\"%s\"\n", device_name); 182 | - goto fail; 183 | + th_model->device_names = th_separate_device_name(ctx->device, &th_model->nb_models); 184 | + if (!th_model->device_names) { 185 | + av_log(ctx, AV_LOG_ERROR, "could not parse devices names\n"); 186 | + return NULL; 187 | } 188 | 189 | - try { 190 | - th_model->jit_model = new torch::jit::Module; 191 | - (*th_model->jit_model) = torch::jit::load(ctx->model_filename); 192 | - th_model->jit_model->to(device); 193 | - } catch (const c10::Error& e) { 194 | - av_log(ctx, AV_LOG_ERROR, "Failed to load torch model\n"); 195 | + th_model->jit_model_queue = ff_safe_queue_create(); 196 | + if (!th_model->jit_model_queue) { 197 | goto fail; 198 | } 199 | - first_param = *th_model->jit_model->named_parameters().begin(); 200 | + for (int i = 0; i < th_model->nb_models; i++) { 201 | + c10::Device *device; 202 | + device = new c10::Device(th_model->device_names[i]); 203 | + if (device && device->is_xpu()) { 204 | + if (!at::hasXPU()) { 205 | + av_log(ctx, AV_LOG_ERROR, "No XPU device found\n"); 206 | + delete device; 207 | + goto fail; 208 | + } 209 | + at::detail::getXPUHooks().initXPU(); 210 | + } 211 | + 212 | + try { 213 | + jit_model = new torch::jit::Module; 214 | + *jit_model = torch::jit::load(ctx->model_filename); 215 | + jit_model->to(*device); 216 | + } catch (const c10::Error& e) { 217 | + av_log(ctx, AV_LOG_ERROR, "Failed to load torch model\n"); 218 | + goto fail; 219 | + } 220 | + if (ff_safe_queue_push_back(th_model->jit_model_queue, jit_model) < 0) { 221 | + delete jit_model; 222 | + goto fail; 223 | + } 224 | + } 225 | + first_param = *jit_model->named_parameters().begin(); 226 | 227 | #if !HAVE_PTHREAD_CANCEL 228 | if (ctx->options.async) { 229 | @@ -527,7 +587,7 @@ static DNNModel *dnn_load_model_th(DnnContext *ctx, DNNFunctionType func_type, A 230 | } 231 | 232 | if (ctx->nireq <= 0) 233 | - ctx->nireq = 1; 234 | + ctx->nireq = th_model->nb_models; 235 | 236 | for (int i = 0; i < ctx->nireq; i++) { 237 | item = (THRequestItem *)av_mallocz(sizeof(THRequestItem)); 238 | -- 239 | 2.34.1 240 | 241 | -------------------------------------------------------------------------------- /patches/0133-libavfilter-raisr_opencl-Add-raisr_opencl-filter.patch: -------------------------------------------------------------------------------- 1 | From d82481fd156db75a3b479167a54663fbe240dcb4 Mon Sep 17 00:00:00 2001 2 | From: Wenbin Chen 3 | Date: Thu, 6 Apr 2023 13:48:23 +0800 4 | Subject: [PATCH] libavfilter/raisr_opencl: Add raisr_opencl filter 5 | 6 | Add raisr_opencl filter. It uses the same algorithm as raisr filter but 7 | it uses cl_mem as input and output, which helps to build high speed 8 | pipeline. 9 | For details please refer to https://github.com/OpenVisualCloud/Video-Super-Resolution-Library 10 | 11 | Signed-off-by: Wenbin Chen 12 | --- 13 | configure | 1 + 14 | libavfilter/Makefile | 1 + 15 | libavfilter/allfilters.c | 1 + 16 | libavfilter/vf_raisr_opencl.c | 266 ++++++++++++++++++++++++++++++++++ 17 | 4 files changed, 269 insertions(+) 18 | create mode 100644 libavfilter/vf_raisr_opencl.c 19 | 20 | diff --git a/configure b/configure 21 | index 993756e6b0..babc199e44 100755 22 | --- a/configure 23 | +++ b/configure 24 | @@ -3921,6 +3921,7 @@ procamp_vaapi_filter_deps="vaapi" 25 | program_opencl_filter_deps="opencl" 26 | pullup_filter_deps="gpl" 27 | raisr_filter_deps="libipp" 28 | +raisr_opencl_filter_deps="opencl libipp" 29 | remap_opencl_filter_deps="opencl" 30 | removelogo_filter_deps="avcodec avformat swscale" 31 | repeatfields_filter_deps="gpl" 32 | diff --git a/libavfilter/Makefile b/libavfilter/Makefile 33 | index bab407b9f9..6d21234699 100644 34 | --- a/libavfilter/Makefile 35 | +++ b/libavfilter/Makefile 36 | @@ -441,6 +441,7 @@ OBJS-$(CONFIG_PULLUP_FILTER) += vf_pullup.o 37 | OBJS-$(CONFIG_QP_FILTER) += vf_qp.o 38 | OBJS-$(CONFIG_QUIRC_FILTER) += vf_quirc.o 39 | OBJS-$(CONFIG_RAISR_FILTER) += vf_raisr.o 40 | +OBJS-$(CONFIG_RAISR_OPENCL_FILTER) += vf_raisr_opencl.o opencl.o 41 | OBJS-$(CONFIG_RANDOM_FILTER) += vf_random.o 42 | OBJS-$(CONFIG_READEIA608_FILTER) += vf_readeia608.o 43 | OBJS-$(CONFIG_READVITC_FILTER) += vf_readvitc.o 44 | diff --git a/libavfilter/allfilters.c b/libavfilter/allfilters.c 45 | index 1e51f68912..98c5bbb768 100644 46 | --- a/libavfilter/allfilters.c 47 | +++ b/libavfilter/allfilters.c 48 | @@ -416,6 +416,7 @@ extern const AVFilter ff_vf_qp; 49 | extern const AVFilter ff_vf_qrencode; 50 | extern const AVFilter ff_vf_quirc; 51 | extern const AVFilter ff_vf_raisr; 52 | +extern const AVFilter ff_vf_raisr_opencl; 53 | extern const AVFilter ff_vf_random; 54 | extern const AVFilter ff_vf_readeia608; 55 | extern const AVFilter ff_vf_readvitc; 56 | diff --git a/libavfilter/vf_raisr_opencl.c b/libavfilter/vf_raisr_opencl.c 57 | new file mode 100644 58 | index 0000000000..6939a15e27 59 | --- /dev/null 60 | +++ b/libavfilter/vf_raisr_opencl.c 61 | @@ -0,0 +1,266 @@ 62 | +/* 63 | + * Intel Library for Video Super Resolution ffmpeg plugin 64 | + * 65 | + * Copyright (c) 2023 Intel Corporation 66 | + * 67 | + * FFmpeg is free software; you can redistribute it and/or 68 | + * modify it under the terms of the GNU Lesser General Public 69 | + * License as published by the Free Software Foundation; either 70 | + * version 2.1 of the License, or (at your option) any later version. 71 | + * 72 | + * FFmpeg is distributed in the hope that it will be useful, 73 | + * but WITHOUT ANY WARRANTY; without even the implied warranty of 74 | + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU 75 | + * Lesser General Public License for more details. 76 | + * 77 | + * You should have received a copy of the GNU Lesser General Public 78 | + * License along with this program; if not, write to the Free Software 79 | + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA 80 | + */ 81 | + 82 | +#include "raisr/RaisrHandler.h" 83 | +#include "raisr/RaisrDefaults.h" 84 | +#include "libavutil/opt.h" 85 | +#include "avfilter.h" 86 | +#include "opencl.h" 87 | +#include "libavutil/pixdesc.h" 88 | +#include "video.h" 89 | + 90 | +#define MIN_RATIO 1 91 | +#define MAX_RATIO 4 92 | +#define DEFAULT_RATIO 2 93 | + 94 | +typedef struct RaisrOpenCLContext { 95 | + OpenCLFilterContext ocf; 96 | + 97 | + int initialised; 98 | + float ratio; 99 | + int bits; 100 | + char *filterfolder; 101 | + BlendingMode blending; 102 | + int passes; 103 | + int mode; 104 | + RangeType range; 105 | + enum AVPixelFormat sw_format; 106 | +} RaisrOpenCLContext; 107 | + 108 | + 109 | +static int raisr_opencl_init(AVFilterContext *avctx) 110 | +{ 111 | + RaisrOpenCLContext *ctx = avctx->priv; 112 | + RNLERRORTYPE err; 113 | + 114 | + err = RNLHandler_SetOpenCLContext(ctx->ocf.hwctx->context, ctx->ocf.hwctx->device_id, 0, 0); 115 | + if (err != RNLErrorNone) { 116 | + av_log(avctx, AV_LOG_ERROR, "RNLHandler_SetExternalOpenCLContext failed\n"); 117 | + return AVERROR(ENAVAIL); 118 | + } 119 | + 120 | + err = RNLHandler_Init(ctx->filterfolder, ctx->ratio, ctx->bits, ctx->range, 1, 121 | + OpenCLExternal, ctx->passes, ctx->mode); 122 | + if (err != RNLErrorNone) { 123 | + av_log(avctx, AV_LOG_ERROR, "RNLInit failed\n"); 124 | + return AVERROR(ENAVAIL); 125 | + } 126 | + return 0; 127 | +} 128 | + 129 | +static int raisr_opencl_filter_frame(AVFilterLink *inlink, AVFrame *input) 130 | +{ 131 | + AVFilterContext *avctx = inlink->dst; 132 | + AVFilterLink *outlink = avctx->outputs[0]; 133 | + RaisrOpenCLContext *ctx = avctx->priv; 134 | + AVFrame *output = NULL; 135 | + const AVPixFmtDescriptor *desc; 136 | + int err, wsub, hsub; 137 | + int nb_planes = 0; 138 | + VideoDataType vdt_in[3] = { 0 }; 139 | + VideoDataType vdt_out[3] = { 0 }; 140 | + 141 | + av_log(ctx, AV_LOG_DEBUG, "Filter input: %s, %ux%u (%"PRId64").\n", 142 | + av_get_pix_fmt_name(input->format), 143 | + input->width, input->height, input->pts); 144 | + 145 | + if (!input->hw_frames_ctx) 146 | + return AVERROR(EINVAL); 147 | + 148 | + output = ff_get_video_buffer(outlink, outlink->w, outlink->h); 149 | + if (!output) { 150 | + err = AVERROR(ENOMEM); 151 | + goto fail; 152 | + } 153 | + desc = av_pix_fmt_desc_get(ctx->sw_format); 154 | + if (!desc) { 155 | + err = AVERROR(EINVAL); 156 | + goto fail; 157 | + } 158 | + 159 | + for(int p = 0; p < desc->nb_components; p++) 160 | + if (desc->comp[p].plane > nb_planes) 161 | + nb_planes = desc->comp[p].plane; 162 | + 163 | + for (int p = 0; p <= nb_planes; p++) { 164 | + wsub = p ? 1 << desc->log2_chroma_w : 1; 165 | + hsub = p ? 1 << desc->log2_chroma_h : 1; 166 | + vdt_in[p].pData = input->data[p]; 167 | + vdt_in[p].width = input->width / wsub; 168 | + vdt_in[p].height = input->height / hsub; 169 | + vdt_in[p].step = input->linesize[p]; 170 | + vdt_in[p].bitShift = desc->comp[p].shift; 171 | + // fill in the output video data type structure 172 | + vdt_out[p].pData = output->data[p]; 173 | + vdt_out[p].width = input->width * ctx->ratio / wsub; 174 | + vdt_out[p].height = input->height * ctx->ratio / hsub; 175 | + vdt_out[p].step = output->linesize[p]; 176 | + vdt_out[p].bitShift = desc->comp[p].shift; 177 | + } 178 | + 179 | + if (!ctx->initialised) { 180 | + err = RNLHandler_SetRes(&vdt_in[0], &vdt_in[1], &vdt_in[2], 181 | + &vdt_out[0], &vdt_out[1], &vdt_out[2]); 182 | + if (err != RNLErrorNone) { 183 | + av_log(ctx, AV_LOG_ERROR, "RNLHandler_SetRes error\n"); 184 | + err = AVERROR(ENOMEM); 185 | + goto fail; 186 | + } 187 | + ctx->initialised = 1; 188 | + } 189 | + 190 | + err = RNLHandler_Process(&vdt_in[0], &vdt_in[1], &vdt_in[2], 191 | + &vdt_out[0], &vdt_out[1], &vdt_out[2], 192 | + ctx->blending); 193 | + if (err != RNLErrorNone) { 194 | + av_log(ctx, AV_LOG_ERROR, "RNLHandler_Process error\n"); 195 | + err = AVERROR(ENOMEM); 196 | + goto fail; 197 | + } 198 | + 199 | + err = av_frame_copy_props(output, input); 200 | + if (err < 0) 201 | + goto fail; 202 | + 203 | + av_frame_free(&input); 204 | + 205 | + av_log(ctx, AV_LOG_DEBUG, "Filter output: %s, %ux%u (%"PRId64").\n", 206 | + av_get_pix_fmt_name(output->format), 207 | + output->width, output->height, output->pts); 208 | + 209 | + return ff_filter_frame(outlink, output); 210 | + 211 | +fail: 212 | + av_frame_free(&input); 213 | + av_frame_free(&output); 214 | + return err; 215 | +} 216 | + 217 | +static int raisr_filter_config_input(AVFilterLink *inlink) 218 | +{ 219 | + AVHWFramesContext *input_frames; 220 | + FilterLink *fl; 221 | + int err; 222 | + 223 | + err = ff_opencl_filter_config_input(inlink); 224 | + if (err < 0) 225 | + return err; 226 | + fl = ff_filter_link(inlink); 227 | + input_frames = (AVHWFramesContext*)fl->hw_frames_ctx->data; 228 | + if (input_frames->format != AV_PIX_FMT_OPENCL) 229 | + return AVERROR(EINVAL); 230 | + 231 | + if (input_frames->sw_format != AV_PIX_FMT_NV12 && 232 | + input_frames->sw_format != AV_PIX_FMT_YUV420P && 233 | + input_frames->sw_format != AV_PIX_FMT_P010) 234 | + return AVERROR(EINVAL); 235 | + 236 | + return 0; 237 | +} 238 | + 239 | +static int raisr_opencl_config_output(AVFilterLink *outlink) 240 | +{ 241 | + AVFilterContext *avctx = outlink->src; 242 | + AVFilterLink *inlink = avctx->inputs[0]; 243 | + RaisrOpenCLContext *ctx = avctx->priv; 244 | + AVHWFramesContext *input_frames; 245 | + const AVPixFmtDescriptor *desc; 246 | + FilterLink *fl; 247 | + int err; 248 | + 249 | + ctx->ocf.output_width = inlink->w * ctx->ratio; 250 | + ctx->ocf.output_height = inlink->h * ctx->ratio; 251 | + 252 | + err = ff_opencl_filter_config_output(outlink); 253 | + if (err < 0) 254 | + return err; 255 | + 256 | + fl = ff_filter_link(inlink); 257 | + input_frames = (AVHWFramesContext*)fl->hw_frames_ctx->data; 258 | + ctx->sw_format = (enum AVPixelFormat)input_frames->sw_format; 259 | + desc = av_pix_fmt_desc_get(ctx->sw_format); 260 | + if (desc && desc->comp[0].depth != ctx->bits) { 261 | + av_log(ctx, AV_LOG_ERROR, "input pixel doesn't match model's bitdepth\n"); 262 | + return AVERROR(EINVAL); 263 | + } 264 | + 265 | + err = raisr_opencl_init(avctx); 266 | + if (err < 0) 267 | + return err; 268 | + 269 | + return 0; 270 | +} 271 | + 272 | +static av_cold void raisr_opencl_uninit(AVFilterContext *avctx) 273 | +{ 274 | + RNLHandler_Deinit(); 275 | + ff_opencl_filter_uninit(avctx); 276 | +} 277 | + 278 | +#define OFFSET(x) offsetof(RaisrOpenCLContext, x) 279 | +#define FLAGS (AV_OPT_FLAG_FILTERING_PARAM | AV_OPT_FLAG_VIDEO_PARAM) 280 | +static const AVOption raisr_opencl_options[] = { 281 | + {"ratio", "ratio of the upscaling, between 1 and 4", OFFSET(ratio), 282 | + AV_OPT_TYPE_FLOAT, {.dbl = DEFAULT_RATIO}, MIN_RATIO, MAX_RATIO, FLAGS}, 283 | + {"bits", "bit depth", OFFSET(bits), AV_OPT_TYPE_INT, {.i64 = 8}, 8, 10, FLAGS}, 284 | + {"range", "input color range", OFFSET(range), AV_OPT_TYPE_INT, {.i64 = VideoRange}, VideoRange, FullRange, FLAGS, "range"}, 285 | + { "video", NULL, 0, AV_OPT_TYPE_CONST, { .i64 = VideoRange }, INT_MIN, INT_MAX, FLAGS, "range" }, 286 | + { "full", NULL, 0, AV_OPT_TYPE_CONST, { .i64 = FullRange }, INT_MIN, INT_MAX, FLAGS, "range" }, 287 | + {"filterfolder", "absolute filter folder path", OFFSET(filterfolder), AV_OPT_TYPE_STRING, {.str = "filters_2x/filters_lowres"}, 0, 0, FLAGS}, 288 | + {"blending", "CT blending mode (1: Randomness, 2: CountOfBitsChanged)", 289 | + OFFSET(blending), AV_OPT_TYPE_INT, {.i64 = CountOfBitsChanged}, Randomness, CountOfBitsChanged, FLAGS, "blending"}, 290 | + { "Randomness", NULL, 0, AV_OPT_TYPE_CONST, { .i64 = Randomness }, INT_MIN, INT_MAX, FLAGS, "blending" }, 291 | + { "CountOfBitsChanged", NULL, 0, AV_OPT_TYPE_CONST, { .i64 = CountOfBitsChanged }, INT_MIN, INT_MAX, FLAGS, "blending" }, 292 | + {"passes", "passes to run (1: one pass, 2: two pass)", OFFSET(passes), AV_OPT_TYPE_INT, {.i64 = 1}, 1, 2, FLAGS}, 293 | + {"mode", "mode for two pass (1: upscale in 1st pass, 2: upscale in 2nd pass)", OFFSET(mode), AV_OPT_TYPE_INT, {.i64 = 1}, 1, 2, FLAGS}, 294 | + {NULL} 295 | +}; 296 | + 297 | +AVFILTER_DEFINE_CLASS(raisr_opencl); 298 | + 299 | +static const AVFilterPad raisr_opencl_inputs[] = { 300 | + { 301 | + .name = "default", 302 | + .type = AVMEDIA_TYPE_VIDEO, 303 | + .filter_frame = &raisr_opencl_filter_frame, 304 | + .config_props = &raisr_filter_config_input, 305 | + }, 306 | +}; 307 | + 308 | +static const AVFilterPad raisr_opencl_outputs[] = { 309 | + { 310 | + .name = "default", 311 | + .type = AVMEDIA_TYPE_VIDEO, 312 | + .config_props = &raisr_opencl_config_output, 313 | + }, 314 | +}; 315 | + 316 | +const AVFilter ff_vf_raisr_opencl = { 317 | + .name = "raisr_opencl", 318 | + .description = NULL_IF_CONFIG_SMALL("Raisr"), 319 | + .priv_size = sizeof(RaisrOpenCLContext), 320 | + .priv_class = &raisr_opencl_class, 321 | + .init = &ff_opencl_filter_init, 322 | + .uninit = &raisr_opencl_uninit, 323 | + FILTER_INPUTS(raisr_opencl_inputs), 324 | + FILTER_OUTPUTS(raisr_opencl_outputs), 325 | + FILTER_SINGLE_PIXFMT(AV_PIX_FMT_OPENCL), 326 | + .flags_internal = FF_FILTER_FLAG_HWFRAME_AWARE, 327 | +}; 328 | -- 329 | 2.34.1 330 | 331 | -------------------------------------------------------------------------------- /patches/0135-avutil-hwcontext_d3d11va-enable-D3D11_RESOURCE_MISC_.patch: -------------------------------------------------------------------------------- 1 | From 6fffdeb521fba34991c527c42d333b8245ba4cf3 Mon Sep 17 00:00:00 2001 2 | From: Tong Wu 3 | Date: Wed, 20 Apr 2022 04:10:29 +0000 4 | Subject: [PATCH] avutil/hwcontext_d3d11va: enable D3D11_RESOURCE_MISC_SHARED 5 | for texture 6 | 7 | Add D3D11_RESOURCE_MISC_SHARED flag for texture to make it shareable. 8 | This can fix the green frames issue when mapping from d3d11va to opencl. 9 | Sample command line: ffmpeg.exe -hwaccel d3d11va -hwaccel_output_format 10 | d3d11 -i input.264 -vf 11 | "hwmap=derive_device=opencl,format=opencl,hwdownload,format=nv12" -c:v 12 | libx264 output.mp4 13 | 14 | Signed-off-by: Tong Wu 15 | --- 16 | libavutil/hwcontext_d3d11va.c | 4 ++-- 17 | 1 file changed, 2 insertions(+), 2 deletions(-) 18 | 19 | diff --git a/libavutil/hwcontext_d3d11va.c b/libavutil/hwcontext_d3d11va.c 20 | index 8963c9fc85..63a3cbdde1 100644 21 | --- a/libavutil/hwcontext_d3d11va.c 22 | +++ b/libavutil/hwcontext_d3d11va.c 23 | @@ -228,7 +228,7 @@ static AVBufferRef *d3d11va_alloc_single(AVHWFramesContext *ctx) 24 | .ArraySize = 1, 25 | .Usage = D3D11_USAGE_DEFAULT, 26 | .BindFlags = hwctx->BindFlags, 27 | - .MiscFlags = hwctx->MiscFlags, 28 | + .MiscFlags = hwctx->MiscFlags | D3D11_RESOURCE_MISC_SHARED, 29 | }; 30 | 31 | hr = ID3D11Device_CreateTexture2D(device_hwctx->device, &texDesc, NULL, &tex); 32 | @@ -292,7 +292,7 @@ static int d3d11va_frames_init(AVHWFramesContext *ctx) 33 | .ArraySize = ctx->initial_pool_size, 34 | .Usage = D3D11_USAGE_DEFAULT, 35 | .BindFlags = hwctx->BindFlags, 36 | - .MiscFlags = hwctx->MiscFlags, 37 | + .MiscFlags = hwctx->MiscFlags | D3D11_RESOURCE_MISC_SHARED, 38 | }; 39 | 40 | if (hwctx->texture) { 41 | -- 42 | 2.41.0.windows.1 43 | 44 | -------------------------------------------------------------------------------- /patches/0151-libavfilter-dnn_backend_openvino-Factor-out-preproce.patch: -------------------------------------------------------------------------------- 1 | From 594684deb2142418a373e14281f1a7c50fd2e25c Mon Sep 17 00:00:00 2001 2 | From: Wenbin Chen 3 | Date: Tue, 5 Mar 2024 14:21:09 +0800 4 | Subject: [PATCH 1/5] libavfilter/dnn_backend_openvino: Factor out preprocess 5 | 6 | Reorganize the code of initializing model part. Prepare for 7 | adding support for hardware frame. 8 | 9 | Signed-off-by: Wenbin Chen 10 | --- 11 | libavfilter/dnn/dnn_backend_openvino.c | 191 +++++++++++++++---------- 12 | 1 file changed, 112 insertions(+), 79 deletions(-) 13 | 14 | diff --git a/libavfilter/dnn/dnn_backend_openvino.c b/libavfilter/dnn/dnn_backend_openvino.c 15 | index 1004b26985..20a5335e35 100644 16 | --- a/libavfilter/dnn/dnn_backend_openvino.c 17 | +++ b/libavfilter/dnn/dnn_backend_openvino.c 18 | @@ -575,46 +575,23 @@ static void dnn_free_model_ov(DNNModel **model) 19 | } 20 | 21 | 22 | -static int init_model_ov(OVModel *ov_model, const char *input_name, const char **output_names, int nb_outputs) 23 | +static int set_model_input_info(OVModel *ov_model, const char *input_name) 24 | { 25 | int ret = 0; 26 | DnnContext *ctx = ov_model->ctx; 27 | -#if HAVE_OPENVINO2 28 | ov_status_e status; 29 | ov_preprocess_input_tensor_info_t* input_tensor_info = NULL; 30 | - ov_preprocess_output_tensor_info_t* output_tensor_info = NULL; 31 | ov_preprocess_input_model_info_t* input_model_info = NULL; 32 | - ov_model_t *tmp_ov_model; 33 | ov_layout_t* NHWC_layout = NULL; 34 | ov_layout_t* NCHW_layout = NULL; 35 | const char* NHWC_desc = "NHWC"; 36 | const char* NCHW_desc = "NCHW"; 37 | - const char* device = ctx->device ? ctx->device : "CPU"; 38 | -#else 39 | - IEStatusCode status; 40 | - ie_available_devices_t a_dev; 41 | - ie_config_t config = {NULL, NULL, NULL}; 42 | - char *all_dev_names = NULL; 43 | -#endif 44 | - // We scale pixel by default when do frame processing. 45 | - if (fabsf(ctx->ov_option.scale) < 1e-6f) 46 | - ctx->ov_option.scale = ov_model->model.func_type == DFT_PROCESS_FRAME ? 255 : 1; 47 | - // batch size 48 | - if (ctx->ov_option.batch_size <= 0) { 49 | - ctx->ov_option.batch_size = 1; 50 | - } 51 | -#if HAVE_OPENVINO2 52 | - if (ctx->ov_option.batch_size > 1) { 53 | - avpriv_report_missing_feature(ctx, "Do not support batch_size > 1 for now," 54 | - "change batch_size to 1.\n"); 55 | - ctx->ov_option.batch_size = 1; 56 | - } 57 | 58 | status = ov_preprocess_prepostprocessor_create(ov_model->ov_model, &ov_model->preprocess); 59 | if (status != OK) { 60 | av_log(ctx, AV_LOG_ERROR, "Failed to create preprocess for ov_model.\n"); 61 | ret = ov2_map_error(status, NULL); 62 | - goto err; 63 | + return ret; 64 | } 65 | 66 | if (input_name) 67 | @@ -624,7 +601,7 @@ static int init_model_ov(OVModel *ov_model, const char *input_name, const char * 68 | if (status != OK) { 69 | av_log(ctx, AV_LOG_ERROR, "Failed to get input info from preprocess.\n"); 70 | ret = ov2_map_error(status, NULL); 71 | - goto err; 72 | + return ret; 73 | } 74 | 75 | status = ov_preprocess_input_info_get_tensor_info(ov_model->input_info, &input_tensor_info); 76 | @@ -672,14 +649,56 @@ static int init_model_ov(OVModel *ov_model, const char *input_name, const char * 77 | ret = ov2_map_error(status, NULL); 78 | goto err; 79 | } 80 | + // set preprocess steps. 81 | + if (fabsf(ctx->ov_option.scale - 1) > 1e-6f || fabsf(ctx->ov_option.mean) > 1e-6f) { 82 | + ov_preprocess_preprocess_steps_t* input_process_steps = NULL; 83 | + status = ov_preprocess_input_info_get_preprocess_steps(ov_model->input_info, &input_process_steps); 84 | + if (status != OK) { 85 | + av_log(ctx, AV_LOG_ERROR, "Failed to get preprocess steps\n"); 86 | + ret = ov2_map_error(status, NULL); 87 | + goto err; 88 | + } 89 | + status = ov_preprocess_preprocess_steps_convert_element_type(input_process_steps, F32); 90 | + status |= ov_preprocess_preprocess_steps_mean(input_process_steps, ctx->ov_option.mean); 91 | + status |= ov_preprocess_preprocess_steps_scale(input_process_steps, ctx->ov_option.scale); 92 | + if (status != OK) { 93 | + av_log(ctx, AV_LOG_ERROR, "Failed to set preprocess steps\n"); 94 | + ov_preprocess_preprocess_steps_free(input_process_steps); 95 | + input_process_steps = NULL; 96 | + ret = ov2_map_error(status, NULL); 97 | + goto err; 98 | + } 99 | + ov_preprocess_preprocess_steps_free(input_process_steps); 100 | + input_process_steps = NULL; 101 | + } 102 | +err: 103 | + if (NCHW_layout) 104 | + ov_layout_free(NCHW_layout); 105 | + if (NHWC_layout) 106 | + ov_layout_free(NHWC_layout); 107 | + if (input_tensor_info) 108 | + ov_preprocess_input_tensor_info_free(input_tensor_info); 109 | + if (input_model_info) 110 | + ov_preprocess_input_model_info_free(input_model_info); 111 | + if (ov_model->input_info) 112 | + ov_preprocess_input_info_free(ov_model->input_info); 113 | + ov_model->input_info = NULL; 114 | + return ret; 115 | +} 116 | 117 | +static int set_model_output_info(OVModel *ov_model, const char **output_names, int nb_outputs) 118 | +{ 119 | + int ret = 0; 120 | + DnnContext *ctx = ov_model->ctx; 121 | + ov_status_e status; 122 | + ov_preprocess_output_tensor_info_t* output_tensor_info = NULL; 123 | if (!nb_outputs) { 124 | size_t output_size; 125 | status = ov_model_outputs_size(ov_model->ov_model, &output_size); 126 | if (status != OK) { 127 | av_log(ctx, AV_LOG_ERROR, "Failed to get output size.\n"); 128 | ret = ov2_map_error(status, NULL); 129 | - goto err; 130 | + return ret; 131 | } 132 | nb_outputs = output_size; 133 | } 134 | @@ -694,13 +713,15 @@ static int init_model_ov(OVModel *ov_model, const char *input_name, const char * 135 | if (status != OK) { 136 | av_log(ctx, AV_LOG_ERROR, "Failed to get output info from preprocess.\n"); 137 | ret = ov2_map_error(status, NULL); 138 | - goto err; 139 | + return ret; 140 | } 141 | status |= ov_preprocess_output_info_get_tensor_info(ov_model->output_info, &output_tensor_info); 142 | + ov_preprocess_output_info_free(ov_model->output_info); 143 | + ov_model->output_info = NULL; 144 | if (status != OK) { 145 | av_log(ctx, AV_LOG_ERROR, "Failed to get tensor info from input/output.\n"); 146 | ret = ov2_map_error(status, NULL); 147 | - goto err; 148 | + return ret; 149 | } 150 | if (ov_model->model.func_type != DFT_PROCESS_FRAME) 151 | status |= ov_preprocess_output_set_element_type(output_tensor_info, F32); 152 | @@ -708,42 +729,35 @@ static int init_model_ov(OVModel *ov_model, const char *input_name, const char * 153 | status |= ov_preprocess_output_set_element_type(output_tensor_info, F32); 154 | else 155 | status |= ov_preprocess_output_set_element_type(output_tensor_info, U8); 156 | - if (status != OK) { 157 | - av_log(ctx, AV_LOG_ERROR, "Failed to set output element type\n"); 158 | - ret = ov2_map_error(status, NULL); 159 | - goto err; 160 | - } 161 | ov_preprocess_output_tensor_info_free(output_tensor_info); 162 | output_tensor_info = NULL; 163 | - ov_preprocess_output_info_free(ov_model->output_info); 164 | - ov_model->output_info = NULL; 165 | - } 166 | - // set preprocess steps. 167 | - if (fabsf(ctx->ov_option.scale - 1) > 1e-6f || fabsf(ctx->ov_option.mean) > 1e-6f) { 168 | - ov_preprocess_preprocess_steps_t* input_process_steps = NULL; 169 | - status = ov_preprocess_input_info_get_preprocess_steps(ov_model->input_info, &input_process_steps); 170 | if (status != OK) { 171 | - av_log(ctx, AV_LOG_ERROR, "Failed to get preprocess steps\n"); 172 | - ret = ov2_map_error(status, NULL); 173 | - goto err; 174 | - } 175 | - status = ov_preprocess_preprocess_steps_convert_element_type(input_process_steps, F32); 176 | - status |= ov_preprocess_preprocess_steps_mean(input_process_steps, ctx->ov_option.mean); 177 | - status |= ov_preprocess_preprocess_steps_scale(input_process_steps, ctx->ov_option.scale); 178 | - if (status != OK) { 179 | - av_log(ctx, AV_LOG_ERROR, "Failed to set preprocess steps\n"); 180 | - ov_preprocess_preprocess_steps_free(input_process_steps); 181 | - input_process_steps = NULL; 182 | + av_log(ctx, AV_LOG_ERROR, "Failed to set output element type\n"); 183 | ret = ov2_map_error(status, NULL); 184 | - goto err; 185 | + return ret; 186 | } 187 | - ov_preprocess_preprocess_steps_free(input_process_steps); 188 | - input_process_steps = NULL; 189 | } 190 | - ov_preprocess_input_tensor_info_free(input_tensor_info); 191 | - input_tensor_info = NULL; 192 | - ov_preprocess_input_info_free(ov_model->input_info); 193 | - ov_model->input_info = NULL; 194 | + return ret; 195 | +} 196 | + 197 | +static int set_model_preprocess(OVModel *ov_model, const char *input_name, const char **output_names, int nb_outputs) 198 | +{ 199 | + int ret = 0; 200 | + DnnContext *ctx = ov_model->ctx; 201 | + ov_status_e status; 202 | + ov_model_t *tmp_ov_model; 203 | + 204 | + ret = set_model_input_info(ov_model, input_name); 205 | + if (ret < 0) { 206 | + av_log(ctx, AV_LOG_ERROR, "Failed to set model input info\n"); 207 | + return ret; 208 | + } 209 | + 210 | + ret = set_model_output_info(ov_model, output_names, nb_outputs); 211 | + if (ret < 0) { 212 | + av_log(ctx, AV_LOG_ERROR, "Failed to set output info\n"); 213 | + return ret; 214 | + } 215 | 216 | //update model 217 | if(ov_model->ov_model) 218 | @@ -754,24 +768,59 @@ static int init_model_ov(OVModel *ov_model, const char *input_name, const char * 219 | ov_model_free(tmp_ov_model); 220 | tmp_ov_model = NULL; 221 | ret = ov2_map_error(status, NULL); 222 | - goto err; 223 | + return ret; 224 | } 225 | ov_model_free(tmp_ov_model); 226 | + return ret; 227 | +} 228 | + 229 | +static int init_model_ov(OVModel *ov_model, const char *input_name, const char **output_names, int nb_outputs) 230 | +{ 231 | + int ret = 0; 232 | + DnnContext *ctx = ov_model->ctx; 233 | +#if HAVE_OPENVINO2 234 | + ov_status_e status; 235 | + const char* device = ctx->device ? ctx->device : "CPU"; 236 | +#else 237 | + IEStatusCode status; 238 | + ie_available_devices_t a_dev; 239 | + ie_config_t config = {NULL, NULL, NULL}; 240 | + char *all_dev_names = NULL; 241 | +#endif 242 | + // We scale pixel by default when do frame processing. 243 | + if (fabsf(ctx->ov_option.scale) < 1e-6f) 244 | + ctx->ov_option.scale = ov_model->model.func_type == DFT_PROCESS_FRAME ? 255 : 1; 245 | + // batch size 246 | + if (ctx->ov_option.batch_size <= 0) { 247 | + ctx->ov_option.batch_size = 1; 248 | + } 249 | +#if HAVE_OPENVINO2 250 | + if (ctx->ov_option.batch_size > 1) { 251 | + avpriv_report_missing_feature(ctx, "Do not support batch_size > 1 for now," 252 | + "change batch_size to 1.\n"); 253 | + ctx->ov_option.batch_size = 1; 254 | + } 255 | + 256 | + ret = set_model_preprocess(ov_model, input_name, output_names, nb_outputs); 257 | + if (ret < 0) { 258 | + av_log(ctx, AV_LOG_ERROR, "Failed to set preprocess\n"); 259 | + goto err; 260 | + } 261 | 262 | //update output_port 263 | if (!ov_model->output_ports) { 264 | - ov_model->output_ports = av_calloc(nb_outputs, sizeof(*ov_model->output_ports)); 265 | + ov_model->output_ports = av_calloc(ov_model->nb_outputs, sizeof(*ov_model->output_ports)); 266 | if (!ov_model->output_ports) { 267 | ret = AVERROR(ENOMEM); 268 | goto err; 269 | } 270 | } else 271 | - for (int i = 0; i < nb_outputs; i++) { 272 | + for (int i = 0; i < ov_model->nb_outputs; i++) { 273 | ov_output_const_port_free(ov_model->output_ports[i]); 274 | ov_model->output_ports[i] = NULL; 275 | } 276 | 277 | - for (int i = 0; i < nb_outputs; i++) { 278 | + for (int i = 0; i < ov_model->nb_outputs; i++) { 279 | char *port_name; 280 | if (output_names) 281 | status = ov_model_const_output_by_name(ov_model->ov_model, output_names[i], 282 | @@ -798,10 +847,6 @@ static int init_model_ov(OVModel *ov_model, const char *input_name, const char * 283 | ret = ov2_map_error(status, NULL); 284 | goto err; 285 | } 286 | - ov_preprocess_input_model_info_free(input_model_info); 287 | - input_model_info = NULL; 288 | - ov_layout_free(NCHW_layout); 289 | - ov_layout_free(NHWC_layout); 290 | #else 291 | if (ctx->ov_option.batch_size > 1) { 292 | input_shapes_t input_shapes; 293 | @@ -947,18 +992,6 @@ static int init_model_ov(OVModel *ov_model, const char *input_name, const char * 294 | return 0; 295 | 296 | err: 297 | -#if HAVE_OPENVINO2 298 | - if (output_tensor_info) 299 | - ov_preprocess_output_tensor_info_free(output_tensor_info); 300 | - if (ov_model->output_info) 301 | - ov_preprocess_output_info_free(ov_model->output_info); 302 | - if (NCHW_layout) 303 | - ov_layout_free(NCHW_layout); 304 | - if (NHWC_layout) 305 | - ov_layout_free(NHWC_layout); 306 | - if (input_model_info) 307 | - ov_preprocess_input_model_info_free(input_model_info); 308 | -#endif 309 | return ret; 310 | } 311 | 312 | -- 313 | 2.34.1 314 | 315 | -------------------------------------------------------------------------------- /patches/0153-libavfilter-vf_dnn_detect-Add-vaapi-into-detect.patch: -------------------------------------------------------------------------------- 1 | From 40584b3f39197987298600e45699f73417f88e77 Mon Sep 17 00:00:00 2001 2 | From: Wenbin Chen 3 | Date: Tue, 5 Mar 2024 16:22:57 +0800 4 | Subject: [PATCH] libavfilter/vf_dnn_detect: Add vaapi into detect 5 | 6 | Signed-off-by: Wenbin Chen 7 | --- 8 | libavfilter/vf_dnn_detect.c | 34 +++++++++++++++++++++++++++++++++- 9 | 1 file changed, 33 insertions(+), 1 deletion(-) 10 | 11 | diff --git a/libavfilter/vf_dnn_detect.c b/libavfilter/vf_dnn_detect.c 12 | index 2a277d4169..85e26bf5a5 100644 13 | --- a/libavfilter/vf_dnn_detect.c 14 | +++ b/libavfilter/vf_dnn_detect.c 15 | @@ -715,6 +715,9 @@ static const enum AVPixelFormat pix_fmts[] = { 16 | AV_PIX_FMT_YUV420P, AV_PIX_FMT_YUV422P, 17 | AV_PIX_FMT_YUV444P, AV_PIX_FMT_YUV410P, AV_PIX_FMT_YUV411P, 18 | AV_PIX_FMT_NV12, 19 | +#if !HAVE_WINDOWS_H && CONFIG_VAAPI 20 | + AV_PIX_FMT_VAAPI, 21 | +#endif 22 | AV_PIX_FMT_NONE 23 | }; 24 | 25 | @@ -840,6 +843,26 @@ static int config_input(AVFilterLink *inlink) 26 | return 0; 27 | } 28 | 29 | +static int config_output(AVFilterLink *outlink) 30 | +{ 31 | + AVFilterContext *context = outlink->src; 32 | + AVFilterLink *inlink = context->inputs[0]; 33 | + DnnDetectContext *detect_ctx = context->priv; 34 | + FilterLink *inl = ff_filter_link(inlink); 35 | + FilterLink *onl = ff_filter_link(outlink); 36 | + if (inl->hw_frames_ctx) { 37 | + if (inlink->format == AV_PIX_FMT_VAAPI && 38 | + detect_ctx->dnnctx.backend_type == DNN_OV) 39 | + onl->hw_frames_ctx = av_buffer_ref(inl->hw_frames_ctx); 40 | + else { 41 | + av_log(detect_ctx, AV_LOG_ERROR, "The dnn_backend doesn't support this pixel format\n"); 42 | + return AVERROR_PATCHWELCOME; 43 | + } 44 | + } 45 | + 46 | + return 0; 47 | +} 48 | + 49 | static const AVFilterPad dnn_detect_inputs[] = { 50 | { 51 | .name = "default", 52 | @@ -848,6 +871,14 @@ static const AVFilterPad dnn_detect_inputs[] = { 53 | }, 54 | }; 55 | 56 | +static const AVFilterPad dnn_detect_outputs[] = { 57 | + { 58 | + .name = "default", 59 | + .type = AVMEDIA_TYPE_VIDEO, 60 | + .config_props = config_output, 61 | + }, 62 | +}; 63 | + 64 | const AVFilter ff_vf_dnn_detect = { 65 | .name = "dnn_detect", 66 | .description = NULL_IF_CONFIG_SMALL("Apply DNN detect filter to the input."), 67 | @@ -856,8 +887,9 @@ const AVFilter ff_vf_dnn_detect = { 68 | .init = dnn_detect_init, 69 | .uninit = dnn_detect_uninit, 70 | FILTER_INPUTS(dnn_detect_inputs), 71 | - FILTER_OUTPUTS(ff_video_default_filterpad), 72 | + FILTER_OUTPUTS(dnn_detect_outputs), 73 | FILTER_PIXFMTS_ARRAY(pix_fmts), 74 | .priv_class = &dnn_detect_class, 75 | .activate = dnn_detect_activate, 76 | + .flags_internal = FF_FILTER_FLAG_HWFRAME_AWARE, 77 | }; 78 | -- 79 | 2.34.1 80 | 81 | -------------------------------------------------------------------------------- /patches/0154-libavfilter-dnn_backend_openvino-Add-vaSurface-crop-.patch: -------------------------------------------------------------------------------- 1 | From e1564c33360588569157f3faf01542013b06e662 Mon Sep 17 00:00:00 2001 2 | From: Wenbin Chen 3 | Date: Tue, 5 Mar 2024 16:34:40 +0800 4 | Subject: [PATCH] libavfilter/dnn_backend_openvino: Add vaSurface crop support 5 | to Openvino 6 | 7 | Add vaSurface crop support to openvino backend. Prepare to add vaSurface 8 | support to classify filter. 9 | 10 | Signed-off-by: Wenbin Chen 11 | --- 12 | libavfilter/dnn/dnn_backend_openvino.c | 98 ++++++++++++++++++++++++++ 13 | 1 file changed, 98 insertions(+) 14 | 15 | diff --git a/libavfilter/dnn/dnn_backend_openvino.c b/libavfilter/dnn/dnn_backend_openvino.c 16 | index f03f80c8a6..a1db2365a2 100644 17 | --- a/libavfilter/dnn/dnn_backend_openvino.c 18 | +++ b/libavfilter/dnn/dnn_backend_openvino.c 19 | @@ -70,6 +70,9 @@ typedef struct OVModel{ 20 | Queue *lltask_queue; // holds LastLevelTaskItem 21 | int nb_outputs; 22 | int use_remote_context; 23 | +#if !HAVE_WINDOWS_H && CONFIG_VAAPI 24 | + VADisplay dpy; 25 | +#endif 26 | } OVModel; 27 | 28 | // one request for one call to openvino 29 | @@ -83,6 +86,9 @@ typedef struct OVRequestItem { 30 | ie_complete_call_back_t callback; 31 | ie_infer_request_t *infer_request; 32 | #endif 33 | +#if !HAVE_WINDOWS_H && CONFIG_VAAPI 34 | + VASurfaceID bbox_surface; 35 | +#endif 36 | } OVRequestItem; 37 | 38 | #define APPEND_STRING(generated_string, iterate_string) \ 39 | @@ -215,6 +221,94 @@ static int fill_model_input_remote_ov(OVModel *ov_model, OVRequestItem *request) 40 | return AVERROR(ENAVAIL); 41 | } 42 | surface_id = (size_t)task->in_frame->data[3]; 43 | + 44 | + if (ov_model->model.func_type == DFT_ANALYTICS_CLASSIFY) { 45 | +#if !HAVE_WINDOWS_H && CONFIG_VAAPI 46 | + AVFrameSideData *sd; 47 | + AVHWFramesContext *hw_frame_ctx = NULL; 48 | + AVVAAPIDeviceContext *vaapi_ctx = NULL; 49 | + const AVDetectionBBox *bbox; 50 | + const AVDetectionBBoxHeader *header; 51 | + FilterLink *fl; 52 | + unsigned int rt_format, fourcc; 53 | + VAImage image; 54 | + VAStatus vas; 55 | + VASurfaceAttrib attributes; 56 | + 57 | + fl = ff_filter_link(ov_model->model.filter_ctx->inputs[0]); 58 | + hw_frame_ctx = (AVHWFramesContext *)fl->hw_frames_ctx->data; 59 | + vaapi_ctx = (AVVAAPIDeviceContext *)hw_frame_ctx->device_ctx->hwctx; 60 | + 61 | + sd = av_frame_get_side_data(task->in_frame, AV_FRAME_DATA_DETECTION_BBOXES); 62 | + av_assert0(sd); 63 | + header = (const AVDetectionBBoxHeader *)sd->data; 64 | + bbox = av_get_detection_bbox(header, lltask->bbox_index); 65 | + 66 | + if (ov_model->input_port) { 67 | + ov_output_const_port_free(ov_model->input_port); 68 | + ov_model->input_port = NULL; 69 | + } 70 | + status = ov_model_const_input_by_index(ov_model->ov_model, 0, &ov_model->input_port); 71 | + if (status != OK) { 72 | + av_log(ctx, AV_LOG_ERROR, "Failed to get input port shape.\n"); 73 | + return ov2_map_error(status, NULL); 74 | + } 75 | + status = ov_const_port_get_shape(ov_model->input_port, &input_shape); 76 | + if (status != OK) { 77 | + av_log(ctx, AV_LOG_ERROR, "Failed to get input port shape.\n"); 78 | + return ov2_map_error(status, NULL); 79 | + } 80 | + ov_model->dpy = vaapi_ctx->display; 81 | + if (!request->bbox_surface) { 82 | + switch (hw_frame_ctx->sw_format) { 83 | + case AV_PIX_FMT_NV12: 84 | + rt_format = VA_RT_FORMAT_YUV420; 85 | + fourcc = VA_FOURCC_NV12; 86 | + } 87 | + attributes = (VASurfaceAttrib) { 88 | + .type = VASurfaceAttribPixelFormat, 89 | + .flags = VA_SURFACE_ATTRIB_SETTABLE, 90 | + .value.type = VAGenericValueTypeInteger, 91 | + .value.value.i = fourcc, 92 | + }; 93 | + vas = vaCreateSurfaces(ov_model->dpy, 94 | + rt_format, 95 | + input_shape.dims[2], 96 | + input_shape.dims[1], 97 | + &request->bbox_surface, 98 | + 1, 99 | + &attributes, 100 | + 1); 101 | + if (vas != VA_STATUS_SUCCESS) { 102 | + av_log(ctx, AV_LOG_ERROR, "Failed to create surface: " 103 | + "%d (%s).\n", vas, vaErrorStr(vas)); 104 | + return AVERROR(EIO); 105 | + } 106 | + } 107 | + vas = vaDeriveImage(ov_model->dpy, (VASurfaceID)(size_t)task->in_frame->data[3], &image); 108 | + if (vas != VA_STATUS_SUCCESS) { 109 | + av_log(ctx, AV_LOG_ERROR, "Failed to derive image from " 110 | + "surface %#x: %d (%s).\n", 111 | + request->bbox_surface, vas, vaErrorStr(vas)); 112 | + return AVERROR(EIO); 113 | + } 114 | + vas = vaPutImage(ov_model->dpy, 115 | + request->bbox_surface, 116 | + image.image_id, 117 | + bbox->x, bbox->y, bbox->w, bbox->h, 118 | + 0, 0, 119 | + input_shape.dims[2], 120 | + input_shape.dims[1]); 121 | + if (vas != VA_STATUS_SUCCESS) { 122 | + av_log(ctx, AV_LOG_ERROR, "Failed to write image to surface " 123 | + "%#x: %d (%s).\n", request->bbox_surface, vas, vaErrorStr(vas)); 124 | + return AVERROR(EIO); 125 | + } 126 | + vaDestroyImage(ov_model->dpy, image.image_id); 127 | + surface_id = request->bbox_surface; 128 | +#endif 129 | + } 130 | + 131 | for (int i = 0; i < nb_planes; i++) { 132 | if (ov_model->input_port) { 133 | ov_output_const_port_free(ov_model->input_port); 134 | @@ -623,6 +717,10 @@ static void dnn_free_model_ov(DNNModel **model) 135 | while (ff_safe_queue_size(ov_model->request_queue) != 0) { 136 | OVRequestItem *item = ff_safe_queue_pop_front(ov_model->request_queue); 137 | if (item && item->infer_request) { 138 | +#if !HAVE_WINDOWS_H && CONFIG_VAAPI 139 | + if (item->bbox_surface) 140 | + vaDestroySurfaces(ov_model->dpy, &item->bbox_surface, 1); 141 | +#endif 142 | #if HAVE_OPENVINO2 143 | ov_infer_request_free(item->infer_request); 144 | #else 145 | -- 146 | 2.34.1 147 | 148 | -------------------------------------------------------------------------------- /patches/0155-libavfilter-vf_dnn_classify-Add-vaapi-into-classify.patch: -------------------------------------------------------------------------------- 1 | From 05bc46d5931cdff56a39d3875a4f12c9e6b93c0c Mon Sep 17 00:00:00 2001 2 | From: Wenbin Chen 3 | Date: Tue, 5 Mar 2024 16:46:12 +0800 4 | Subject: [PATCH] libavfilter/vf_dnn_classify: Add vaapi into classify 5 | 6 | Signed-off-by: Wenbin Chen 7 | --- 8 | libavfilter/vf_dnn_classify.c | 33 ++++++++++++++++++++++++++++++++- 9 | 1 file changed, 32 insertions(+), 1 deletion(-) 10 | 11 | diff --git a/libavfilter/vf_dnn_classify.c b/libavfilter/vf_dnn_classify.c 12 | index f6d3678796..6b0ff408b7 100644 13 | --- a/libavfilter/vf_dnn_classify.c 14 | +++ b/libavfilter/vf_dnn_classify.c 15 | @@ -200,6 +200,9 @@ static const enum AVPixelFormat pix_fmts[] = { 16 | AV_PIX_FMT_YUV420P, AV_PIX_FMT_YUV422P, 17 | AV_PIX_FMT_YUV444P, AV_PIX_FMT_YUV410P, AV_PIX_FMT_YUV411P, 18 | AV_PIX_FMT_NV12, 19 | +#if !HAVE_WINDOWS_H && CONFIG_VAAPI 20 | + AV_PIX_FMT_VAAPI, 21 | +#endif 22 | AV_PIX_FMT_NONE 23 | }; 24 | 25 | @@ -294,6 +297,33 @@ static av_cold void dnn_classify_uninit(AVFilterContext *context) 26 | free_classify_labels(ctx); 27 | } 28 | 29 | +static int config_output(AVFilterLink *outlink) 30 | +{ 31 | + AVFilterContext *context = outlink->src; 32 | + AVFilterLink *inlink = context->inputs[0]; 33 | + DnnClassifyContext *classify_ctx = context->priv; 34 | + FilterLink *inl = ff_filter_link(inlink); 35 | + FilterLink *onl = ff_filter_link(outlink); 36 | + if (inl->hw_frames_ctx) { 37 | + if (inlink->format == AV_PIX_FMT_VAAPI && 38 | + classify_ctx->dnnctx.backend_type == DNN_OV) 39 | + onl->hw_frames_ctx = av_buffer_ref(inl->hw_frames_ctx); 40 | + else { 41 | + av_log(classify_ctx, AV_LOG_ERROR, "The dnn_backend doesn't support this pixel format\n"); 42 | + return AVERROR_PATCHWELCOME; 43 | + } 44 | + } 45 | + return 0; 46 | +} 47 | + 48 | +static const AVFilterPad dnn_classify_outputs[] = { 49 | + { 50 | + .name = "default", 51 | + .type = AVMEDIA_TYPE_VIDEO, 52 | + .config_props = config_output, 53 | + }, 54 | +}; 55 | + 56 | const AVFilter ff_vf_dnn_classify = { 57 | .name = "dnn_classify", 58 | .description = NULL_IF_CONFIG_SMALL("Apply DNN classify filter to the input."), 59 | @@ -302,8 +332,9 @@ const AVFilter ff_vf_dnn_classify = { 60 | .init = dnn_classify_init, 61 | .uninit = dnn_classify_uninit, 62 | FILTER_INPUTS(ff_video_default_filterpad), 63 | - FILTER_OUTPUTS(ff_video_default_filterpad), 64 | + FILTER_OUTPUTS(dnn_classify_outputs), 65 | FILTER_PIXFMTS_ARRAY(pix_fmts), 66 | .priv_class = &dnn_classify_class, 67 | .activate = dnn_classify_activate, 68 | + .flags_internal = FF_FILTER_FLAG_HWFRAME_AWARE, 69 | }; 70 | -- 71 | 2.34.1 72 | 73 | -------------------------------------------------------------------------------- /patches/0165-libavcodec-qsvenc-enable-Alpha-Encode-for-HEVC.patch: -------------------------------------------------------------------------------- 1 | From 3e97199ea06e27baedbab94881c8e561288f8a28 Mon Sep 17 00:00:00 2001 2 | From: Fei Wang 3 | Date: Fri, 16 Aug 2024 21:22:33 +0800 4 | Subject: [PATCH 1/4] libavcodec/qsvenc: enable Alpha Encode for HEVC 5 | 6 | This support alpha encode for HEVC introduced by Apple: 7 | https://developer.apple.com/videos/play/wwdc2019/506/ 8 | 9 | Currently, it only support RGBA video memory as input. RGB and alpha 10 | channel will be encoded in different layers with 4:2:0 color format. 11 | 12 | Example cmdline: 13 | ffmpeg.exe -v verbose -hwaccel qsv -hwaccel_output_format qsv -f rawvideo \ 14 | -pix_fmt bgra -s:v 1920x1080 -r:v 25 -i input.argb -vf \ 15 | 'format=bgra,hwupload=extra_hw_frames=120' -an -c:v hevc_qsv \ 16 | -alpha_encode 1 -y out.mp4 17 | 18 | Signed-off-by: Fei Wang 19 | --- 20 | doc/encoders.texi | 4 ++ 21 | libavcodec/qsvenc.c | 78 +++++++++++++++++++++++++++++++++++++-- 22 | libavcodec/qsvenc.h | 10 ++++- 23 | libavcodec/qsvenc_hevc.c | 3 ++ 24 | libavutil/hwcontext_qsv.c | 2 +- 25 | 5 files changed, 91 insertions(+), 6 deletions(-) 26 | 27 | diff --git a/doc/encoders.texi b/doc/encoders.texi 28 | index 496852faeb..1ed4a0aa74 100644 29 | --- a/doc/encoders.texi 30 | +++ b/doc/encoders.texi 31 | @@ -4036,6 +4036,10 @@ skip_frame metadata indicates the number of missed frames before the current 32 | frame. 33 | @end table 34 | 35 | +@item @var{alpha_encode} 36 | +Encode Alpha and RGB into different layers introduced by Apple: 37 | +https://developer.apple.com/videos/play/wwdc2019/506/. Only support on Windows 38 | +with RGBA video memory as input. 39 | @end table 40 | 41 | @subsection MPEG2 Options 42 | diff --git a/libavcodec/qsvenc.c b/libavcodec/qsvenc.c 43 | index 77c592a24b..bc8c53577b 100644 44 | --- a/libavcodec/qsvenc.c 45 | +++ b/libavcodec/qsvenc.c 46 | @@ -203,6 +203,9 @@ static void dump_video_param(AVCodecContext *avctx, QSVEncContext *q, 47 | #if QSV_HAVE_HE 48 | mfxExtHyperModeParam *exthypermodeparam = NULL; 49 | #endif 50 | +#if QSV_HAVE_AC 51 | + mfxExtAlphaChannelEncCtrl *extalphachannel = NULL; 52 | +#endif 53 | 54 | const char *tmp_str = NULL; 55 | mfxExtHEVCParam *exthevcparam = NULL; 56 | @@ -221,6 +224,11 @@ static void dump_video_param(AVCodecContext *avctx, QSVEncContext *q, 57 | exthypermodeparam = (mfxExtHyperModeParam *)coding_opts[q->exthypermodeparam_idx]; 58 | #endif 59 | 60 | +#if QSV_HAVE_AC 61 | + if (q->extaplhachannel_idx > 0) 62 | + extalphachannel = (mfxExtAlphaChannelEncCtrl *)coding_opts[q->extaplhachannel_idx]; 63 | +#endif 64 | + 65 | if (q->exthevcparam_idx > 0) 66 | exthevcparam = (mfxExtHEVCParam *)coding_opts[q->exthevcparam_idx]; 67 | av_log(avctx, AV_LOG_VERBOSE, "profile: %s; level: %"PRIu16"\n", 68 | @@ -408,6 +416,23 @@ static void dump_video_param(AVCodecContext *avctx, QSVEncContext *q, 69 | avctx->codec_id == AV_CODEC_ID_HEVC && 70 | info->CodecProfile == MFX_PROFILE_HEVC_MAIN10) 71 | av_log(avctx, AV_LOG_VERBOSE, "Main10sp (Main10 profile and one_pic_only flag): enable\n"); 72 | + 73 | +#if QSV_HAVE_AC 74 | + if (extalphachannel) { 75 | + av_log(avctx, AV_LOG_VERBOSE, "AlphaChannel Encode: %s; ", print_threestate(extalphachannel->EnableAlphaChannelEncoding)); 76 | + 77 | + av_log(avctx, AV_LOG_VERBOSE, "Mode: "); 78 | + if (extalphachannel->AlphaChannelMode == MFX_ALPHA_MODE_PREMULTIPLIED) 79 | + av_log(avctx, AV_LOG_VERBOSE, "PREMULTIPLIED; "); 80 | + else if (extalphachannel->AlphaChannelMode == MFX_ALPHA_MODE_STRAIGHT) 81 | + av_log(avctx, AV_LOG_VERBOSE, "STRAIGHT; "); 82 | + else 83 | + av_log(avctx, AV_LOG_VERBOSE, "unknown; "); 84 | + av_log(avctx, AV_LOG_VERBOSE, "BitrateRatio: %d", extalphachannel->AlphaChannelBitrateRatio); 85 | + 86 | + av_log(avctx, AV_LOG_VERBOSE, "\n"); 87 | + } 88 | +#endif 89 | } 90 | 91 | static void dump_video_vp9_param(AVCodecContext *avctx, QSVEncContext *q, 92 | @@ -1158,7 +1183,8 @@ static int init_video_param(AVCodecContext *avctx, QSVEncContext *q) 93 | q->extco3.MaxFrameSizeP = q->max_frame_size_p; 94 | if (sw_format == AV_PIX_FMT_BGRA && 95 | (q->profile == MFX_PROFILE_HEVC_REXT || 96 | - q->profile == MFX_PROFILE_UNKNOWN)) 97 | + q->profile == MFX_PROFILE_UNKNOWN) && 98 | + !q->alpha_encode) 99 | q->extco3.TargetChromaFormatPlus1 = MFX_CHROMAFORMAT_YUV444 + 1; 100 | 101 | q->extco3.ScenarioInfo = q->scenario; 102 | @@ -1302,6 +1328,37 @@ static int init_video_param(AVCodecContext *avctx, QSVEncContext *q) 103 | } 104 | #endif 105 | 106 | +#if QSV_HAVE_AC 107 | + if (q->alpha_encode) { 108 | + if (QSV_RUNTIME_VERSION_ATLEAST(q->ver, 2, 13)) { 109 | + mfxIMPL impl; 110 | + MFXQueryIMPL(q->session, &impl); 111 | + 112 | + if (MFX_IMPL_VIA_MASK(impl) != MFX_IMPL_VIA_D3D11) { 113 | + av_log(avctx, AV_LOG_ERROR, "Alpha Channel Encode requires D3D11VA \n"); 114 | + return AVERROR_UNKNOWN; 115 | + } 116 | + 117 | + if (q->param.mfx.CodecId != MFX_CODEC_HEVC) { 118 | + av_log(avctx, AV_LOG_ERROR, "Not supported encoder for Alpha Channel Encode. " 119 | + "Supported: hevc_qsv \n"); 120 | + return AVERROR_UNKNOWN; 121 | + } 122 | + 123 | + q->extaplhachannelparam.Header.BufferId = MFX_EXTBUFF_ALPHA_CHANNEL_ENC_CTRL; 124 | + q->extaplhachannelparam.Header.BufferSz = sizeof(q->extaplhachannelparam); 125 | + q->extaplhachannelparam.EnableAlphaChannelEncoding = MFX_CODINGOPTION_ON; 126 | + q->extaplhachannelparam.AlphaChannelBitrateRatio = 25; 127 | + q->extaplhachannelparam.AlphaChannelMode = MFX_ALPHA_MODE_PREMULTIPLIED; 128 | + q->extparam_internal[q->nb_extparam_internal++] = (mfxExtBuffer *)&q->extaplhachannelparam; 129 | + } else { 130 | + av_log(avctx, AV_LOG_ERROR, 131 | + "This version of runtime doesn't support Alpha Channel Encode\n"); 132 | + return AVERROR_UNKNOWN; 133 | + } 134 | + } 135 | +#endif 136 | + 137 | if (!check_enc_param(avctx,q)) { 138 | av_log(avctx, AV_LOG_ERROR, 139 | "some encoding parameters are not supported by the QSV " 140 | @@ -1483,17 +1540,24 @@ static int qsv_retrieve_enc_params(AVCodecContext *avctx, QSVEncContext *q) 141 | }; 142 | #endif 143 | 144 | +#if QSV_HAVE_AC 145 | + mfxExtAlphaChannelEncCtrl alpha_encode_buf = { 146 | + .Header.BufferId = MFX_EXTBUFF_ALPHA_CHANNEL_ENC_CTRL, 147 | + .Header.BufferSz = sizeof(alpha_encode_buf), 148 | + }; 149 | +#endif 150 | + 151 | mfxExtHEVCParam hevc_param_buf = { 152 | .Header.BufferId = MFX_EXTBUFF_HEVC_PARAM, 153 | .Header.BufferSz = sizeof(hevc_param_buf), 154 | }; 155 | 156 | - mfxExtBuffer *ext_buffers[7 + QSV_HAVE_HE]; 157 | + mfxExtBuffer *ext_buffers[7 + QSV_HAVE_HE + QSV_HAVE_AC]; 158 | int need_pps = avctx->codec_id != AV_CODEC_ID_MPEG2VIDEO; 159 | int ret, ext_buf_num = 0, extradata_offset = 0; 160 | 161 | q->co2_idx = q->co3_idx = q->exthevctiles_idx = q->exthypermodeparam_idx = -1; 162 | - q->exthevcparam_idx = -1; 163 | + q->exthevcparam_idx = q->extaplhachannel_idx = -1; 164 | ext_buffers[ext_buf_num++] = (mfxExtBuffer*)&extradata; 165 | ext_buffers[ext_buf_num++] = (mfxExtBuffer*)&co; 166 | 167 | @@ -1521,6 +1585,14 @@ static int qsv_retrieve_enc_params(AVCodecContext *avctx, QSVEncContext *q) 168 | ext_buffers[ext_buf_num++] = (mfxExtBuffer*)&hyper_mode_param_buf; 169 | } 170 | #endif 171 | + 172 | +#if QSV_HAVE_AC 173 | + if (q->alpha_encode && QSV_RUNTIME_VERSION_ATLEAST(q->ver, 2, 13)) { 174 | + q->extaplhachannel_idx = ext_buf_num; 175 | + ext_buffers[ext_buf_num++] = (mfxExtBuffer*)&alpha_encode_buf; 176 | + } 177 | +#endif 178 | + 179 | if (avctx->codec_id == AV_CODEC_ID_HEVC && QSV_RUNTIME_VERSION_ATLEAST(q->ver, 2, 0)) { 180 | q->exthevcparam_idx = ext_buf_num; 181 | ext_buffers[ext_buf_num++] = (mfxExtBuffer*)&hevc_param_buf; 182 | diff --git a/libavcodec/qsvenc.h b/libavcodec/qsvenc.h 183 | index 522f3b5b55..b91326f907 100644 184 | --- a/libavcodec/qsvenc.h 185 | +++ b/libavcodec/qsvenc.h 186 | @@ -46,11 +46,13 @@ 187 | #define QSV_HAVE_VCM 1 188 | #define QSV_HAVE_MF 0 189 | #define QSV_HAVE_HE QSV_VERSION_ATLEAST(2, 4) 190 | +#define QSV_HAVE_AC QSV_VERSION_ATLEAST(2, 13) 191 | #else 192 | #define QSV_HAVE_AVBR 0 193 | #define QSV_HAVE_VCM 0 194 | #define QSV_HAVE_MF !QSV_ONEVPL 195 | #define QSV_HAVE_HE 0 196 | +#define QSV_HAVE_AC 0 197 | #endif 198 | 199 | #define QSV_COMMON_OPTS \ 200 | @@ -191,10 +193,13 @@ typedef struct QSVEncContext { 201 | mfxFrameSurface1 **opaque_surfaces; 202 | AVBufferRef *opaque_alloc_buf; 203 | #endif 204 | +#if QSV_HAVE_AC 205 | + mfxExtAlphaChannelEncCtrl extaplhachannelparam; 206 | +#endif 207 | 208 | mfxExtVideoSignalInfo extvsi; 209 | 210 | - mfxExtBuffer *extparam_internal[6 + (QSV_HAVE_MF * 2) + (QSV_HAVE_EXT_AV1_PARAM * 2) + QSV_HAVE_HE]; 211 | + mfxExtBuffer *extparam_internal[6 + (QSV_HAVE_MF * 2) + (QSV_HAVE_EXT_AV1_PARAM * 2) + QSV_HAVE_HE + QSV_HAVE_AC]; 212 | int nb_extparam_internal; 213 | 214 | mfxExtBuffer **extparam_str; 215 | @@ -272,6 +277,7 @@ typedef struct QSVEncContext { 216 | 217 | int co2_idx; 218 | int co3_idx; 219 | + int extaplhachannel_idx; 220 | int exthevctiles_idx; 221 | int exthypermodeparam_idx; 222 | int vp9_idx; 223 | @@ -324,7 +330,7 @@ typedef struct QSVEncContext { 224 | AVDictionary *qsv_params; 225 | int exthevcparam_idx; 226 | int main10sp; 227 | - 228 | + int alpha_encode; 229 | } QSVEncContext; 230 | 231 | int ff_qsv_enc_init(AVCodecContext *avctx, QSVEncContext *q); 232 | diff --git a/libavcodec/qsvenc_hevc.c b/libavcodec/qsvenc_hevc.c 233 | index 3997757d48..fa3ddb18bb 100644 234 | --- a/libavcodec/qsvenc_hevc.c 235 | +++ b/libavcodec/qsvenc_hevc.c 236 | @@ -367,6 +367,9 @@ static const AVOption options[] = { 237 | 238 | #if QSV_ONEVPL 239 | { "main10sp", "This profile allow to encode 10 bit single still picture", OFFSET(qsv.main10sp), AV_OPT_TYPE_BOOL, { .i64 = 0 }, 0, 1, VE}, 240 | +#endif 241 | +#if QSV_HAVE_AC 242 | + { "alpha_encode", "Encode with alpha channel", OFFSET(qsv.alpha_encode), AV_OPT_TYPE_INT, { .i64 = 0 }, 0, 1, VE}, 243 | #endif 244 | { NULL }, 245 | }; 246 | diff --git a/libavutil/hwcontext_qsv.c b/libavutil/hwcontext_qsv.c 247 | index 24ff173cbb..71ef22aa5e 100644 248 | --- a/libavutil/hwcontext_qsv.c 249 | +++ b/libavutil/hwcontext_qsv.c 250 | @@ -236,7 +236,7 @@ static uint32_t qsv_get_d3d11va_bind_flags(int mem_type) 251 | bind_flags = D3D11_BIND_DECODER; 252 | 253 | if ((MFX_MEMTYPE_FROM_VPPOUT & mem_type) || (MFX_MEMTYPE_VIDEO_MEMORY_PROCESSOR_TARGET & mem_type)) 254 | - bind_flags = D3D11_BIND_RENDER_TARGET; 255 | + bind_flags = D3D11_BIND_RENDER_TARGET | D3D11_BIND_SHADER_RESOURCE; 256 | 257 | return bind_flags; 258 | } 259 | -- 260 | 2.34.1 261 | 262 | -------------------------------------------------------------------------------- /patches/0166-libavcodec-qsvenc-enable-Screen-Content-Tool-Encode-.patch: -------------------------------------------------------------------------------- 1 | From 6d2fcb8a36d51b9316c4fb743152d5b2b2dc7c0a Mon Sep 17 00:00:00 2001 2 | From: Fei Wang 3 | Date: Fri, 16 Aug 2024 21:47:40 +0800 4 | Subject: [PATCH] libavcodec/qsvenc: enable Screen Content Tool Encode for AV1 5 | 6 | Screen Content Tool provides Intra Block Copy and Palette Mode when encoding. 7 | --- 8 | doc/encoders.texi | 6 ++++++ 9 | libavcodec/qsvenc.c | 43 +++++++++++++++++++++++++++++++++++++++++ 10 | libavcodec/qsvenc.h | 9 ++++++++- 11 | libavcodec/qsvenc_av1.c | 4 ++++ 12 | 4 files changed, 61 insertions(+), 1 deletion(-) 13 | 14 | diff --git a/doc/encoders.texi b/doc/encoders.texi 15 | index 6094434ed4..85be976a46 100644 16 | --- a/doc/encoders.texi 17 | +++ b/doc/encoders.texi 18 | @@ -4121,6 +4121,12 @@ than zero, then for I frames the value set by max_frame_size is ignored. 19 | @item @var{max_frame_size_p} 20 | Maximum encoded frame size for P frames in bytes. If this value is set as larger 21 | than zero, then for P frames the value set by max_frame_size is ignored. 22 | + 23 | +@item @var{palette_mode} 24 | +Use palette mode in Screen Content Tool. 25 | + 26 | +@item @var{intrabc} 27 | +Use intra block copy in Screen Content Tool. 28 | @end table 29 | 30 | @section snow 31 | diff --git a/libavcodec/qsvenc.c b/libavcodec/qsvenc.c 32 | index bc8c53577b..ee5f7548b6 100644 33 | --- a/libavcodec/qsvenc.c 34 | +++ b/libavcodec/qsvenc.c 35 | @@ -527,6 +527,9 @@ static void dump_video_av1_param(AVCodecContext *avctx, QSVEncContext *q, 36 | mfxExtAV1BitstreamParam *av1_bs_param = (mfxExtAV1BitstreamParam *)coding_opts[1]; 37 | mfxExtCodingOption2 *co2 = (mfxExtCodingOption2*)coding_opts[2]; 38 | mfxExtCodingOption3 *co3 = (mfxExtCodingOption3*)coding_opts[3]; 39 | +#if QSV_HAVE_EXT_AV1_SCC 40 | + mfxExtAV1ScreenContentTools *scc = (mfxExtAV1ScreenContentTools*)coding_opts[4]; 41 | +#endif 42 | 43 | av_log(avctx, AV_LOG_VERBOSE, "profile: %s; level: %"PRIu16"\n", 44 | print_profile(avctx->codec_id, info->CodecProfile), info->CodecLevel); 45 | @@ -599,6 +602,14 @@ static void dump_video_av1_param(AVCodecContext *avctx, QSVEncContext *q, 46 | print_threestate(av1_bs_param->WriteIVFHeaders)); 47 | av_log(avctx, AV_LOG_VERBOSE, "LowDelayBRC: %s\n", print_threestate(co3->LowDelayBRC)); 48 | av_log(avctx, AV_LOG_VERBOSE, "MaxFrameSize: %d;\n", co2->MaxFrameSize); 49 | + 50 | +#if QSV_HAVE_EXT_AV1_SCC 51 | + if (scc) { 52 | + av_log(avctx, AV_LOG_VERBOSE, 53 | + "Palette: %s; IntraBlockCopy: %s\n", 54 | + print_threestate(scc->Palette), print_threestate(scc->IntraBlockCopy)); 55 | + } 56 | +#endif 57 | } 58 | #endif 59 | 60 | @@ -1359,6 +1370,28 @@ static int init_video_param(AVCodecContext *avctx, QSVEncContext *q) 61 | } 62 | #endif 63 | 64 | +#if QSV_HAVE_EXT_AV1_SCC 65 | + if (q->palette_mode || q->intrabc) { 66 | + if (QSV_RUNTIME_VERSION_ATLEAST(q->ver, 2, 13)) { 67 | + if (q->param.mfx.CodecId != MFX_CODEC_AV1) { 68 | + av_log(avctx, AV_LOG_ERROR, "Not supported encoder for Screen Content Tool Encode. " 69 | + "Supported: av1_qsv \n"); 70 | + return AVERROR_UNKNOWN; 71 | + } 72 | + 73 | + q->extsccparam.Header.BufferId = MFX_EXTBUFF_AV1_SCREEN_CONTENT_TOOLS; 74 | + q->extsccparam.Header.BufferSz = sizeof(q->extsccparam); 75 | + q->extsccparam.Palette = q->palette_mode ? MFX_CODINGOPTION_ON : MFX_CODINGOPTION_OFF; 76 | + q->extsccparam.IntraBlockCopy = q->intrabc ? MFX_CODINGOPTION_ON : MFX_CODINGOPTION_OFF; 77 | + q->extparam_internal[q->nb_extparam_internal++] = (mfxExtBuffer *)&q->extsccparam; 78 | + } else { 79 | + av_log(avctx, AV_LOG_ERROR, 80 | + "This version of runtime doesn't support Screen Content Tool Encode\n"); 81 | + return AVERROR_UNKNOWN; 82 | + } 83 | + } 84 | +#endif 85 | + 86 | if (!check_enc_param(avctx,q)) { 87 | av_log(avctx, AV_LOG_ERROR, 88 | "some encoding parameters are not supported by the QSV " 89 | @@ -1466,11 +1499,21 @@ static int qsv_retrieve_enc_av1_params(AVCodecContext *avctx, QSVEncContext *q) 90 | .Header.BufferSz = sizeof(co3), 91 | }; 92 | 93 | +#if QSV_HAVE_EXT_AV1_SCC 94 | + mfxExtAV1ScreenContentTools scc_buf = { 95 | + .Header.BufferId = MFX_EXTBUFF_AV1_SCREEN_CONTENT_TOOLS, 96 | + .Header.BufferSz = sizeof(scc_buf), 97 | + }; 98 | +#endif 99 | + 100 | mfxExtBuffer *ext_buffers[] = { 101 | (mfxExtBuffer*)&av1_extend_tile_buf, 102 | (mfxExtBuffer*)&av1_bs_param, 103 | (mfxExtBuffer*)&co2, 104 | (mfxExtBuffer*)&co3, 105 | +#if QSV_HAVE_EXT_AV1_SCC 106 | + (mfxExtBuffer*)&scc_buf, 107 | +#endif 108 | }; 109 | 110 | if (!QSV_RUNTIME_VERSION_ATLEAST(q->ver, 2, 5)) { 111 | diff --git a/libavcodec/qsvenc.h b/libavcodec/qsvenc.h 112 | index d7a094e251..f67113ca6c 100644 113 | --- a/libavcodec/qsvenc.h 114 | +++ b/libavcodec/qsvenc.h 115 | @@ -38,6 +38,7 @@ 116 | 117 | #define QSV_HAVE_EXT_VP9_TILES QSV_VERSION_ATLEAST(1, 29) 118 | #define QSV_HAVE_EXT_AV1_PARAM QSV_VERSION_ATLEAST(2, 5) 119 | +#define QSV_HAVE_EXT_AV1_SCC QSV_VERSION_ATLEAST(2, 13) 120 | 121 | #if defined(_WIN32) || defined(__CYGWIN__) 122 | #define QSV_HAVE_AVBR 1 123 | @@ -194,10 +195,14 @@ typedef struct QSVEncContext { 124 | #if QSV_HAVE_AC 125 | mfxExtAlphaChannelEncCtrl extaplhachannelparam; 126 | #endif 127 | +#if QSV_HAVE_EXT_AV1_SCC 128 | + mfxExtAV1ScreenContentTools extsccparam; 129 | +#endif 130 | 131 | mfxExtVideoSignalInfo extvsi; 132 | 133 | - mfxExtBuffer *extparam_internal[6 + (QSV_HAVE_MF * 2) + (QSV_HAVE_EXT_AV1_PARAM * 2) + QSV_HAVE_HE + QSV_HAVE_AC]; 134 | + mfxExtBuffer *extparam_internal[6 + (QSV_HAVE_MF * 2) + (QSV_HAVE_EXT_AV1_PARAM * 2) + 135 | + QSV_HAVE_HE + QSV_HAVE_AC + QSV_HAVE_EXT_AV1_SCC]; 136 | int nb_extparam_internal; 137 | 138 | mfxExtBuffer **extparam_str; 139 | @@ -329,6 +334,8 @@ typedef struct QSVEncContext { 140 | int exthevcparam_idx; 141 | int main10sp; 142 | int alpha_encode; 143 | + int palette_mode; 144 | + int intrabc; 145 | } QSVEncContext; 146 | 147 | int ff_qsv_enc_init(AVCodecContext *avctx, QSVEncContext *q); 148 | diff --git a/libavcodec/qsvenc_av1.c b/libavcodec/qsvenc_av1.c 149 | index f7505a69f8..612772557a 100644 150 | --- a/libavcodec/qsvenc_av1.c 151 | +++ b/libavcodec/qsvenc_av1.c 152 | @@ -189,6 +189,10 @@ static const AVOption options[] = { 153 | { "tile_cols", "Number of columns for tiled encoding", OFFSET(qsv.tile_cols), AV_OPT_TYPE_INT, { .i64 = 0 }, 0, UINT16_MAX, VE }, 154 | { "tile_rows", "Number of rows for tiled encoding", OFFSET(qsv.tile_rows), AV_OPT_TYPE_INT, { .i64 = 0 }, 0, UINT16_MAX, VE }, 155 | { "look_ahead_depth", "Depth of look ahead in number frames, available when extbrc option is enabled", OFFSET(qsv.look_ahead_depth), AV_OPT_TYPE_INT, { .i64 = 0 }, 0, 100, VE }, 156 | +#if QSV_HAVE_EXT_AV1_SCC 157 | + { "palette_mode", "Enable palette mode of Screen Content Tool for encodeing", OFFSET(qsv.palette_mode), AV_OPT_TYPE_BOOL, { .i64 = 0}, 0, 1, VE}, 158 | + { "intrabc", "Enable intra block copy of Screen Content Tool for encodeing", OFFSET(qsv.intrabc), AV_OPT_TYPE_BOOL, { .i64 = 0}, 0, 1, VE}, 159 | +#endif 160 | { NULL }, 161 | }; 162 | 163 | -- 164 | 2.34.1 165 | 166 | -------------------------------------------------------------------------------- /patches/0168-lavc-qsvenc-Make-ffmpeg-compilable-with-experimental.patch: -------------------------------------------------------------------------------- 1 | From 47a98b701fe52417e64a46d416954142e307d0b5 Mon Sep 17 00:00:00 2001 2 | From: Fei Wang 3 | Date: Fri, 16 Aug 2024 13:20:40 +0800 4 | Subject: [PATCH 4/4] lavc/qsvenc: Make ffmpeg compilable with experimental API 5 | of libvpl 6 | 7 | This is a temporary patch to allow ffmpeg-qsv to use hevc alpha encode, 8 | av1 scc encode and encode frame quality features which are under 9 | experimental of libvpl. Since the runtime version check been removed, 10 | please make sure your vpl runtime can support them when you use it. 11 | 12 | Signed-off-by: Fei Wang 13 | --- 14 | configure | 8 ++++++++ 15 | libavcodec/qsvenc.c | 15 ++++++++++----- 16 | libavcodec/qsvenc.h | 16 ++++++++++++---- 17 | 3 files changed, 30 insertions(+), 9 deletions(-) 18 | 19 | diff --git a/configure b/configure 20 | index daf358b95b..f8765f11a1 100755 21 | --- a/configure 22 | +++ b/configure 23 | @@ -6930,6 +6930,14 @@ elif enabled libvpl; then 24 | die "ERROR: libvpl >= 2.6 not found" 25 | add_cflags -DMFX_DEPRECATED_OFF 26 | check_type "vpl/mfxdefs.h vpl/mfxvideo.h" "struct mfxConfigInterface" 27 | + add_cflags -DONEVPL_EXPERIMENTAL 28 | + check_cc AV1_SCC_ENCODE "mfxstructures.h" "MFX_EXTBUFF_AV1_SCREEN_CONTENT_TOOLS" && add_cflags -DQSV_HAVE_EXT_AV1_SCC 29 | + case $target_os in 30 | + win32|win64) 31 | + check_cc HEVC_ALPHA_ENCODE "mfxstructures.h" "MFX_EXTBUFF_ALPHA_CHANNEL_ENC_CTRL" && add_cflags -DQSV_HAVE_AC 32 | + check_cc FRAME_QUALITY_ENCODE "mfxstructures.h" "MFX_EXTBUFF_ENCODED_QUALITY_INFO_MODE" && add_cflags -DQSV_HAVE_EXT_MSE 33 | + ;; 34 | + esac 35 | fi 36 | 37 | if enabled libmfx; then 38 | diff --git a/libavcodec/qsvenc.c b/libavcodec/qsvenc.c 39 | index 14b32abd50..22a1949722 100644 40 | --- a/libavcodec/qsvenc.c 41 | +++ b/libavcodec/qsvenc.c 42 | @@ -1386,7 +1386,8 @@ static int init_video_param(AVCodecContext *avctx, QSVEncContext *q) 43 | 44 | #if QSV_HAVE_AC 45 | if (q->alpha_encode) { 46 | - if (QSV_RUNTIME_VERSION_ATLEAST(q->ver, 2, 13)) { 47 | + // if (QSV_RUNTIME_VERSION_ATLEAST(q->ver, 2, 13)) { 48 | + if (1) { 49 | mfxIMPL impl; 50 | MFXQueryIMPL(q->session, &impl); 51 | 52 | @@ -1417,7 +1418,8 @@ static int init_video_param(AVCodecContext *avctx, QSVEncContext *q) 53 | 54 | #if QSV_HAVE_EXT_AV1_SCC 55 | if (q->palette_mode || q->intrabc) { 56 | - if (QSV_RUNTIME_VERSION_ATLEAST(q->ver, 2, 13)) { 57 | + // if (QSV_RUNTIME_VERSION_ATLEAST(q->ver, 2, 13)) { 58 | + if (1) { 59 | if (q->param.mfx.CodecId != MFX_CODEC_AV1) { 60 | av_log(avctx, AV_LOG_ERROR, "Not supported encoder for Screen Content Tool Encode. " 61 | "Supported: av1_qsv \n"); 62 | @@ -1439,7 +1441,8 @@ static int init_video_param(AVCodecContext *avctx, QSVEncContext *q) 63 | 64 | #if QSV_HAVE_EXT_MSE 65 | if (q->mse) { 66 | - if (QSV_RUNTIME_VERSION_ATLEAST(q->ver, 2, 13)) { 67 | + // if (QSV_RUNTIME_VERSION_ATLEAST(q->ver, 2, 13)) { 68 | + if (1) { 69 | q->extmseparam.Header.BufferId = MFX_EXTBUFF_ENCODED_QUALITY_INFO_MODE; 70 | q->extmseparam.Header.BufferSz = sizeof(q->extmseparam); 71 | q->extmseparam.QualityInfoMode = MFX_QUALITY_INFO_LEVEL_FRAME; 72 | @@ -1706,14 +1709,16 @@ static int qsv_retrieve_enc_params(AVCodecContext *avctx, QSVEncContext *q) 73 | #endif 74 | 75 | #if QSV_HAVE_AC 76 | - if (q->alpha_encode && QSV_RUNTIME_VERSION_ATLEAST(q->ver, 2, 13)) { 77 | + // if (q->alpha_encode && QSV_RUNTIME_VERSION_ATLEAST(q->ver, 2, 13)) { 78 | + if (q->alpha_encode) { 79 | q->extaplhachannel_idx = ext_buf_num; 80 | ext_buffers[ext_buf_num++] = (mfxExtBuffer*)&alpha_encode_buf; 81 | } 82 | #endif 83 | 84 | #if QSV_HAVE_EXT_MSE 85 | - if (q->mse && QSV_RUNTIME_VERSION_ATLEAST(q->ver, 2, 13)) { 86 | + // if (q->mse && QSV_RUNTIME_VERSION_ATLEAST(q->ver, 2, 13)) { 87 | + if (q->mse) { 88 | q->extmse_idx = ext_buf_num; 89 | ext_buffers[ext_buf_num++] = (mfxExtBuffer*)&mse_buf; 90 | } 91 | diff --git a/libavcodec/qsvenc.h b/libavcodec/qsvenc.h 92 | index 56308ed844..6ace0cbe8e 100644 93 | --- a/libavcodec/qsvenc.h 94 | +++ b/libavcodec/qsvenc.h 95 | @@ -40,21 +40,29 @@ 96 | 97 | #define QSV_HAVE_EXT_VP9_TILES QSV_VERSION_ATLEAST(1, 29) 98 | #define QSV_HAVE_EXT_AV1_PARAM QSV_VERSION_ATLEAST(2, 5) 99 | -#define QSV_HAVE_EXT_AV1_SCC QSV_VERSION_ATLEAST(2, 13) 100 | -#define QSV_HAVE_EXT_MSE QSV_VERSION_ATLEAST(2, 13) 101 | + 102 | +#ifndef QSV_HAVE_EXT_AV1_SCC 103 | +#define QSV_HAVE_EXT_AV1_SCC 0 104 | +#endif 105 | + 106 | +#ifndef QSV_HAVE_EXT_MSE 107 | +#define QSV_HAVE_EXT_MSE 0 108 | +#endif 109 | + 110 | +#ifndef QSV_HAVE_AC 111 | +#define QSV_HAVE_AC 0 112 | +#endif 113 | 114 | #if defined(_WIN32) || defined(__CYGWIN__) 115 | #define QSV_HAVE_AVBR 1 116 | #define QSV_HAVE_VCM 1 117 | #define QSV_HAVE_MF 0 118 | #define QSV_HAVE_HE QSV_VERSION_ATLEAST(2, 4) 119 | -#define QSV_HAVE_AC QSV_VERSION_ATLEAST(2, 13) 120 | #else 121 | #define QSV_HAVE_AVBR 0 122 | #define QSV_HAVE_VCM 0 123 | #define QSV_HAVE_MF !QSV_ONEVPL 124 | #define QSV_HAVE_HE 0 125 | -#define QSV_HAVE_AC 0 126 | #endif 127 | 128 | #define QSV_COMMON_OPTS \ 129 | -- 130 | 2.34.1 131 | 132 | -------------------------------------------------------------------------------- /patches/0169-lavc-qsvenc-Add-sliding-window-bitrate-control-for-C.patch: -------------------------------------------------------------------------------- 1 | From 8c276d52ac27b9f6a2cede6dd99f8c3ac5c92262 Mon Sep 17 00:00:00 2001 2 | From: Fei Wang 3 | Date: Mon, 19 Aug 2024 16:43:10 +0800 4 | Subject: [PATCH] lavc/qsvenc: Add sliding window bitrate control for CBR 5 | 6 | Sliding window bitrate control will provide a more stable bitrate when 7 | use CBR bitrate control mode. 8 | 9 | Signed-off-by: Fei Wang 10 | --- 11 | doc/encoders.texi | 9 +++++++++ 12 | libavcodec/qsvenc.c | 25 +++++++++++++++++++++++++ 13 | libavcodec/qsvenc.h | 11 +++++++++++ 14 | libavcodec/qsvenc_av1.c | 3 +++ 15 | libavcodec/qsvenc_h264.c | 3 +++ 16 | libavcodec/qsvenc_hevc.c | 3 +++ 17 | 6 files changed, 54 insertions(+) 18 | 19 | diff --git a/doc/encoders.texi b/doc/encoders.texi 20 | index 7d1373e0e0..7cec23a353 100644 21 | --- a/doc/encoders.texi 22 | +++ b/doc/encoders.texi 23 | @@ -3628,6 +3628,15 @@ This option allows fine-grained control over various encoder-specific settings p 24 | @item @var{mse} 25 | Supported in h264_qsv, hevc_qsv, and av1_qsv on Windows. Output encoded 26 | frame's quality(MSE/PSNR) information in VERBOSE log. 27 | + 28 | +@item @var{sw_size} 29 | +Number of frames used for sliding window(Only available for CBR bitrate 30 | +control mode). Range from 30 to 60 for AVC and HEVC. 30 to 120 for AV1. 31 | + 32 | +@item @var{sw_max_bitrate_factor} 33 | +Factor between bitrate and max bitrate for frames in sliding window. 34 | +Range from 1.1 to 2.0. 35 | + 36 | @end table 37 | 38 | @subsection H264 options 39 | diff --git a/libavcodec/qsvenc.c b/libavcodec/qsvenc.c 40 | index 22a1949722..51f226ded1 100644 41 | --- a/libavcodec/qsvenc.c 42 | +++ b/libavcodec/qsvenc.c 43 | @@ -401,6 +401,11 @@ static void dump_video_param(AVCodecContext *avctx, QSVEncContext *q, 44 | av_log(avctx, AV_LOG_VERBOSE, "MaxFrameSizeI: %d; ", co3->MaxFrameSizeI); 45 | av_log(avctx, AV_LOG_VERBOSE, "MaxFrameSizeP: %d\n", co3->MaxFrameSizeP); 46 | av_log(avctx, AV_LOG_VERBOSE, "ScenarioInfo: %"PRId16"\n", co3->ScenarioInfo); 47 | +#if QSV_HAVE_SLIDING_WINDOW 48 | + av_log(avctx, AV_LOG_VERBOSE, 49 | + "WinBRCSize: %"PRIu16"; WinBRCMaxAvgKbps: %"PRIu16"\n", 50 | + co3->WinBRCSize, co3->WinBRCMaxAvgKbps); 51 | +#endif 52 | } 53 | 54 | if (exthevctiles) { 55 | @@ -655,6 +660,12 @@ static void dump_video_av1_param(AVCodecContext *avctx, QSVEncContext *q, 56 | av_log(avctx, AV_LOG_VERBOSE, "\n"); 57 | } 58 | #endif 59 | + 60 | +#if QSV_HAVE_SLIDING_WINDOW 61 | + av_log(avctx, AV_LOG_VERBOSE, 62 | + "WinBRCSize: %"PRIu16"; WinBRCMaxAvgKbps: %"PRIu16"\n", 63 | + co3->WinBRCSize, co3->WinBRCMaxAvgKbps); 64 | +#endif 65 | } 66 | #endif 67 | 68 | @@ -1258,6 +1269,20 @@ static int init_video_param(AVCodecContext *avctx, QSVEncContext *q) 69 | q->extco3.TransformSkip = MFX_CODINGOPTION_UNKNOWN; 70 | q->extco3.GPB = q->gpb ? MFX_CODINGOPTION_ON : MFX_CODINGOPTION_OFF; 71 | } 72 | + 73 | +#if QSV_HAVE_SLIDING_WINDOW 74 | + if ((avctx->codec_id == AV_CODEC_ID_H264 || avctx->codec_id == AV_CODEC_ID_HEVC || 75 | + avctx->codec_id == AV_CODEC_ID_AV1) && q->sw_size) { 76 | + if (QSV_RUNTIME_VERSION_ATLEAST(q->ver, 2, 13)) { 77 | + q->extco3.WinBRCSize = q->sw_size; 78 | + q->extco3.WinBRCMaxAvgKbps = (int)(q->sw_max_bitrate_factor * q->param.mfx.TargetKbps); 79 | + } else { 80 | + av_log(avctx, AV_LOG_ERROR, 81 | + "This version of runtime doesn't support sliding windows bitrate control\n"); 82 | + return AVERROR_UNKNOWN; 83 | + } 84 | + } 85 | +#endif 86 | q->extparam_internal[q->nb_extparam_internal++] = (mfxExtBuffer *)&q->extco3; 87 | } 88 | 89 | diff --git a/libavcodec/qsvenc.h b/libavcodec/qsvenc.h 90 | index 98e4fa313b..a8ec5c2fb2 100644 91 | --- a/libavcodec/qsvenc.h 92 | +++ b/libavcodec/qsvenc.h 93 | @@ -24,6 +24,7 @@ 94 | #define AVCODEC_QSVENC_H 95 | 96 | #include 97 | +#include 98 | #include 99 | 100 | #include "libavutil/common.h" 101 | @@ -56,11 +57,13 @@ 102 | #define QSV_HAVE_VCM 1 103 | #define QSV_HAVE_MF 0 104 | #define QSV_HAVE_HE QSV_VERSION_ATLEAST(2, 4) 105 | +#define QSV_HAVE_SLIDING_WINDOW 1 106 | #else 107 | #define QSV_HAVE_AVBR 0 108 | #define QSV_HAVE_VCM 0 109 | #define QSV_HAVE_MF !QSV_ONEVPL 110 | #define QSV_HAVE_HE 0 111 | +#define QSV_HAVE_SLIDING_WINDOW 0 112 | #endif 113 | 114 | #define QSV_COMMON_OPTS \ 115 | @@ -165,6 +168,12 @@ 116 | { "mse", "Enable output MSE(Mean Squared Error) of each frame", OFFSET(qsv.mse), AV_OPT_TYPE_BOOL, { .i64 = 0 }, 0, 1, VE }, 117 | #endif 118 | 119 | +#if QSV_HAVE_SLIDING_WINDOW 120 | +#define QSV_SLIDING_WINDOW_OPTIONS \ 121 | +{ "sw_size", "Number of frames used for sliding window(Only available for CBR bitrate control mode)", OFFSET(qsv.sw_size), AV_OPT_TYPE_INT, { .i64 = 0 }, 0, INT_MAX, VE }, \ 122 | +{ "sw_max_bitrate_factor", "Factor between bitrate and max bitrate for frames in sliding window(Only available when sw_size is set)", OFFSET(qsv.sw_max_bitrate_factor), AV_OPT_TYPE_FLOAT, { .i64 = 0 }, 0, FLT_MAX, VE }, 123 | +#endif 124 | + 125 | extern const AVCodecHWConfigInternal *const ff_qsv_enc_hw_configs[]; 126 | 127 | typedef int SetEncodeCtrlCB (AVCodecContext *avctx, 128 | @@ -355,6 +364,8 @@ typedef struct QSVEncContext { 129 | int palette_mode; 130 | int intrabc; 131 | int mse; 132 | + int sw_size; 133 | + float sw_max_bitrate_factor; 134 | } QSVEncContext; 135 | 136 | int ff_qsv_enc_init(AVCodecContext *avctx, QSVEncContext *q); 137 | diff --git a/libavcodec/qsvenc_av1.c b/libavcodec/qsvenc_av1.c 138 | index 2bfcdceac0..0dd92c5fd8 100644 139 | --- a/libavcodec/qsvenc_av1.c 140 | +++ b/libavcodec/qsvenc_av1.c 141 | @@ -185,6 +185,9 @@ static const AVOption options[] = { 142 | QSV_OPTION_MAX_FRAME_SIZE 143 | #if QSV_HAVE_EXT_MSE 144 | QSV_MSE_OPTIONS 145 | +#endif 146 | +#if QSV_HAVE_SLIDING_WINDOW 147 | + QSV_SLIDING_WINDOW_OPTIONS 148 | #endif 149 | { "profile", NULL, OFFSET(qsv.profile), AV_OPT_TYPE_INT, { .i64 = MFX_PROFILE_UNKNOWN }, 0, INT_MAX, VE, .unit = "profile" }, 150 | { "unknown" , NULL, 0, AV_OPT_TYPE_CONST, { .i64 = MFX_PROFILE_UNKNOWN }, INT_MIN, INT_MAX, VE, .unit = "profile" }, 151 | diff --git a/libavcodec/qsvenc_h264.c b/libavcodec/qsvenc_h264.c 152 | index 9a3806ab56..4e728038ba 100644 153 | --- a/libavcodec/qsvenc_h264.c 154 | +++ b/libavcodec/qsvenc_h264.c 155 | @@ -123,6 +123,9 @@ static const AVOption options[] = { 156 | #if QSV_HAVE_EXT_MSE 157 | QSV_MSE_OPTIONS 158 | #endif 159 | +#if QSV_HAVE_SLIDING_WINDOW 160 | + QSV_SLIDING_WINDOW_OPTIONS 161 | +#endif 162 | 163 | { "cavlc", "Enable CAVLC", OFFSET(qsv.cavlc), AV_OPT_TYPE_BOOL, { .i64 = 0 }, 0, 1, VE }, 164 | #if QSV_HAVE_VCM 165 | diff --git a/libavcodec/qsvenc_hevc.c b/libavcodec/qsvenc_hevc.c 166 | index 0d771f246b..bc00dcc5e5 100644 167 | --- a/libavcodec/qsvenc_hevc.c 168 | +++ b/libavcodec/qsvenc_hevc.c 169 | @@ -326,6 +326,9 @@ static const AVOption options[] = { 170 | #if QSV_HAVE_EXT_MSE 171 | QSV_MSE_OPTIONS 172 | #endif 173 | +#if QSV_HAVE_SLIDING_WINDOW 174 | + QSV_SLIDING_WINDOW_OPTIONS 175 | +#endif 176 | 177 | { "idr_interval", "Distance (in I-frames) between IDR frames", OFFSET(qsv.idr_interval), AV_OPT_TYPE_INT, { .i64 = 0 }, -1, INT_MAX, VE, .unit = "idr_interval" }, 178 | { "begin_only", "Output an IDR-frame only at the beginning of the stream", 0, AV_OPT_TYPE_CONST, { .i64 = -1 }, 0, 0, VE, .unit = "idr_interval" }, 179 | -- 180 | 2.34.1 181 | 182 | --------------------------------------------------------------------------------