From 2d0722270b1f92a78bc210f23a2b5697cb0fd6bc Mon Sep 17 00:00:00 2001 From: Philipp Zabel Date: Fri, 9 Aug 2019 17:11:25 +0200 Subject: [PATCH 01/13] v4l2: introduce v4l2_set_controls This can be used to reduce number of issued ioctls, by setting multiple controls at once. Signed-off-by: Philipp Zabel --- src/v4l2.c | 31 ++++++++++++++++++++----------- src/v4l2.h | 3 +++ 2 files changed, 23 insertions(+), 11 deletions(-) diff --git a/src/v4l2.c b/src/v4l2.c index d5000ac..352047b 100644 --- a/src/v4l2.c +++ b/src/v4l2.c @@ -428,22 +428,17 @@ int v4l2_export_buffer(int video_fd, unsigned int type, unsigned int index, return 0; } -int v4l2_set_control(int video_fd, int request_fd, unsigned int id, void *data, - unsigned int size) +int v4l2_set_controls(int video_fd, int request_fd, + struct v4l2_ext_control *control_array, + unsigned int num_controls) { - struct v4l2_ext_control control; struct v4l2_ext_controls controls; int rc; - memset(&control, 0, sizeof(control)); memset(&controls, 0, sizeof(controls)); - control.id = id; - control.ptr = data; - control.size = size; - - controls.controls = &control; - controls.count = 1; + controls.controls = control_array; + controls.count = num_controls; if (request_fd >= 0) { controls.which = V4L2_CTRL_WHICH_REQUEST_VAL; @@ -452,13 +447,27 @@ int v4l2_set_control(int video_fd, int request_fd, unsigned int id, void *data, rc = ioctl(video_fd, VIDIOC_S_EXT_CTRLS, &controls); if (rc < 0) { - request_log("Unable to set control: %s\n", strerror(errno)); + request_log("Unable to set control(s): %s\n", strerror(errno)); return -1; } return 0; } +int v4l2_set_control(int video_fd, int request_fd, unsigned int id, void *data, + unsigned int size) +{ + struct v4l2_ext_control control; + + memset(&control, 0, sizeof(control)); + + control.id = id; + control.ptr = data; + control.size = size; + + return v4l2_set_controls(video_fd, request_fd, &control, 1); +} + int v4l2_set_stream(int video_fd, unsigned int type, bool enable) { enum v4l2_buf_type buf_type = type; diff --git a/src/v4l2.h b/src/v4l2.h index 73e9a42..3bccb23 100644 --- a/src/v4l2.h +++ b/src/v4l2.h @@ -54,6 +54,9 @@ int v4l2_dequeue_buffer(int video_fd, int request_fd, unsigned int type, int v4l2_export_buffer(int video_fd, unsigned int type, unsigned int index, unsigned int flags, int *export_fds, unsigned int export_fds_count); +int v4l2_set_controls(int video_fd, int request_fd, + struct v4l2_ext_control *controls, + unsigned int num_controls); int v4l2_set_control(int video_fd, int request_fd, unsigned int id, void *data, unsigned int size); int v4l2_set_stream(int video_fd, unsigned int type, bool enable); From c1261cce0fa0c0fad87327995f980c6aec564c98 Mon Sep 17 00:00:00 2001 From: Philipp Zabel Date: Fri, 23 Aug 2019 14:37:00 +0200 Subject: [PATCH 02/13] v4l2: introduce v4l2_get_controls This can be used to query codec mode controls, such as decode mode and start code for h.264. Signed-off-by: Philipp Zabel --- src/v4l2.c | 35 ++++++++++++++++++++++++++++++----- src/v4l2.h | 3 +++ 2 files changed, 33 insertions(+), 5 deletions(-) diff --git a/src/v4l2.c b/src/v4l2.c index 352047b..3addb33 100644 --- a/src/v4l2.c +++ b/src/v4l2.c @@ -428,12 +428,11 @@ int v4l2_export_buffer(int video_fd, unsigned int type, unsigned int index, return 0; } -int v4l2_set_controls(int video_fd, int request_fd, - struct v4l2_ext_control *control_array, - unsigned int num_controls) +static int v4l2_ioctl_controls(int video_fd, int request_fd, unsigned long ioc, + struct v4l2_ext_control *control_array, + unsigned int num_controls) { struct v4l2_ext_controls controls; - int rc; memset(&controls, 0, sizeof(controls)); @@ -445,7 +444,33 @@ int v4l2_set_controls(int video_fd, int request_fd, controls.request_fd = request_fd; } - rc = ioctl(video_fd, VIDIOC_S_EXT_CTRLS, &controls); + return ioctl(video_fd, ioc, &controls); +} + +int v4l2_get_controls(int video_fd, int request_fd, + struct v4l2_ext_control *control_array, + unsigned int num_controls) +{ + int rc; + + rc = v4l2_ioctl_controls(video_fd, request_fd, VIDIOC_G_EXT_CTRLS, + control_array, num_controls); + if (rc < 0) { + request_log("Unable to get control(s): %s\n", strerror(errno)); + return -1; + } + + return 0; +} + +int v4l2_set_controls(int video_fd, int request_fd, + struct v4l2_ext_control *control_array, + unsigned int num_controls) +{ + int rc; + + rc = v4l2_ioctl_controls(video_fd, request_fd, VIDIOC_S_EXT_CTRLS, + control_array, num_controls); if (rc < 0) { request_log("Unable to set control(s): %s\n", strerror(errno)); return -1; diff --git a/src/v4l2.h b/src/v4l2.h index 3bccb23..24c12a0 100644 --- a/src/v4l2.h +++ b/src/v4l2.h @@ -54,6 +54,9 @@ int v4l2_dequeue_buffer(int video_fd, int request_fd, unsigned int type, int v4l2_export_buffer(int video_fd, unsigned int type, unsigned int index, unsigned int flags, int *export_fds, unsigned int export_fds_count); +int v4l2_get_controls(int video_fd, int request_fd, + struct v4l2_ext_control *controls, + unsigned int num_controls); int v4l2_set_controls(int video_fd, int request_fd, struct v4l2_ext_control *controls, unsigned int num_controls); From 0923e901e57fb393d205532828788f8bd129182f Mon Sep 17 00:00:00 2001 From: Philipp Zabel Date: Wed, 7 Aug 2019 17:05:47 +0200 Subject: [PATCH 03/13] h264: update to merged h.264 kernel interface Update to the merged stateless h.264 kernel interface, as of commit c3adb85745ca ("media: uapi: h264: Get rid of the p0/b0/b1 ref-lists"). Signed-off-by: Philipp Zabel --- include/h264-ctrls.h | 21 +++++++++++++++++---- src/config.c | 2 +- src/context.c | 2 +- 3 files changed, 19 insertions(+), 6 deletions(-) diff --git a/include/h264-ctrls.h b/include/h264-ctrls.h index e1404d7..e877bf1 100644 --- a/include/h264-ctrls.h +++ b/include/h264-ctrls.h @@ -14,7 +14,7 @@ #include /* Our pixel format isn't stable at the moment */ -#define V4L2_PIX_FMT_H264_SLICE_RAW v4l2_fourcc('S', '2', '6', '4') /* H264 parsed slices */ +#define V4L2_PIX_FMT_H264_SLICE v4l2_fourcc('S', '2', '6', '4') /* H264 parsed slices */ /* * This is put insanely high to avoid conflicting with controls that @@ -26,6 +26,8 @@ #define V4L2_CID_MPEG_VIDEO_H264_SCALING_MATRIX (V4L2_CID_MPEG_BASE+1002) #define V4L2_CID_MPEG_VIDEO_H264_SLICE_PARAMS (V4L2_CID_MPEG_BASE+1003) #define V4L2_CID_MPEG_VIDEO_H264_DECODE_PARAMS (V4L2_CID_MPEG_BASE+1004) +#define V4L2_CID_MPEG_VIDEO_H264_DECODE_MODE (V4L2_CID_MPEG_BASE+1005) +#define V4L2_CID_MPEG_VIDEO_H264_START_CODE (V4L2_CID_MPEG_BASE+1006) /* enum v4l2_ctrl_type type values */ #define V4L2_CTRL_TYPE_H264_SPS 0x0110 @@ -34,6 +36,16 @@ #define V4L2_CTRL_TYPE_H264_SLICE_PARAMS 0x0113 #define V4L2_CTRL_TYPE_H264_DECODE_PARAMS 0x0114 +enum v4l2_mpeg_video_h264_decode_mode { + V4L2_MPEG_VIDEO_H264_DECODE_MODE_SLICE_BASED, + V4L2_MPEG_VIDEO_H264_DECODE_MODE_FRAME_BASED, +}; + +enum v4l2_mpeg_video_h264_start_code { + V4L2_MPEG_VIDEO_H264_START_CODE_NONE, + V4L2_MPEG_VIDEO_H264_START_CODE_ANNEX_B, +}; + #define V4L2_H264_SPS_CONSTRAINT_SET0_FLAG 0x01 #define V4L2_H264_SPS_CONSTRAINT_SET1_FLAG 0x02 #define V4L2_H264_SPS_CONSTRAINT_SET2_FLAG 0x04 @@ -125,6 +137,10 @@ struct v4l2_h264_pred_weight_table { struct v4l2_ctrl_h264_slice_params { /* Size in bytes, including header */ __u32 size; + + /* Offset in bytes to the start of slice in the OUTPUT buffer. */ + __u32 start_byte_offset; + /* Offset in bits to slice_data() from the beginning of this slice. */ __u32 header_bit_size; @@ -186,9 +202,6 @@ struct v4l2_ctrl_h264_decode_params { struct v4l2_h264_dpb_entry dpb[16]; __u16 num_slices; __u16 nal_ref_idc; - __u8 ref_pic_list_p0[32]; - __u8 ref_pic_list_b0[32]; - __u8 ref_pic_list_b1[32]; __s32 top_field_order_cnt; __s32 bottom_field_order_cnt; __u32 flags; /* V4L2_H264_DECODE_PARAM_FLAG_* */ diff --git a/src/config.c b/src/config.c index e396268..8c08148 100644 --- a/src/config.c +++ b/src/config.c @@ -128,7 +128,7 @@ VAStatus RequestQueryConfigProfiles(VADriverContextP context, found = v4l2_find_format(driver_data->video_fd, V4L2_BUF_TYPE_VIDEO_OUTPUT, - V4L2_PIX_FMT_H264_SLICE_RAW); + V4L2_PIX_FMT_H264_SLICE); if (found && index < (V4L2_REQUEST_MAX_CONFIG_ATTRIBUTES - 5)) { profiles[index++] = VAProfileH264Main; profiles[index++] = VAProfileH264High; diff --git a/src/context.c b/src/context.c index 04ba9a6..13662ed 100644 --- a/src/context.c +++ b/src/context.c @@ -104,7 +104,7 @@ VAStatus RequestCreateContext(VADriverContextP context, VAConfigID config_id, case VAProfileH264ConstrainedBaseline: case VAProfileH264MultiviewHigh: case VAProfileH264StereoHigh: - pixelformat = V4L2_PIX_FMT_H264_SLICE_RAW; + pixelformat = V4L2_PIX_FMT_H264_SLICE; break; case VAProfileHEVCMain: From fbde9f6add1844775a870cf447e9a4fabe06a3d6 Mon Sep 17 00:00:00 2001 From: Philipp Zabel Date: Fri, 9 Aug 2019 17:11:25 +0200 Subject: [PATCH 04/13] h264: use v4l2_set_controls to reduce number of issued ioctls Signed-off-by: Philipp Zabel --- src/h264.c | 49 +++++++++++++++++++++++++------------------------ 1 file changed, 25 insertions(+), 24 deletions(-) diff --git a/src/h264.c b/src/h264.c index 25bc8cb..ee6fe33 100644 --- a/src/h264.c +++ b/src/h264.c @@ -435,31 +435,32 @@ int h264_set_controls(struct request_data *driver_data, &surface->params.h264.slice, &surface->params.h264.picture, &slice); - rc = v4l2_set_control(driver_data->video_fd, surface->request_fd, - V4L2_CID_MPEG_VIDEO_H264_DECODE_PARAMS, &decode, - sizeof(decode)); - if (rc < 0) - return VA_STATUS_ERROR_OPERATION_FAILED; - - rc = v4l2_set_control(driver_data->video_fd, surface->request_fd, - V4L2_CID_MPEG_VIDEO_H264_SLICE_PARAMS, &slice, - sizeof(slice)); - if (rc < 0) - return VA_STATUS_ERROR_OPERATION_FAILED; - - rc = v4l2_set_control(driver_data->video_fd, surface->request_fd, - V4L2_CID_MPEG_VIDEO_H264_PPS, &pps, sizeof(pps)); - if (rc < 0) - return VA_STATUS_ERROR_OPERATION_FAILED; - - rc = v4l2_set_control(driver_data->video_fd, surface->request_fd, - V4L2_CID_MPEG_VIDEO_H264_SPS, &sps, sizeof(sps)); - if (rc < 0) - return VA_STATUS_ERROR_OPERATION_FAILED; + struct v4l2_ext_control controls[5] = { + { + .id = V4L2_CID_MPEG_VIDEO_H264_SPS, + .ptr = &sps, + .size = sizeof(sps), + }, { + .id = V4L2_CID_MPEG_VIDEO_H264_PPS, + .ptr = &pps, + .size = sizeof(pps), + }, { + .id = V4L2_CID_MPEG_VIDEO_H264_SCALING_MATRIX, + .ptr = &matrix, + .size = sizeof(matrix), + }, { + .id = V4L2_CID_MPEG_VIDEO_H264_SLICE_PARAMS, + .ptr = &slice, + .size = sizeof(slice), + }, { + .id = V4L2_CID_MPEG_VIDEO_H264_DECODE_PARAMS, + .ptr = &decode, + .size = sizeof(decode), + } + }; - rc = v4l2_set_control(driver_data->video_fd, surface->request_fd, - V4L2_CID_MPEG_VIDEO_H264_SCALING_MATRIX, &matrix, - sizeof(matrix)); + rc = v4l2_set_controls(driver_data->video_fd, surface->request_fd, + controls, 5); if (rc < 0) return VA_STATUS_ERROR_OPERATION_FAILED; From c7385a6bf44d7aac5657ad8df742f090d1f5c764 Mon Sep 17 00:00:00 2001 From: Philipp Zabel Date: Fri, 23 Aug 2019 15:33:17 +0200 Subject: [PATCH 05/13] h264: use v4l2_get_controls to query decode mode and start code Signed-off-by: Philipp Zabel --- src/context.c | 2 ++ src/context.h | 1 + src/h264.c | 40 ++++++++++++++++++++++++++++++++++++++++ src/h264.h | 2 ++ 4 files changed, 45 insertions(+) diff --git a/src/context.c b/src/context.c index 13662ed..c32eacb 100644 --- a/src/context.c +++ b/src/context.c @@ -105,6 +105,8 @@ VAStatus RequestCreateContext(VADriverContextP context, VAConfigID config_id, case VAProfileH264MultiviewHigh: case VAProfileH264StereoHigh: pixelformat = V4L2_PIX_FMT_H264_SLICE; + /* Query decode mode and start code */ + h264_get_controls(driver_data, context_object); break; case VAProfileHEVCMain: diff --git a/src/context.h b/src/context.h index cd0910a..8f4f70f 100644 --- a/src/context.h +++ b/src/context.h @@ -50,6 +50,7 @@ struct object_context { /* H264 only */ struct h264_dpb dpb; + bool h264_start_code; }; VAStatus RequestCreateContext(VADriverContextP context, VAConfigID config_id, diff --git a/src/h264.c b/src/h264.c index ee6fe33..a59d9c1 100644 --- a/src/h264.c +++ b/src/h264.c @@ -405,6 +405,46 @@ static void h264_va_slice_to_v4l2(struct request_data *driver_data, VASlice->chroma_offset_l1); } +int h264_get_controls(struct request_data *driver_data, + struct object_context *context) +{ + struct v4l2_ext_control controls[2] = { + { + .id = V4L2_CID_MPEG_VIDEO_H264_DECODE_MODE, + }, { + .id = V4L2_CID_MPEG_VIDEO_H264_START_CODE, + } + }; + int rc; + + rc = v4l2_get_controls(driver_data->video_fd, -1, controls, 2); + if (rc < 0) + return VA_STATUS_ERROR_OPERATION_FAILED; + + switch (controls[0].value) { + case V4L2_MPEG_VIDEO_H264_DECODE_MODE_SLICE_BASED: + break; + case V4L2_MPEG_VIDEO_H264_DECODE_MODE_FRAME_BASED: + break; + default: + request_log("Unsupported decode mode\n"); + return VA_STATUS_ERROR_OPERATION_FAILED; + } + + switch (controls[1].value) { + case V4L2_MPEG_VIDEO_H264_START_CODE_NONE: + break; + case V4L2_MPEG_VIDEO_H264_START_CODE_ANNEX_B: + context->h264_start_code = true; + break; + default: + request_log("Unsupported start code\n"); + return VA_STATUS_ERROR_OPERATION_FAILED; + } + + return VA_STATUS_SUCCESS; +} + int h264_set_controls(struct request_data *driver_data, struct object_context *context, struct object_surface *surface) diff --git a/src/h264.h b/src/h264.h index 35ef31d..004a416 100644 --- a/src/h264.h +++ b/src/h264.h @@ -51,6 +51,8 @@ struct h264_dpb { unsigned int age; }; +int h264_get_controls(struct request_data *driver_data, + struct object_context *context); int h264_set_controls(struct request_data *data, struct object_context *context, struct object_surface *surface); From b7aadc5a63e42a5a37e20c3ef20bd85033ab7e44 Mon Sep 17 00:00:00 2001 From: Philipp Zabel Date: Fri, 9 Aug 2019 17:59:03 +0200 Subject: [PATCH 06/13] h264: add H.264 Annex B start codes if required If the driver reports that it expects H.264 Annex B start codes, provide them. Signed-off-by: Philipp Zabel --- src/h264.c | 2 ++ src/picture.c | 12 +++++++++++- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/src/h264.c b/src/h264.c index a59d9c1..509c876 100644 --- a/src/h264.c +++ b/src/h264.c @@ -330,6 +330,8 @@ static void h264_va_slice_to_v4l2(struct request_data *driver_data, struct v4l2_ctrl_h264_slice_params *slice) { slice->size = VASlice->slice_data_size; + if (context->h264_start_code) + slice->size += 3; slice->header_bit_size = VASlice->slice_data_bit_offset; slice->first_mb_in_slice = VASlice->first_mb_in_slice; slice->slice_type = VASlice->slice_type; diff --git a/src/picture.c b/src/picture.c index aa86265..819fdbd 100644 --- a/src/picture.c +++ b/src/picture.c @@ -51,6 +51,7 @@ #include "autoconfig.h" static VAStatus codec_store_buffer(struct request_data *driver_data, + struct object_context *context, VAProfile profile, struct object_surface *surface_object, struct object_buffer *buffer_object) @@ -63,6 +64,14 @@ static VAStatus codec_store_buffer(struct request_data *driver_data, * RenderPicture), we can't use a V4L2 buffer directly * and have to copy from a regular buffer. */ + if (context->h264_start_code) { + static const char start_code[3] = { 0x00, 0x00, 0x01 }; + + memcpy(surface_object->source_data + + surface_object->slices_size, + start_code, sizeof(start_code)); + surface_object->slices_size += sizeof(start_code); + } memcpy(surface_object->source_data + surface_object->slices_size, buffer_object->data, @@ -255,7 +264,8 @@ VAStatus RequestRenderPicture(VADriverContextP context, VAContextID context_id, if (buffer_object == NULL) return VA_STATUS_ERROR_INVALID_BUFFER; - rc = codec_store_buffer(driver_data, config_object->profile, + rc = codec_store_buffer(driver_data, context_object, + config_object->profile, surface_object, buffer_object); if (rc != VA_STATUS_SUCCESS) return rc; From 97a013ca9891ee341814ce49a70e8381f133c1cd Mon Sep 17 00:00:00 2001 From: Philipp Zabel Date: Thu, 29 Aug 2019 17:08:37 +0200 Subject: [PATCH 07/13] h264: set pic_num in dpb Signed-off-by: Philipp Zabel --- src/h264.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/h264.c b/src/h264.c index 509c876..bd3b6ab 100644 --- a/src/h264.c +++ b/src/h264.c @@ -197,6 +197,7 @@ static void h264_fill_dpb(struct request_data *data, } dpb->frame_num = entry->pic.frame_idx; + dpb->pic_num = entry->pic.picture_id; dpb->top_field_order_cnt = entry->pic.TopFieldOrderCnt; dpb->bottom_field_order_cnt = entry->pic.BottomFieldOrderCnt; From a33da99cd385e66c1401842c4ea03d878db46915 Mon Sep 17 00:00:00 2001 From: Philipp Zabel Date: Thu, 29 Aug 2019 17:11:08 +0200 Subject: [PATCH 08/13] h264: set frame_num in slice_params Signed-off-by: Philipp Zabel --- src/h264.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/h264.c b/src/h264.c index bd3b6ab..ecbb3d9 100644 --- a/src/h264.c +++ b/src/h264.c @@ -336,6 +336,7 @@ static void h264_va_slice_to_v4l2(struct request_data *driver_data, slice->header_bit_size = VASlice->slice_data_bit_offset; slice->first_mb_in_slice = VASlice->first_mb_in_slice; slice->slice_type = VASlice->slice_type; + slice->frame_num = VAPicture->frame_num; slice->cabac_init_idc = VASlice->cabac_init_idc; slice->slice_qp_delta = VASlice->slice_qp_delta; slice->disable_deblocking_filter_idc = From a42274220a5d76bee6dec951426329090134b671 Mon Sep 17 00:00:00 2001 From: Philipp Zabel Date: Tue, 3 Sep 2019 17:35:16 +0200 Subject: [PATCH 09/13] h264: extract nal_ref_idc and nal_unit_type Signed-off-by: Philipp Zabel --- src/h264.c | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/h264.c b/src/h264.c index ecbb3d9..a0cf687 100644 --- a/src/h264.c +++ b/src/h264.c @@ -219,9 +219,23 @@ static void h264_va_picture_to_v4l2(struct request_data *driver_data, struct v4l2_ctrl_h264_pps *pps, struct v4l2_ctrl_h264_sps *sps) { + unsigned char *b; + unsigned char nal_ref_idc; + unsigned char nal_unit_type; + + /* Extract missing nal_ref_idc and nal_unit_type */ + b = surface->source_data; + if (context->h264_start_code) + b += 3; + nal_ref_idc = (b[0] >> 5) & 0x3; + nal_unit_type = b[0] & 0x1f; + h264_fill_dpb(driver_data, context, decode); decode->num_slices = surface->slices_count; + decode->nal_ref_idc = nal_ref_idc; + if (nal_unit_type == 5) + decode->flags = V4L2_H264_DECODE_PARAM_FLAG_IDR_PIC; decode->top_field_order_cnt = VAPicture->CurrPic.TopFieldOrderCnt; decode->bottom_field_order_cnt = VAPicture->CurrPic.BottomFieldOrderCnt; From 6d59904c3c8bf80ee271fc042a146970c0a7a1d9 Mon Sep 17 00:00:00 2001 From: Philipp Zabel Date: Thu, 5 Sep 2019 10:34:01 +0200 Subject: [PATCH 10/13] h264: set max_num_ref_frames in SPS Signed-off-by: Philipp Zabel --- src/h264.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/h264.c b/src/h264.c index a0cf687..c09fa72 100644 --- a/src/h264.c +++ b/src/h264.c @@ -270,6 +270,7 @@ static void h264_va_picture_to_v4l2(struct request_data *driver_data, if (VAPicture->pic_fields.bits.redundant_pic_cnt_present_flag) pps->flags |= V4L2_H264_PPS_FLAG_REDUNDANT_PIC_CNT_PRESENT; + sps->max_num_ref_frames = VAPicture->num_ref_frames; sps->chroma_format_idc = VAPicture->seq_fields.bits.chroma_format_idc; sps->bit_depth_luma_minus8 = VAPicture->bit_depth_luma_minus8; sps->bit_depth_chroma_minus8 = VAPicture->bit_depth_chroma_minus8; From a74198aa571ff4ff1172b786e207519c0afa3b55 Mon Sep 17 00:00:00 2001 From: Philipp Zabel Date: Thu, 5 Sep 2019 10:33:44 +0200 Subject: [PATCH 11/13] h264: set profile_idc in SPS Signed-off-by: Philipp Zabel --- src/h264.c | 21 +++++++++++++++++++++ src/h264.h | 1 + src/picture.c | 3 ++- 3 files changed, 24 insertions(+), 1 deletion(-) diff --git a/src/h264.c b/src/h264.c index c09fa72..1209c8f 100644 --- a/src/h264.c +++ b/src/h264.c @@ -464,8 +464,27 @@ int h264_get_controls(struct request_data *driver_data, return VA_STATUS_SUCCESS; } +static inline __u8 h264_profile_to_idc(VAProfile profile) +{ + switch (profile) { + case VAProfileH264Main: + return 77; + case VAProfileH264High: + return 100; + case VAProfileH264ConstrainedBaseline: + return 66; + case VAProfileH264MultiviewHigh: + return 118; + case VAProfileH264StereoHigh: + return 128; + default: + return 0; + } +} + int h264_set_controls(struct request_data *driver_data, struct object_context *context, + VAProfile profile, struct object_surface *surface) { struct v4l2_ctrl_h264_scaling_matrix matrix = { 0 }; @@ -494,6 +513,8 @@ int h264_set_controls(struct request_data *driver_data, &surface->params.h264.slice, &surface->params.h264.picture, &slice); + sps.profile_idc = h264_profile_to_idc(profile); + struct v4l2_ext_control controls[5] = { { .id = V4L2_CID_MPEG_VIDEO_H264_SPS, diff --git a/src/h264.h b/src/h264.h index 004a416..da0b87f 100644 --- a/src/h264.h +++ b/src/h264.h @@ -55,6 +55,7 @@ int h264_get_controls(struct request_data *driver_data, struct object_context *context); int h264_set_controls(struct request_data *data, struct object_context *context, + VAProfile profile, struct object_surface *surface); #endif diff --git a/src/picture.c b/src/picture.c index 819fdbd..a65dd7f 100644 --- a/src/picture.c +++ b/src/picture.c @@ -193,7 +193,8 @@ static VAStatus codec_set_controls(struct request_data *driver_data, case VAProfileH264ConstrainedBaseline: case VAProfileH264MultiviewHigh: case VAProfileH264StereoHigh: - rc = h264_set_controls(driver_data, context, surface_object); + rc = h264_set_controls(driver_data, context, profile, + surface_object); if (rc < 0) return VA_STATUS_ERROR_OPERATION_FAILED; break; From acbdfb1caa20f21a04a8ea488c9e9c81ac99137b Mon Sep 17 00:00:00 2001 From: "Andy Chun @noneucat" Date: Thu, 12 Aug 2021 12:06:50 +0000 Subject: [PATCH 12/13] Update backend for kernel 5.14 (POC) This commit reworks the codebase so that it builds and functions in Linux 5.14. This comes with the following caveat: - V4L2_PIX_FMT_H264_SLICE is currently hardcoded as the pixel format for all output streams in surface.c . You will need to do some significant refactoring to get it working with the rest of the codecs. --- src/config.c | 1 - src/config.h | 1 + src/context.c | 35 ------------------ src/h264.c | 95 +++++++++++++++++++++++++++++-------------------- src/image.c | 4 +++ src/surface.c | 28 ++++++++++++++- src/surface.h | 2 +- src/tiled_yuv.S | 2 +- 8 files changed, 91 insertions(+), 77 deletions(-) diff --git a/src/config.c b/src/config.c index 8c08148..60a63ff 100644 --- a/src/config.c +++ b/src/config.c @@ -35,7 +35,6 @@ #include #include -#include #include #include "utils.h" diff --git a/src/config.h b/src/config.h index b7b12e7..1377e49 100644 --- a/src/config.h +++ b/src/config.h @@ -43,6 +43,7 @@ struct object_config { VAEntrypoint entrypoint; VAConfigAttrib attributes[V4L2_REQUEST_MAX_CONFIG_ATTRIBUTES]; int attributes_count; + unsigned int pixelformat; }; VAStatus RequestCreateConfig(VADriverContextP context, VAProfile profile, diff --git a/src/context.c b/src/context.c index c32eacb..d07cea6 100644 --- a/src/context.c +++ b/src/context.c @@ -40,7 +40,6 @@ #include #include -#include #include #include "utils.h" @@ -65,7 +64,6 @@ VAStatus RequestCreateContext(VADriverContextP context, VAConfigID config_id, VAContextID id; VAStatus status; unsigned int output_type, capture_type; - unsigned int pixelformat; unsigned int index_base; unsigned int index; unsigned int i; @@ -92,39 +90,6 @@ VAStatus RequestCreateContext(VADriverContextP context, VAConfigID config_id, } memset(&context_object->dpb, 0, sizeof(context_object->dpb)); - switch (config_object->profile) { - - case VAProfileMPEG2Simple: - case VAProfileMPEG2Main: - pixelformat = V4L2_PIX_FMT_MPEG2_SLICE; - break; - - case VAProfileH264Main: - case VAProfileH264High: - case VAProfileH264ConstrainedBaseline: - case VAProfileH264MultiviewHigh: - case VAProfileH264StereoHigh: - pixelformat = V4L2_PIX_FMT_H264_SLICE; - /* Query decode mode and start code */ - h264_get_controls(driver_data, context_object); - break; - - case VAProfileHEVCMain: - pixelformat = V4L2_PIX_FMT_HEVC_SLICE; - break; - - default: - status = VA_STATUS_ERROR_UNSUPPORTED_PROFILE; - goto error; - } - - rc = v4l2_set_format(driver_data->video_fd, output_type, pixelformat, - picture_width, picture_height); - if (rc < 0) { - status = VA_STATUS_ERROR_OPERATION_FAILED; - goto error; - } - rc = v4l2_create_buffers(driver_data->video_fd, output_type, surfaces_count, &index_base); if (rc < 0) { diff --git a/src/h264.c b/src/h264.c index 1209c8f..f3f08b0 100644 --- a/src/h264.c +++ b/src/h264.c @@ -33,7 +33,6 @@ #include #include -#include #include "request.h" #include "surface.h" @@ -95,7 +94,8 @@ static struct h264_dpb_entry *dpb_find_entry(struct object_context *context) } static struct h264_dpb_entry *dpb_lookup(struct object_context *context, - VAPictureH264 *pic, unsigned int *idx) + VAPictureH264 *pic, unsigned int *idx, + unsigned char *fields) { unsigned int i; @@ -109,6 +109,16 @@ static struct h264_dpb_entry *dpb_lookup(struct object_context *context, if (idx) *idx = i; + if (fields) { + //if (entry->pic.TopFieldOrderCnt < entry->pic.BottomFieldOrderCnt) { + // *fields = V4L2_H264_TOP_FIELD_REF; + //} else if (entry->pic.TopFieldOrderCnt > entry->pic.BottomFieldOrderCnt) { + // *fields = V4L2_H264_BOTTOM_FIELD_REF; + //} else { + *fields = V4L2_H264_FRAME_REF; + //} + } + return entry; } } @@ -130,7 +140,7 @@ static void dpb_insert(struct object_context *context, VAPictureH264 *pic, if (is_picture_null(pic)) return; - if (dpb_lookup(context, pic, NULL)) + if (dpb_lookup(context, pic, NULL, NULL)) return; if (!entry) @@ -165,7 +175,7 @@ static void dpb_update(struct object_context *context, if (is_picture_null(pic)) continue; - entry = dpb_lookup(context, pic, NULL); + entry = dpb_lookup(context, pic, NULL, NULL); if (entry) { entry->age = context->dpb.age; entry->used = true; @@ -232,7 +242,7 @@ static void h264_va_picture_to_v4l2(struct request_data *driver_data, h264_fill_dpb(driver_data, context, decode); - decode->num_slices = surface->slices_count; + //decode->num_slices = surface->slices_count; decode->nal_ref_idc = nal_ref_idc; if (nal_unit_type == 5) decode->flags = V4L2_H264_DECODE_PARAM_FLAG_IDR_PIC; @@ -343,15 +353,14 @@ static void h264_va_slice_to_v4l2(struct request_data *driver_data, struct object_context *context, VASliceParameterBufferH264 *VASlice, VAPictureParameterBufferH264 *VAPicture, - struct v4l2_ctrl_h264_slice_params *slice) + struct v4l2_ctrl_h264_slice_params *slice, + struct v4l2_ctrl_h264_pred_weights *weights) { - slice->size = VASlice->slice_data_size; - if (context->h264_start_code) - slice->size += 3; slice->header_bit_size = VASlice->slice_data_bit_offset; + //if (context->h264_start_code) + // slice->header_bit_size += 3 * 8; slice->first_mb_in_slice = VASlice->first_mb_in_slice; slice->slice_type = VASlice->slice_type; - slice->frame_num = VAPicture->frame_num; slice->cabac_init_idc = VASlice->cabac_init_idc; slice->slice_qp_delta = VASlice->slice_qp_delta; slice->disable_deblocking_filter_idc = @@ -370,12 +379,14 @@ static void h264_va_slice_to_v4l2(struct request_data *driver_data, VAPictureH264 *pic = &VASlice->RefPicList0[i]; struct h264_dpb_entry *entry; unsigned int idx; + unsigned char fields; - entry = dpb_lookup(context, pic, &idx); + entry = dpb_lookup(context, pic, &idx, &fields); if (!entry) continue; - slice->ref_pic_list0[i] = idx; + slice->ref_pic_list0[i].index = idx; + slice->ref_pic_list0[i].fields = fields; } } @@ -389,26 +400,28 @@ static void h264_va_slice_to_v4l2(struct request_data *driver_data, VAPictureH264 *pic = &VASlice->RefPicList1[i]; struct h264_dpb_entry *entry; unsigned int idx; + unsigned char fields; - entry = dpb_lookup(context, pic, &idx); + entry = dpb_lookup(context, pic, &idx, &fields); if (!entry) continue; - slice->ref_pic_list1[i] = idx; + slice->ref_pic_list1[i].index = idx; + slice->ref_pic_list0[i].fields = fields; } } if (VASlice->direct_spatial_mv_pred_flag) slice->flags |= V4L2_H264_SLICE_FLAG_DIRECT_SPATIAL_MV_PRED; - slice->pred_weight_table.chroma_log2_weight_denom = + weights->chroma_log2_weight_denom = VASlice->chroma_log2_weight_denom; - slice->pred_weight_table.luma_log2_weight_denom = + weights->luma_log2_weight_denom = VASlice->luma_log2_weight_denom; if (((VASlice->slice_type % 5) == H264_SLICE_P) || ((VASlice->slice_type % 5) == H264_SLICE_B)) - h264_copy_pred_table(&slice->pred_weight_table.weight_factors[0], + h264_copy_pred_table(&weights->weight_factors[0], slice->num_ref_idx_l0_active_minus1 + 1, VASlice->luma_weight_l0, VASlice->luma_offset_l0, @@ -416,7 +429,7 @@ static void h264_va_slice_to_v4l2(struct request_data *driver_data, VASlice->chroma_offset_l0); if ((VASlice->slice_type % 5) == H264_SLICE_B) - h264_copy_pred_table(&slice->pred_weight_table.weight_factors[1], + h264_copy_pred_table(&weights->weight_factors[1], slice->num_ref_idx_l1_active_minus1 + 1, VASlice->luma_weight_l1, VASlice->luma_offset_l1, @@ -429,9 +442,9 @@ int h264_get_controls(struct request_data *driver_data, { struct v4l2_ext_control controls[2] = { { - .id = V4L2_CID_MPEG_VIDEO_H264_DECODE_MODE, + .id = V4L2_CID_STATELESS_H264_DECODE_MODE, }, { - .id = V4L2_CID_MPEG_VIDEO_H264_START_CODE, + .id = V4L2_CID_STATELESS_H264_START_CODE, } }; int rc; @@ -441,9 +454,9 @@ int h264_get_controls(struct request_data *driver_data, return VA_STATUS_ERROR_OPERATION_FAILED; switch (controls[0].value) { - case V4L2_MPEG_VIDEO_H264_DECODE_MODE_SLICE_BASED: + case V4L2_STATELESS_H264_DECODE_MODE_SLICE_BASED: break; - case V4L2_MPEG_VIDEO_H264_DECODE_MODE_FRAME_BASED: + case V4L2_STATELESS_H264_DECODE_MODE_FRAME_BASED: break; default: request_log("Unsupported decode mode\n"); @@ -451,9 +464,10 @@ int h264_get_controls(struct request_data *driver_data, } switch (controls[1].value) { - case V4L2_MPEG_VIDEO_H264_START_CODE_NONE: + case V4L2_STATELESS_H264_START_CODE_NONE: + context->h264_start_code = false; break; - case V4L2_MPEG_VIDEO_H264_START_CODE_ANNEX_B: + case V4L2_STATELESS_H264_START_CODE_ANNEX_B: context->h264_start_code = true; break; default: @@ -490,13 +504,14 @@ int h264_set_controls(struct request_data *driver_data, struct v4l2_ctrl_h264_scaling_matrix matrix = { 0 }; struct v4l2_ctrl_h264_decode_params decode = { 0 }; struct v4l2_ctrl_h264_slice_params slice = { 0 }; + struct v4l2_ctrl_h264_pred_weights weights = { 0 }; struct v4l2_ctrl_h264_pps pps = { 0 }; struct v4l2_ctrl_h264_sps sps = { 0 }; struct h264_dpb_entry *output; int rc; output = dpb_lookup(context, &surface->params.h264.picture.CurrPic, - NULL); + NULL, NULL); if (!output) output = dpb_find_entry(context); @@ -511,36 +526,40 @@ int h264_set_controls(struct request_data *driver_data, &surface->params.h264.matrix, &matrix); h264_va_slice_to_v4l2(driver_data, context, &surface->params.h264.slice, - &surface->params.h264.picture, &slice); + &surface->params.h264.picture, &slice, &weights); sps.profile_idc = h264_profile_to_idc(profile); - struct v4l2_ext_control controls[5] = { + struct v4l2_ext_control controls[6] = { { - .id = V4L2_CID_MPEG_VIDEO_H264_SPS, - .ptr = &sps, + .id = V4L2_CID_STATELESS_H264_SPS, + .p_h264_sps = &sps, .size = sizeof(sps), }, { - .id = V4L2_CID_MPEG_VIDEO_H264_PPS, - .ptr = &pps, + .id = V4L2_CID_STATELESS_H264_PPS, + .p_h264_pps = &pps, .size = sizeof(pps), }, { - .id = V4L2_CID_MPEG_VIDEO_H264_SCALING_MATRIX, - .ptr = &matrix, + .id = V4L2_CID_STATELESS_H264_SCALING_MATRIX, + .p_h264_scaling_matrix = &matrix, .size = sizeof(matrix), }, { - .id = V4L2_CID_MPEG_VIDEO_H264_SLICE_PARAMS, - .ptr = &slice, + .id = V4L2_CID_STATELESS_H264_SLICE_PARAMS, + .p_h264_slice_params = &slice, .size = sizeof(slice), }, { - .id = V4L2_CID_MPEG_VIDEO_H264_DECODE_PARAMS, - .ptr = &decode, + .id = V4L2_CID_STATELESS_H264_DECODE_PARAMS, + .p_h264_decode_params = &decode, .size = sizeof(decode), + }, { + .id = V4L2_CID_STATELESS_H264_PRED_WEIGHTS, + .ptr = &weights, + .size = sizeof(weights), } }; rc = v4l2_set_controls(driver_data->video_fd, surface->request_fd, - controls, 5); + controls, 6); if (rc < 0) return VA_STATUS_ERROR_OPERATION_FAILED; diff --git a/src/image.c b/src/image.c index fa8b0ea..05ed455 100644 --- a/src/image.c +++ b/src/image.c @@ -155,6 +155,7 @@ static VAStatus copy_surface_to_image (struct request_data *driver_data, return VA_STATUS_ERROR_INVALID_BUFFER; for (i = 0; i < surface_object->destination_planes_count; i++) { +#ifdef __arm__ if (!video_format_is_linear(driver_data->video_format)) tiled_to_planar(surface_object->destination_data[i], buffer_object->data + image->offsets[i], @@ -162,10 +163,13 @@ static VAStatus copy_surface_to_image (struct request_data *driver_data, i == 0 ? image->height : image->height / 2); else { +#endif memcpy(buffer_object->data + image->offsets[i], surface_object->destination_data[i], surface_object->destination_sizes[i]); +#ifdef __arm__ } +#endif } return VA_STATUS_SUCCESS; diff --git a/src/surface.c b/src/surface.c index a6abb9b..478f0cc 100644 --- a/src/surface.c +++ b/src/surface.c @@ -46,6 +46,8 @@ #include "v4l2.h" #include "video.h" +bool SET_FORMAT_OF_OUTPUT_ONCE = false; + VAStatus RequestCreateSurfaces2(VADriverContextP context, unsigned int format, unsigned int width, unsigned int height, VASurfaceID *surfaces_ids, @@ -68,10 +70,34 @@ VAStatus RequestCreateSurfaces2(VADriverContextP context, unsigned int format, bool found; int rc; + //////////// HACK: this portion of the code should get cleaned up. + + // v4l2_set_format needs to be called BEFORE we create any buffers + // + // we originally did this for the output stream in context.c, but + // RequestCreateSurfaces2 gets called multiple times before RequestCreateContext + // to allocate & map buffers. this doesn't seem to work in recent kernel versions. + // + // we declare SET_FORMAT_OF_OUTPUT_ONCE to ensure v4l2_set_format only gets called once + // (in the first RequestCreateSurfaces2 call BEFORE any buffers are created later on) + unsigned int pixelformat = V4L2_PIX_FMT_H264_SLICE; + unsigned int output_type = v4l2_type_video_output(false); + + if (!SET_FORMAT_OF_OUTPUT_ONCE) { + rc = v4l2_set_format(driver_data->video_fd, output_type, pixelformat, + width, height); + if (rc < 0) { + return VA_STATUS_ERROR_OPERATION_FAILED; + } + + SET_FORMAT_OF_OUTPUT_ONCE = true; + } + + /////////// ENDHACK + if (format != VA_RT_FORMAT_YUV420) return VA_STATUS_ERROR_UNSUPPORTED_RT_FORMAT; - if (!driver_data->video_format) { found = v4l2_find_format(driver_data->video_fd, V4L2_BUF_TYPE_VIDEO_CAPTURE, diff --git a/src/surface.h b/src/surface.h index 41007f8..a9be915 100644 --- a/src/surface.h +++ b/src/surface.h @@ -40,7 +40,7 @@ struct object_surface { struct object_base base; - VAStatus status; + VASurfaceStatus status; int width; int height; diff --git a/src/tiled_yuv.S b/src/tiled_yuv.S index c6fa833..4bce33f 100644 --- a/src/tiled_yuv.S +++ b/src/tiled_yuv.S @@ -27,7 +27,7 @@ .section .note.GNU-stack,"",%progbits /* mark stack as non-executable */ #endif -#ifndef __aarch64__ +#ifdef __arm__ .text .syntax unified From 1c5f2cad21dff3b56d35355082867c24e4f191c6 Mon Sep 17 00:00:00 2001 From: "Andy Chun @noneucat" Date: Thu, 12 Aug 2021 12:22:55 +0000 Subject: [PATCH 13/13] Don't advertise broken profiles --- src/config.c | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/src/config.c b/src/config.c index 60a63ff..2d5e202 100644 --- a/src/config.c +++ b/src/config.c @@ -53,18 +53,17 @@ VAStatus RequestCreateConfig(VADriverContextP context, VAProfile profile, int i, index; switch (profile) { - case VAProfileMPEG2Simple: - case VAProfileMPEG2Main: + case VAProfileH264Main: case VAProfileH264High: case VAProfileH264ConstrainedBaseline: case VAProfileH264MultiviewHigh: case VAProfileH264StereoHigh: - case VAProfileHEVCMain: - if (entrypoint != VAEntrypointVLD) - return VA_STATUS_ERROR_UNSUPPORTED_ENTRYPOINT; + // FIXME break; - + case VAProfileMPEG2Simple: + case VAProfileMPEG2Main: + case VAProfileHEVCMain: default: return VA_STATUS_ERROR_UNSUPPORTED_PROFILE; }