avcodec/cbs_h2645: Avoid function pointer casts, fix UB

The SEI message read/write functions are called
via function pointers where the SEI message-specific
context is passed as void*. But the actual function
definitions use a pointer to their proper context
in place of void*, making the calls undefined behaviour.
Clang UBSan 17 warns about this.

This commit fixes this by adding wrapper functions
(created via macros) that have the right type that
call the actual functions. This reduced the number of failing
FATE tests with UBSan from 164 to 85 here.

Reviewed-by: Mark Thompson <sw@jkqxz.net>
Signed-off-by: Andreas Rheinhardt <andreas.rheinhardt@outlook.com>

libavcodec/cbs_h2645.c

@@ -235,6 +235,16 @@ static int cbs_h265_payload_extension_present(GetBitContext *gbc, uint32_t paylo
 #define FUNC_H266(name) FUNC_NAME1(READWRITE, h266, name)
 #define FUNC_SEI(name)  FUNC_NAME1(READWRITE, sei,  name)
 
+#define SEI_FUNC(name, args) \
+static int FUNC(name) args;  \
+static int FUNC(name ## _internal)(CodedBitstreamContext *ctx, \
+                                   RWContext *rw, void *cur,   \
+                                   SEIMessageState *state)     \
+{ \
+    return FUNC(name)(ctx, rw, cur, state); \
+} \
+static int FUNC(name) args
+
 #define SUBSCRIPTS(subs, ...) (subs > 0 ? ((int[subs + 1]){ subs, __VA_ARGS__ }) : NULL)
 
 #define u(width, name, range_min, range_max) \
@@ -2070,6 +2080,11 @@ const CodedBitstreamType ff_cbs_type_h266 = {
     .close             = &cbs_h266_close,
 };
 
+// Macro for the read/write pair.
+#define SEI_MESSAGE_RW(codec, name) \
+    .read  = cbs_ ## codec ## _read_  ## name ## _internal, \
+    .write = cbs_ ## codec ## _write_ ## name ## _internal
+
 static const SEIMessageTypeDescriptor cbs_sei_common_types[] = {
     {
         SEI_TYPE_FILLER_PAYLOAD,

libavcodec/cbs_h264_syntax_template.c

@@ -510,9 +510,9 @@ static int FUNC(pps)(CodedBitstreamContext *ctx, RWContext *rw,
     return 0;
 }
 
-static int FUNC(sei_buffering_period)(CodedBitstreamContext *ctx, RWContext *rw,
-                                      H264RawSEIBufferingPeriod *current,
-                                      SEIMessageState *sei)
+SEI_FUNC(sei_buffering_period, (CodedBitstreamContext *ctx, RWContext *rw,
+                                H264RawSEIBufferingPeriod *current,
+                                SEIMessageState *sei))
 {
     CodedBitstreamH264Context *h264 = ctx->priv_data;
     const H264RawSPS *sps;
@@ -604,9 +604,8 @@ static int FUNC(sei_pic_timestamp)(CodedBitstreamContext *ctx, RWContext *rw,
     return 0;
 }
 
-static int FUNC(sei_pic_timing)(CodedBitstreamContext *ctx, RWContext *rw,
-                                H264RawSEIPicTiming *current,
-                                SEIMessageState *sei)
+SEI_FUNC(sei_pic_timing, (CodedBitstreamContext *ctx, RWContext *rw,
+                          H264RawSEIPicTiming *current, SEIMessageState *sei))
 {
     CodedBitstreamH264Context *h264 = ctx->priv_data;
     const H264RawSPS *sps;
@@ -676,9 +675,9 @@ static int FUNC(sei_pic_timing)(CodedBitstreamContext *ctx, RWContext *rw,
     return 0;
 }
 
-static int FUNC(sei_pan_scan_rect)(CodedBitstreamContext *ctx, RWContext *rw,
-                                   H264RawSEIPanScanRect *current,
-                                   SEIMessageState *sei)
+SEI_FUNC(sei_pan_scan_rect, (CodedBitstreamContext *ctx, RWContext *rw,
+                             H264RawSEIPanScanRect *current,
+                             SEIMessageState *sei))
 {
     int err, i;
 
@@ -703,9 +702,9 @@ static int FUNC(sei_pan_scan_rect)(CodedBitstreamContext *ctx, RWContext *rw,
     return 0;
 }
 
-static int FUNC(sei_recovery_point)(CodedBitstreamContext *ctx, RWContext *rw,
-                                    H264RawSEIRecoveryPoint *current,
-                                    SEIMessageState *sei)
+SEI_FUNC(sei_recovery_point, (CodedBitstreamContext *ctx, RWContext *rw,
+                              H264RawSEIRecoveryPoint *current,
+                              SEIMessageState *sei))
 {
     int err;
 
@@ -719,9 +718,9 @@ static int FUNC(sei_recovery_point)(CodedBitstreamContext *ctx, RWContext *rw,
     return 0;
 }
 
-static int FUNC(film_grain_characteristics)(CodedBitstreamContext *ctx, RWContext *rw,
-                                            H264RawFilmGrainCharacteristics *current,
-                                            SEIMessageState *state)
+SEI_FUNC(film_grain_characteristics, (CodedBitstreamContext *ctx, RWContext *rw,
+                                      H264RawFilmGrainCharacteristics *current,
+                                      SEIMessageState *state))
 {
     CodedBitstreamH264Context *h264 = ctx->priv_data;
     const H264RawSPS *sps;
@@ -802,9 +801,9 @@ static int FUNC(film_grain_characteristics)(CodedBitstreamContext *ctx, RWContex
     return 0;
 }
 
-static int FUNC(sei_display_orientation)(CodedBitstreamContext *ctx, RWContext *rw,
-                                         H264RawSEIDisplayOrientation *current,
-                                         SEIMessageState *sei)
+SEI_FUNC(sei_display_orientation, (CodedBitstreamContext *ctx, RWContext *rw,
+                                   H264RawSEIDisplayOrientation *current,
+                                   SEIMessageState *sei))
 {
     int err;
 

libavcodec/cbs_h265_syntax_template.c

@@ -1618,9 +1618,9 @@ static int FUNC(slice_segment_header)(CodedBitstreamContext *ctx, RWContext *rw,
     return 0;
 }
 
-static int FUNC(sei_buffering_period)
-    (CodedBitstreamContext *ctx, RWContext *rw,
-     H265RawSEIBufferingPeriod *current, SEIMessageState *sei)
+SEI_FUNC(sei_buffering_period, (CodedBitstreamContext *ctx, RWContext *rw,
+                                H265RawSEIBufferingPeriod *current,
+                                SEIMessageState *sei))
 {
     CodedBitstreamH265Context *h265 = ctx->priv_data;
     const H265RawSPS *sps;
@@ -1728,9 +1728,8 @@ static int FUNC(sei_buffering_period)
     return 0;
 }
 
-static int FUNC(sei_pic_timing)
-    (CodedBitstreamContext *ctx, RWContext *rw,
-     H265RawSEIPicTiming *current, SEIMessageState *sei)
+SEI_FUNC(sei_pic_timing, (CodedBitstreamContext *ctx, RWContext *rw,
+                          H265RawSEIPicTiming *current, SEIMessageState *sei))
 {
     CodedBitstreamH265Context *h265 = ctx->priv_data;
     const H265RawSPS *sps;
@@ -1804,9 +1803,9 @@ static int FUNC(sei_pic_timing)
     return 0;
 }
 
-static int FUNC(sei_pan_scan_rect)
-    (CodedBitstreamContext *ctx, RWContext *rw,
-     H265RawSEIPanScanRect *current, SEIMessageState *sei)
+SEI_FUNC(sei_pan_scan_rect, (CodedBitstreamContext *ctx, RWContext *rw,
+                             H265RawSEIPanScanRect *current,
+                             SEIMessageState *sei))
 {
     int err, i;
 
@@ -1831,9 +1830,9 @@ static int FUNC(sei_pan_scan_rect)
     return 0;
 }
 
-static int FUNC(sei_recovery_point)
-    (CodedBitstreamContext *ctx, RWContext *rw,
-     H265RawSEIRecoveryPoint *current, SEIMessageState *sei)
+SEI_FUNC(sei_recovery_point, (CodedBitstreamContext *ctx, RWContext *rw,
+                              H265RawSEIRecoveryPoint *current,
+                              SEIMessageState *sei))
 {
     int err;
 
@@ -1847,9 +1846,9 @@ static int FUNC(sei_recovery_point)
     return 0;
 }
 
-static int FUNC(film_grain_characteristics)(CodedBitstreamContext *ctx, RWContext *rw,
-                                            H265RawFilmGrainCharacteristics *current,
-                                            SEIMessageState *state)
+SEI_FUNC(film_grain_characteristics, (CodedBitstreamContext *ctx, RWContext *rw,
+                                      H265RawFilmGrainCharacteristics *current,
+                                      SEIMessageState *state))
 {
     CodedBitstreamH265Context *h265 = ctx->priv_data;
     const H265RawSPS *sps = h265->active_sps;
@@ -1912,9 +1911,9 @@ static int FUNC(film_grain_characteristics)(CodedBitstreamContext *ctx, RWContex
     return 0;
 }
 
-static int FUNC(sei_display_orientation)
-    (CodedBitstreamContext *ctx, RWContext *rw,
-     H265RawSEIDisplayOrientation *current, SEIMessageState *sei)
+SEI_FUNC(sei_display_orientation, (CodedBitstreamContext *ctx, RWContext *rw,
+                                   H265RawSEIDisplayOrientation *current,
+                                   SEIMessageState *sei))
 {
     int err;
 
@@ -1931,9 +1930,9 @@ static int FUNC(sei_display_orientation)
     return 0;
 }
 
-static int FUNC(sei_active_parameter_sets)
-    (CodedBitstreamContext *ctx, RWContext *rw,
-     H265RawSEIActiveParameterSets *current, SEIMessageState *sei)
+SEI_FUNC(sei_active_parameter_sets, (CodedBitstreamContext *ctx, RWContext *rw,
+                                     H265RawSEIActiveParameterSets *current,
+                                     SEIMessageState *sei))
 {
     CodedBitstreamH265Context *h265 = ctx->priv_data;
     const H265RawVPS *vps;
@@ -1968,9 +1967,9 @@ static int FUNC(sei_active_parameter_sets)
     return 0;
 }
 
-static int FUNC(sei_decoded_picture_hash)
-    (CodedBitstreamContext *ctx, RWContext *rw,
-     H265RawSEIDecodedPictureHash *current, SEIMessageState *sei)
+SEI_FUNC(sei_decoded_picture_hash, (CodedBitstreamContext *ctx, RWContext *rw,
+                                    H265RawSEIDecodedPictureHash *current,
+                                    SEIMessageState *sei))
 {
     CodedBitstreamH265Context *h265 = ctx->priv_data;
     const H265RawSPS *sps = h265->active_sps;
@@ -2000,9 +1999,8 @@ static int FUNC(sei_decoded_picture_hash)
     return 0;
 }
 
-static int FUNC(sei_time_code)
-    (CodedBitstreamContext *ctx, RWContext *rw,
-     H265RawSEITimeCode *current, SEIMessageState *sei)
+SEI_FUNC(sei_time_code, (CodedBitstreamContext *ctx, RWContext *rw,
+                         H265RawSEITimeCode *current, SEIMessageState *sei))
 {
     int err, i;
 
@@ -2051,9 +2049,9 @@ static int FUNC(sei_time_code)
     return 0;
 }
 
-static int FUNC(sei_alpha_channel_info)
-    (CodedBitstreamContext *ctx, RWContext *rw,
-     H265RawSEIAlphaChannelInfo *current, SEIMessageState *sei)
+SEI_FUNC(sei_alpha_channel_info, (CodedBitstreamContext *ctx, RWContext *rw,
+                                  H265RawSEIAlphaChannelInfo *current,
+                                  SEIMessageState *sei))
 {
     int err, length;
 

libavcodec/cbs_h266_syntax_template.c

@@ -3427,10 +3427,10 @@ static int FUNC(slice_header) (CodedBitstreamContext *ctx, RWContext *rw,
     return 0;
 }
 
-static int FUNC(sei_decoded_picture_hash) (CodedBitstreamContext *ctx,
-                                           RWContext *rw,
-                                           H266RawSEIDecodedPictureHash *
-                                           current, SEIMessageState *unused)
+SEI_FUNC(sei_decoded_picture_hash, (CodedBitstreamContext *ctx,
+                                    RWContext *rw,
+                                    H266RawSEIDecodedPictureHash *current,
+                                    SEIMessageState *unused))
 {
     int err, c_idx, i;
 

libavcodec/cbs_sei.h

@@ -126,13 +126,6 @@ typedef struct SEIMessageTypeDescriptor {
     SEIMessageWriteFunction write;
 } SEIMessageTypeDescriptor;
 
-// Macro for the read/write pair.  The clumsy cast is needed because the
-// current pointer is typed in all of the read/write functions but has to
-// be void here to fit all cases.
-#define SEI_MESSAGE_RW(codec, name) \
-    .read  = (SEIMessageReadFunction) cbs_ ## codec ## _read_  ## name, \
-    .write = (SEIMessageWriteFunction)cbs_ ## codec ## _write_ ## name
-
 // End-of-list sentinel element.
 #define SEI_MESSAGE_TYPE_END { .type = -1 }
 

libavcodec/cbs_sei_syntax_template.c

@@ -16,9 +16,9 @@
  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
  */
 
-static int FUNC(filler_payload)
-    (CodedBitstreamContext *ctx, RWContext *rw,
-     SEIRawFillerPayload *current, SEIMessageState *state)
+SEI_FUNC(filler_payload, (CodedBitstreamContext *ctx, RWContext *rw,
+                          SEIRawFillerPayload *current,
+                          SEIMessageState *state))
 {
     int err, i;
 
@@ -34,9 +34,9 @@ static int FUNC(filler_payload)
     return 0;
 }
 
-static int FUNC(user_data_registered)
-    (CodedBitstreamContext *ctx, RWContext *rw,
-     SEIRawUserDataRegistered *current, SEIMessageState *state)
+SEI_FUNC(user_data_registered, (CodedBitstreamContext *ctx, RWContext *rw,
+                                SEIRawUserDataRegistered *current,
+                                SEIMessageState *state))
 {
     int err, i, j;
 
@@ -66,9 +66,9 @@ static int FUNC(user_data_registered)
     return 0;
 }
 
-static int FUNC(user_data_unregistered)
-    (CodedBitstreamContext *ctx, RWContext *rw,
-     SEIRawUserDataUnregistered *current, SEIMessageState *state)
+SEI_FUNC(user_data_unregistered, (CodedBitstreamContext *ctx, RWContext *rw,
+                                  SEIRawUserDataUnregistered *current,
+                                  SEIMessageState *state))
 {
     int err, i;
 
@@ -94,9 +94,10 @@ static int FUNC(user_data_unregistered)
     return 0;
 }
 
-static int FUNC(mastering_display_colour_volume)
-    (CodedBitstreamContext *ctx, RWContext *rw,
-     SEIRawMasteringDisplayColourVolume *current, SEIMessageState *state)
+SEI_FUNC(mastering_display_colour_volume,
+         (CodedBitstreamContext *ctx, RWContext *rw,
+          SEIRawMasteringDisplayColourVolume *current,
+          SEIMessageState *state))
 {
     int err, c;
 
@@ -116,9 +117,9 @@ static int FUNC(mastering_display_colour_volume)
     return 0;
 }
 
-static int FUNC(content_light_level_info)
-    (CodedBitstreamContext *ctx, RWContext *rw,
-     SEIRawContentLightLevelInfo *current, SEIMessageState *state)
+SEI_FUNC(content_light_level_info, (CodedBitstreamContext *ctx, RWContext *rw,
+                                    SEIRawContentLightLevelInfo *current,
+                                    SEIMessageState *state))
 {
     int err;
 
@@ -130,10 +131,10 @@ static int FUNC(content_light_level_info)
     return 0;
 }
 
-static int FUNC(alternative_transfer_characteristics)
-    (CodedBitstreamContext *ctx, RWContext *rw,
-     SEIRawAlternativeTransferCharacteristics *current,
-     SEIMessageState *state)
+SEI_FUNC(alternative_transfer_characteristics,
+         (CodedBitstreamContext *ctx, RWContext *rw,
+          SEIRawAlternativeTransferCharacteristics *current,
+          SEIMessageState *state))
 {
     int err;
 
@@ -144,10 +145,10 @@ static int FUNC(alternative_transfer_characteristics)
     return 0;
 }
 
-static int FUNC(ambient_viewing_environment)
-    (CodedBitstreamContext *ctx, RWContext *rw,
-     SEIRawAmbientViewingEnvironment *current,
-     SEIMessageState *state)
+SEI_FUNC(ambient_viewing_environment,
+         (CodedBitstreamContext *ctx, RWContext *rw,
+          SEIRawAmbientViewingEnvironment *current,
+          SEIMessageState *state))
 {
     static const uint16_t max_ambient_light_value = 50000;
     int err;