avcodec/h2645: Fix SEI->display matrix transformation

The earlier code did not account for the fact that
av_display_rotation_set() wants the angle in the anticlockwise
direction (despite what its documentation stated for a long time);
furthermore, the H.2645 spec wants the flips applied first,
whereas our code did it the other way around. This can be fixed
by negating the angle once for every flip.

Signed-off-by: Andreas Rheinhardt <andreas.rheinhardt@outlook.com>

libavcodec/h264_metadata_bsf.c

@@ -341,15 +341,24 @@ static int h264_metadata_handle_display_orientation(AVBSFContext *bsf,
                                    SEI_TYPE_DISPLAY_ORIENTATION,
                                    &message) == 0) {
         H264RawSEIDisplayOrientation *disp = message->payload;
+        double angle = disp->anticlockwise_rotation * 180.0 / 65536.0;
         int32_t *matrix;
 
         matrix = av_malloc(9 * sizeof(int32_t));
         if (!matrix)
             return AVERROR(ENOMEM);
 
-        av_display_rotation_set(matrix,
-                                disp->anticlockwise_rotation *
-                                180.0 / 65536.0);
+        /* av_display_rotation_set() expects the angle in the clockwise
+         * direction, hence the first minus.
+         * The below code applies the flips after the rotation, yet
+         * the H.2645 specs require flipping to be applied first.
+         * Because of R O(phi) = O(-phi) R (where R is flipping around
+         * an arbitatry axis and O(phi) is the proper rotation by phi)
+         * we can create display matrices as desired by negating
+         * the degree once for every flip applied. */
+        angle = -angle * (1 - 2 * !!disp->hor_flip) * (1 - 2 * !!disp->ver_flip);
+
+        av_display_rotation_set(matrix, angle);
         av_display_matrix_flip(matrix, disp->hor_flip, disp->ver_flip);
 
         // If there are multiple display orientation messages in an

libavcodec/h264_slice.c

@@ -1305,6 +1305,15 @@ static int h264_export_frame_props(H264Context *h)
                                                            AV_FRAME_DATA_DISPLAYMATRIX,
                                                            sizeof(int32_t) * 9);
         if (rotation) {
+            /* av_display_rotation_set() expects the angle in the clockwise
+             * direction, hence the first minus.
+             * The below code applies the flips after the rotation, yet
+             * the H.2645 specs require flipping to be applied first.
+             * Because of R O(phi) = O(-phi) R (where R is flipping around
+             * an arbitatry axis and O(phi) is the proper rotation by phi)
+             * we can create display matrices as desired by negating
+             * the degree once for every flip applied. */
+            angle = -angle * (1 - 2 * !!o->hflip) * (1 - 2 * !!o->vflip);
             av_display_rotation_set((int32_t *)rotation->data, angle);
             av_display_matrix_flip((int32_t *)rotation->data,
                                    o->hflip, o->vflip);

libavcodec/hevcdec.c

@@ -2769,6 +2769,16 @@ static int set_side_data(HEVCContext *s)
         if (!rotation)
             return AVERROR(ENOMEM);
 
+        /* av_display_rotation_set() expects the angle in the clockwise
+         * direction, hence the first minus.
+         * The below code applies the flips after the rotation, yet
+         * the H.2645 specs require flipping to be applied first.
+         * Because of R O(phi) = O(-phi) R (where R is flipping around
+         * an arbitatry axis and O(phi) is the proper rotation by phi)
+         * we can create display matrices as desired by negating
+         * the degree once for every flip applied. */
+        angle = -angle * (1 - 2 * !!s->sei.display_orientation.hflip)
+                       * (1 - 2 * !!s->sei.display_orientation.vflip);
         av_display_rotation_set((int32_t *)rotation->data, angle);
         av_display_matrix_flip((int32_t *)rotation->data,
                                s->sei.display_orientation.hflip,