avcodec/smcenc: more y<height checks to fix invalid reads

libavcodec/smcenc.c

@@ -302,11 +302,12 @@ static void smc_encode_stream(SMCContext *s, const AVFrame *frame,
             }
 
             for (int i = 0; i < blocks; i++) {
+                const int y_size = FFMIN(4, height - cur_y);
                 uint8_t value = s->color_pairs[color_table_index][1];
                 uint16_t flags = 0;
                 int shift = 15;
 
-                for (int y = 0; y < 4; y++) {
+                for (int y = 0; y < y_size; y++) {
                     for (int x = 0; x < 4; x++) {
                         flags |= (value == pixel_ptr[x + y * stride]) << shift;
                         shift--;
@@ -350,6 +351,7 @@ static void smc_encode_stream(SMCContext *s, const AVFrame *frame,
             }
 
             for (int i = 0; i < blocks; i++) {
+                const int y_size = FFMIN(4, height - cur_y);
                 uint32_t flags = 0;
                 uint8_t quad[4];
                 int shift = 30;
@@ -357,7 +359,7 @@ static void smc_encode_stream(SMCContext *s, const AVFrame *frame,
                 for (int k = 0; k < 4; k++)
                     quad[k] = s->color_quads[color_table_index][k];
 
-                for (int y = 0; y < 4; y++) {
+                for (int y = 0; y < y_size; y++) {
                     for (int x = 0; x < 4; x++) {
                         int pixel = pixel_ptr[x + y * stride];
                         uint32_t idx = 0;
@@ -417,6 +419,7 @@ static void smc_encode_stream(SMCContext *s, const AVFrame *frame,
             }
 
             for (int i = 0; i < blocks; i++) {
+                const int y_size = FFMIN(4, height - cur_y);
                 uint64_t flags = 0;
                 uint8_t octet[8];
                 int shift = 45;
@@ -424,7 +427,7 @@ static void smc_encode_stream(SMCContext *s, const AVFrame *frame,
                 for (int k = 0; k < 8; k++)
                     octet[k] = s->color_octets[color_table_index][k];
 
-                for (int y = 0; y < 4; y++) {
+                for (int y = 0; y < y_size; y++) {
                     for (int x = 0; x < 4; x++) {
                         int pixel = pixel_ptr[x + y * stride];
                         uint64_t idx = 0;
@@ -451,11 +454,17 @@ static void smc_encode_stream(SMCContext *s, const AVFrame *frame,
         default:
             bytestream2_put_byte(pb, 0xE0 | (blocks - 1));
             for (int i = 0; i < blocks; i++) {
-                for (int y = 0; y < 4; y++) {
+                const int y_size = FFMIN(4, height - cur_y);
+                for (int y = 0; y < y_size; y++) {
                     for (int x = 0; x < 4; x++)
                         bytestream2_put_byte(pb, pixel_ptr[x + y * stride]);
                 }
 
+                for (int y = y_size; y < 4; y++) {
+                    for (int x = 0; x < 4; x++)
+                        bytestream2_put_byte(pb, 0);
+                }
+
                 ADVANCE_BLOCK(pixel_ptr, row_ptr, 1)
             }
             break;