avcodec/jpeg2000: Fix undefined behaviour in left shift operations

Signed-off-by: Pierre-Anthony Lemieux <pal@palemieux.com>
pull/391/head
Osamu Watanabe 2 months ago committed by Pierre-Anthony Lemieux
parent 8050a7955e
commit c893dcce31
No known key found for this signature in database
GPG Key ID: 698FA1F0F8D4EED4
  1. 6
      libavcodec/jpeg2000dec.c
  2. 6
      libavcodec/jpeg2000htdec.c
  3. 2
      libavcodec/tests/jpeg2000dwt.c

@ -1886,10 +1886,10 @@ static void decode_sigpass(Jpeg2000T1Context *t1, int width, int height,
if (ff_mqc_decode(&t1->mqc, t1->mqc.cx_states + ff_jpeg2000_getsigctxno(t1->flags[(y+1) * t1->stride + x+1] & flags_mask, bandno))) {
int xorbit, ctxno = ff_jpeg2000_getsgnctxno(t1->flags[(y+1) * t1->stride + x+1] & flags_mask, &xorbit);
if (t1->mqc.raw) {
t1->data[(y) * t1->stride + x] |= ff_mqc_decode(&t1->mqc, t1->mqc.cx_states + ctxno) << 31;
t1->data[(y) * t1->stride + x] |= (uint32_t)(ff_mqc_decode(&t1->mqc, t1->mqc.cx_states + ctxno)) << 31;
t1->data[(y) * t1->stride + x] |= mask;
} else {
t1->data[(y) * t1->stride + x] |= (ff_mqc_decode(&t1->mqc, t1->mqc.cx_states + ctxno) ^ xorbit) << 31;
t1->data[(y) * t1->stride + x] |= (uint32_t)(ff_mqc_decode(&t1->mqc, t1->mqc.cx_states + ctxno) ^ xorbit) << 31;
t1->data[(y) * t1->stride + x] |= mask;
}
ff_jpeg2000_set_significance(t1, x, y,
@ -1969,7 +1969,7 @@ static void decode_clnpass(const Jpeg2000DecoderContext *s, Jpeg2000T1Context *t
int xorbit;
int ctxno = ff_jpeg2000_getsgnctxno(t1->flags[(y + 1) * t1->stride + x + 1] & flags_mask,
&xorbit);
t1->data[(y) * t1->stride + x] |= (ff_mqc_decode(&t1->mqc, t1->mqc.cx_states + ctxno) ^ xorbit) << 31;
t1->data[(y) * t1->stride + x] |= (uint32_t)(ff_mqc_decode(&t1->mqc, t1->mqc.cx_states + ctxno) ^ xorbit) << 31;
t1->data[(y) * t1->stride + x] |= mask;
ff_jpeg2000_set_significance(t1, x, y, t1->data[(y) * t1->stride + x] & INT32_MIN);
}

@ -1070,7 +1070,7 @@ static void jpeg2000_process_stripes_block(StateVars *sig_prop, int i_s, int j_s
uint8_t *state_p = block_states + (i + 1) * stride + (j + 1);
if ((state_p[0] >> HT_SHIFT_REF) & 1) {
bit = jpeg2000_peek_bit(sig_prop, magref_segment, magref_length);
*sp |= (int32_t)bit << 31;
*sp |= (uint32_t)bit << 31;
}
}
}
@ -1160,7 +1160,7 @@ jpeg2000_decode_magref_segment( uint16_t width, uint16_t block_height, const int
jpeg2000_modify_state(i, j, stride, 1 << HT_SHIFT_REF_IND, block_states);
bit = jpeg2000_import_magref_bit(&mag_ref, magref_segment, magref_length);
tmp = 0xFFFFFFFE | (uint32_t)bit;
tmp <<= pLSB;
tmp = (uint32_t)tmp << pLSB;
sp[0] &= tmp;
sp[0] |= 1 << (pLSB - 1); // Add 0.5 (reconstruction parameter = 1/2)
}
@ -1176,7 +1176,7 @@ jpeg2000_decode_magref_segment( uint16_t width, uint16_t block_height, const int
jpeg2000_modify_state(i, j, stride, 1 << HT_SHIFT_REF_IND, block_states);
bit = jpeg2000_import_magref_bit(&mag_ref, magref_segment, magref_length);
tmp = 0xFFFFFFFE | (uint32_t)bit;
tmp <<= pLSB;
tmp = (uint32_t)tmp << pLSB;
sp[0] &= tmp;
sp[0] |= 1 << (pLSB - 1); // Add 0.5 (reconstruction parameter = 1/2)
}

@ -49,7 +49,7 @@ static int test_dwt(int *array, int *ref, int border[2][2], int decomp_levels, i
if (type == FF_DWT97_INT) {
// pre-scaling to simulate dequantization which places the binary point at 1 bit above from LSB
for (j = 0; j< s->linelen[decomp_levels-1][0] * s->linelen[decomp_levels-1][1]; j++)
array[j] <<= I_PRESHIFT;
array[j] = (uint32_t)array[j] << I_PRESHIFT;
}
ret = ff_dwt_decode(s, array);
if (ret < 0) {

Loading…
Cancel
Save