avcodec/vc1dec: Split VC-1 decoders from msmpeg4

The only msmpeg4 code that is ever executed by the VC-1 based
decoders is ff_msmpeg4_decode_init() and what is directly
reachable from it. This is:
a) A call to av_image_check_size(), then ff_h263_decode_init(),
b) followed by setting [yc]_dc_scale_table and initializing
scantable/permutations.
c) Afterwards, some static tables are initialized.
d) Finally, slice_height is set.

The replacement for ff_msmpeg4_decode_init() performs a)
just like now; it also sets [yc]_dc_scale_table,
but it only initializes inter_scantable and intra_scantable
and not permutated_intra_[hv]_scantable: The latter are only
used inside decode_mb callbacks which are only called
in ff_h263_decode_frame() which is unused for VC-1.*

The static tables initialized in c) are not used at all by
VC-1 (the ones that are used have been factored out in
previous commits); this avoids touching 327KiB of .bss.

slice_height is also not used by the VC-1 decoder (setting
it in ff_msmpeg4_decode_init() is probably redundant after
b34397b4cd).

*: It follows from this that the VC-1 decoder is not really
based upon the H.263 decoder either; changing this will
be done in a future commit.

Signed-off-by: Andreas Rheinhardt <andreas.rheinhardt@outlook.com>

configure

@@ -2970,7 +2970,7 @@ utvideo_encoder_select="bswapdsp huffman llvidencdsp"
 vble_decoder_select="llviddsp"
 vbn_decoder_select="texturedsp"
 vbn_encoder_select="texturedspenc"
-vc1_decoder_select="blockdsp h264qpel intrax8 mpegvideodec msmpeg4dec vc1dsp"
+vc1_decoder_select="blockdsp h263_decoder h264qpel intrax8 mpegvideodec vc1dsp"
 vc1image_decoder_select="vc1_decoder"
 vorbis_encoder_select="audio_frame_queue"
 vp3_decoder_select="hpeldsp vp3dsp videodsp"

libavcodec/msmpeg4.c

@@ -27,8 +27,6 @@
  * MSMPEG4 backend for encoder and decoder
  */
 
-#include "config_components.h"
-
 #include "libavutil/thread.h"
 
 #include "avcodec.h"
@@ -40,8 +38,6 @@
 #include "msmpeg4data.h"
 #include "msmpeg4_vc1_data.h"
 #include "mpegvideodata.h"
-#include "vc1data.h"
-#include "libavutil/imgutils.h"
 
 /*
  * You can also call this codec: MPEG-4 with a twist!
@@ -139,16 +135,8 @@ av_cold void ff_msmpeg4_common_init(MpegEncContext *s)
         s->y_dc_scale_table= ff_wmv1_y_dc_scale_table;
         s->c_dc_scale_table= ff_wmv1_c_dc_scale_table;
         break;
-#if CONFIG_VC1_DECODER
-    case 6:
-        s->y_dc_scale_table= ff_wmv3_dc_scale_table;
-        s->c_dc_scale_table= ff_wmv3_dc_scale_table;
-        break;
-#endif
-
     }
 
-
     if(s->msmpeg4_version>=4){
         ff_init_scantable(s->idsp.idct_permutation, &s->intra_scantable,   ff_wmv1_scantable[1]);
         ff_init_scantable(s->idsp.idct_permutation, &s->inter_scantable,   ff_wmv1_scantable[0]);

libavcodec/msmpeg4dec.c

@@ -373,9 +373,6 @@ av_cold int ff_msmpeg4_decode_init(AVCodecContext *avctx)
         break;
     case 5:
         break;
-    case 6:
-        //FIXME + TODO VC1 decode mb
-        break;
     }
 
     s->slice_height= s->mb_height; //to avoid 1/0 if the first frame is not a keyframe

libavcodec/vc1dec.c

@@ -33,12 +33,12 @@
 #include "codec_internal.h"
 #include "decode.h"
 #include "get_bits.h"
+#include "h263dec.h"
 #include "hwconfig.h"
 #include "mpeg_er.h"
 #include "mpegvideo.h"
 #include "mpegvideodec.h"
 #include "msmpeg4_vc1_data.h"
-#include "msmpeg4dec.h"
 #include "profiles.h"
 #include "simple_idct.h"
 #include "vc1.h"
@@ -46,6 +46,7 @@
 #include "vc1_vlc_data.h"
 #include "libavutil/attributes.h"
 #include "libavutil/avassert.h"
+#include "libavutil/imgutils.h"
 #include "libavutil/thread.h"
 
 
@@ -406,11 +407,26 @@ static av_cold int vc1_decode_init_alloc_tables(VC1Context *v)
 
 av_cold int ff_vc1_decode_init(AVCodecContext *avctx)
 {
-    int ret = ff_msmpeg4_decode_init(avctx);
     VC1Context *const v = avctx->priv_data;
+    MpegEncContext *const s = &v->s;
+    int ret;
+
+    ret = av_image_check_size(avctx->width, avctx->height, 0, avctx);
     if (ret < 0)
         return ret;
 
+    ret = ff_h263_decode_init(avctx);
+    if (ret < 0)
+        return ret;
+
+    s->y_dc_scale_table = ff_wmv3_dc_scale_table;
+    s->c_dc_scale_table = ff_wmv3_dc_scale_table;
+
+    ff_init_scantable(s->idsp.idct_permutation, &s->inter_scantable,
+                      ff_wmv1_scantable[0]);
+    ff_init_scantable(s->idsp.idct_permutation, &s->intra_scantable,
+                      ff_wmv1_scantable[1]);
+
     ret = vc1_decode_init_alloc_tables(v);
     if (ret < 0) {
         ff_vc1_decode_end(avctx);