avcodec/agm: Check for too many too short codes in make_new_tree()

Fixes: SEGV on unknown address
Fixes: 14198/clusterfuzz-testcase-minimized-ffmpeg_AV_CODEC_ID_AGM_fuzzer-5723579234123776

Found-by: continuous fuzzing process https://github.com/google/oss-fuzz/tree/master/projects/ffmpeg
Reviewed-by: Paul B Mahol <onemda@gmail.com>
Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>

libavcodec/agm.c

@@ -919,7 +919,7 @@ static void get_tree_codes(uint32_t *codes, Node *nodes, int idx, uint32_t pfx,
     }
 }
 
-static void make_new_tree(const uint8_t *bitlens, uint32_t *codes)
+static int make_new_tree(const uint8_t *bitlens, uint32_t *codes)
 {
     int zlcount = 0, curlen, idx, nindex, last, llast;
     int blcounts[32] = { 0 };
@@ -959,6 +959,9 @@ static void make_new_tree(const uint8_t *bitlens, uint32_t *codes)
                 int p = node_idx[nindex - 1 + 512];
                 int ch = syms[256 * curlen + i];
 
+                if (nindex <= 0)
+                    return AVERROR_INVALIDDATA;
+
                 if (nodes[p].child[0] == -1) {
                     nodes[p].child[0] = ch;
                 } else {
@@ -998,6 +1001,7 @@ static void make_new_tree(const uint8_t *bitlens, uint32_t *codes)
 next:
 
     get_tree_codes(codes, nodes, 256, 0, 0);
+    return 0;
 }
 
 static int build_huff(const uint8_t *bitlen, VLC *vlc)
@@ -1008,7 +1012,9 @@ static int build_huff(const uint8_t *bitlen, VLC *vlc)
     uint32_t codes[256];
     int nb_codes = 0;
 
-    make_new_tree(bitlen, new_codes);
+    int ret = make_new_tree(bitlen, new_codes);
+    if (ret < 0)
+        return ret;
 
     for (int i = 0; i < 256; i++) {
         if (bitlen[i]) {