avutil/tx: add null pointer check after av_mallocz

Fix CID: 1497863
there will get null pointer in attempt to initialize each if alloc memory failed.

Signed-off-by: Steven Liu <liuqi05@kuaishou.com>

libavutil/tx.c

@@ -565,8 +565,13 @@ av_cold int ff_tx_init_subtx(AVTXContext *s, enum AVTXType type,
         print_cd_info(cd_matches[i].cd, cd_matches[i].prio, 1);
     }
 
-    if (!s->sub)
+    if (!s->sub) {
         s->sub = sub = av_mallocz(TX_MAX_SUB*sizeof(*sub));
+        if (!sub) {
+            ret = AVERROR(ENOMEM);
+            goto end;
+        }
+    }
 
     /* Attempt to initialize each */
     for (int i = 0; i < nb_cd_matches; i++) {