swscale/utils: return better error code from initFilter()

Reviewed-by: Michael Niedermayer <michael@niedermayer.cc>
Signed-off-by: Limin Wang <lance.lmwang@gmail.com>

libswscale/utils.c

@@ -612,7 +612,7 @@ static av_cold int initFilter(int16_t **outFilter, int32_t **filterPos,
     av_assert0(filterSize > 0);
     filter = av_malloc_array(dstW, filterSize * sizeof(*filter));
     if (!filter)
-        goto fail;
+        goto nomem;
     if (filterSize >= MAX_FILTER_SIZE * 16 /
                       ((flags & SWS_ACCURATE_RND) ? APCK_SIZE : 16)) {
         ret = RETCODE_USE_CASCADE;
@@ -1491,7 +1491,7 @@ av_cold int sws_init_context(SwsContext *c, SwsFilter *srcFilter,
                                                 srcW, srcH, tmpFmt,
                                                 flags, NULL, NULL, c->param);
         if (!c->cascaded_context[0]) {
-            return -1;
+            return AVERROR(ENOMEM);
         }
 
         c->cascaded_context[1] = sws_getContext(srcW, srcH, tmpFmt,
@@ -1499,7 +1499,7 @@ av_cold int sws_init_context(SwsContext *c, SwsFilter *srcFilter,
                                                 flags, srcFilter, dstFilter, c->param);
 
         if (!c->cascaded_context[1])
-            return -1;
+            return AVERROR(ENOMEM);
 
         c2 = c->cascaded_context[1];
         c2->is_internal_gamma = 1;
@@ -1512,10 +1512,10 @@ av_cold int sws_init_context(SwsContext *c, SwsFilter *srcFilter,
         // to properly create the gamma convert FilterDescriptor
         // we have to re-initialize it
         ff_free_filters(c2);
-        if (ff_init_filters(c2) < 0) {
+        if ((ret = ff_init_filters(c2)) < 0) {
             sws_freeContext(c2);
             c->cascaded_context[1] = NULL;
-            return -1;
+            return ret;
         }
 
         c->cascaded_context[2] = NULL;
@@ -1529,7 +1529,7 @@ av_cold int sws_init_context(SwsContext *c, SwsFilter *srcFilter,
                                                 dstW, dstH, dstFormat,
                                                 flags, NULL, NULL, c->param);
             if (!c->cascaded_context[2])
-                return -1;
+                return AVERROR(ENOMEM);
         }
         return 0;
     }
@@ -1548,13 +1548,13 @@ av_cold int sws_init_context(SwsContext *c, SwsFilter *srcFilter,
                                                     srcW, srcH, tmpFormat,
                                                     flags, srcFilter, NULL, c->param);
             if (!c->cascaded_context[0])
-                return -1;
+                return AVERROR(ENOMEM);
 
             c->cascaded_context[1] = sws_getContext(srcW, srcH, tmpFormat,
                                                     dstW, dstH, dstFormat,
                                                     flags, NULL, dstFilter, c->param);
             if (!c->cascaded_context[1])
-                return -1;
+                return AVERROR(ENOMEM);
             return 0;
         }
     }
@@ -1591,7 +1591,7 @@ av_cold int sws_init_context(SwsContext *c, SwsFilter *srcFilter,
                                                             srcW, srcH, tmpFormat,
                                                             flags, c->param);
                 if (!c->cascaded_context[0])
-                    return -1;
+                    return AVERROR(EINVAL);
                 c->cascaded_context[0]->alphablend = c->alphablend;
                 ret = sws_init_context(c->cascaded_context[0], NULL , NULL);
                 if (ret < 0)
@@ -1601,7 +1601,7 @@ av_cold int sws_init_context(SwsContext *c, SwsFilter *srcFilter,
                                                             dstW, dstH, dstFormat,
                                                             flags, c->param);
                 if (!c->cascaded_context[1])
-                    return -1;
+                    return AVERROR(EINVAL);
 
                 c->cascaded_context[1]->srcRange = c->srcRange;
                 c->cascaded_context[1]->dstRange = c->dstRange;
@@ -1678,6 +1678,7 @@ av_cold int sws_init_context(SwsContext *c, SwsFilter *srcFilter,
             if (   mprotect(c->lumMmxextFilterCode, c->lumMmxextFilterCodeSize, PROT_EXEC | PROT_READ) == -1
                 || mprotect(c->chrMmxextFilterCode, c->chrMmxextFilterCodeSize, PROT_EXEC | PROT_READ) == -1) {
                 av_log(c, AV_LOG_ERROR, "mprotect failed, cannot use fast bilinear scaler\n");
+                ret = AVERROR(EINVAL);
                 goto fail;
             }
 #endif
@@ -1870,16 +1871,16 @@ fail: // FIXME replace things by appropriate error codes
                                                 tmpW, tmpH, tmpFormat,
                                                 flags, srcFilter, NULL, c->param);
         if (!c->cascaded_context[0])
-            return -1;
+            return AVERROR(ENOMEM);
 
         c->cascaded_context[1] = sws_getContext(tmpW, tmpH, tmpFormat,
                                                 dstW, dstH, dstFormat,
                                                 flags, NULL, dstFilter, c->param);
         if (!c->cascaded_context[1])
-            return -1;
+            return AVERROR(ENOMEM);
         return 0;
     }
-    return -1;
+    return ret;
 }
 
 SwsContext *sws_alloc_set_opts(int srcW, int srcH, enum AVPixelFormat srcFormat,