check sample_fmt in avcodec_open

check AVCodecContext->sample_fmt against AVCodec->sample_fmts[] to ensure
that the encoder supports the specified sample format. Error out if it doesn't.
Previously, it would continue and output garbage. Fixes issue 2587.
(cherry picked from commit 2cfa2d9258)

libavcodec/utils.c

@@ -551,6 +551,16 @@ int attribute_align_arg avcodec_open(AVCodecContext *avctx, AVCodec *codec)
                avctx->codec->max_lowres);
         goto free_and_end;
     }
+    if (avctx->codec->sample_fmts && avctx->codec->encode) {
+        int i;
+        for (i = 0; avctx->codec->sample_fmts[i] != AV_SAMPLE_FMT_NONE; i++)
+            if (avctx->sample_fmt == avctx->codec->sample_fmts[i])
+                break;
+        if (avctx->codec->sample_fmts[i] == AV_SAMPLE_FMT_NONE) {
+            av_log(avctx, AV_LOG_ERROR, "Specified sample_fmt is not supported.\n");
+            goto free_and_end;
+        }
+    }
 
     avctx->pts_correction_num_faulty_pts =
     avctx->pts_correction_num_faulty_dts = 0;