From 7b915a40458f2cd01db29f9979fda39c7958980e Mon Sep 17 00:00:00 2001 From: Mohamed Naufal Date: Thu, 19 Apr 2012 00:02:53 +0530 Subject: [PATCH 2/5] libstagefright: explicitly set positive timestamps as stagefright expects them so --- configure | 2 +- libavcodec/libstagefright.cpp | 28 +++++++++++++++++++++++----- tools/build_libstagefright | 6 +++--- 3 files changed, 27 insertions(+), 9 deletions(-) diff --git a/configure b/configure index 6f23af3969..8b87ad4824 100755 --- a/configure +++ b/configure @@ -2951,7 +2951,7 @@ enabled libschroedinger && require_pkg_config schroedinger-1.0 schroedinger/schr enabled libspeex && require libspeex speex/speex.h speex_decoder_init -lspeex enabled libstagefright_h264 && require_cpp libstagefright_h264 "binder/ProcessState.h media/stagefright/MetaData.h media/stagefright/MediaBufferGroup.h media/stagefright/MediaDebug.h media/stagefright/MediaDefs.h - media/stagefright/OMXClient.h media/stagefright/OMXCodec.h" android::OMXClient -lstagefright -lmedia -lutils -lbinder + media/stagefright/OMXClient.h media/stagefright/OMXCodec.h" android::OMXClient -lstagefright -lmedia -lutils -lbinder -lgnustl_static enabled libtheora && require libtheora theora/theoraenc.h th_info_init -ltheoraenc -ltheoradec -logg enabled libvo_aacenc && require libvo_aacenc vo-aacenc/voAAC.h voGetAACEncAPI -lvo-aacenc enabled libvo_amrwbenc && require libvo_amrwbenc vo-amrwbenc/enc_if.h E_IF_init -lvo-amrwbenc diff --git a/libavcodec/libstagefright.cpp b/libavcodec/libstagefright.cpp index 9ffe27c0d9..f373a10fa5 100644 --- a/libavcodec/libstagefright.cpp +++ b/libavcodec/libstagefright.cpp @@ -31,6 +31,7 @@ #include #include #include +#include extern "C" { #include "avcodec.h" @@ -51,6 +52,11 @@ struct Frame { int32_t w, h; }; +struct TimeStamp { + int64_t pts; + int64_t reordered_opaque; +}; + class CustomSource; struct StagefrightContext { @@ -69,6 +75,8 @@ struct StagefrightContext { volatile sig_atomic_t thread_started, thread_exited, stop_decode; AVFrame ret_frame; + std::map *ts_map; + int64_t frame_index; uint8_t *dummy_buf; int dummy_bufsize; @@ -234,10 +242,11 @@ static av_cold int Stagefright_init(AVCodecContext *avctx) *s->source = new CustomSource(avctx, meta); s->in_queue = new List; s->out_queue = new List; + s->ts_map = new std::map; s->client = new OMXClient; s->end_frame = (Frame*)av_mallocz(sizeof(Frame)); if (s->source == NULL || !s->in_queue || !s->out_queue || !s->client || - !s->end_frame) { + !s->ts_map || !s->end_frame) { ret = AVERROR(ENOMEM); goto fail; } @@ -282,6 +291,7 @@ fail: av_freep(&s->end_frame); delete s->in_queue; delete s->out_queue; + delete s->ts_map; delete s->client; return ret; } @@ -300,6 +310,7 @@ static int Stagefright_decode_frame(AVCodecContext *avctx, void *data, int src_linesize[3]; int orig_size = avpkt->size; AVPacket pkt = *avpkt; + int64_t out_frame_index = 0; int ret; if (!s->thread_started) { @@ -326,10 +337,6 @@ static int Stagefright_decode_frame(AVCodecContext *avctx, void *data, if (avpkt->data) { frame->status = OK; frame->size = avpkt->size; - // Stagefright can't handle negative timestamps - - // if needed, work around this by offsetting them manually? - if (avpkt->pts >= 0) - frame->time = avpkt->pts; frame->key = avpkt->flags & AV_PKT_FLAG_KEY ? 1 : 0; frame->buffer = (uint8_t*)av_malloc(avpkt->size); if (!frame->buffer) { @@ -343,6 +350,10 @@ static int Stagefright_decode_frame(AVCodecContext *avctx, void *data, frame->size = orig_size; } memcpy(frame->buffer, ptr, orig_size); + + frame->time = ++s->frame_index; + (*s->ts_map)[s->frame_index].pts = avpkt->pts; + (*s->ts_map)[s->frame_index].reordered_opaque = avctx->reordered_opaque; } else { frame->status = ERROR_END_OF_STREAM; s->source_done = true; @@ -431,6 +442,12 @@ static int Stagefright_decode_frame(AVCodecContext *avctx, void *data, src_data, src_linesize, avctx->pix_fmt, avctx->width, avctx->height); + mbuffer->meta_data()->findInt64(kKeyTime, &out_frame_index); + if (out_frame_index && s->ts_map->count(out_frame_index) > 0) { + s->ret_frame.pts = (*s->ts_map)[out_frame_index].pts; + s->ret_frame.reordered_opaque = (*s->ts_map)[out_frame_index].reordered_opaque; + s->ts_map->erase(out_frame_index); + } *data_size = sizeof(AVFrame); *(AVFrame*)data = s->ret_frame; ret = orig_size; @@ -523,6 +540,7 @@ static av_cold int Stagefright_close(AVCodecContext *avctx) delete s->in_queue; delete s->out_queue; + delete s->ts_map; delete s->client; delete s->decoder; delete s->source; diff --git a/tools/build_libstagefright b/tools/build_libstagefright index 78c20841ee..8b3a0930f9 100755 --- a/tools/build_libstagefright +++ b/tools/build_libstagefright @@ -28,6 +28,7 @@ TOOLCHAIN=`echo $NDK/toolchains/arm-linux-androideabi-4.4.3/prebuilt/*-x86` export PATH=$TOOLCHAIN/bin:$PATH ANDROID_SOURCE=../android-source ANDROID_LIBS=../android-libs +ABI="armeabi-v7a" rm -rf ../build/stagefright mkdir -p ../build/stagefright @@ -40,12 +41,11 @@ FLAGS="$FLAGS --disable-avdevice --disable-decoder=h264 --disable-decoder=h264_v EXTRA_CFLAGS="-I$ANDROID_SOURCE/frameworks/base/include -I$ANDROID_SOURCE/system/core/include" EXTRA_CFLAGS="$EXTRA_CFLAGS -I$ANDROID_SOURCE/frameworks/base/media/libstagefright" EXTRA_CFLAGS="$EXTRA_CFLAGS -I$ANDROID_SOURCE/frameworks/base/include/media/stagefright/openmax" -EXTRA_CFLAGS="$EXTRA_CFLAGS -I$NDK/sources/cxx-stl/system/include" +EXTRA_CFLAGS="$EXTRA_CFLAGS -I$NDK/sources/cxx-stl/gnu-libstdc++/include -I$NDK/sources/cxx-stl/gnu-libstdc++/libs/$ABI/include" EXTRA_CFLAGS="$EXTRA_CFLAGS -march=armv7-a -mfloat-abi=softfp -mfpu=neon" -EXTRA_LDFLAGS="-Wl,--fix-cortex-a8 -L$ANDROID_LIBS -Wl,-rpath-link,$ANDROID_LIBS" +EXTRA_LDFLAGS="-Wl,--fix-cortex-a8 -L$ANDROID_LIBS -Wl,-rpath-link,$ANDROID_LIBS -L$NDK/sources/cxx-stl/gnu-libstdc++/libs/$ABI" EXTRA_CXXFLAGS="-Wno-multichar -fno-exceptions -fno-rtti" -ABI="armeabi-v7a" DEST="$DEST/$ABI" FLAGS="$FLAGS --prefix=$DEST" From 1d48e88d4148a271ac56de706265e863e53da94b Mon Sep 17 00:00:00 2001 From: Mohamed Naufal Date: Thu, 19 Apr 2012 01:08:50 +0530 Subject: [PATCH 3/5] libstagefright: avoid potential deadlock on output MediaBuffer Maintain an output queue of AVFrames instead of MediaBuffers so that the latter can be released early. This avoids a potential deadlock between the stagefright decoder::read() and Stagefright_decode_frame() --- libavcodec/libstagefright.cpp | 159 ++++++++++++++++++---------------- 1 file changed, 86 insertions(+), 73 deletions(-) diff --git a/libavcodec/libstagefright.cpp b/libavcodec/libstagefright.cpp index f373a10fa5..9206170d3f 100644 --- a/libavcodec/libstagefright.cpp +++ b/libavcodec/libstagefright.cpp @@ -48,8 +48,7 @@ struct Frame { int64_t time; int key; uint8_t *buffer; - MediaBuffer* mbuffer; - int32_t w, h; + AVFrame *vframe; }; struct TimeStamp { @@ -74,7 +73,7 @@ struct StagefrightContext { bool source_done; volatile sig_atomic_t thread_started, thread_exited, stop_decode; - AVFrame ret_frame; + AVFrame *prev_frame; std::map *ts_map; int64_t frame_index; @@ -156,7 +155,13 @@ void* decode_thread(void *arg) StagefrightContext *s = (StagefrightContext*)avctx->priv_data; Frame* frame; MediaBuffer *buffer; + int32_t w, h; int decode_done = 0; + int ret; + int src_linesize[3]; + const uint8_t *src_data[3]; + int64_t out_frame_index = 0; + do { buffer = NULL; frame = (Frame*)av_mallocz(sizeof(Frame)); @@ -165,14 +170,63 @@ void* decode_thread(void *arg) frame->status = AVERROR(ENOMEM); decode_done = 1; s->end_frame = NULL; - } else { - frame->status = (*s->decoder)->read(&buffer); - if (frame->status == OK) { - sp outFormat = (*s->decoder)->getFormat(); - outFormat->findInt32(kKeyWidth , &frame->w); - outFormat->findInt32(kKeyHeight, &frame->h); - frame->size = buffer->range_length(); - frame->mbuffer = buffer; + goto push_frame; + } + frame->status = (*s->decoder)->read(&buffer); + if (frame->status == OK) { + sp outFormat = (*s->decoder)->getFormat(); + outFormat->findInt32(kKeyWidth , &w); + outFormat->findInt32(kKeyHeight, &h); + frame->vframe = (AVFrame*)av_mallocz(sizeof(AVFrame)); + if (!frame->vframe) { + frame->status = AVERROR(ENOMEM); + decode_done = 1; + buffer->release(); + goto push_frame; + } + ret = avctx->get_buffer(avctx, frame->vframe); + if (ret < 0) { + av_log(avctx, AV_LOG_ERROR, "get_buffer() failed\n"); + frame->status = ret; + decode_done = 1; + buffer->release(); + goto push_frame; + } + + // The OMX.SEC decoder doesn't signal the modified width/height + if (s->decoder_component && !strncmp(s->decoder_component, "OMX.SEC", 7) && + (w & 15 || h & 15)) { + if (((w + 15)&~15) * ((h + 15)&~15) * 3/2 == buffer->range_length()) { + w = (w + 15)&~15; + h = (h + 15)&~15; + } + } + + if (!avctx->width || !avctx->height || avctx->width > w || avctx->height > h) { + avctx->width = w; + avctx->height = h; + } + + src_linesize[0] = w; + if (avctx->pix_fmt == PIX_FMT_YUV420P) + src_linesize[1] = src_linesize[2] = w/2; + else if (avctx->pix_fmt == PIX_FMT_NV21) + src_linesize[1] = w; + + src_data[0] = (uint8_t*)buffer->data(); + src_data[1] = src_data[0] + src_linesize[0] * h; + src_data[2] = src_data[1] + src_linesize[1] * h/2; + av_image_copy(frame->vframe->data, frame->vframe->linesize, + src_data, src_linesize, + avctx->pix_fmt, avctx->width, avctx->height); + + buffer->meta_data()->findInt64(kKeyTime, &out_frame_index); + if (out_frame_index && s->ts_map->count(out_frame_index) > 0) { + frame->vframe->pts = (*s->ts_map)[out_frame_index].pts; + frame->vframe->reordered_opaque = (*s->ts_map)[out_frame_index].reordered_opaque; + s->ts_map->erase(out_frame_index); + } + buffer->release(); } else if (frame->status == INFO_FORMAT_CHANGED) { if (buffer) buffer->release(); @@ -181,7 +235,7 @@ void* decode_thread(void *arg) } else { decode_done = 1; } - } +push_frame: while (true) { pthread_mutex_lock(&s->out_mutex); if (s->out_queue->size() >= 10) { @@ -301,17 +355,10 @@ static int Stagefright_decode_frame(AVCodecContext *avctx, void *data, { StagefrightContext *s = (StagefrightContext*)avctx->priv_data; Frame *frame; - MediaBuffer *mbuffer; status_t status; - size_t size; - uint8_t *buf; - const uint8_t *src_data[3]; - int w, h; - int src_linesize[3]; int orig_size = avpkt->size; AVPacket pkt = *avpkt; - int64_t out_frame_index = 0; - int ret; + AVFrame *ret_frame; if (!s->thread_started) { pthread_create(&s->decode_thread_id, NULL, &decode_thread, avctx); @@ -392,11 +439,8 @@ static int Stagefright_decode_frame(AVCodecContext *avctx, void *data, s->out_queue->erase(s->out_queue->begin()); pthread_mutex_unlock(&s->out_mutex); - mbuffer = frame->mbuffer; + ret_frame = frame->vframe; status = frame->status; - size = frame->size; - w = frame->w; - h = frame->h; av_freep(&frame); if (status == ERROR_END_OF_STREAM) @@ -408,52 +452,15 @@ static int Stagefright_decode_frame(AVCodecContext *avctx, void *data, return -1; } - // The OMX.SEC decoder doesn't signal the modified width/height - if (s->decoder_component && !strncmp(s->decoder_component, "OMX.SEC", 7) && - (w & 15 || h & 15)) { - if (((w + 15)&~15) * ((h + 15)&~15) * 3/2 == size) { - w = (w + 15)&~15; - h = (h + 15)&~15; - } - } - - if (!avctx->width || !avctx->height || avctx->width > w || avctx->height > h) { - avctx->width = w; - avctx->height = h; - } - - ret = avctx->reget_buffer(avctx, &s->ret_frame); - if (ret < 0) { - av_log(avctx, AV_LOG_ERROR, "reget buffer() failed\n"); - goto end; + if (s->prev_frame) { + avctx->release_buffer(avctx, s->prev_frame); + av_freep(&s->prev_frame); } + s->prev_frame = ret_frame; - src_linesize[0] = w; - if (avctx->pix_fmt == PIX_FMT_YUV420P) - src_linesize[1] = src_linesize[2] = w/2; - else if (avctx->pix_fmt == PIX_FMT_NV21) - src_linesize[1] = w; - - buf = (uint8_t*)mbuffer->data(); - src_data[0] = buf; - src_data[1] = buf + src_linesize[0] * h; - src_data[2] = src_data[1] + src_linesize[1] * h/2; - av_image_copy(s->ret_frame.data, s->ret_frame.linesize, - src_data, src_linesize, - avctx->pix_fmt, avctx->width, avctx->height); - - mbuffer->meta_data()->findInt64(kKeyTime, &out_frame_index); - if (out_frame_index && s->ts_map->count(out_frame_index) > 0) { - s->ret_frame.pts = (*s->ts_map)[out_frame_index].pts; - s->ret_frame.reordered_opaque = (*s->ts_map)[out_frame_index].reordered_opaque; - s->ts_map->erase(out_frame_index); - } *data_size = sizeof(AVFrame); - *(AVFrame*)data = s->ret_frame; - ret = orig_size; -end: - mbuffer->release(); - return ret; + *(AVFrame*)data = *ret_frame; + return orig_size; } static av_cold int Stagefright_close(AVCodecContext *avctx) @@ -470,8 +477,10 @@ static av_cold int Stagefright_close(AVCodecContext *avctx) while (!s->out_queue->empty()) { frame = *s->out_queue->begin(); s->out_queue->erase(s->out_queue->begin()); - if (frame->size) - frame->mbuffer->release(); + if (frame->vframe) { + avctx->release_buffer(avctx, frame->vframe); + av_freep(&frame->vframe); + } av_freep(&frame); } pthread_mutex_unlock(&s->out_mutex); @@ -501,8 +510,10 @@ static av_cold int Stagefright_close(AVCodecContext *avctx) pthread_join(s->decode_thread_id, NULL); - if (s->ret_frame.data[0]) - avctx->release_buffer(avctx, &s->ret_frame); + if (s->prev_frame) { + avctx->release_buffer(avctx, s->prev_frame); + av_freep(&s->prev_frame); + } s->thread_started = false; } @@ -518,8 +529,10 @@ static av_cold int Stagefright_close(AVCodecContext *avctx) while (!s->out_queue->empty()) { frame = *s->out_queue->begin(); s->out_queue->erase(s->out_queue->begin()); - if (frame->size) - frame->mbuffer->release(); + if (frame->vframe) { + avctx->release_buffer(avctx, frame->vframe); + av_freep(&frame->vframe); + } av_freep(&frame); } From 2343a99cf2d7a7d9c10fde3c7687be5d1dd297d7 Mon Sep 17 00:00:00 2001 From: Mohamed Naufal Date: Thu, 19 Apr 2012 01:42:12 +0530 Subject: [PATCH 4/5] libstagefright: support more output pixel formats --- libavcodec/libstagefright.cpp | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/libavcodec/libstagefright.cpp b/libavcodec/libstagefright.cpp index 9206170d3f..ef45fae9f0 100644 --- a/libavcodec/libstagefright.cpp +++ b/libavcodec/libstagefright.cpp @@ -153,6 +153,7 @@ void* decode_thread(void *arg) { AVCodecContext *avctx = (AVCodecContext*)arg; StagefrightContext *s = (StagefrightContext*)avctx->priv_data; + const AVPixFmtDescriptor *pix_desc = &av_pix_fmt_descriptors[avctx->pix_fmt]; Frame* frame; MediaBuffer *buffer; int32_t w, h; @@ -207,15 +208,13 @@ void* decode_thread(void *arg) avctx->height = h; } - src_linesize[0] = w; - if (avctx->pix_fmt == PIX_FMT_YUV420P) - src_linesize[1] = src_linesize[2] = w/2; - else if (avctx->pix_fmt == PIX_FMT_NV21) - src_linesize[1] = w; + src_linesize[0] = av_image_get_linesize(avctx->pix_fmt, w, 0); + src_linesize[1] = av_image_get_linesize(avctx->pix_fmt, w, 1); + src_linesize[2] = av_image_get_linesize(avctx->pix_fmt, w, 2); src_data[0] = (uint8_t*)buffer->data(); src_data[1] = src_data[0] + src_linesize[0] * h; - src_data[2] = src_data[1] + src_linesize[1] * h/2; + src_data[2] = src_data[1] + src_linesize[1] * -(-h>>pix_desc->log2_chroma_h); av_image_copy(frame->vframe->data, frame->vframe->linesize, src_data, src_linesize, avctx->pix_fmt, avctx->width, avctx->height); @@ -327,6 +326,10 @@ static av_cold int Stagefright_init(AVCodecContext *avctx) if (colorFormat == OMX_QCOM_COLOR_FormatYVU420SemiPlanar || colorFormat == OMX_COLOR_FormatYUV420SemiPlanar) avctx->pix_fmt = PIX_FMT_NV21; + else if (colorFormat == OMX_COLOR_FormatYCbYCr) + avctx->pix_fmt = PIX_FMT_YUYV422; + else if (colorFormat == OMX_COLOR_FormatCbYCrY) + avctx->pix_fmt = PIX_FMT_UYVY422; else avctx->pix_fmt = PIX_FMT_YUV420P; From f51b7e52a6b31e98a82ce48dddac9cabc1e62e04 Mon Sep 17 00:00:00 2001 From: Mohamed Naufal Date: Thu, 19 Apr 2012 03:00:41 +0530 Subject: [PATCH 5/5] libstagefright: avoid memory leak --- libavcodec/libstagefright.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/libavcodec/libstagefright.cpp b/libavcodec/libstagefright.cpp index ef45fae9f0..a74c5a3a42 100644 --- a/libavcodec/libstagefright.cpp +++ b/libavcodec/libstagefright.cpp @@ -400,6 +400,8 @@ static int Stagefright_decode_frame(AVCodecContext *avctx, void *data, frame->size = orig_size; } memcpy(frame->buffer, ptr, orig_size); + if (avpkt == &pkt) + av_free(avpkt->data); frame->time = ++s->frame_index; (*s->ts_map)[s->frame_index].pts = avpkt->pts;