2 * Interface to the Android Stagefright library for
3 * H/W accelerated H.264 decoding
5 * Copyright (C) 2011 Mohamed Naufal
6 * Copyright (C) 2011 Martin Storsjö
8 * This file is part of FFmpeg.
10 * FFmpeg is free software; you can redistribute it and/or
11 * modify it under the terms of the GNU Lesser General Public
12 * License as published by the Free Software Foundation; either
13 * version 2.1 of the License, or (at your option) any later version.
15 * FFmpeg is distributed in the hope that it will be useful,
16 * but WITHOUT ANY WARRANTY; without even the implied warranty of
17 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
18 * Lesser General Public License for more details.
20 * You should have received a copy of the GNU Lesser General Public
21 * License along with FFmpeg; if not, write to the Free Software
22 * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
25 #include <binder/ProcessState.h>
26 #include <media/stagefright/MetaData.h>
27 #include <media/stagefright/MediaBufferGroup.h>
28 #include <media/stagefright/MediaDebug.h>
29 #include <media/stagefright/MediaDefs.h>
30 #include <media/stagefright/OMXClient.h>
31 #include <media/stagefright/OMXCodec.h>
32 #include <utils/List.h>
38 #include "libavutil/imgutils.h"
42 #define OMX_QCOM_COLOR_FormatYVU420SemiPlanar 0x7FA30C00
44 using namespace android
;
57 int64_t reordered_opaque
;
62 struct StagefrightContext
{
63 AVCodecContext
*avctx
;
64 AVBitStreamFilterContext
*bsfc
;
65 uint8_t* orig_extradata
;
66 int orig_extradata_size
;
67 sp
<MediaSource
> *source
;
68 List
<Frame
*> *in_queue
, *out_queue
;
69 pthread_mutex_t in_mutex
, out_mutex
;
70 pthread_cond_t condition
;
71 pthread_t decode_thread_id
;
75 volatile sig_atomic_t thread_started
, thread_exited
, stop_decode
;
78 std::map
<int64_t, TimeStamp
> *ts_map
;
85 sp
<MediaSource
> *decoder
;
86 const char *decoder_component
;
89 class CustomSource
: public MediaSource
{
91 CustomSource(AVCodecContext
*avctx
, sp
<MetaData
> meta
) {
92 s
= (StagefrightContext
*)avctx
->priv_data
;
94 frame_size
= (avctx
->width
* avctx
->height
* 3) / 2;
95 buf_group
.add_buffer(new MediaBuffer(frame_size
));
98 virtual sp
<MetaData
> getFormat() {
102 virtual status_t
start(MetaData
*params
) {
106 virtual status_t
stop() {
110 virtual status_t
read(MediaBuffer
**buffer
,
111 const MediaSource::ReadOptions
*options
) {
115 if (s
->thread_exited
)
116 return ERROR_END_OF_STREAM
;
117 pthread_mutex_lock(&s
->in_mutex
);
119 while (s
->in_queue
->empty())
120 pthread_cond_wait(&s
->condition
, &s
->in_mutex
);
122 frame
= *s
->in_queue
->begin();
126 ret
= buf_group
.acquire_buffer(buffer
);
128 memcpy((*buffer
)->data(), frame
->buffer
, frame
->size
);
129 (*buffer
)->set_range(0, frame
->size
);
130 (*buffer
)->meta_data()->clear();
131 (*buffer
)->meta_data()->setInt32(kKeyIsSyncFrame
,frame
->key
);
132 (*buffer
)->meta_data()->setInt64(kKeyTime
, frame
->time
);
134 av_log(s
->avctx
, AV_LOG_ERROR
, "Failed to acquire MediaBuffer\n");
136 av_freep(&frame
->buffer
);
139 s
->in_queue
->erase(s
->in_queue
->begin());
140 pthread_mutex_unlock(&s
->in_mutex
);
147 MediaBufferGroup buf_group
;
148 sp
<MetaData
> source_meta
;
149 StagefrightContext
*s
;
153 void* decode_thread(void *arg
)
155 AVCodecContext
*avctx
= (AVCodecContext
*)arg
;
156 StagefrightContext
*s
= (StagefrightContext
*)avctx
->priv_data
;
157 const AVPixFmtDescriptor
*pix_desc
= av_pix_fmt_desc_get(avctx
->pix_fmt
);
164 const uint8_t *src_data
[3];
165 int64_t out_frame_index
= 0;
169 frame
= (Frame
*)av_mallocz(sizeof(Frame
));
171 frame
= s
->end_frame
;
172 frame
->status
= AVERROR(ENOMEM
);
177 frame
->status
= (*s
->decoder
)->read(&buffer
);
178 if (frame
->status
== OK
) {
179 sp
<MetaData
> outFormat
= (*s
->decoder
)->getFormat();
180 outFormat
->findInt32(kKeyWidth
, &w
);
181 outFormat
->findInt32(kKeyHeight
, &h
);
182 frame
->vframe
= av_frame_alloc();
183 if (!frame
->vframe
) {
184 frame
->status
= AVERROR(ENOMEM
);
189 ret
= ff_get_buffer(avctx
, frame
->vframe
, AV_GET_BUFFER_FLAG_REF
);
197 // The OMX.SEC decoder doesn't signal the modified width/height
198 if (s
->decoder_component
&& !strncmp(s
->decoder_component
, "OMX.SEC", 7) &&
199 (w
& 15 || h
& 15)) {
200 if (((w
+ 15)&~15) * ((h
+ 15)&~15) * 3/2 == buffer
->range_length()) {
206 if (!avctx
->width
|| !avctx
->height
|| avctx
->width
> w
|| avctx
->height
> h
) {
211 src_linesize
[0] = av_image_get_linesize(avctx
->pix_fmt
, w
, 0);
212 src_linesize
[1] = av_image_get_linesize(avctx
->pix_fmt
, w
, 1);
213 src_linesize
[2] = av_image_get_linesize(avctx
->pix_fmt
, w
, 2);
215 src_data
[0] = (uint8_t*)buffer
->data();
216 src_data
[1] = src_data
[0] + src_linesize
[0] * h
;
217 src_data
[2] = src_data
[1] + src_linesize
[1] * -(-h
>>pix_desc
->log2_chroma_h
);
218 av_image_copy(frame
->vframe
->data
, frame
->vframe
->linesize
,
219 src_data
, src_linesize
,
220 avctx
->pix_fmt
, avctx
->width
, avctx
->height
);
222 buffer
->meta_data()->findInt64(kKeyTime
, &out_frame_index
);
223 if (out_frame_index
&& s
->ts_map
->count(out_frame_index
) > 0) {
224 frame
->vframe
->pts
= (*s
->ts_map
)[out_frame_index
].pts
;
225 frame
->vframe
->reordered_opaque
= (*s
->ts_map
)[out_frame_index
].reordered_opaque
;
226 s
->ts_map
->erase(out_frame_index
);
229 } else if (frame
->status
== INFO_FORMAT_CHANGED
) {
239 pthread_mutex_lock(&s
->out_mutex
);
240 if (s
->out_queue
->size() >= 10) {
241 pthread_mutex_unlock(&s
->out_mutex
);
247 s
->out_queue
->push_back(frame
);
248 pthread_mutex_unlock(&s
->out_mutex
);
249 } while (!decode_done
&& !s
->stop_decode
);
251 s
->thread_exited
= true;
256 static av_cold
int Stagefright_init(AVCodecContext
*avctx
)
258 StagefrightContext
*s
= (StagefrightContext
*)avctx
->priv_data
;
259 sp
<MetaData
> meta
, outFormat
;
260 int32_t colorFormat
= 0;
263 if (!avctx
->extradata
|| !avctx
->extradata_size
|| avctx
->extradata
[0] != 1)
267 s
->bsfc
= av_bitstream_filter_init("h264_mp4toannexb");
269 av_log(avctx
, AV_LOG_ERROR
, "Cannot open the h264_mp4toannexb BSF!\n");
273 s
->orig_extradata_size
= avctx
->extradata_size
;
274 s
->orig_extradata
= (uint8_t*) av_mallocz(avctx
->extradata_size
+
275 FF_INPUT_BUFFER_PADDING_SIZE
);
276 if (!s
->orig_extradata
) {
277 ret
= AVERROR(ENOMEM
);
280 memcpy(s
->orig_extradata
, avctx
->extradata
, avctx
->extradata_size
);
284 ret
= AVERROR(ENOMEM
);
287 meta
->setCString(kKeyMIMEType
, MEDIA_MIMETYPE_VIDEO_AVC
);
288 meta
->setInt32(kKeyWidth
, avctx
->width
);
289 meta
->setInt32(kKeyHeight
, avctx
->height
);
290 meta
->setData(kKeyAVCC
, kTypeAVCC
, avctx
->extradata
, avctx
->extradata_size
);
292 android::ProcessState::self()->startThreadPool();
294 s
->source
= new sp
<MediaSource
>();
295 *s
->source
= new CustomSource(avctx
, meta
);
296 s
->in_queue
= new List
<Frame
*>;
297 s
->out_queue
= new List
<Frame
*>;
298 s
->ts_map
= new std::map
<int64_t, TimeStamp
>;
299 s
->client
= new OMXClient
;
300 s
->end_frame
= (Frame
*)av_mallocz(sizeof(Frame
));
301 if (s
->source
== NULL
|| !s
->in_queue
|| !s
->out_queue
|| !s
->client
||
302 !s
->ts_map
|| !s
->end_frame
) {
303 ret
= AVERROR(ENOMEM
);
307 if (s
->client
->connect() != OK
) {
308 av_log(avctx
, AV_LOG_ERROR
, "Cannot connect OMX client\n");
313 s
->decoder
= new sp
<MediaSource
>();
314 *s
->decoder
= OMXCodec::Create(s
->client
->interface(), meta
,
315 false, *s
->source
, NULL
,
316 OMXCodec::kClientNeedsFramebuffer
);
317 if ((*s
->decoder
)->start() != OK
) {
318 av_log(avctx
, AV_LOG_ERROR
, "Cannot start decoder\n");
320 s
->client
->disconnect();
324 outFormat
= (*s
->decoder
)->getFormat();
325 outFormat
->findInt32(kKeyColorFormat
, &colorFormat
);
326 if (colorFormat
== OMX_QCOM_COLOR_FormatYVU420SemiPlanar
||
327 colorFormat
== OMX_COLOR_FormatYUV420SemiPlanar
)
328 avctx
->pix_fmt
= AV_PIX_FMT_NV21
;
329 else if (colorFormat
== OMX_COLOR_FormatYCbYCr
)
330 avctx
->pix_fmt
= AV_PIX_FMT_YUYV422
;
331 else if (colorFormat
== OMX_COLOR_FormatCbYCrY
)
332 avctx
->pix_fmt
= AV_PIX_FMT_UYVY422
;
334 avctx
->pix_fmt
= AV_PIX_FMT_YUV420P
;
336 outFormat
->findCString(kKeyDecoderComponent
, &s
->decoder_component
);
337 if (s
->decoder_component
)
338 s
->decoder_component
= av_strdup(s
->decoder_component
);
340 pthread_mutex_init(&s
->in_mutex
, NULL
);
341 pthread_mutex_init(&s
->out_mutex
, NULL
);
342 pthread_cond_init(&s
->condition
, NULL
);
346 av_bitstream_filter_close(s
->bsfc
);
347 av_freep(&s
->orig_extradata
);
348 av_freep(&s
->end_frame
);
356 static int Stagefright_decode_frame(AVCodecContext
*avctx
, void *data
,
357 int *got_frame
, AVPacket
*avpkt
)
359 StagefrightContext
*s
= (StagefrightContext
*)avctx
->priv_data
;
362 int orig_size
= avpkt
->size
;
363 AVPacket pkt
= *avpkt
;
366 if (!s
->thread_started
) {
367 pthread_create(&s
->decode_thread_id
, NULL
, &decode_thread
, avctx
);
368 s
->thread_started
= true;
371 if (avpkt
&& avpkt
->data
) {
372 av_bitstream_filter_filter(s
->bsfc
, avctx
, NULL
, &pkt
.data
, &pkt
.size
,
373 avpkt
->data
, avpkt
->size
, avpkt
->flags
& AV_PKT_FLAG_KEY
);
377 if (!s
->source_done
) {
379 s
->dummy_buf
= (uint8_t*)av_malloc(avpkt
->size
);
381 return AVERROR(ENOMEM
);
382 s
->dummy_bufsize
= avpkt
->size
;
383 memcpy(s
->dummy_buf
, avpkt
->data
, avpkt
->size
);
386 frame
= (Frame
*)av_mallocz(sizeof(Frame
));
389 frame
->size
= avpkt
->size
;
390 frame
->key
= avpkt
->flags
& AV_PKT_FLAG_KEY
? 1 : 0;
391 frame
->buffer
= (uint8_t*)av_malloc(avpkt
->size
);
392 if (!frame
->buffer
) {
394 return AVERROR(ENOMEM
);
396 uint8_t *ptr
= avpkt
->data
;
397 // The OMX.SEC decoder fails without this.
398 if (avpkt
->size
== orig_size
+ avctx
->extradata_size
) {
399 ptr
+= avctx
->extradata_size
;
400 frame
->size
= orig_size
;
402 memcpy(frame
->buffer
, ptr
, orig_size
);
404 av_free(avpkt
->data
);
406 frame
->time
= ++s
->frame_index
;
407 (*s
->ts_map
)[s
->frame_index
].pts
= avpkt
->pts
;
408 (*s
->ts_map
)[s
->frame_index
].reordered_opaque
= avctx
->reordered_opaque
;
410 frame
->status
= ERROR_END_OF_STREAM
;
411 s
->source_done
= true;
415 if (s
->thread_exited
) {
416 s
->source_done
= true;
419 pthread_mutex_lock(&s
->in_mutex
);
420 if (s
->in_queue
->size() >= 10) {
421 pthread_mutex_unlock(&s
->in_mutex
);
425 s
->in_queue
->push_back(frame
);
426 pthread_cond_signal(&s
->condition
);
427 pthread_mutex_unlock(&s
->in_mutex
);
432 pthread_mutex_lock(&s
->out_mutex
);
433 if (!s
->out_queue
->empty()) break;
434 pthread_mutex_unlock(&s
->out_mutex
);
435 if (s
->source_done
) {
443 frame
= *s
->out_queue
->begin();
444 s
->out_queue
->erase(s
->out_queue
->begin());
445 pthread_mutex_unlock(&s
->out_mutex
);
447 ret_frame
= frame
->vframe
;
448 status
= frame
->status
;
451 if (status
== ERROR_END_OF_STREAM
)
454 if (status
== AVERROR(ENOMEM
))
456 av_log(avctx
, AV_LOG_ERROR
, "Decode failed: %x\n", status
);
461 av_frame_free(&s
->prev_frame
);
462 s
->prev_frame
= ret_frame
;
465 *(AVFrame
*)data
= *ret_frame
;
469 static av_cold
int Stagefright_close(AVCodecContext
*avctx
)
471 StagefrightContext
*s
= (StagefrightContext
*)avctx
->priv_data
;
474 if (s
->thread_started
) {
475 if (!s
->thread_exited
) {
478 // Make sure decode_thread() doesn't get stuck
479 pthread_mutex_lock(&s
->out_mutex
);
480 while (!s
->out_queue
->empty()) {
481 frame
= *s
->out_queue
->begin();
482 s
->out_queue
->erase(s
->out_queue
->begin());
484 av_frame_free(&frame
->vframe
);
487 pthread_mutex_unlock(&s
->out_mutex
);
489 // Feed a dummy frame prior to signalling EOF.
490 // This is required to terminate the decoder(OMX.SEC)
491 // when only one frame is read during stream info detection.
492 if (s
->dummy_buf
&& (frame
= (Frame
*)av_mallocz(sizeof(Frame
)))) {
494 frame
->size
= s
->dummy_bufsize
;
496 frame
->buffer
= s
->dummy_buf
;
497 pthread_mutex_lock(&s
->in_mutex
);
498 s
->in_queue
->push_back(frame
);
499 pthread_cond_signal(&s
->condition
);
500 pthread_mutex_unlock(&s
->in_mutex
);
504 pthread_mutex_lock(&s
->in_mutex
);
505 s
->end_frame
->status
= ERROR_END_OF_STREAM
;
506 s
->in_queue
->push_back(s
->end_frame
);
507 pthread_cond_signal(&s
->condition
);
508 pthread_mutex_unlock(&s
->in_mutex
);
512 pthread_join(s
->decode_thread_id
, NULL
);
515 av_frame_free(&s
->prev_frame
);
517 s
->thread_started
= false;
520 while (!s
->in_queue
->empty()) {
521 frame
= *s
->in_queue
->begin();
522 s
->in_queue
->erase(s
->in_queue
->begin());
524 av_freep(&frame
->buffer
);
528 while (!s
->out_queue
->empty()) {
529 frame
= *s
->out_queue
->begin();
530 s
->out_queue
->erase(s
->out_queue
->begin());
532 av_frame_free(&frame
->vframe
);
536 (*s
->decoder
)->stop();
537 s
->client
->disconnect();
539 if (s
->decoder_component
)
540 av_freep(&s
->decoder_component
);
541 av_freep(&s
->dummy_buf
);
542 av_freep(&s
->end_frame
);
544 // Reset the extradata back to the original mp4 format, so that
545 // the next invocation (both when decoding and when called from
546 // av_find_stream_info) get the original mp4 format extradata.
547 av_freep(&avctx
->extradata
);
548 avctx
->extradata
= s
->orig_extradata
;
549 avctx
->extradata_size
= s
->orig_extradata_size
;
558 pthread_mutex_destroy(&s
->in_mutex
);
559 pthread_mutex_destroy(&s
->out_mutex
);
560 pthread_cond_destroy(&s
->condition
);
561 av_bitstream_filter_close(s
->bsfc
);
565 AVCodec ff_libstagefright_h264_decoder
= {
566 "libstagefright_h264",
567 NULL_IF_CONFIG_SMALL("libstagefright H.264"),
571 NULL
, //supported_framerates
573 NULL
, //supported_samplerates
575 NULL
, //channel_layouts
579 sizeof(StagefrightContext
),
581 NULL
, //init_thread_copy
582 NULL
, //update_thread_context
584 NULL
, //init_static_data
588 Stagefright_decode_frame
,