From a47ead72e315cdd497af174e81eb72aa9361d598 Mon Sep 17 00:00:00 2001 From: "sw.multimedia" Date: Wed, 25 May 2022 20:05:11 +0800 Subject: [PATCH 1/8] avcodec add omx decoder support Signed-off-by: sw.multimedia --- configure | 1 + libavcodec/Makefile | 1 + libavcodec/allcodecs.c | 1 + libavcodec/omxdec.c | 893 +++++++++++++++++++++++++++++++++++++++++ 4 files changed, 896 insertions(+) create mode 100644 libavcodec/omxdec.c --- a/configure +++ b/configure @@ -3176,6 +3176,7 @@ h264_nvenc_encoder_deps="nvenc" h264_nvenc_encoder_select="atsc_a53" h264_omx_encoder_deps="omx" +h264_omx_decoder_deps="omx" h264_qsv_decoder_select="h264_mp4toannexb_bsf qsvdec" h264_qsv_encoder_select="atsc_a53 qsvenc" h264_rkmpp_decoder_deps="rkmpp" --- a/libavcodec/Makefile +++ b/libavcodec/Makefile @@ -392,6 +392,7 @@ OBJS-$(CONFIG_H264_MMAL_DECODER) += mmaldec.o OBJS-$(CONFIG_H264_NVENC_ENCODER) += nvenc_h264.o nvenc.o OBJS-$(CONFIG_H264_OMX_ENCODER) += omx.o +OBJS-$(CONFIG_H264_OMX_DECODER) += omxdec.o OBJS-$(CONFIG_H264_QSV_DECODER) += qsvdec.o OBJS-$(CONFIG_H264_QSV_ENCODER) += qsvenc_h264.o OBJS-$(CONFIG_H264_RKMPP_DECODER) += rkmppdec.o --- /dev/null +++ b/libavcodec/omxdec.c @@ -0,0 +1,893 @@ +/* + * OMX Video encoder + * Copyright (C) 2011 Martin Storsjo + * + * This file is part of FFmpeg. + * + * FFmpeg is free software; you can redistribute it and/or + * modify it under the terms of the GNU Lesser General Public + * License as published by the Free Software Foundation; either + * version 2.1 of the License, or (at your option) any later version. + * + * FFmpeg is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU + * Lesser General Public License for more details. + * + * You should have received a copy of the GNU Lesser General Public + * License along with FFmpeg; if not, write to the Free Software + * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA + */ + +#include "config.h" + +#if CONFIG_OMX_RPI +#define OMX_SKIP64BIT +#endif + +#include +#include +#include +#include +#include +#include +#include + +#include "libavutil/avstring.h" +#include "libavutil/avutil.h" +#include "libavutil/common.h" +#include "libavutil/imgutils.h" +#include "libavutil/log.h" +#include "libavutil/opt.h" +#include "libavutil/time.h" + +#include "avcodec.h" +#include "h264.h" +#include "internal.h" + +#ifdef OMX_SKIP64BIT +static OMX_TICKS to_omx_ticks(int64_t value) +{ + OMX_TICKS s; + s.nLowPart = value & 0xffffffff; + s.nHighPart = value >> 32; + return s; +} +static int64_t from_omx_ticks(OMX_TICKS value) +{ + return (((int64_t)value.nHighPart) << 32) | value.nLowPart; +} +#else +#define to_omx_ticks(x) (x) +#define from_omx_ticks(x) (x) +#endif + +#define INIT_STRUCT(x) do { \ + x.nSize = sizeof(x); \ + x.nVersion = s->version; \ + } while (0) +#define CHECK(x) do { \ + if (x != OMX_ErrorNone) { \ + av_log(avctx, AV_LOG_ERROR, \ + "err %x (%d) on line %d\n", x, x, __LINE__); \ + return AVERROR_UNKNOWN; \ + } \ + } while (0) + +typedef struct OMXContext { + void *lib; + void *lib2; + OMX_ERRORTYPE (*ptr_Init)(void); + OMX_ERRORTYPE (*ptr_Deinit)(void); + OMX_ERRORTYPE (*ptr_ComponentNameEnum)(OMX_STRING, OMX_U32, OMX_U32); + OMX_ERRORTYPE (*ptr_GetHandle)(OMX_HANDLETYPE*, OMX_STRING, OMX_PTR, OMX_CALLBACKTYPE*); + OMX_ERRORTYPE (*ptr_FreeHandle)(OMX_HANDLETYPE); + OMX_ERRORTYPE (*ptr_GetComponentsOfRole)(OMX_STRING, OMX_U32*, OMX_U8**); + OMX_ERRORTYPE (*ptr_GetRolesOfComponent)(OMX_STRING, OMX_U32*, OMX_U8**); + void (*host_init)(void); +} OMXContext; + +static av_cold void *dlsym_prefixed(void *handle, const char *symbol, const char *prefix) +{ + char buf[50]; + snprintf(buf, sizeof(buf), "%s%s", prefix ? prefix : "", symbol); + return dlsym(handle, buf); +} + +static av_cold int omx_try_load(OMXContext *s, void *logctx, + const char *libname, const char *prefix, + const char *libname2) +{ + if (libname2) { + s->lib2 = dlopen(libname2, RTLD_NOW | RTLD_GLOBAL); + if (!s->lib2) { + av_log(logctx, AV_LOG_WARNING, "%s not found\n", libname2); + return AVERROR_ENCODER_NOT_FOUND; + } + s->host_init = dlsym(s->lib2, "bcm_host_init"); + if (!s->host_init) { + av_log(logctx, AV_LOG_WARNING, "bcm_host_init not found\n"); + dlclose(s->lib2); + s->lib2 = NULL; + return AVERROR_ENCODER_NOT_FOUND; + } + } + s->lib = dlopen(libname, RTLD_NOW | RTLD_GLOBAL); + if (!s->lib) { + av_log(logctx, AV_LOG_WARNING, "%s not found\n", libname); + return AVERROR_ENCODER_NOT_FOUND; + } + s->ptr_Init = dlsym_prefixed(s->lib, "OMX_Init", prefix); + s->ptr_Deinit = dlsym_prefixed(s->lib, "OMX_Deinit", prefix); + s->ptr_ComponentNameEnum = dlsym_prefixed(s->lib, "OMX_ComponentNameEnum", prefix); + s->ptr_GetHandle = dlsym_prefixed(s->lib, "OMX_GetHandle", prefix); + s->ptr_FreeHandle = dlsym_prefixed(s->lib, "OMX_FreeHandle", prefix); + s->ptr_GetComponentsOfRole = dlsym_prefixed(s->lib, "OMX_GetComponentsOfRole", prefix); + s->ptr_GetRolesOfComponent = dlsym_prefixed(s->lib, "OMX_GetRolesOfComponent", prefix); + if (!s->ptr_Init || !s->ptr_Deinit || !s->ptr_ComponentNameEnum || + !s->ptr_GetHandle || !s->ptr_FreeHandle || + !s->ptr_GetComponentsOfRole || !s->ptr_GetRolesOfComponent) { + av_log(logctx, AV_LOG_WARNING, "Not all functions found in %s\n", libname); + dlclose(s->lib); + s->lib = NULL; + if (s->lib2) + dlclose(s->lib2); + s->lib2 = NULL; + return AVERROR_ENCODER_NOT_FOUND; + } + return 0; +} + +static av_cold OMXContext *omx_init(void *logctx, const char *libname, const char *prefix) +{ + static const char * const libnames[] = { +#if CONFIG_OMX_RPI + "/opt/vc/lib/libopenmaxil.so", "/opt/vc/lib/libbcm_host.so", +#else + "libOMX_Core.so", NULL, + "libOmxCore.so", NULL, +#endif + NULL + }; + const char* const* nameptr; + int ret = AVERROR_DECODER_NOT_FOUND; + OMXContext *omx_context; + + omx_context = av_mallocz(sizeof(*omx_context)); + if (!omx_context) + return NULL; + if (libname) { + ret = omx_try_load(omx_context, logctx, libname, prefix, NULL); + if (ret < 0) { + av_free(omx_context); + return NULL; + } + } else { + for (nameptr = libnames; *nameptr; nameptr += 2) + if (!(ret = omx_try_load(omx_context, logctx, nameptr[0], prefix, nameptr[1]))) + break; + if (!*nameptr) { + av_free(omx_context); + return NULL; + } + } + + if (omx_context->host_init) + omx_context->host_init(); + omx_context->ptr_Init(); + return omx_context; +} + +static av_cold void omx_deinit(OMXContext *omx_context) +{ + if (!omx_context) + return; + + omx_context->ptr_Deinit(); + dlclose(omx_context->lib); + av_free(omx_context); +} + +typedef struct OMXCodecContext { + const AVClass *class; + char *libname; + char *libprefix; + OMXContext *omx_context; + + AVCodecContext *avctx; + + char component_name[OMX_MAX_STRINGNAME_SIZE]; + OMX_VERSIONTYPE version; + OMX_HANDLETYPE handle; + int in_port, out_port; + OMX_COLOR_FORMATTYPE color_format; + int stride, plane_size; + + int num_in_buffers, num_out_buffers; + OMX_BUFFERHEADERTYPE **in_buffer_headers; + OMX_BUFFERHEADERTYPE **out_buffer_headers; + int num_free_in_buffers; + OMX_BUFFERHEADERTYPE **free_in_buffers; + int num_done_out_buffers; + OMX_BUFFERHEADERTYPE **done_out_buffers; + pthread_mutex_t input_mutex; + pthread_cond_t input_cond; + pthread_mutex_t output_mutex; + pthread_cond_t output_cond; + + pthread_mutex_t state_mutex; + pthread_cond_t state_cond; + OMX_STATETYPE state; + OMX_ERRORTYPE error; + + int mutex_cond_inited; + + int eos_sent, got_eos, first_get_outbuffer; + + int extradata_sent; + + uint8_t *output_buf; + int output_buf_size; + + int input_zerocopy; + int profile; +} OMXCodecContext; + +static void append_buffer(pthread_mutex_t *mutex, pthread_cond_t *cond, + int* array_size, OMX_BUFFERHEADERTYPE **array, + OMX_BUFFERHEADERTYPE *buffer) +{ + pthread_mutex_lock(mutex); + array[(*array_size)++] = buffer; + pthread_cond_broadcast(cond); + pthread_mutex_unlock(mutex); +} + +static OMX_BUFFERHEADERTYPE *get_buffer(pthread_mutex_t *mutex, pthread_cond_t *cond, + int* array_size, OMX_BUFFERHEADERTYPE **array, + int wait) +{ + OMX_BUFFERHEADERTYPE *buffer; + pthread_mutex_lock(mutex); + if (wait) { + while (!*array_size) + { + pthread_cond_wait(cond, mutex); + } + } + if (*array_size > 0) { + buffer = array[0]; + (*array_size)--; + memmove(&array[0], &array[1], (*array_size) * sizeof(OMX_BUFFERHEADERTYPE*)); + } else { + buffer = NULL; + } + pthread_mutex_unlock(mutex); + return buffer; +} + +static OMX_ERRORTYPE event_handler(OMX_HANDLETYPE component, OMX_PTR app_data, OMX_EVENTTYPE event, + OMX_U32 data1, OMX_U32 data2, OMX_PTR event_data) +{ + OMXCodecContext *s = app_data; + // This uses casts in the printfs, since OMX_U32 actually is a typedef for + // unsigned long in official header versions (but there are also modified + // versions where it is something else). + switch (event) { + case OMX_EventError: + pthread_mutex_lock(&s->state_mutex); + av_log(s->avctx, AV_LOG_ERROR, "OMX error %"PRIx32"\n", (uint32_t) data1); + s->error = data1; + pthread_cond_broadcast(&s->state_cond); + pthread_mutex_unlock(&s->state_mutex); + break; + case OMX_EventCmdComplete: + if (data1 == OMX_CommandStateSet) { + pthread_mutex_lock(&s->state_mutex); + s->state = data2; + av_log(s->avctx, AV_LOG_VERBOSE, "OMX state changed to %"PRIu32"\n", (uint32_t) data2); + pthread_cond_broadcast(&s->state_cond); + pthread_mutex_unlock(&s->state_mutex); + } else if (data1 == OMX_CommandPortDisable) { + av_log(s->avctx, AV_LOG_VERBOSE, "OMX port %"PRIu32" disabled\n", (uint32_t) data2); + } else if (data1 == OMX_CommandPortEnable) { + av_log(s->avctx, AV_LOG_VERBOSE, "OMX port %"PRIu32" enabled\n", (uint32_t) data2); + } else { + av_log(s->avctx, AV_LOG_VERBOSE, "OMX command complete, command %"PRIu32", value %"PRIu32"\n", + (uint32_t) data1, (uint32_t) data2); + } + break; + case OMX_EventPortSettingsChanged: + av_log(s->avctx, AV_LOG_VERBOSE, "OMX port %"PRIu32" settings changed\n", (uint32_t) data1); + break; + default: + av_log(s->avctx, AV_LOG_VERBOSE, "OMX event %d %"PRIx32" %"PRIx32"\n", + event, (uint32_t) data1, (uint32_t) data2); + break; + } + return OMX_ErrorNone; +} + +static OMX_ERRORTYPE empty_buffer_done(OMX_HANDLETYPE component, OMX_PTR app_data, + OMX_BUFFERHEADERTYPE *buffer) +{ + + OMXCodecContext *s = app_data; + if (s->input_zerocopy) { + if (buffer->pAppPrivate) { + if (buffer->pOutputPortPrivate) + av_free(buffer->pAppPrivate); + else + av_frame_free((AVFrame**)&buffer->pAppPrivate); + buffer->pAppPrivate = NULL; + } + } + append_buffer(&s->input_mutex, &s->input_cond, + &s->num_free_in_buffers, s->free_in_buffers, buffer); + return OMX_ErrorNone; +} + +static OMX_ERRORTYPE fill_buffer_done(OMX_HANDLETYPE component, OMX_PTR app_data, + OMX_BUFFERHEADERTYPE *buffer) +{ + OMXCodecContext *s = app_data; + append_buffer(&s->output_mutex, &s->output_cond, + &s->num_done_out_buffers, s->done_out_buffers, buffer); + return OMX_ErrorNone; +} + +static const OMX_CALLBACKTYPE callbacks = { + event_handler, + empty_buffer_done, + fill_buffer_done +}; + +static av_cold int find_component(OMXContext *omx_context, void *logctx, + const char *role, char *str, int str_size) +{ + OMX_U32 i, num = 0; + char **components; + int ret = 0; + +#if CONFIG_OMX_RPI + if (av_strstart(role, "video_decoder.", NULL)) { + av_strlcpy(str, "OMX.broadcom.video_decode", str_size); + return 0; + } +#endif + omx_context->ptr_GetComponentsOfRole((OMX_STRING) role, &num, NULL); + if (!num) { + av_log(logctx, AV_LOG_WARNING, "No component for role %s found\n", role); + return AVERROR_DECODER_NOT_FOUND; + } + components = av_mallocz_array(num, sizeof(*components)); + if (!components) + return AVERROR(ENOMEM); + for (i = 0; i < num; i++) { + components[i] = av_mallocz(OMX_MAX_STRINGNAME_SIZE); + if (!components[i]) { + ret = AVERROR(ENOMEM); + goto end; + } + } + omx_context->ptr_GetComponentsOfRole((OMX_STRING) role, &num, (OMX_U8**) components); + av_strlcpy(str, components[0], str_size); +end: + for (i = 0; i < num; i++) + av_free(components[i]); + av_free(components); + return ret; +} + +static av_cold int wait_for_state(OMXCodecContext *s, OMX_STATETYPE state) +{ + int ret = 0; + pthread_mutex_lock(&s->state_mutex); + while (s->state != state && s->error == OMX_ErrorNone) + pthread_cond_wait(&s->state_cond, &s->state_mutex); + if (s->error != OMX_ErrorNone) + ret = AVERROR_DECODER_NOT_FOUND; + pthread_mutex_unlock(&s->state_mutex); + return ret; +} + +static av_cold int omx_component_init(AVCodecContext *avctx, const char *role) +{ + OMXCodecContext *s = avctx->priv_data; + OMX_PARAM_COMPONENTROLETYPE role_params = { 0 }; + OMX_PORT_PARAM_TYPE video_port_params = { 0 }; + OMX_PARAM_PORTDEFINITIONTYPE in_port_params = { 0 }, out_port_params = { 0 }; + //OMX_VIDEO_PARAM_PORTFORMATTYPE video_port_format = { 0 }; + //OMX_VIDEO_PARAM_BITRATETYPE vid_param_bitrate = { 0 }; + OMX_ERRORTYPE err; + int i; + + s->version.s.nVersionMajor = 1; + s->version.s.nVersionMinor = 1; + s->version.s.nRevision = 2; + + err = s->omx_context->ptr_GetHandle(&s->handle, s->component_name, s, (OMX_CALLBACKTYPE*) &callbacks); + if (err != OMX_ErrorNone) { + av_log(avctx, AV_LOG_ERROR, "OMX_GetHandle(%s) failed: %x\n", s->component_name, err); + return AVERROR_UNKNOWN; + } + + // This one crashes the mediaserver on qcom, if used over IOMX + INIT_STRUCT(role_params); + av_strlcpy(role_params.cRole, role, sizeof(role_params.cRole)); + // Intentionally ignore errors on this one + OMX_SetParameter(s->handle, OMX_IndexParamStandardComponentRole, &role_params); + + INIT_STRUCT(video_port_params); + err = OMX_GetParameter(s->handle, OMX_IndexParamVideoInit, &video_port_params); + CHECK(err); + + s->in_port = s->out_port = -1; + for (i = 0; i < video_port_params.nPorts; i++) { + int port = video_port_params.nStartPortNumber + i; + OMX_PARAM_PORTDEFINITIONTYPE port_params = { 0 }; + INIT_STRUCT(port_params); + port_params.nPortIndex = port; + err = OMX_GetParameter(s->handle, OMX_IndexParamPortDefinition, &port_params); + if (err != OMX_ErrorNone) { + av_log(avctx, AV_LOG_WARNING, "port %d error %x\n", port, err); + break; + } + if (port_params.eDir == OMX_DirInput && s->in_port < 0) { + in_port_params = port_params; + s->in_port = port; + } else if (port_params.eDir == OMX_DirOutput && s->out_port < 0) { + out_port_params = port_params; + s->out_port = port; + } + } + if (s->in_port < 0 || s->out_port < 0) { + av_log(avctx, AV_LOG_ERROR, "No in or out port found (in %d out %d)\n", s->in_port, s->out_port); + return AVERROR_UNKNOWN; + } + + in_port_params.bEnabled = OMX_TRUE; + in_port_params.bPopulated = OMX_FALSE; + in_port_params.eDomain = OMX_PortDomainVideo; + + in_port_params.format.video.pNativeRender = NULL; + in_port_params.format.video.bFlagErrorConcealment = OMX_FALSE; + //in_port_params.format.video.eColorFormat = s->color_format; + s->stride = avctx->width; + s->plane_size = avctx->height; + // If specific codecs need to manually override the stride/plane_size, + // that can be done here. + in_port_params.format.video.nStride = s->stride; + in_port_params.format.video.nSliceHeight = s->plane_size; + in_port_params.format.video.nFrameWidth = avctx->width; + in_port_params.format.video.nFrameHeight = avctx->height; + + if (avctx->framerate.den > 0 && avctx->framerate.num > 0) + in_port_params.format.video.xFramerate = (1LL << 16) * avctx->framerate.num / avctx->framerate.den; + else + in_port_params.format.video.xFramerate = (1LL << 16) * avctx->time_base.den / avctx->time_base.num; + + err = OMX_SetParameter(s->handle, OMX_IndexParamPortDefinition, &in_port_params); + CHECK(err); + err = OMX_GetParameter(s->handle, OMX_IndexParamPortDefinition, &in_port_params); + CHECK(err); + +#if 1 + s->stride = in_port_params.format.video.nStride; + s->plane_size = in_port_params.format.video.nSliceHeight; + s->num_in_buffers = in_port_params.nBufferCountActual; +#endif + err = OMX_GetParameter(s->handle, OMX_IndexParamPortDefinition, &out_port_params); + out_port_params.bEnabled = OMX_TRUE; + out_port_params.bPopulated = OMX_FALSE; + out_port_params.eDomain = OMX_PortDomainVideo; + out_port_params.format.video.pNativeRender = NULL; + out_port_params.format.video.bFlagErrorConcealment = OMX_FALSE; + out_port_params.format.video.nFrameWidth = avctx->width; + out_port_params.format.video.nFrameHeight = avctx->height; + out_port_params.format.video.nStride = 0; + out_port_params.format.video.nSliceHeight = 0; + out_port_params.format.video.nBitrate = avctx->bit_rate; + out_port_params.format.video.xFramerate = in_port_params.format.video.xFramerate; + out_port_params.format.video.bFlagErrorConcealment = OMX_FALSE; + if (avctx->codec->id == AV_CODEC_ID_MPEG4) + out_port_params.format.video.eCompressionFormat = OMX_VIDEO_CodingMPEG4; + else if (avctx->codec->id == AV_CODEC_ID_H264) + out_port_params.format.video.eCompressionFormat = OMX_VIDEO_CodingAVC; + + err = OMX_SetParameter(s->handle, OMX_IndexParamPortDefinition, &out_port_params); + CHECK(err); + err = OMX_GetParameter(s->handle, OMX_IndexParamPortDefinition, &out_port_params); + CHECK(err); + s->num_out_buffers = out_port_params.nBufferCountActual; + + + err = OMX_SendCommand(s->handle, OMX_CommandStateSet, OMX_StateIdle, NULL); + CHECK(err); + + s->in_buffer_headers = av_mallocz(sizeof(OMX_BUFFERHEADERTYPE*) * s->num_in_buffers); + s->free_in_buffers = av_mallocz(sizeof(OMX_BUFFERHEADERTYPE*) * s->num_in_buffers); + s->out_buffer_headers = av_mallocz(sizeof(OMX_BUFFERHEADERTYPE*) * s->num_out_buffers); + s->done_out_buffers = av_mallocz(sizeof(OMX_BUFFERHEADERTYPE*) * s->num_out_buffers); + if (!s->in_buffer_headers || !s->free_in_buffers || !s->out_buffer_headers || !s->done_out_buffers) + return AVERROR(ENOMEM); + for (i = 0; i < s->num_in_buffers && err == OMX_ErrorNone; i++) { + if (s->input_zerocopy) + err = OMX_UseBuffer(s->handle, &s->in_buffer_headers[i], s->in_port, s, in_port_params.nBufferSize, NULL); + else + err = OMX_AllocateBuffer(s->handle, &s->in_buffer_headers[i], s->in_port, s, in_port_params.nBufferSize); + if (err == OMX_ErrorNone) + s->in_buffer_headers[i]->pAppPrivate = s->in_buffer_headers[i]->pOutputPortPrivate = NULL; + } + CHECK(err); + s->num_in_buffers = i; + for (i = 0; i < s->num_out_buffers && err == OMX_ErrorNone; i++) + err = OMX_AllocateBuffer(s->handle, &s->out_buffer_headers[i], s->out_port, s, out_port_params.nBufferSize); + CHECK(err); + s->num_out_buffers = i; + + if (wait_for_state(s, OMX_StateIdle) < 0) { + av_log(avctx, AV_LOG_ERROR, "Didn't get OMX_StateIdle\n"); + return AVERROR_UNKNOWN; + } + err = OMX_SendCommand(s->handle, OMX_CommandStateSet, OMX_StateExecuting, NULL); + CHECK(err); + if (wait_for_state(s, OMX_StateExecuting) < 0) { + av_log(avctx, AV_LOG_ERROR, "Didn't get OMX_StateExecuting\n"); + return AVERROR_UNKNOWN; + } + + for (i = 0; i < s->num_out_buffers && err == OMX_ErrorNone; i++) + err = OMX_FillThisBuffer(s->handle, s->out_buffer_headers[i]); + if (err != OMX_ErrorNone) { + for (; i < s->num_out_buffers; i++) + s->done_out_buffers[s->num_done_out_buffers++] = s->out_buffer_headers[i]; + } + for (i = 0; i < s->num_in_buffers; i++) + s->free_in_buffers[s->num_free_in_buffers++] = s->in_buffer_headers[i]; + return err != OMX_ErrorNone ? AVERROR_UNKNOWN : 0; +} + +static av_cold void cleanup(OMXCodecContext *s) +{ + int i, executing; + + pthread_mutex_lock(&s->state_mutex); + executing = s->state == OMX_StateExecuting; + pthread_mutex_unlock(&s->state_mutex); + + if (executing) { + OMX_SendCommand(s->handle, OMX_CommandStateSet, OMX_StateIdle, NULL); + wait_for_state(s, OMX_StateIdle); + OMX_SendCommand(s->handle, OMX_CommandStateSet, OMX_StateLoaded, NULL); + for (i = 0; i < s->num_in_buffers; i++) { + OMX_BUFFERHEADERTYPE *buffer = get_buffer(&s->input_mutex, &s->input_cond, + &s->num_free_in_buffers, s->free_in_buffers, 1); + if (s->input_zerocopy) + buffer->pBuffer = NULL; + OMX_FreeBuffer(s->handle, s->in_port, buffer); + } + for (i = 0; i < s->num_out_buffers; i++) { + OMX_BUFFERHEADERTYPE *buffer = get_buffer(&s->output_mutex, &s->output_cond, + &s->num_done_out_buffers, s->done_out_buffers, 1); + OMX_FreeBuffer(s->handle, s->out_port, buffer); + } + wait_for_state(s, OMX_StateLoaded); + } + if (s->handle) { + s->omx_context->ptr_FreeHandle(s->handle); + s->handle = NULL; + } + + omx_deinit(s->omx_context); + s->omx_context = NULL; + if (s->mutex_cond_inited) { + pthread_cond_destroy(&s->state_cond); + pthread_mutex_destroy(&s->state_mutex); + pthread_cond_destroy(&s->input_cond); + pthread_mutex_destroy(&s->input_mutex); + pthread_cond_destroy(&s->output_cond); + pthread_mutex_destroy(&s->output_mutex); + s->mutex_cond_inited = 0; + } + av_freep(&s->in_buffer_headers); + av_freep(&s->out_buffer_headers); + av_freep(&s->free_in_buffers); + av_freep(&s->done_out_buffers); + av_freep(&s->output_buf); +} + +static av_cold int omx_decode_init(AVCodecContext *avctx) +{ + OMXCodecContext *s = avctx->priv_data; + int ret = AVERROR_ENCODER_NOT_FOUND; + const char *role; + //OMX_BUFFERHEADERTYPE *buffer; + //OMX_ERRORTYPE err; + + s->omx_context = omx_init(avctx, s->libname, s->libprefix); + if (!s->omx_context) + return AVERROR_ENCODER_NOT_FOUND; + + pthread_mutex_init(&s->state_mutex, NULL); + pthread_cond_init(&s->state_cond, NULL); + pthread_mutex_init(&s->input_mutex, NULL); + pthread_cond_init(&s->input_cond, NULL); + pthread_mutex_init(&s->output_mutex, NULL); + pthread_cond_init(&s->output_cond, NULL); + s->mutex_cond_inited = 1; + s->avctx = avctx; + s->state = OMX_StateLoaded; + s->error = OMX_ErrorNone; + + switch (avctx->codec->id) { + case AV_CODEC_ID_MPEG4: + role = "video_decoder.mpeg4"; + break; + case AV_CODEC_ID_H264: + role = "video_decoder.avc"; + break; + default: + return AVERROR(ENOSYS); + } + + if ((ret = find_component(s->omx_context, avctx, role, s->component_name, sizeof(s->component_name))) < 0) + goto fail; + + av_log(avctx, AV_LOG_INFO, "Using %s\n", s->component_name); + + if ((ret = omx_component_init(avctx, role)) < 0) + goto fail; + +#if 0 + if (avctx->flags & AV_CODEC_FLAG_GLOBAL_HEADER) { + while (1) { + buffer = get_buffer(&s->output_mutex, &s->output_cond, + &s->num_done_out_buffers, s->done_out_buffers, 1); + if (buffer->nFlags & OMX_BUFFERFLAG_CODECCONFIG) { + if ((ret = av_reallocp(&avctx->extradata, avctx->extradata_size + buffer->nFilledLen + AV_INPUT_BUFFER_PADDING_SIZE)) < 0) { + avctx->extradata_size = 0; + goto fail; + } + memcpy(avctx->extradata + avctx->extradata_size, buffer->pBuffer + buffer->nOffset, buffer->nFilledLen); + avctx->extradata_size += buffer->nFilledLen; + memset(avctx->extradata + avctx->extradata_size, 0, AV_INPUT_BUFFER_PADDING_SIZE); + } + err = OMX_FillThisBuffer(s->handle, buffer); + if (err != OMX_ErrorNone) { + append_buffer(&s->output_mutex, &s->output_cond, + &s->num_done_out_buffers, s->done_out_buffers, buffer); + av_log(avctx, AV_LOG_ERROR, "OMX_FillThisBuffer failed: %x\n", err); + ret = AVERROR_UNKNOWN; + goto fail; + } + if (avctx->codec->id == AV_CODEC_ID_H264) { + // For H.264, the extradata can be returned in two separate buffers + // (the videocore encoder on raspberry pi does this); + // therefore check that we have got both SPS and PPS before continuing. + int nals[32] = { 0 }; + int i; + for (i = 0; i + 4 < avctx->extradata_size; i++) { + if (!avctx->extradata[i + 0] && + !avctx->extradata[i + 1] && + !avctx->extradata[i + 2] && + avctx->extradata[i + 3] == 1) { + nals[avctx->extradata[i + 4] & 0x1f]++; + } + } + if (nals[H264_NAL_SPS] && nals[H264_NAL_PPS]) + break; + } else { + if (avctx->extradata_size > 0) + break; + } + } + } +#endif + + return 0; +fail: + return ret; +} + + +static int omx_decode_frame(AVCodecContext *avctx, void *data, + int *got_packet, AVPacket *pkt) +{ + OMXCodecContext *s = avctx->priv_data; + int ret = 0; + OMX_BUFFERHEADERTYPE* buffer; + OMX_ERRORTYPE err; + int had_partial = 0; + + AVFrame *avframe = data; + + uint8_t *dst[4]; + int linesize[4]; + + if (pkt->size) { + + //VPU init and fill buffer slow, so empty buf sleep to send before get vpu fill buf. + if(!s->first_get_outbuffer) + av_usleep(100000); + + buffer = get_buffer(&s->input_mutex, &s->input_cond, + &s->num_free_in_buffers, s->free_in_buffers, 1); + + if (!buffer) { + av_log(avctx, AV_LOG_ERROR, "get_buffer NULL\n"); + return AVERROR(ENOMEM); + } + + //cpy the extradata + if(!s->extradata_sent) { + + memcpy(buffer->pBuffer + buffer->nOffset, avctx->extradata, avctx->extradata_size); + memcpy(buffer->pBuffer + buffer->nOffset + avctx->extradata_size, pkt->data, pkt->size); + buffer->nFilledLen = pkt->size + avctx->extradata_size; + s->extradata_sent = 1; + + } + else { + memcpy(buffer->pBuffer + buffer->nOffset, pkt->data, pkt->size); + buffer->nFilledLen = pkt->size; + } + + /* reduce memcpy. point it addr*/ + //buffer->pAppPrivate = pkt; + //buffer->pBuffer = pkt->data; + //buffer->nFilledLen = pkt->size; + + buffer->pOutputPortPrivate = NULL; + buffer->pAppPrivate = avctx->priv_data; + + err = OMX_EmptyThisBuffer(s->handle, buffer); + if (err != OMX_ErrorNone) { + append_buffer(&s->input_mutex, &s->input_cond, &s->num_free_in_buffers, s->free_in_buffers, buffer); + av_log(avctx, AV_LOG_ERROR, "OMX_EmptyThisBuffer failed: %x\n", err); + return AVERROR_UNKNOWN; + } + } else if (!s->eos_sent) { + + if(!s->first_get_outbuffer) + av_usleep(1000000); + + buffer = get_buffer(&s->input_mutex, &s->input_cond, + &s->num_free_in_buffers, s->free_in_buffers, 1); + + buffer->nFilledLen = 0; + buffer->nFlags = OMX_BUFFERFLAG_EOS; + buffer->pAppPrivate = buffer->pOutputPortPrivate = NULL; + err = OMX_EmptyThisBuffer(s->handle, buffer); + if (err != OMX_ErrorNone) { + append_buffer(&s->input_mutex, &s->input_cond, &s->num_free_in_buffers, s->free_in_buffers, buffer); + av_log(avctx, AV_LOG_ERROR, "OMX_EmptyThisBuffer failed: %x\n", err); + return AVERROR_UNKNOWN; + } + s->eos_sent = 1; + } + + while (!*got_packet && ret == 0 && !s->got_eos) { + // If not flushing, just poll the queue if there's finished packets. + // If flushing, do a blocking wait until we either get a completed + // packet, or get EOS. + buffer = get_buffer(&s->output_mutex, &s->output_cond, + &s->num_done_out_buffers, s->done_out_buffers, + !pkt || had_partial); + + if (!buffer) { + /*eos is sent but fill buf still can't get then continue*/ + if(!s->first_get_outbuffer && s->eos_sent) + continue; + else + break; + } + //if (!buffer) + // break; + + if(!buffer->nFilledLen) + goto end; + + if(!s->first_get_outbuffer) + s->first_get_outbuffer = 1; + + if (buffer->nFlags & OMX_BUFFERFLAG_EOS) + s->got_eos = 1; + + if ((ret = ff_get_buffer(avctx, avframe, 0)) < 0) { + av_log(avctx, AV_LOG_ERROR, "Unable to allocate buffer\n"); + goto end; + } + + ret = av_image_fill_arrays(dst, linesize, buffer->pBuffer, + avctx->pix_fmt, s->stride, s->plane_size, 1); + if (ret < 0) + goto end; + av_image_copy(avframe->data, avframe->linesize, (const uint8_t**)dst, linesize, + avctx->pix_fmt, avctx->width, avctx->height); + + //avframe->pts = buffer->nTimeStamp; + //avframe->pkt_dts = AV_NOPTS_VALUE; + //avframe->pict_type= AV_PICTURE_TYPE_I; + //avframe->key_frame= 1; + + *got_packet = 1; + + /* + if ((ret = av_frame_ref(data, avframe)) < 0) + goto end; + */ + +end: + err = OMX_FillThisBuffer(s->handle, buffer); + if (err != OMX_ErrorNone) { + append_buffer(&s->output_mutex, &s->output_cond, &s->num_done_out_buffers, s->done_out_buffers, buffer); + av_log(avctx, AV_LOG_ERROR, "OMX_FillThisBuffer failed: %x\n", err); + ret = AVERROR_UNKNOWN; + } + } + return ret; +} + +static av_cold int omx_decode_end(AVCodecContext *avctx) +{ + OMXCodecContext *s = avctx->priv_data; + + cleanup(s); + return 0; +} + +#define OFFSET(x) offsetof(OMXCodecContext, x) +#define VDE AV_OPT_FLAG_VIDEO_PARAM | AV_OPT_FLAG_DECODING_PARAM | AV_OPT_FLAG_ENCODING_PARAM +#define VE AV_OPT_FLAG_VIDEO_PARAM | AV_OPT_FLAG_ENCODING_PARAM +static const AVOption options[] = { + { "omx_libname", "OpenMAX library name", OFFSET(libname), AV_OPT_TYPE_STRING, { 0 }, 0, 0, VDE }, + { "omx_libprefix", "OpenMAX library prefix", OFFSET(libprefix), AV_OPT_TYPE_STRING, { 0 }, 0, 0, VDE }, + { "zerocopy", "Try to avoid copying input frames if possible", OFFSET(input_zerocopy), AV_OPT_TYPE_INT, { .i64 = CONFIG_OMX_RPI }, 0, 1, VE }, + { "profile", "Set the encoding profile", OFFSET(profile), AV_OPT_TYPE_INT, { .i64 = FF_PROFILE_UNKNOWN }, FF_PROFILE_UNKNOWN, FF_PROFILE_H264_HIGH, VE, "profile" }, + { "baseline", "", 0, AV_OPT_TYPE_CONST, { .i64 = FF_PROFILE_H264_BASELINE }, 0, 0, VE, "profile" }, + { "main", "", 0, AV_OPT_TYPE_CONST, { .i64 = FF_PROFILE_H264_MAIN }, 0, 0, VE, "profile" }, + { "high", "", 0, AV_OPT_TYPE_CONST, { .i64 = FF_PROFILE_H264_HIGH }, 0, 0, VE, "profile" }, + { NULL } +}; + + +static const AVClass omx_mpeg4dec_class = { + .class_name = "mpeg4_omx", + .item_name = av_default_item_name, + .option = options, + .version = LIBAVUTIL_VERSION_INT, +}; +AVCodec ff_mpeg4_omx_decoder = { + .name = "mpeg4_omx", + .long_name = NULL_IF_CONFIG_SMALL("OpenMAX IL MPEG-4 video decoder"), + .type = AVMEDIA_TYPE_VIDEO, + .id = AV_CODEC_ID_MPEG4, + .priv_data_size = sizeof(OMXCodecContext), + .init = omx_decode_init, + .decode = omx_decode_frame, + .close = omx_decode_end, + .capabilities = AV_CODEC_CAP_DELAY, + .caps_internal = FF_CODEC_CAP_INIT_THREADSAFE | FF_CODEC_CAP_INIT_CLEANUP, + .priv_class = &omx_mpeg4dec_class, +}; + +static const AVClass omx_h264dec_class = { + .class_name = "h264_omx", + .item_name = av_default_item_name, + .option = options, + .version = LIBAVUTIL_VERSION_INT, +}; +AVCodec ff_h264_omx_decoder = { + .name = "h264_omx", + .long_name = NULL_IF_CONFIG_SMALL("OpenMAX IL H.264 video decoder"), + .type = AVMEDIA_TYPE_VIDEO, + .id = AV_CODEC_ID_H264, + .priv_data_size = sizeof(OMXCodecContext), + .init = omx_decode_init, + .decode = omx_decode_frame, + .close = omx_decode_end, + .capabilities = AV_CODEC_CAP_DELAY, + .caps_internal = FF_CODEC_CAP_INIT_THREADSAFE | FF_CODEC_CAP_INIT_CLEANUP, + .priv_class = &omx_h264dec_class, +}; --- a/libavcodec/allcodecs.c +++ b/libavcodec/allcodecs.c @@ -823,6 +823,7 @@ extern const FFCodec ff_h264_mf_encoder; extern const FFCodec ff_h264_nvenc_encoder; extern const FFCodec ff_h264_omx_encoder; +extern const FFCodec ff_h264_omx_decoder; extern const FFCodec ff_h264_qsv_encoder; extern const FFCodec ff_h264_v4l2m2m_encoder; extern const FFCodec ff_h264_vaapi_encoder;