0009-ffmpeg-add-mjpeg-decoder-support-and-fix-some-bug.patch 8.6 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209
  1. From fff5724489cd0a503d206c6f513b1d8572247484 Mon Sep 17 00:00:00 2001
  2. From: "sw.multimedia" <se.multimedia@starfivetech.com>
  3. Date: Wed, 25 May 2022 21:28:23 +0800
  4. Subject: [PATCH] ffmpeg: add mjpeg decoder support and fix some bug
  5. Signed-off-by: sw.multimedia <se.multimedia@starfivetech.com>
  6. ---
  7. configure | 1 +
  8. libavcodec/allcodecs.c | 1 +
  9. libavcodec/omx.c | 25 +++++++++++++++++++++---
  10. libavcodec/omxdec.c | 44 +++++++++++++++++++++++++++++++++++++-----
  11. libavformat/utils.c | 7 +++++++
  12. 5 files changed, 70 insertions(+), 8 deletions(-)
  13. --- a/configure
  14. +++ b/configure
  15. @@ -3187,6 +3187,7 @@
  16. h264_v4l2m2m_encoder_deps="v4l2_m2m h264_v4l2_m2m"
  17. hevc_omx_encoder_deps="omx"
  18. hevc_omx_decoder_deps="omx"
  19. +mjpeg_omx_decoder_deps="omx"
  20. hevc_amf_encoder_deps="amf"
  21. hevc_cuvid_decoder_deps="cuvid"
  22. hevc_cuvid_decoder_select="hevc_mp4toannexb_bsf"
  23. --- a/libavcodec/allcodecs.c
  24. +++ b/libavcodec/allcodecs.c
  25. @@ -827,6 +827,7 @@
  26. extern const FFCodec ff_hevc_omx_encoder;
  27. extern const FFCodec ff_hevc_omx_decoder;
  28. extern const FFCodec ff_h264_qsv_encoder;
  29. +extern const FFCodec ff_mjpeg_omx_decoder;
  30. extern const FFCodec ff_h264_v4l2m2m_encoder;
  31. extern const FFCodec ff_h264_vaapi_encoder;
  32. extern const FFCodec ff_h264_videotoolbox_encoder;
  33. --- a/libavcodec/omx.c
  34. +++ b/libavcodec/omx.c
  35. @@ -43,6 +43,7 @@
  36. #include "avcodec.h"
  37. #include "codec_internal.h"
  38. #include "h264.h"
  39. +#include "hevc.h"
  40. #include "pthread_internal.h"
  41. #include "profiles.h"
  42. @@ -481,9 +482,11 @@
  43. in_port_params.format.video.nFrameWidth = avctx->width;
  44. in_port_params.format.video.nFrameHeight = avctx->height;
  45. if (avctx->framerate.den > 0 && avctx->framerate.num > 0)
  46. - in_port_params.format.video.xFramerate = (1LL << 16) * avctx->framerate.num / avctx->framerate.den;
  47. + //in_port_params.format.video.xFramerate = (1LL << 16) * avctx->framerate.num / avctx->framerate.den;
  48. + in_port_params.format.video.xFramerate = avctx->framerate.num / avctx->framerate.den;
  49. else
  50. - in_port_params.format.video.xFramerate = (1LL << 16) * avctx->time_base.den / avctx->time_base.num;
  51. + //in_port_params.format.video.xFramerate = (1LL << 16) * avctx->time_base.den / avctx->time_base.num;
  52. + in_port_params.format.video.xFramerate = avctx->time_base.den / avctx->time_base.num;
  53. err = OMX_SetParameter(s->handle, OMX_IndexParamPortDefinition, &in_port_params);
  54. CHECK(err);
  55. @@ -721,7 +724,23 @@
  56. }
  57. if (nals[H264_NAL_SPS] && nals[H264_NAL_PPS])
  58. break;
  59. - } else {
  60. + } else if (avctx->codec->id == AV_CODEC_ID_HEVC) {
  61. + // For H.265, the extradata can be returned in two separate buffers
  62. + // (the videocore encoder on raspberry pi does this);
  63. + // therefore check that we have got both SPS and PPS before continuing.
  64. + int nals[128] = { 0 };
  65. + int i;
  66. + for (i = 0; i + 4 < avctx->extradata_size; i++) {
  67. + if (!avctx->extradata[i + 0] &&
  68. + !avctx->extradata[i + 1] &&
  69. + !avctx->extradata[i + 2] &&
  70. + avctx->extradata[i + 3] == 1) {
  71. + nals[(avctx->extradata[i + 4] & 0x7E) >> 1]++;
  72. + }
  73. + }
  74. + if (nals[HEVC_NAL_SPS] && nals[HEVC_NAL_PPS] && nals[HEVC_NAL_VPS])
  75. + break;
  76. + } else {
  77. if (avctx->extradata_size > 0)
  78. break;
  79. }
  80. --- a/libavcodec/omxdec.c
  81. +++ b/libavcodec/omxdec.c
  82. @@ -106,6 +106,7 @@
  83. { OMX_COLOR_FormatYUV420Planar, AV_PIX_FMT_YUV420P },
  84. { OMX_COLOR_FormatYUV420SemiPlanar, AV_PIX_FMT_NV12 },
  85. { OMX_COLOR_FormatYUV420PackedSemiPlanar, AV_PIX_FMT_NV21 },
  86. + { OMX_COLOR_FormatYUV444Interleaved, AV_PIX_FMT_YUV444P },
  87. { 0 }
  88. };
  89. @@ -366,6 +367,7 @@
  90. dec_out_height = out_port_params.format.video.nFrameHeight;
  91. dec_pix_fmt = out_port_params.format.video.eColorFormat;
  92. + av_log(s->avctx, AV_LOG_VERBOSE, "w:%d, h:%d, fmt:%d\n", dec_out_width, dec_out_height, dec_pix_fmt);
  93. }
  94. }
  95. break;
  96. @@ -535,9 +537,11 @@
  97. in_port_params.format.video.nFrameHeight = avctx->height;
  98. if (avctx->framerate.den > 0 && avctx->framerate.num > 0)
  99. - in_port_params.format.video.xFramerate = (1LL << 16) * avctx->framerate.num / avctx->framerate.den;
  100. + //in_port_params.format.video.xFramerate = (1LL << 16) * avctx->framerate.num / avctx->framerate.den;
  101. + in_port_params.format.video.xFramerate = avctx->framerate.num / avctx->framerate.den;
  102. else
  103. - in_port_params.format.video.xFramerate = (1LL << 16) * avctx->time_base.den / avctx->time_base.num;
  104. + //in_port_params.format.video.xFramerate = (1LL << 16) * avctx->time_base.den / avctx->time_base.num;
  105. + in_port_params.format.video.xFramerate = avctx->time_base.den / avctx->time_base.num;
  106. err = OMX_SetParameter(s->handle, OMX_IndexParamPortDefinition, &in_port_params);
  107. CHECK(err);
  108. @@ -567,6 +571,8 @@
  109. out_port_params.format.video.eCompressionFormat = OMX_VIDEO_CodingAVC;
  110. else if (avctx->codec->id == AV_CODEC_ID_HEVC)
  111. out_port_params.format.video.eCompressionFormat = OMX_VIDEO_CodingHEVC;
  112. + else if (avctx->codec->id == AV_CODEC_ID_MJPEG)
  113. + out_port_params.format.video.eCompressionFormat = OMX_VIDEO_CodingMJPEG;
  114. err = OMX_SetParameter(s->handle, OMX_IndexParamPortDefinition, &out_port_params);
  115. CHECK(err);
  116. @@ -703,6 +709,9 @@
  117. case AV_CODEC_ID_HEVC:
  118. role = "video_decoder.hevc";
  119. break;
  120. + case AV_CODEC_ID_MJPEG:
  121. + role = "video_decoder.mjpeg";
  122. + break;
  123. default:
  124. return AVERROR(ENOSYS);
  125. }
  126. @@ -852,11 +861,13 @@
  127. !pkt || had_partial);
  128. if (!buffer) {
  129. - /*eos is sent wait for vpu evnet_bufferflag to get all frames*/
  130. -+ if(s->eos_sent && !evnet_bufferflag){}
  131. + /*eos is sent wait for vpu evnet_bufferflag to get all frames
  132. + mjpeg: sent a frame, then wait for a decoder frame
  133. + */
  134. + if((s->eos_sent && !evnet_bufferflag) || (avctx->codec_id == AV_CODEC_ID_MJPEG )) {
  135. continue;
  136. }
  137. - break;
  138. + break;
  139. }
  140. //if (!buffer)
  141. // break;
  142. @@ -924,6 +935,7 @@
  143. #define OFFSET(x) offsetof(OMXCodecContext, x)
  144. #define VDE AV_OPT_FLAG_VIDEO_PARAM | AV_OPT_FLAG_DECODING_PARAM | AV_OPT_FLAG_ENCODING_PARAM
  145. #define VE AV_OPT_FLAG_VIDEO_PARAM | AV_OPT_FLAG_ENCODING_PARAM
  146. +#define VD AV_OPT_FLAG_VIDEO_PARAM | AV_OPT_FLAG_DECODING_PARAM
  147. static const AVOption options[] = {
  148. { "omx_libname", "OpenMAX library name", OFFSET(libname), AV_OPT_TYPE_STRING, { 0 }, 0, 0, VDE },
  149. { "omx_libprefix", "OpenMAX library prefix", OFFSET(libprefix), AV_OPT_TYPE_STRING, { 0 }, 0, 0, VDE },
  150. @@ -980,6 +992,7 @@
  151. .capabilities = AV_CODEC_CAP_DELAY,
  152. .caps_internal = FF_CODEC_CAP_INIT_THREADSAFE | FF_CODEC_CAP_INIT_CLEANUP,
  153. .priv_class = &omx_h264dec_class,
  154. + .bsfs = "h264_mp4toannexb",
  155. };
  156. static const AVClass omx_hevcdec_class = {
  157. @@ -1001,4 +1014,25 @@
  158. .capabilities = AV_CODEC_CAP_DELAY,
  159. .caps_internal = FF_CODEC_CAP_INIT_THREADSAFE | FF_CODEC_CAP_INIT_CLEANUP,
  160. .priv_class = &omx_hevcdec_class,
  161. + .bsfs = "hevc_mp4toannexb",
  162. +};
  163. +
  164. +static const AVClass omx_mjpegdec_class = {
  165. + .class_name = "mjpeg_omx",
  166. + .item_name = av_default_item_name,
  167. + .version = LIBAVUTIL_VERSION_INT,
  168. +};
  169. +AVCodec ff_mjpeg_omx_decoder = {
  170. + .name = "mjpeg_omx",
  171. + .long_name = NULL_IF_CONFIG_SMALL("OpenMAX IL mjpeg video decoder"),
  172. + .type = AVMEDIA_TYPE_VIDEO,
  173. + .id = AV_CODEC_ID_MJPEG,
  174. + .priv_data_size = sizeof(OMXCodecContext),
  175. + .init = omx_decode_init,
  176. + .decode = omx_decode_frame,
  177. + .close = omx_decode_end,
  178. + .capabilities = AV_CODEC_CAP_DR1,
  179. + .max_lowres = 3,
  180. + .caps_internal = FF_CODEC_CAP_INIT_THREADSAFE,
  181. + .priv_class = &omx_mjpegdec_class,
  182. };
  183. --- a/libavformat/demux.c
  184. +++ b/libavformat/demux.c
  185. @@ -84,6 +84,13 @@
  186. return avcodec_find_decoder_by_name("hevc");
  187. #endif
  188. +#if CONFIG_MJPEG_DECODER
  189. + /* Other parts of the code assume this decoder to be used for mjpeg,
  190. + * so force it if possible. */
  191. + if (codec_id == AV_CODEC_ID_MJPEG)
  192. + return avcodec_find_decoder_by_name("mjpeg");
  193. +#endif
  194. +
  195. codec = ff_find_decoder(s, st, codec_id);
  196. if (!codec)
  197. return NULL;