Browse Source

Add media - gstreamer and ffmpeg source code with patches

Clement 1 year ago
parent
commit
652c1f3785
58 changed files with 7102 additions and 0 deletions
  1. 138 0
      multimedia/README.md
  2. 186 0
      multimedia/build.sh
  3. BIN
      multimedia/ffmpeg-4.4.1.tar.xz
  4. BIN
      multimedia/gst-omx-1.18.5.tar.xz
  5. BIN
      multimedia/gst-plugins-bad-1.18.5.tar.xz
  6. BIN
      multimedia/gst-plugins-base-1.18.5.tar.xz
  7. BIN
      multimedia/gst-plugins-good-1.18.5.tar.xz
  8. BIN
      multimedia/gst-plugins-ugly-1.18.5.tar.xz
  9. BIN
      multimedia/gstreamer-1.18.5.tar.xz
  10. BIN
      multimedia/mm_libs-VF2_515_v2.4.1.tar.xz
  11. 43 0
      multimedia/patch/ffmpeg/0001-swscale-x86-yuv2rgb-Fix-build-without-SSSE3.patch
  12. 36 0
      multimedia/patch/ffmpeg/0002-avcodec-vaapi_h264-skip-decode-if-pic-has-no-slices.patch
  13. 72 0
      multimedia/patch/ffmpeg/0003-libavutil-Fix-mips-build.patch
  14. 47 0
      multimedia/patch/ffmpeg/0004-configure-add-extralibs-to-extralibs_xxx.patch
  15. 952 0
      multimedia/patch/ffmpeg/0005-avcodec-add-omx-decoder-support.patch
  16. 210 0
      multimedia/patch/ffmpeg/0006-add-hevc-decoder-and-encoder-support.patch
  17. 307 0
      multimedia/patch/ffmpeg/0007-avcoder-fix-decoder-bug.patch
  18. 83 0
      multimedia/patch/ffmpeg/0008-fix-omx-decoder-setting-pix-fmt-bug.patch
  19. 222 0
      multimedia/patch/ffmpeg/0009-ffmpeg-add-mjpeg-decoder-support-and-fix-some-bug.patch
  20. 48 0
      multimedia/patch/ffmpeg/0010-ffmpeg-support-to-change-gop.patch
  21. 59 0
      multimedia/patch/ffmpeg/0011-ffmpeg-add-delay-for-decoding.patch
  22. 120 0
      multimedia/patch/ffmpeg/0012-ffmpeg-add-omx_pix_fmt.patch
  23. 433 0
      multimedia/patch/ffmpeg/0013-ffmpeg-replace-the-indent-with-space.patch
  24. 159 0
      multimedia/patch/ffmpeg/0014-ffmpeg-add-pixel-foramt.patch
  25. 52 0
      multimedia/patch/ffmpeg/0015-ffmpeg-remove-delay.patch
  26. 209 0
      multimedia/patch/ffmpeg/0016-FFmpeg-omx-add-scale-option-for-omx.patch
  27. 265 0
      multimedia/patch/ffmpeg/0017-FFmpeg-omxdec-add-mirror-rotation-and-crop-option.patch
  28. 35 0
      multimedia/patch/ffmpeg/0018-FFmpeg-omx-gop-parameter-of-hevc-encoder.patch
  29. 34 0
      multimedia/patch/ffmpeg/0019-FFmpeg-omcdec-fix-mjpeg_omx-decoder-scale-option-bug.patch
  30. 72 0
      multimedia/patch/gstreamer1/gst1-plugins-bad/0001-add-I420.patch
  31. 1027 0
      multimedia/patch/gstreamer1/gst1-plugins-bad/0002-reset-drm-in-atomic-api.patch
  32. 25 0
      multimedia/patch/gstreamer1/gst1-plugins-bad/0003-add-default-param-for-kmssink.patch
  33. 45 0
      multimedia/patch/gstreamer1/gst1-plugins-base/0001-gst-libs-gst-video-gstvideoaggregator.c-fix-build-wi.patch
  34. 106 0
      multimedia/patch/gstreamer1/gst1-plugins-base/0002-quit-when-pipeline-interrupted.patch
  35. 69 0
      multimedia/patch/gstreamer1/sf-gst-omx/0001-add-starfive-support.patch
  36. 70 0
      multimedia/patch/gstreamer1/sf-gst-omx/0002-Fix-gst-omx-Enable-the-gst-omx-VPU-decoding-and-enco.patch
  37. 50 0
      multimedia/patch/gstreamer1/sf-gst-omx/0003-add-video-scale-support.patch
  38. 24 0
      multimedia/patch/gstreamer1/sf-gst-omx/0004-add-encoder-support.patch
  39. 19 0
      multimedia/patch/gstreamer1/sf-gst-omx/0005-rank-257-for-sf-codecs.patch
  40. 19 0
      multimedia/patch/gstreamer1/sf-gst-omx/0006-dont-invoke-USE_BUFFER-if-no-dmabuffer.patch
  41. 16 0
      multimedia/patch/gstreamer1/sf-gst-omx/0007-add-omxmjpegdec-support.patch
  42. 100 0
      multimedia/patch/gstreamer1/sf-gst-omx/0008-support-nv21-i422-y444-for-omxmjpegdec.patch
  43. 247 0
      multimedia/patch/gstreamer1/sf-gst-omx/0009-suport-usebuffer-mode-for-encoding.patch
  44. 144 0
      multimedia/patch/gstreamer1/sf-gst-omx/0010-add-property-framerate.patch
  45. 38 0
      multimedia/patch/gstreamer1/sf-gst-omx/0011-hanle-some-extra-profile-for-avc.patch
  46. 67 0
      multimedia/patch/gstreamer1/sf-gst-omx/0012-combine-sps-pps-header-to-idr.patch
  47. 62 0
      multimedia/patch/gstreamer1/sf-gst-omx/0013-Modify-sf-component-name-to-in-std-format.patch
  48. 16 0
      multimedia/patch/gstreamer1/sf-gst-omx/0014-support-nv21-for-omxh264_5dec.patch
  49. 72 0
      multimedia/patch/gstreamer1/sf-gst-omx/0015_Add_NV21_for_gstomxvideoenc_class.patch
  50. 86 0
      multimedia/patch/gstreamer1/sf-gst-omx/0016-Modify-gstomxmjpegdec-format.patch
  51. 388 0
      multimedia/patch/gstreamer1/sf-gst-omx/0017-support-mirror-rotation-scale-for-gstomxmjpegdec.patch
  52. 194 0
      multimedia/patch/gstreamer1/sf-gst-omx/0018-support-cut-for-gstomxmjpegdec.patch
  53. 125 0
      multimedia/patch/mm_libs/0001-wave511-fix-building-issue.patch
  54. 124 0
      multimedia/patch/mm_libs/0002-wave420l-fix-building-issue.patch
  55. 58 0
      multimedia/patch/mm_libs/0003-codaj12-fix-building-issue.patch
  56. 88 0
      multimedia/patch/mm_libs/0004-omx-il-fix-building-issue.patch
  57. 41 0
      multimedia/run_env.sh
  58. BIN
      multimedia/target.tar.xz

+ 138 - 0
multimedia/README.md

@@ -0,0 +1,138 @@
+
+
+## FFmpeg and GStreamer with OMX Plugin Building Guide
+
+
+
+### Building Files Struct:
+
+```
+├── build.sh                 # building script
+├── ffmpeg-4.4.1.tar.xz      # ffmpeg src from offical site
+├── gst-omx-1.18.5.tar.xz    
+├── gst-plugins-bad-1.18.5.tar.xz
+├── gst-plugins-base-1.18.5.tar.xz
+├── gst-plugins-good-1.18.5.tar.xz
+├── gst-plugins-ugly-1.18.5.tar.xz
+├── gstreamer-1.18.5.tar.xz  # gst offical source code
+├── mm_libs-VF2_515_vx.x.x.tar.xz   # tarball version include wave511, wave420l, codaj12, omx-il
+├── patch
+│   ├── ffmpeg               # ffmpeg patch of tag VF2_515_vx.x.x
+│   ├── gstreamer1           # gstreamer patch  of tag VF2_515_vx.x.x
+│   └── mm_libs              # wave511, wave420l, codaj12, omx-il patchs to fix the building issue on debian
+├── README.md
+├── run_env.sh               # run env script after build completed
+
+```
+
+### Prepare:
+
+Before build, the debian need to install the below packages:
+
+```
+# apt install build-essential libchromaprint-dev libaom-dev liblilv-dev libiec61883-dev libass-dev \
+    libbluray-dev libbs2b-dev libcodec2-dev libdav1d-dev flite1-dev libgme-dev \
+    libgsm1-dev libmp3lame-dev libmysofa-dev libopenjp2-7-dev libopenmpt-dev \
+    libopus-dev librabbitmq-dev libssl-dev librubberband-dev libsamplerate0-dev \
+    libshine-dev libsnappy-dev libsoxr-dev libsoxr-lsr0 libspeex-dev libssh-4 \
+    libssh-dev libv4l-0 libv4l-dev libtheora-dev libtwolame-dev libvidstab-dev \
+    libvpx-dev libwebp-dev libwebpdemux2 libx264-dev libx265-dev libxvidcore-dev \
+    libzimg-dev libzvbi-dev libopenal-dev ocl-icd-opencl-dev opencl-c-headers \
+    opencl-clhpp-headers javascript-common libjs-jquery libpocketsphinx-dev \
+    libsphinxbase-dev libgnutls28-dev libsrt-openssl-dev libsrt-gnutls-dev \
+    libsrt1.4-openssl libcaca-dev libdc1394-dev \
+    libjack-dev libcdio-dev libcdparanoia-dev libcdio-paranoia-dev librsvg2-dev libzmq3-dev \
+    meson ninja-build flex bison \
+    libudev-dev libgudev-1.0-dev \
+    libx11-dev libxext-dev libxv-dev libxfixes-dev libxdamage-dev \
+    libasound2-dev libflac-dev
+ 
+ Note: Need to remove the gstreamer installed from upstream repo
+ # apt purge libgstreamer1.0-0 libgstreamer-plugins-base1.0-0 gstreamer1.0-plugins-base
+ 
+```
+
+The `mm_libs-JH7110_MM_vxxxx.tar.xz` could be generated by the below, first git clone the usdk code
+
+```
+# git clone git@gitlab.starfivetech.com:sbc/visionfive.git
+# git checkout VF2_515_vx.x.x    # checkout to the tag VF2_515_vx.x.x
+# cd visionfive/soft_3rdpart
+# tar cvJf mm_libs-VF2_515_vx.x.x.tar.xz --exclude=codaj12/stream --exclude=codaj12/yuv --exclude=wave420l/code/yuv --exclude=wave511/media --exclude=wave511/code/stream --exclude=wave511/code/yuv codaj12 omx-il wave420l wave511
+```
+
+### How to Build:
+
+Run the below:
+
+```
+# ./build.sh
+```
+
+the ffmpeg and gstreamer will be built and installed to path `target`
+
+```
+# tree  target/ -L 2
+target/
+├── lib
+│   └── firmware
+├── root
+│   ├── codaj12
+│   ├── wave420l
+│   └── wave511
+└── usr
+    ├── bin
+    ├── etc
+    ├── include
+    ├── lib
+    ├── libexec
+    └── share
+```
+
+The ffmpeg and gstreamer will be both built by default. If want to only build ffmpeg or only build gstreamer, can modify the below in the `build.sh`
+
+only ffmpeg:
+
+```
+Building_FFmpeg="enable"
+Building_GStreamer="disable"
+```
+
+only gstreamer:
+
+```
+Building_FFmpeg="disable"
+Building_GStreamer="enable"
+```
+
+### How to Run:
+
+Note the wave511 driver vdec.ko, wave420l driver venc.ko, codaj12 driver jpu.ko not built in this. These *.ko driver should match the corresponding linux kernel image. They can be built under the usdk path `work/buildroot_initramfs_sysroot/root/wave511/` ,`work/buildroot_initramfs_sysroot/root/wave420l/`, `work/buildroot_initramfs_sysroot/root/codaj12/`
+
+Then copy these *.ko driver to `target/root/wave511`,  `target/root/wave420l`, `target/root/codaj12`
+
+Then run the below:
+
+for ffmpeg:
+
+```
+# source run_env.sh
+# target/usr/bin/ffmpeg  -vcodec h264_omx -i Sintel_720_10s_10MB.mp4 -pix_fmt yuv420p 720p_i420.yuv
+# ffplay  -vcodec h264_omx -i Sintel_720_10s_10MB.mp4
+Note: ffplay had problem, will fix it asap
+
+
+```
+
+or for gstreamer:
+
+```
+# source run_env.sh
+# gst-launch-1.0 filesrc location=Audio_Video_Sync_Test_1_1920x1080.mp4 ! qtdemux name=demux demux.video_0 ! queue ! h264parse ! omxh264dec ! xvimagesink
+# gst-launch-1.0 filesrc location=4K_30FPS_AVC_MainL5_2.h265 ! h265parse ! omxh265dec ! videoconvert ! videoscale ! xvimagesink
+# gst-launch-1.0 filesrc location=youtube001_1080p_h264_aac.mp4 ! qtdemux name=demux demux.video_0 ! queue ! h264parse ! omxh264dec ! xvimagesink
+# gst-launch-1.0 filesrc location=2k.18fps.mp4 ! qtdemux name=demux demux.video_0 ! queue ! h265parse ! omxh265dec ! videoconvert ! videoscale ! xvimagesink
+# gst-launch-1.0 filesrc location=youtube002_1080p_h264_aac.mp4 ! qtdemux name=demux demux.video_0 ! queue ! h264parse ! omxh264dec ! xvimagesink
+# gst-launch-1.0 filesrc location=file_example_MP4_640_3MG.mp4 ! qtdemux name=demux demux.video_0 ! queue ! h264parse ! omxh264dec ! xvimagesink
+```
+

+ 186 - 0
multimedia/build.sh

@@ -0,0 +1,186 @@
+#!/bin/bash
+##################################################################
+##                                                              ##
+##      SPDX-License-Identifier: GPL-2.0-or-later               ##
+##                                                              ##
+##      Copyright (C) 2018-2022   Starfive Technology           ##
+##                                                              ##
+##################################################################
+set -e
+CURDIR=$(cd `dirname $0`; pwd)
+cd $CURDIR
+rm -rf $CURDIR/target/
+
+Building_FFmpeg="enable"
+Building_GStreamer="enable"
+
+function apply_patch {
+    local i
+    local srcdir="${1}"
+    local patchdir="${2}"
+
+    for i in `cd $patchdir; ls -d *.patch 2> /dev/null` ; do
+        echo "Applying $patchdir/$i "
+        cat $patchdir/$i | patch -g0 -p1 -E --no-backup-if-mismatch -d $srcdir -t -N
+        if [ $? != 0 ] ; then
+            echo "Patch failed!  Please fix $patchdir/$i!"
+            exit 1
+        fi
+    done
+}
+
+##################################################################
+## Install third-party packages depended by ffmpeg and gstreamer
+if [ 1 != 1 ] ; then
+    apt install build-essential libchromaprint-dev libaom-dev liblilv-dev libiec61883-dev libass-dev \
+    libbluray-dev libbs2b-dev libcodec2-dev libdav1d-dev flite1-dev libgme-dev \
+    libgsm1-dev libmp3lame-dev libmysofa-dev libopenjp2-7-dev libopenmpt-dev \
+    libopus-dev librabbitmq-dev libssl-dev librubberband-dev libsamplerate0-dev \
+    libshine-dev libsnappy-dev libsoxr-dev libsoxr-lsr0 libspeex-dev libssh-4 \
+    libssh-dev libv4l-0 libv4l-dev libtheora-dev libtwolame-dev libvidstab-dev \
+    libvpx-dev libwebp-dev libwebpdemux2 libx264-dev libx265-dev libxvidcore-dev \
+    libzimg-dev libzvbi-dev libopenal-dev ocl-icd-opencl-dev opencl-c-headers \
+    opencl-clhpp-headers javascript-common libjs-jquery libpocketsphinx-dev \
+    libsphinxbase-dev libgnutls28-dev libsrt-openssl-dev libsrt-gnutls-dev \
+    libsrt1.4-openssl libcaca-dev libdc1394-dev \
+    libjack-dev libcdio-dev libcdparanoia-dev libcdio-paranoia-dev librsvg2-dev libzmq3-dev \
+    meson ninja-build flex bison \
+    libudev-dev libgudev-1.0-dev \
+    libx11-dev libxext-dev libxv-dev libxfixes-dev libxdamage-dev \
+    libasound2-dev libflac-dev
+    
+    apt purge libgstreamer1.0-0 libgstreamer-plugins-base1.0-0 gstreamer1.0-plugins-base
+    sync
+fi
+
+##################################################################
+## Extract and patch for wave511, wave420l, codaj12, omx-il
+echo "Extract and patch wave511, wave420l, codaj12, omx-il..."
+mm_libs_TARBALL=mm_libs-VF2_515_v2.4.1.tar.xz
+mm_libs_PATCH=patch/mm_libs
+rm -rf wave511/ wave420l/ codaj12/ omx-il/
+tar xf ${mm_libs_TARBALL}
+mv wave511/code wave511/wave511
+sync
+apply_patch ./ $mm_libs_PATCH
+
+## Extract and patch for gstreamer and ffmpeg
+GST_TARBALLS=("gstreamer-1.18.5" "gst-plugins-base-1.18.5" "gst-plugins-good-1.18.5" "gst-plugins-bad-1.18.5" "gst-plugins-ugly-1.18.5" "gst-omx-1.18.5")
+GST_PATCHS=("gstreamer1/gstreamer1" "gstreamer1/gst1-plugins-base" "gstreamer1/gst1-plugins-good" "gstreamer1/gst1-plugins-bad" "gstreamer1/gst1-plugins-ugly" "gstreamer1/sf-gst-omx")
+PKG_TARBALLS=(${GST_TARBALLS[@]} "ffmpeg-4.4.1")
+PKG_PATCHS=(${GST_PATCHS[@]} "ffmpeg")
+PKG_NUMS=${#PKG_TARBALLS[@]}
+for ((i=0; i<$PKG_NUMS; i++)); do
+    pkg_src=${PKG_TARBALLS[$i]}
+    pkg_tarball=$pkg_src.tar.xz
+    pkg_patchs=patch/${PKG_PATCHS[$i]}
+
+    rm -rf $pkg_src
+    echo "Extract $pkg_tarball and patch $pkg_patchs"
+    tar xf $pkg_tarball
+    sync
+    apply_patch $pkg_src $pkg_patchs
+done
+
+##################################################################
+## build wave511
+pushd wave511/wave511/
+echo "Building video decoder wave511..."
+make -f WaveDecode_buildroot.mak clean
+make -f WaveDecode_buildroot.mak
+make -f WaveDecode_buildroot.mak install
+sync
+popd
+
+## build wave420l
+pushd wave420l/code/
+echo "Building video encoder wave420l..."
+make -f WaveEncoder_buildroot.mak clean
+make -f WaveEncoder_buildroot.mak
+make -f WaveEncoder_buildroot.mak install
+sync
+popd
+
+## build codaj12
+pushd codaj12/
+echo "Building jpeg decoder codaj12..."
+make -f codaj12_buildroot.mak clean
+make -f codaj12_buildroot.mak
+make -f codaj12_buildroot.mak install
+sync
+popd
+
+## build omx-il
+pushd omx-il/
+echo "Building omx-il library..."
+make clean && make
+make install
+sync
+popd
+
+## install system
+## install chagall.bin/encoder_defconfig.cfg/monet.bin to /lib/firmware/
+#sudo cp -ar target/lib/firmware/* /lib/firmware/
+
+##################################################################
+## build ffmpeg
+if [ "$Building_FFmpeg"X = "enable"X ]; then
+    ffmpeg_src=ffmpeg-4.4.1
+    pushd $ffmpeg_src
+    echo "Building ffmpeg start..."
+    ./configure --prefix=$CURDIR/target/usr  --arch="riscv64" --target-os="linux" \
+    --enable-gpl --disable-stripping --disable-static --enable-shared --enable-avfilter \
+    --disable-version3 --enable-logging --disable-extra-warnings --enable-avdevice \
+    --enable-avcodec --enable-avformat --enable-network --disable-gray --enable-swscale-alpha \
+    --disable-small --enable-dct --enable-fft --enable-mdct --enable-rdft --enable-libv4l2 \
+    --enable-alsa --enable-outdevs --enable-pthreads --enable-zlib --enable-indevs \
+    --enable-runtime-cpudetect --enable-pic --cpu="rv64imafd" --enable-libaom --enable-libass \
+    --enable-libbluray --enable-libbs2b --enable-libcaca --enable-libcdio --enable-libcodec2 \
+    --enable-libdav1d --enable-libflite --enable-libfontconfig --enable-libfreetype \
+    --enable-libfribidi --enable-libgme --enable-libgsm --enable-libjack --enable-libmp3lame \
+    --enable-libmysofa --enable-libopenjpeg --enable-libopenmpt --enable-libopus \
+    --enable-libpulse --enable-librabbitmq --enable-librubberband --enable-libshine \
+    --enable-libsnappy --enable-libsoxr --enable-libspeex --enable-libsrt --enable-libssh \
+    --enable-libtheora --enable-libtwolame --enable-libvidstab --enable-libvorbis \
+    --enable-libvpx --enable-libwebp --enable-libx265 --enable-libxml2 --enable-libxvid \
+    --enable-libzimg --enable-libzmq --enable-libzvbi --enable-lv2 --enable-openal \
+    --enable-opencl --enable-opengl --enable-sdl2 --enable-pocketsphinx --enable-librsvg \
+    --enable-libdc1394 --enable-libdrm  --enable-chromaprint --enable-libx264 \
+    --disable-frei0r --disable-gnutls --disable-ladspa --disable-libiec61883 --enable-omx \
+    --extra-ldflags="-L$CURDIR/target/usr/lib" \
+    --extra-cflags="-I$CURDIR/target/usr/include/omx-il" \
+    --extra-libs="-lOMX_Core"
+    make -j$(nproc)
+    make install
+    sync
+    echo "Building ffmpeg end..."
+    popd
+fi
+
+##################################################################
+## build GStreamer
+if [ "$Building_GStreamer"X = "enable"X ]; then
+    GST_CONFIGS=("--libdir=lib --default-library=shared --buildtype=release -Dstrip=true" \
+        "--libdir=lib --default-library=shared --buildtype=release -Dstrip=true" \
+        "--libdir=lib --default-library=shared --buildtype=release -Dstrip=true -Dcairo=disabled" \
+        "--libdir=lib --default-library=shared --buildtype=release -Dstrip=true" \
+        "--libdir=lib --default-library=shared --buildtype=release -Dstrip=true" \
+        "--libdir=lib --default-library=shared --buildtype=release -Dstrip=true -Dtarget=stf -Dheader_path=$CURDIR/target/usr/include/omx-il" \
+        )
+
+    echo "Building GStreamer start..."
+    GST_NUMS=${#GST_TARBALLS[@]}
+    for ((i=0; i<$GST_NUMS; i++)); do
+        gst_src=${GST_TARBALLS[$i]}
+        gst_config=${GST_CONFIGS[$i]}
+
+        pushd $gst_src
+        mkdir -p $gst_src/build
+        PKG_CONFIG_PATH=$CURDIR/target/usr/lib/pkgconfig meson --prefix=$CURDIR/target/usr $gst_config $CURDIR/$gst_src $CURDIR/$gst_src/build
+        ninja -C $CURDIR/$gst_src/build -j $(nproc)
+        ninja -C $CURDIR/$gst_src/build install
+        sync
+        popd
+    done
+    echo "Building GStreamer end..."
+fi

BIN
multimedia/ffmpeg-4.4.1.tar.xz


BIN
multimedia/gst-omx-1.18.5.tar.xz


BIN
multimedia/gst-plugins-bad-1.18.5.tar.xz


BIN
multimedia/gst-plugins-base-1.18.5.tar.xz


BIN
multimedia/gst-plugins-good-1.18.5.tar.xz


BIN
multimedia/gst-plugins-ugly-1.18.5.tar.xz


BIN
multimedia/gstreamer-1.18.5.tar.xz


BIN
multimedia/mm_libs-VF2_515_v2.4.1.tar.xz


+ 43 - 0
multimedia/patch/ffmpeg/0001-swscale-x86-yuv2rgb-Fix-build-without-SSSE3.patch

@@ -0,0 +1,43 @@
+From bf8eae2d30a205bb76ac625e04c0351106981b9e Mon Sep 17 00:00:00 2001
+From: Parker Ernest <@>
+Date: Sun, 23 Feb 2020 13:22:42 +0100
+Subject: [PATCH] swscale/x86/yuv2rgb: Fix build without SSSE3
+
+commit fc6a5883d6af8cae0e96af84dda0ad74b360a084 breaks build on
+x86_64 CPUs which do not have SSSE3, e.g. AMD Phenom-II
+
+Signed-off-by: Michael Niedermayer <michael@niedermayer.cc>
+[bernd.kuhls@t-online.de:
+  - retrieved from upstream patchwork:
+    https://patchwork.ffmpeg.org/project/ffmpeg/patch/20200223122256.23402-1-michael@niedermayer.cc/
+]
+Signed-off-by: Bernd Kuhls <bernd.kuhls@t-online.de>
+[yann.morin.1998@free.fr: fix reference top patchwork and SoB order]
+Signed-off-by: Yann E. MORIN <yann.morin.1998@free.fr>
+---
+ libswscale/x86/yuv2rgb.c | 2 ++
+ 1 file changed, 2 insertions(+)
+
+diff --git a/libswscale/x86/yuv2rgb.c b/libswscale/x86/yuv2rgb.c
+index c12e88cbb5..4791e5b93a 100644
+--- a/libswscale/x86/yuv2rgb.c
++++ b/libswscale/x86/yuv2rgb.c
+@@ -83,6 +83,7 @@ av_cold SwsFunc ff_yuv2rgb_init_x86(SwsContext *c)
+ #if HAVE_X86ASM
+     int cpu_flags = av_get_cpu_flags();
+ 
++#if HAVE_SSSE3
+     if (EXTERNAL_SSSE3(cpu_flags)) {
+         switch (c->dstFormat) {
+         case AV_PIX_FMT_RGB32:
+@@ -111,6 +112,7 @@ av_cold SwsFunc ff_yuv2rgb_init_x86(SwsContext *c)
+             return yuv420_rgb15_ssse3;
+         }
+     }
++#endif
+ 
+     if (EXTERNAL_MMXEXT(cpu_flags)) {
+         switch (c->dstFormat) {
+-- 
+2.26.2
+

+ 36 - 0
multimedia/patch/ffmpeg/0002-avcodec-vaapi_h264-skip-decode-if-pic-has-no-slices.patch

@@ -0,0 +1,36 @@
+From 2c6b3f357331e203ad87214984661c40704aceb7 Mon Sep 17 00:00:00 2001
+From: Rainer Hochecker <fernetmenta@online.de>
+Date: Sat, 26 Jan 2019 19:48:35 +0100
+Subject: [PATCH] avcodec/vaapi_h264: skip decode if pic has no slices
+
+This fixes / workarounds https://bugs.freedesktop.org/show_bug.cgi?id=105368.
+It was hit frequently when watching h264 channels received via DVB-X.
+Corresponding kodi bug: https://github.com/xbmc/xbmc/issues/15704
+
+Downloaded from Kodi ffmpeg repo:
+https://github.com/xbmc/FFmpeg/commit/2c6b3f357331e203ad87214984661c40704aceb7
+
+Patch was sent upstream:
+http://ffmpeg.org/pipermail/ffmpeg-devel/2019-March/240863.html
+
+Signed-off-by: Bernd Kuhls <bernd.kuhls@t-online.de>
+---
+ libavcodec/vaapi_h264.c | 5 +++++
+ 1 file changed, 5 insertions(+)
+
+diff --git a/libavcodec/vaapi_h264.c b/libavcodec/vaapi_h264.c
+index dd2a6571604..e521a05c4ff 100644
+--- a/libavcodec/vaapi_h264.c
++++ b/libavcodec/vaapi_h264.c
+@@ -314,6 +314,11 @@ static int vaapi_h264_end_frame(AVCodecContext *avctx)
+     H264SliceContext *sl = &h->slice_ctx[0];
+     int ret;
+ 
++    if (pic->nb_slices == 0) {
++        ret = AVERROR_INVALIDDATA;
++        goto finish;
++    }
++
+     ret = ff_vaapi_decode_issue(avctx, pic);
+     if (ret < 0)
+         goto finish;

+ 72 - 0
multimedia/patch/ffmpeg/0003-libavutil-Fix-mips-build.patch

@@ -0,0 +1,72 @@
+From 6e8daf0d502a2a822f1f08f42368d7d676dc1a9e Mon Sep 17 00:00:00 2001
+From: Bernd Kuhls <bernd.kuhls@t-online.de>
+Date: Sun, 6 Jun 2021 08:54:53 +0200
+Subject: [PATCH] libavutil: Fix mips build
+
+Check for sys/auxv.h because not all toolchains contain this header.
+
+Fixes https://trac.ffmpeg.org/ticket/9138
+
+Patch sent upstream:
+http://ffmpeg.org/pipermail/ffmpeg-devel/2021-June/281272.html
+
+Signed-off-by: Bernd Kuhls <bernd.kuhls@t-online.de>
+---
+ configure            | 2 ++
+ libavutil/mips/cpu.c | 6 +++---
+ 2 files changed, 5 insertions(+), 3 deletions(-)
+
+diff --git a/configure b/configure
+index 6bfd98b384..773a7d516c 100755
+--- a/configure
++++ b/configure
+@@ -2161,6 +2161,7 @@ HEADERS_LIST="
+     opencv2_core_core_c_h
+     OpenGL_gl3_h
+     poll_h
++    sys_auxv_h
+     sys_param_h
+     sys_resource_h
+     sys_select_h
+@@ -6218,6 +6219,7 @@ check_func_headers VideoToolbox/VTCompressionSession.h VTCompressionSessionPrepa
+ check_headers windows.h
+ check_headers X11/extensions/XvMClib.h
+ check_headers asm/types.h
++check_headers sys/auxv.h
+ 
+ # it seems there are versions of clang in some distros that try to use the
+ # gcc headers, which explodes for stdatomic
+diff --git a/libavutil/mips/cpu.c b/libavutil/mips/cpu.c
+index 59619d54de..19196de50b 100644
+--- a/libavutil/mips/cpu.c
++++ b/libavutil/mips/cpu.c
+@@ -19,7 +19,7 @@
+ #include "libavutil/cpu.h"
+ #include "libavutil/cpu_internal.h"
+ #include "config.h"
+-#if defined __linux__ || defined __ANDROID__
++#if (defined __linux__ || defined __ANDROID__) && HAVE_SYS_AUXV_H
+ #include <stdint.h>
+ #include <stdio.h>
+ #include <string.h>
+@@ -28,7 +28,7 @@
+ #include "libavutil/avstring.h"
+ #endif
+ 
+-#if defined __linux__ || defined __ANDROID__
++#if (defined __linux__ || defined __ANDROID__) && HAVE_SYS_AUXV_H
+ 
+ #define HWCAP_LOONGSON_CPUCFG (1 << 14)
+ 
+@@ -105,7 +105,7 @@ static int cpu_flags_cpuinfo(void)
+ 
+ int ff_get_cpu_flags_mips(void)
+ {
+-#if defined __linux__ || defined __ANDROID__
++#if (defined __linux__ || defined __ANDROID__) && HAVE_SYS_AUXV_H
+     if (cpucfg_available())
+         return cpu_flags_cpucfg();
+     else
+-- 
+2.29.2
+

+ 47 - 0
multimedia/patch/ffmpeg/0004-configure-add-extralibs-to-extralibs_xxx.patch

@@ -0,0 +1,47 @@
+From 0c288853630b7b4e004774c39945d4a804afcfa8 Mon Sep 17 00:00:00 2001
+From: Fabrice Fontaine <fontaine.fabrice@gmail.com>
+Date: Fri, 6 Aug 2021 09:17:20 +0200
+Subject: [PATCH] configure: add extralibs to extralibs_xxx
+
+Add extralibs to extralibs_xxx (e.g. extralibs_avformat) to allow
+applications such as motion to retrieve ffmpeg dependencies such as
+-latomic through pkg-config
+
+Signed-off-by: Fabrice Fontaine <fontaine.fabrice@gmail.com>
+[Upstream status: not upstreamable]
+---
+ configure | 18 +++++++++---------
+ 1 file changed, 9 insertions(+), 9 deletions(-)
+
+diff --git a/configure b/configure
+index 0bb3a7cf2b..3bda99e415 100755
+--- a/configure
++++ b/configure
+@@ -7602,15 +7602,15 @@ rpath=$(enabled rpath && echo "-Wl,-rpath,\${libdir}")
+ source_path=${source_path}
+ LIBPREF=${LIBPREF}
+ LIBSUF=${LIBSUF}
+-extralibs_avutil="$avutil_extralibs"
+-extralibs_avcodec="$avcodec_extralibs"
+-extralibs_avformat="$avformat_extralibs"
+-extralibs_avdevice="$avdevice_extralibs"
+-extralibs_avfilter="$avfilter_extralibs"
+-extralibs_avresample="$avresample_extralibs"
+-extralibs_postproc="$postproc_extralibs"
+-extralibs_swscale="$swscale_extralibs"
+-extralibs_swresample="$swresample_extralibs"
++extralibs_avutil="$avutil_extralibs $extralibs"
++extralibs_avcodec="$avcodec_extralibs $extralibs"
++extralibs_avformat="$avformat_extralibs $extralibs"
++extralibs_avdevice="$avdevice_extralibs $extralibs"
++extralibs_avfilter="$avfilter_extralibs $extralibs"
++extralibs_avresample="$avresample_extralibs $extralibs"
++extralibs_postproc="$postproc_extralibs $extralibs"
++extralibs_swscale="$swscale_extralibs $extralibs"
++extralibs_swresample="$swresample_extralibs $extralibs"
+ EOF
+ 
+ for lib in $LIBRARY_LIST; do
+-- 
+2.30.2
+

+ 952 - 0
multimedia/patch/ffmpeg/0005-avcodec-add-omx-decoder-support.patch

@@ -0,0 +1,952 @@
+From a47ead72e315cdd497af174e81eb72aa9361d598 Mon Sep 17 00:00:00 2001
+From: "sw.multimedia" <se.multimedia@starfivetech.com>
+Date: Wed, 25 May 2022 20:05:11 +0800
+Subject: [PATCH 1/8] avcodec add omx decoder support
+
+Signed-off-by: sw.multimedia <se.multimedia@starfivetech.com>
+---
+ configure              |   1 +
+ libavcodec/Makefile    |   1 +
+ libavcodec/allcodecs.c |   1 +
+ libavcodec/omxdec.c    | 893 +++++++++++++++++++++++++++++++++++++++++
+ 4 files changed, 896 insertions(+)
+ create mode 100644 libavcodec/omxdec.c
+
+diff --git a/configure b/configure
+index cdf6bb1..e3976df 100755
+--- a/configure
++++ b/configure
+@@ -3084,6 +3084,7 @@ h264_mmal_decoder_deps="mmal"
+ h264_nvenc_encoder_deps="nvenc"
+ h264_nvenc_encoder_select="atsc_a53"
+ h264_omx_encoder_deps="omx"
++h264_omx_decoder_deps="omx"
+ h264_qsv_decoder_select="h264_mp4toannexb_bsf qsvdec"
+ h264_qsv_encoder_select="atsc_a53 qsvenc"
+ h264_rkmpp_decoder_deps="rkmpp"
+diff --git a/libavcodec/Makefile b/libavcodec/Makefile
+index 33a280c..372d7de 100644
+--- a/libavcodec/Makefile
++++ b/libavcodec/Makefile
+@@ -379,6 +379,7 @@ OBJS-$(CONFIG_H264_NVENC_ENCODER)      += nvenc_h264.o
+ OBJS-$(CONFIG_NVENC_ENCODER)           += nvenc_h264.o
+ OBJS-$(CONFIG_NVENC_H264_ENCODER)      += nvenc_h264.o
+ OBJS-$(CONFIG_H264_OMX_ENCODER)        += omx.o
++OBJS-$(CONFIG_H264_OMX_DECODER)        += omxdec.o
+ OBJS-$(CONFIG_H264_QSV_DECODER)        += qsvdec.o
+ OBJS-$(CONFIG_H264_QSV_ENCODER)        += qsvenc_h264.o
+ OBJS-$(CONFIG_H264_RKMPP_DECODER)      += rkmppdec.o
+diff --git a/libavcodec/allcodecs.c b/libavcodec/allcodecs.c
+index 2e9a358..e42fa30 100644
+--- a/libavcodec/allcodecs.c
++++ b/libavcodec/allcodecs.c
+@@ -792,6 +792,7 @@ extern AVCodec ff_h264_cuvid_decoder;
+ extern AVCodec ff_h264_mf_encoder;
+ extern AVCodec ff_h264_nvenc_encoder;
+ extern AVCodec ff_h264_omx_encoder;
++extern AVCodec ff_h264_omx_decoder;
+ extern AVCodec ff_h264_qsv_encoder;
+ extern AVCodec ff_h264_v4l2m2m_encoder;
+ extern AVCodec ff_h264_vaapi_encoder;
+diff --git a/libavcodec/omxdec.c b/libavcodec/omxdec.c
+new file mode 100644
+index 0000000..7f76ec3
+--- /dev/null
++++ b/libavcodec/omxdec.c
+@@ -0,0 +1,893 @@
++/*
++ * OMX Video encoder
++ * Copyright (C) 2011 Martin Storsjo
++ *
++ * This file is part of FFmpeg.
++ *
++ * FFmpeg is free software; you can redistribute it and/or
++ * modify it under the terms of the GNU Lesser General Public
++ * License as published by the Free Software Foundation; either
++ * version 2.1 of the License, or (at your option) any later version.
++ *
++ * FFmpeg is distributed in the hope that it will be useful,
++ * but WITHOUT ANY WARRANTY; without even the implied warranty of
++ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
++ * Lesser General Public License for more details.
++ *
++ * You should have received a copy of the GNU Lesser General Public
++ * License along with FFmpeg; if not, write to the Free Software
++ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
++ */
++
++#include "config.h"
++
++#if CONFIG_OMX_RPI
++#define OMX_SKIP64BIT
++#endif
++
++#include <dlfcn.h>
++#include <OMX_Core.h>
++#include <OMX_Component.h>
++#include <pthread.h>
++#include <stdio.h>
++#include <stdlib.h>
++#include <sys/time.h>
++
++#include "libavutil/avstring.h"
++#include "libavutil/avutil.h"
++#include "libavutil/common.h"
++#include "libavutil/imgutils.h"
++#include "libavutil/log.h"
++#include "libavutil/opt.h"
++#include "libavutil/time.h"
++
++#include "avcodec.h"
++#include "h264.h"
++#include "internal.h"
++
++#ifdef OMX_SKIP64BIT
++static OMX_TICKS to_omx_ticks(int64_t value)
++{
++    OMX_TICKS s;
++    s.nLowPart  = value & 0xffffffff;
++    s.nHighPart = value >> 32;
++    return s;
++}
++static int64_t from_omx_ticks(OMX_TICKS value)
++{
++    return (((int64_t)value.nHighPart) << 32) | value.nLowPart;
++}
++#else
++#define to_omx_ticks(x) (x)
++#define from_omx_ticks(x) (x)
++#endif
++
++#define INIT_STRUCT(x) do {                                               \
++        x.nSize = sizeof(x);                                              \
++        x.nVersion = s->version;                                          \
++    } while (0)
++#define CHECK(x) do {                                                     \
++        if (x != OMX_ErrorNone) {                                         \
++            av_log(avctx, AV_LOG_ERROR,                                   \
++                   "err %x (%d) on line %d\n", x, x, __LINE__);           \
++            return AVERROR_UNKNOWN;                                       \
++        }                                                                 \
++    } while (0)
++
++typedef struct OMXContext {
++    void *lib;
++    void *lib2;
++    OMX_ERRORTYPE (*ptr_Init)(void);
++    OMX_ERRORTYPE (*ptr_Deinit)(void);
++    OMX_ERRORTYPE (*ptr_ComponentNameEnum)(OMX_STRING, OMX_U32, OMX_U32);
++    OMX_ERRORTYPE (*ptr_GetHandle)(OMX_HANDLETYPE*, OMX_STRING, OMX_PTR, OMX_CALLBACKTYPE*);
++    OMX_ERRORTYPE (*ptr_FreeHandle)(OMX_HANDLETYPE);
++    OMX_ERRORTYPE (*ptr_GetComponentsOfRole)(OMX_STRING, OMX_U32*, OMX_U8**);
++    OMX_ERRORTYPE (*ptr_GetRolesOfComponent)(OMX_STRING, OMX_U32*, OMX_U8**);
++    void (*host_init)(void);
++} OMXContext;
++
++static av_cold void *dlsym_prefixed(void *handle, const char *symbol, const char *prefix)
++{
++    char buf[50];
++    snprintf(buf, sizeof(buf), "%s%s", prefix ? prefix : "", symbol);
++    return dlsym(handle, buf);
++}
++
++static av_cold int omx_try_load(OMXContext *s, void *logctx,
++                                const char *libname, const char *prefix,
++                                const char *libname2)
++{
++    if (libname2) {
++        s->lib2 = dlopen(libname2, RTLD_NOW | RTLD_GLOBAL);
++        if (!s->lib2) {
++            av_log(logctx, AV_LOG_WARNING, "%s not found\n", libname2);
++            return AVERROR_ENCODER_NOT_FOUND;
++        }
++        s->host_init = dlsym(s->lib2, "bcm_host_init");
++        if (!s->host_init) {
++            av_log(logctx, AV_LOG_WARNING, "bcm_host_init not found\n");
++            dlclose(s->lib2);
++            s->lib2 = NULL;
++            return AVERROR_ENCODER_NOT_FOUND;
++        }
++    }
++    s->lib = dlopen(libname, RTLD_NOW | RTLD_GLOBAL);
++    if (!s->lib) {
++        av_log(logctx, AV_LOG_WARNING, "%s not found\n", libname);
++        return AVERROR_ENCODER_NOT_FOUND;
++    }
++    s->ptr_Init                = dlsym_prefixed(s->lib, "OMX_Init", prefix);
++    s->ptr_Deinit              = dlsym_prefixed(s->lib, "OMX_Deinit", prefix);
++    s->ptr_ComponentNameEnum   = dlsym_prefixed(s->lib, "OMX_ComponentNameEnum", prefix);
++    s->ptr_GetHandle           = dlsym_prefixed(s->lib, "OMX_GetHandle", prefix);
++    s->ptr_FreeHandle          = dlsym_prefixed(s->lib, "OMX_FreeHandle", prefix);
++    s->ptr_GetComponentsOfRole = dlsym_prefixed(s->lib, "OMX_GetComponentsOfRole", prefix);
++    s->ptr_GetRolesOfComponent = dlsym_prefixed(s->lib, "OMX_GetRolesOfComponent", prefix);
++    if (!s->ptr_Init || !s->ptr_Deinit || !s->ptr_ComponentNameEnum ||
++        !s->ptr_GetHandle || !s->ptr_FreeHandle ||
++        !s->ptr_GetComponentsOfRole || !s->ptr_GetRolesOfComponent) {
++        av_log(logctx, AV_LOG_WARNING, "Not all functions found in %s\n", libname);
++        dlclose(s->lib);
++        s->lib = NULL;
++        if (s->lib2)
++            dlclose(s->lib2);
++        s->lib2 = NULL;
++        return AVERROR_ENCODER_NOT_FOUND;
++    }
++    return 0;
++}
++
++static av_cold OMXContext *omx_init(void *logctx, const char *libname, const char *prefix)
++{
++    static const char * const libnames[] = {
++#if CONFIG_OMX_RPI
++        "/opt/vc/lib/libopenmaxil.so", "/opt/vc/lib/libbcm_host.so",
++#else
++        "libOMX_Core.so", NULL,
++        "libOmxCore.so", NULL,
++#endif
++        NULL
++    };
++    const char* const* nameptr;
++    int ret = AVERROR_DECODER_NOT_FOUND;
++    OMXContext *omx_context;
++
++    omx_context = av_mallocz(sizeof(*omx_context));
++    if (!omx_context)
++        return NULL;
++    if (libname) {
++        ret = omx_try_load(omx_context, logctx, libname, prefix, NULL);
++        if (ret < 0) {
++            av_free(omx_context);
++            return NULL;
++        }
++    } else {
++        for (nameptr = libnames; *nameptr; nameptr += 2)
++            if (!(ret = omx_try_load(omx_context, logctx, nameptr[0], prefix, nameptr[1])))
++                break;
++        if (!*nameptr) {
++            av_free(omx_context);
++            return NULL;
++        }
++    }
++
++    if (omx_context->host_init)
++        omx_context->host_init();
++    omx_context->ptr_Init();
++    return omx_context;
++}
++
++static av_cold void omx_deinit(OMXContext *omx_context)
++{
++    if (!omx_context)
++        return;
++
++    omx_context->ptr_Deinit();
++    dlclose(omx_context->lib);
++    av_free(omx_context);
++}
++
++typedef struct OMXCodecContext {
++    const AVClass *class;
++    char *libname;
++    char *libprefix;
++    OMXContext *omx_context;
++
++    AVCodecContext *avctx;
++
++    char component_name[OMX_MAX_STRINGNAME_SIZE];
++    OMX_VERSIONTYPE version;
++    OMX_HANDLETYPE handle;
++    int in_port, out_port;
++    OMX_COLOR_FORMATTYPE color_format;
++    int stride, plane_size;
++
++    int num_in_buffers, num_out_buffers;
++    OMX_BUFFERHEADERTYPE **in_buffer_headers;
++    OMX_BUFFERHEADERTYPE **out_buffer_headers;
++    int num_free_in_buffers;
++    OMX_BUFFERHEADERTYPE **free_in_buffers;
++    int num_done_out_buffers;
++    OMX_BUFFERHEADERTYPE **done_out_buffers;
++    pthread_mutex_t input_mutex;
++    pthread_cond_t input_cond;
++    pthread_mutex_t output_mutex;
++    pthread_cond_t output_cond;
++
++    pthread_mutex_t state_mutex;
++    pthread_cond_t state_cond;
++    OMX_STATETYPE state;
++    OMX_ERRORTYPE error;
++
++    int mutex_cond_inited;
++
++    int eos_sent, got_eos, first_get_outbuffer;
++
++	int extradata_sent;
++
++    uint8_t *output_buf;
++    int output_buf_size;
++
++    int input_zerocopy;
++    int profile;
++} OMXCodecContext;
++
++static void append_buffer(pthread_mutex_t *mutex, pthread_cond_t *cond,
++                          int* array_size, OMX_BUFFERHEADERTYPE **array,
++                          OMX_BUFFERHEADERTYPE *buffer)
++{
++    pthread_mutex_lock(mutex);
++    array[(*array_size)++] = buffer;
++    pthread_cond_broadcast(cond);
++    pthread_mutex_unlock(mutex);
++}
++
++static OMX_BUFFERHEADERTYPE *get_buffer(pthread_mutex_t *mutex, pthread_cond_t *cond,
++                                        int* array_size, OMX_BUFFERHEADERTYPE **array,
++                                        int wait)
++{
++    OMX_BUFFERHEADERTYPE *buffer;
++    pthread_mutex_lock(mutex);
++    if (wait) {
++        while (!*array_size)
++        {
++           pthread_cond_wait(cond, mutex);
++        }
++    }
++    if (*array_size > 0) {
++        buffer = array[0];
++        (*array_size)--;
++        memmove(&array[0], &array[1], (*array_size) * sizeof(OMX_BUFFERHEADERTYPE*));
++    } else {
++        buffer = NULL;
++    }
++    pthread_mutex_unlock(mutex);
++    return buffer;
++}
++
++static OMX_ERRORTYPE event_handler(OMX_HANDLETYPE component, OMX_PTR app_data, OMX_EVENTTYPE event,
++                                   OMX_U32 data1, OMX_U32 data2, OMX_PTR event_data)
++{
++    OMXCodecContext *s = app_data;
++    // This uses casts in the printfs, since OMX_U32 actually is a typedef for
++    // unsigned long in official header versions (but there are also modified
++    // versions where it is something else).
++    switch (event) {
++    case OMX_EventError:
++        pthread_mutex_lock(&s->state_mutex);
++        av_log(s->avctx, AV_LOG_ERROR, "OMX error %"PRIx32"\n", (uint32_t) data1);
++        s->error = data1;
++        pthread_cond_broadcast(&s->state_cond);
++        pthread_mutex_unlock(&s->state_mutex);
++        break;
++    case OMX_EventCmdComplete:
++        if (data1 == OMX_CommandStateSet) {
++            pthread_mutex_lock(&s->state_mutex);
++            s->state = data2;
++            av_log(s->avctx, AV_LOG_VERBOSE, "OMX state changed to %"PRIu32"\n", (uint32_t) data2);
++            pthread_cond_broadcast(&s->state_cond);
++            pthread_mutex_unlock(&s->state_mutex);
++        } else if (data1 == OMX_CommandPortDisable) {
++            av_log(s->avctx, AV_LOG_VERBOSE, "OMX port %"PRIu32" disabled\n", (uint32_t) data2);
++        } else if (data1 == OMX_CommandPortEnable) {
++            av_log(s->avctx, AV_LOG_VERBOSE, "OMX port %"PRIu32" enabled\n", (uint32_t) data2);
++        } else {
++            av_log(s->avctx, AV_LOG_VERBOSE, "OMX command complete, command %"PRIu32", value %"PRIu32"\n",
++                                             (uint32_t) data1, (uint32_t) data2);
++        }
++        break;
++    case OMX_EventPortSettingsChanged:
++        av_log(s->avctx, AV_LOG_VERBOSE, "OMX port %"PRIu32" settings changed\n", (uint32_t) data1);
++        break;
++    default:
++        av_log(s->avctx, AV_LOG_VERBOSE, "OMX event %d %"PRIx32" %"PRIx32"\n",
++                                         event, (uint32_t) data1, (uint32_t) data2);
++        break;
++    }
++    return OMX_ErrorNone;
++}
++
++static OMX_ERRORTYPE empty_buffer_done(OMX_HANDLETYPE component, OMX_PTR app_data,
++                                       OMX_BUFFERHEADERTYPE *buffer)
++{
++
++    OMXCodecContext *s = app_data;
++    if (s->input_zerocopy) {
++        if (buffer->pAppPrivate) {
++            if (buffer->pOutputPortPrivate)
++                av_free(buffer->pAppPrivate);
++            else
++                av_frame_free((AVFrame**)&buffer->pAppPrivate);
++            buffer->pAppPrivate = NULL;
++        }
++    }
++    append_buffer(&s->input_mutex, &s->input_cond,
++                  &s->num_free_in_buffers, s->free_in_buffers, buffer);
++    return OMX_ErrorNone;
++}
++
++static OMX_ERRORTYPE fill_buffer_done(OMX_HANDLETYPE component, OMX_PTR app_data,
++                                      OMX_BUFFERHEADERTYPE *buffer)
++{
++    OMXCodecContext *s = app_data;
++    append_buffer(&s->output_mutex, &s->output_cond,
++                  &s->num_done_out_buffers, s->done_out_buffers, buffer);
++    return OMX_ErrorNone;
++}
++
++static const OMX_CALLBACKTYPE callbacks = {
++    event_handler,
++    empty_buffer_done,
++    fill_buffer_done
++};
++
++static av_cold int find_component(OMXContext *omx_context, void *logctx,
++                                  const char *role, char *str, int str_size)
++{
++    OMX_U32 i, num = 0;
++    char **components;
++    int ret = 0;
++
++#if CONFIG_OMX_RPI
++    if (av_strstart(role, "video_decoder.", NULL)) {
++        av_strlcpy(str, "OMX.broadcom.video_decode", str_size);
++        return 0;
++    }
++#endif
++    omx_context->ptr_GetComponentsOfRole((OMX_STRING) role, &num, NULL);
++    if (!num) {
++        av_log(logctx, AV_LOG_WARNING, "No component for role %s found\n", role);
++        return AVERROR_DECODER_NOT_FOUND;
++    }
++    components = av_mallocz_array(num, sizeof(*components));
++    if (!components)
++        return AVERROR(ENOMEM);
++    for (i = 0; i < num; i++) {
++        components[i] = av_mallocz(OMX_MAX_STRINGNAME_SIZE);
++        if (!components[i]) {
++            ret = AVERROR(ENOMEM);
++            goto end;
++        }
++    }
++    omx_context->ptr_GetComponentsOfRole((OMX_STRING) role, &num, (OMX_U8**) components);
++    av_strlcpy(str, components[0], str_size);
++end:
++    for (i = 0; i < num; i++)
++        av_free(components[i]);
++    av_free(components);
++    return ret;
++}
++
++static av_cold int wait_for_state(OMXCodecContext *s, OMX_STATETYPE state)
++{
++    int ret = 0;
++    pthread_mutex_lock(&s->state_mutex);
++    while (s->state != state && s->error == OMX_ErrorNone)
++        pthread_cond_wait(&s->state_cond, &s->state_mutex);
++    if (s->error != OMX_ErrorNone)
++        ret = AVERROR_DECODER_NOT_FOUND;
++    pthread_mutex_unlock(&s->state_mutex);
++    return ret;
++}
++
++static av_cold int omx_component_init(AVCodecContext *avctx, const char *role)
++{
++    OMXCodecContext *s = avctx->priv_data;
++    OMX_PARAM_COMPONENTROLETYPE role_params = { 0 };
++    OMX_PORT_PARAM_TYPE video_port_params = { 0 };
++    OMX_PARAM_PORTDEFINITIONTYPE in_port_params = { 0 }, out_port_params = { 0 };
++    //OMX_VIDEO_PARAM_PORTFORMATTYPE video_port_format = { 0 };
++    //OMX_VIDEO_PARAM_BITRATETYPE vid_param_bitrate = { 0 };
++    OMX_ERRORTYPE err;
++    int i;
++
++    s->version.s.nVersionMajor = 1;
++    s->version.s.nVersionMinor = 1;
++    s->version.s.nRevision     = 2;
++
++    err = s->omx_context->ptr_GetHandle(&s->handle, s->component_name, s, (OMX_CALLBACKTYPE*) &callbacks);
++    if (err != OMX_ErrorNone) {
++        av_log(avctx, AV_LOG_ERROR, "OMX_GetHandle(%s) failed: %x\n", s->component_name, err);
++        return AVERROR_UNKNOWN;
++    }
++
++    // This one crashes the mediaserver on qcom, if used over IOMX
++    INIT_STRUCT(role_params);
++    av_strlcpy(role_params.cRole, role, sizeof(role_params.cRole));
++    // Intentionally ignore errors on this one
++    OMX_SetParameter(s->handle, OMX_IndexParamStandardComponentRole, &role_params);
++
++    INIT_STRUCT(video_port_params);
++    err = OMX_GetParameter(s->handle, OMX_IndexParamVideoInit, &video_port_params);
++    CHECK(err);
++
++    s->in_port = s->out_port = -1;
++    for (i = 0; i < video_port_params.nPorts; i++) {
++        int port = video_port_params.nStartPortNumber + i;
++        OMX_PARAM_PORTDEFINITIONTYPE port_params = { 0 };
++        INIT_STRUCT(port_params);
++        port_params.nPortIndex = port;
++        err = OMX_GetParameter(s->handle, OMX_IndexParamPortDefinition, &port_params);
++        if (err != OMX_ErrorNone) {
++            av_log(avctx, AV_LOG_WARNING, "port %d error %x\n", port, err);
++            break;
++        }
++        if (port_params.eDir == OMX_DirInput && s->in_port < 0) {
++            in_port_params = port_params;
++            s->in_port = port;
++        } else if (port_params.eDir == OMX_DirOutput && s->out_port < 0) {
++            out_port_params = port_params;
++            s->out_port = port;
++        }
++    }
++    if (s->in_port < 0 || s->out_port < 0) {
++        av_log(avctx, AV_LOG_ERROR, "No in or out port found (in %d out %d)\n", s->in_port, s->out_port);
++        return AVERROR_UNKNOWN;
++    }
++
++    in_port_params.bEnabled   = OMX_TRUE;
++    in_port_params.bPopulated = OMX_FALSE;
++    in_port_params.eDomain    = OMX_PortDomainVideo;
++
++    in_port_params.format.video.pNativeRender         = NULL;
++    in_port_params.format.video.bFlagErrorConcealment = OMX_FALSE;
++    //in_port_params.format.video.eColorFormat          = s->color_format;
++    s->stride     = avctx->width;
++    s->plane_size = avctx->height;
++    // If specific codecs need to manually override the stride/plane_size,
++    // that can be done here.
++    in_port_params.format.video.nStride      = s->stride;
++    in_port_params.format.video.nSliceHeight = s->plane_size;
++    in_port_params.format.video.nFrameWidth  = avctx->width;
++    in_port_params.format.video.nFrameHeight = avctx->height;
++
++    if (avctx->framerate.den > 0 && avctx->framerate.num > 0)
++        in_port_params.format.video.xFramerate = (1LL << 16) * avctx->framerate.num / avctx->framerate.den;
++    else
++        in_port_params.format.video.xFramerate = (1LL << 16) * avctx->time_base.den / avctx->time_base.num;
++
++    err = OMX_SetParameter(s->handle, OMX_IndexParamPortDefinition, &in_port_params);
++    CHECK(err);
++    err = OMX_GetParameter(s->handle, OMX_IndexParamPortDefinition, &in_port_params);
++    CHECK(err);
++
++#if 1
++    s->stride         = in_port_params.format.video.nStride;
++    s->plane_size     = in_port_params.format.video.nSliceHeight;
++    s->num_in_buffers = in_port_params.nBufferCountActual;
++#endif
++    err = OMX_GetParameter(s->handle, OMX_IndexParamPortDefinition, &out_port_params);
++    out_port_params.bEnabled   = OMX_TRUE;
++    out_port_params.bPopulated = OMX_FALSE;
++    out_port_params.eDomain    = OMX_PortDomainVideo;
++    out_port_params.format.video.pNativeRender = NULL;
++	out_port_params.format.video.bFlagErrorConcealment  = OMX_FALSE;
++    out_port_params.format.video.nFrameWidth   = avctx->width;
++    out_port_params.format.video.nFrameHeight  = avctx->height;
++    out_port_params.format.video.nStride       = 0;
++    out_port_params.format.video.nSliceHeight  = 0;
++    out_port_params.format.video.nBitrate      = avctx->bit_rate;
++    out_port_params.format.video.xFramerate    = in_port_params.format.video.xFramerate;
++    out_port_params.format.video.bFlagErrorConcealment  = OMX_FALSE;
++    if (avctx->codec->id == AV_CODEC_ID_MPEG4)
++        out_port_params.format.video.eCompressionFormat = OMX_VIDEO_CodingMPEG4;
++    else if (avctx->codec->id == AV_CODEC_ID_H264)
++        out_port_params.format.video.eCompressionFormat = OMX_VIDEO_CodingAVC;
++
++    err = OMX_SetParameter(s->handle, OMX_IndexParamPortDefinition, &out_port_params);
++    CHECK(err);
++    err = OMX_GetParameter(s->handle, OMX_IndexParamPortDefinition, &out_port_params);
++    CHECK(err);
++    s->num_out_buffers = out_port_params.nBufferCountActual;
++
++
++    err = OMX_SendCommand(s->handle, OMX_CommandStateSet, OMX_StateIdle, NULL);
++    CHECK(err);
++
++    s->in_buffer_headers  = av_mallocz(sizeof(OMX_BUFFERHEADERTYPE*) * s->num_in_buffers);
++    s->free_in_buffers    = av_mallocz(sizeof(OMX_BUFFERHEADERTYPE*) * s->num_in_buffers);
++    s->out_buffer_headers = av_mallocz(sizeof(OMX_BUFFERHEADERTYPE*) * s->num_out_buffers);
++    s->done_out_buffers   = av_mallocz(sizeof(OMX_BUFFERHEADERTYPE*) * s->num_out_buffers);
++    if (!s->in_buffer_headers || !s->free_in_buffers || !s->out_buffer_headers || !s->done_out_buffers)
++        return AVERROR(ENOMEM);
++    for (i = 0; i < s->num_in_buffers && err == OMX_ErrorNone; i++) {
++        if (s->input_zerocopy)
++            err = OMX_UseBuffer(s->handle, &s->in_buffer_headers[i], s->in_port, s, in_port_params.nBufferSize, NULL);
++        else
++            err = OMX_AllocateBuffer(s->handle, &s->in_buffer_headers[i],  s->in_port,  s, in_port_params.nBufferSize);
++        if (err == OMX_ErrorNone)
++            s->in_buffer_headers[i]->pAppPrivate = s->in_buffer_headers[i]->pOutputPortPrivate = NULL;
++    }
++    CHECK(err);
++    s->num_in_buffers = i;
++    for (i = 0; i < s->num_out_buffers && err == OMX_ErrorNone; i++)
++        err = OMX_AllocateBuffer(s->handle, &s->out_buffer_headers[i], s->out_port, s, out_port_params.nBufferSize);
++    CHECK(err);
++    s->num_out_buffers = i;
++
++    if (wait_for_state(s, OMX_StateIdle) < 0) {
++        av_log(avctx, AV_LOG_ERROR, "Didn't get OMX_StateIdle\n");
++        return AVERROR_UNKNOWN;
++    }
++    err = OMX_SendCommand(s->handle, OMX_CommandStateSet, OMX_StateExecuting, NULL);
++    CHECK(err);
++    if (wait_for_state(s, OMX_StateExecuting) < 0) {
++        av_log(avctx, AV_LOG_ERROR, "Didn't get OMX_StateExecuting\n");
++        return AVERROR_UNKNOWN;
++    }
++
++    for (i = 0; i < s->num_out_buffers && err == OMX_ErrorNone; i++)
++        err = OMX_FillThisBuffer(s->handle, s->out_buffer_headers[i]);
++    if (err != OMX_ErrorNone) {
++        for (; i < s->num_out_buffers; i++)
++            s->done_out_buffers[s->num_done_out_buffers++] = s->out_buffer_headers[i];
++    }
++    for (i = 0; i < s->num_in_buffers; i++)
++        s->free_in_buffers[s->num_free_in_buffers++] = s->in_buffer_headers[i];
++    return err != OMX_ErrorNone ? AVERROR_UNKNOWN : 0;
++}
++
++static av_cold void cleanup(OMXCodecContext *s)
++{
++    int i, executing;
++
++    pthread_mutex_lock(&s->state_mutex);
++    executing = s->state == OMX_StateExecuting;
++    pthread_mutex_unlock(&s->state_mutex);
++
++    if (executing) {
++        OMX_SendCommand(s->handle, OMX_CommandStateSet, OMX_StateIdle, NULL);
++        wait_for_state(s, OMX_StateIdle);
++        OMX_SendCommand(s->handle, OMX_CommandStateSet, OMX_StateLoaded, NULL);
++        for (i = 0; i < s->num_in_buffers; i++) {
++            OMX_BUFFERHEADERTYPE *buffer = get_buffer(&s->input_mutex, &s->input_cond,
++                                                      &s->num_free_in_buffers, s->free_in_buffers, 1);
++            if (s->input_zerocopy)
++                buffer->pBuffer = NULL;
++            OMX_FreeBuffer(s->handle, s->in_port, buffer);
++        }
++        for (i = 0; i < s->num_out_buffers; i++) {
++            OMX_BUFFERHEADERTYPE *buffer = get_buffer(&s->output_mutex, &s->output_cond,
++                                                      &s->num_done_out_buffers, s->done_out_buffers, 1);
++            OMX_FreeBuffer(s->handle, s->out_port, buffer);
++        }
++        wait_for_state(s, OMX_StateLoaded);
++    }
++    if (s->handle) {
++        s->omx_context->ptr_FreeHandle(s->handle);
++        s->handle = NULL;
++    }
++
++    omx_deinit(s->omx_context);
++    s->omx_context = NULL;
++    if (s->mutex_cond_inited) {
++        pthread_cond_destroy(&s->state_cond);
++        pthread_mutex_destroy(&s->state_mutex);
++        pthread_cond_destroy(&s->input_cond);
++        pthread_mutex_destroy(&s->input_mutex);
++        pthread_cond_destroy(&s->output_cond);
++        pthread_mutex_destroy(&s->output_mutex);
++        s->mutex_cond_inited = 0;
++    }
++    av_freep(&s->in_buffer_headers);
++    av_freep(&s->out_buffer_headers);
++    av_freep(&s->free_in_buffers);
++    av_freep(&s->done_out_buffers);
++    av_freep(&s->output_buf);
++}
++
++static av_cold int omx_decode_init(AVCodecContext *avctx)
++{
++    OMXCodecContext *s = avctx->priv_data;
++    int ret = AVERROR_ENCODER_NOT_FOUND;
++    const char *role;
++    //OMX_BUFFERHEADERTYPE *buffer;
++    //OMX_ERRORTYPE err;
++
++    s->omx_context = omx_init(avctx, s->libname, s->libprefix);
++    if (!s->omx_context)
++        return AVERROR_ENCODER_NOT_FOUND;
++
++    pthread_mutex_init(&s->state_mutex, NULL);
++    pthread_cond_init(&s->state_cond, NULL);
++    pthread_mutex_init(&s->input_mutex, NULL);
++    pthread_cond_init(&s->input_cond, NULL);
++    pthread_mutex_init(&s->output_mutex, NULL);
++    pthread_cond_init(&s->output_cond, NULL);
++    s->mutex_cond_inited = 1;
++    s->avctx = avctx;
++    s->state = OMX_StateLoaded;
++    s->error = OMX_ErrorNone;
++
++    switch (avctx->codec->id) {
++    case AV_CODEC_ID_MPEG4:
++        role = "video_decoder.mpeg4";
++        break;
++    case AV_CODEC_ID_H264:
++        role = "video_decoder.avc";
++        break;
++    default:
++        return AVERROR(ENOSYS);
++    }
++
++    if ((ret = find_component(s->omx_context, avctx, role, s->component_name, sizeof(s->component_name))) < 0)
++        goto fail;
++
++    av_log(avctx, AV_LOG_INFO, "Using %s\n", s->component_name);
++
++    if ((ret = omx_component_init(avctx, role)) < 0)
++        goto fail;
++
++#if 0
++    if (avctx->flags & AV_CODEC_FLAG_GLOBAL_HEADER) {
++        while (1) {
++            buffer = get_buffer(&s->output_mutex, &s->output_cond,
++                                &s->num_done_out_buffers, s->done_out_buffers, 1);
++            if (buffer->nFlags & OMX_BUFFERFLAG_CODECCONFIG) {
++                if ((ret = av_reallocp(&avctx->extradata, avctx->extradata_size + buffer->nFilledLen + AV_INPUT_BUFFER_PADDING_SIZE)) < 0) {
++                    avctx->extradata_size = 0;
++                    goto fail;
++                }
++                memcpy(avctx->extradata + avctx->extradata_size, buffer->pBuffer + buffer->nOffset, buffer->nFilledLen);
++                avctx->extradata_size += buffer->nFilledLen;
++                memset(avctx->extradata + avctx->extradata_size, 0, AV_INPUT_BUFFER_PADDING_SIZE);
++            }
++            err = OMX_FillThisBuffer(s->handle, buffer);
++            if (err != OMX_ErrorNone) {
++                append_buffer(&s->output_mutex, &s->output_cond,
++                              &s->num_done_out_buffers, s->done_out_buffers, buffer);
++                av_log(avctx, AV_LOG_ERROR, "OMX_FillThisBuffer failed: %x\n", err);
++                ret = AVERROR_UNKNOWN;
++                goto fail;
++            }
++            if (avctx->codec->id == AV_CODEC_ID_H264) {
++                // For H.264, the extradata can be returned in two separate buffers
++                // (the videocore encoder on raspberry pi does this);
++                // therefore check that we have got both SPS and PPS before continuing.
++                int nals[32] = { 0 };
++                int i;
++                for (i = 0; i + 4 < avctx->extradata_size; i++) {
++                     if (!avctx->extradata[i + 0] &&
++                         !avctx->extradata[i + 1] &&
++                         !avctx->extradata[i + 2] &&
++                         avctx->extradata[i + 3] == 1) {
++                         nals[avctx->extradata[i + 4] & 0x1f]++;
++                     }
++                }
++                if (nals[H264_NAL_SPS] && nals[H264_NAL_PPS])
++                    break;
++            } else {
++                if (avctx->extradata_size > 0)
++                    break;
++            }
++        }
++    }
++#endif
++	
++    return 0;
++fail:
++    return ret;
++}
++
++
++static int omx_decode_frame(AVCodecContext *avctx, void *data,
++                           int *got_packet, AVPacket *pkt)
++{
++    OMXCodecContext *s = avctx->priv_data;
++    int ret = 0;
++    OMX_BUFFERHEADERTYPE* buffer;
++    OMX_ERRORTYPE err;
++    int had_partial = 0;
++
++	AVFrame *avframe = data;
++	
++	uint8_t *dst[4];
++	int linesize[4];
++
++    if (pkt->size) {
++
++		//VPU init and fill buffer slow, so empty buf sleep to send before get vpu fill buf.
++		if(!s->first_get_outbuffer)
++			av_usleep(100000);
++	
++        buffer = get_buffer(&s->input_mutex, &s->input_cond,
++                            &s->num_free_in_buffers, s->free_in_buffers, 1);
++
++		if (!buffer) {
++        	av_log(avctx, AV_LOG_ERROR, "get_buffer NULL\n");
++            return AVERROR(ENOMEM);
++        }
++
++		//cpy the extradata
++		if(!s->extradata_sent) {
++
++			memcpy(buffer->pBuffer + buffer->nOffset, avctx->extradata, avctx->extradata_size);	
++			memcpy(buffer->pBuffer + buffer->nOffset + avctx->extradata_size, pkt->data, pkt->size);
++			buffer->nFilledLen = pkt->size + avctx->extradata_size;
++			s->extradata_sent = 1;
++		
++		}
++		else {
++			memcpy(buffer->pBuffer + buffer->nOffset, pkt->data, pkt->size);
++			buffer->nFilledLen = pkt->size;
++		}
++
++		/* reduce memcpy. point it addr*/
++		//buffer->pAppPrivate = pkt;
++		//buffer->pBuffer = pkt->data;
++		//buffer->nFilledLen = pkt->size;
++		
++		buffer->pOutputPortPrivate = NULL;
++		buffer->pAppPrivate = avctx->priv_data;
++
++	    err = OMX_EmptyThisBuffer(s->handle, buffer);
++	    if (err != OMX_ErrorNone) {
++	        append_buffer(&s->input_mutex, &s->input_cond, &s->num_free_in_buffers, s->free_in_buffers, buffer);
++	        av_log(avctx, AV_LOG_ERROR, "OMX_EmptyThisBuffer failed: %x\n", err);
++	        return AVERROR_UNKNOWN;
++	    }
++    } else if (!s->eos_sent) {
++
++		if(!s->first_get_outbuffer)
++			av_usleep(1000000);
++		
++        buffer = get_buffer(&s->input_mutex, &s->input_cond,
++                            &s->num_free_in_buffers, s->free_in_buffers, 1);
++
++        buffer->nFilledLen = 0;
++        buffer->nFlags = OMX_BUFFERFLAG_EOS;
++        buffer->pAppPrivate = buffer->pOutputPortPrivate = NULL;
++        err = OMX_EmptyThisBuffer(s->handle, buffer);
++        if (err != OMX_ErrorNone) {
++            append_buffer(&s->input_mutex, &s->input_cond, &s->num_free_in_buffers, s->free_in_buffers, buffer);
++            av_log(avctx, AV_LOG_ERROR, "OMX_EmptyThisBuffer failed: %x\n", err);
++            return AVERROR_UNKNOWN;
++        }
++        s->eos_sent = 1;
++    }
++
++    while (!*got_packet && ret == 0 && !s->got_eos) {
++        // If not flushing, just poll the queue if there's finished packets.
++        // If flushing, do a blocking wait until we either get a completed
++        // packet, or get EOS.
++        buffer = get_buffer(&s->output_mutex, &s->output_cond,
++                            &s->num_done_out_buffers, s->done_out_buffers,
++                            !pkt || had_partial);
++
++        if (!buffer) {
++            /*eos is sent but fill buf still can't get then continue*/
++            if(!s->first_get_outbuffer && s->eos_sent)
++				continue; 
++			else
++				break;
++        }
++		//if (!buffer)
++           // break;
++
++		if(!buffer->nFilledLen)
++			goto end;
++		
++		if(!s->first_get_outbuffer)
++			s->first_get_outbuffer = 1;
++		
++        if (buffer->nFlags & OMX_BUFFERFLAG_EOS)
++            s->got_eos = 1;
++       
++		if ((ret = ff_get_buffer(avctx, avframe, 0)) < 0) {
++			av_log(avctx, AV_LOG_ERROR, "Unable to allocate buffer\n");
++			goto end;
++		}
++		
++		ret = av_image_fill_arrays(dst, linesize, buffer->pBuffer,
++								   avctx->pix_fmt, s->stride, s->plane_size, 1);
++		if (ret < 0)
++			goto end;
++		av_image_copy(avframe->data, avframe->linesize, (const uint8_t**)dst, linesize, 
++							avctx->pix_fmt, avctx->width, avctx->height);
++	
++		//avframe->pts	 = buffer->nTimeStamp;
++		//avframe->pkt_dts = AV_NOPTS_VALUE;
++		//avframe->pict_type= AV_PICTURE_TYPE_I;
++    	//avframe->key_frame= 1;
++
++		*got_packet = 1;
++
++	/*
++		if ((ret = av_frame_ref(data, avframe)) < 0)
++         	goto end;
++         	*/
++
++end:     
++        err = OMX_FillThisBuffer(s->handle, buffer);
++        if (err != OMX_ErrorNone) {
++            append_buffer(&s->output_mutex, &s->output_cond, &s->num_done_out_buffers, s->done_out_buffers, buffer);
++            av_log(avctx, AV_LOG_ERROR, "OMX_FillThisBuffer failed: %x\n", err);
++            ret = AVERROR_UNKNOWN;
++        }
++    }
++    return ret;
++}
++
++static av_cold int omx_decode_end(AVCodecContext *avctx)
++{
++    OMXCodecContext *s = avctx->priv_data;
++
++    cleanup(s);
++    return 0;
++}
++
++#define OFFSET(x) offsetof(OMXCodecContext, x)
++#define VDE AV_OPT_FLAG_VIDEO_PARAM | AV_OPT_FLAG_DECODING_PARAM | AV_OPT_FLAG_ENCODING_PARAM
++#define VE  AV_OPT_FLAG_VIDEO_PARAM | AV_OPT_FLAG_ENCODING_PARAM
++static const AVOption options[] = {
++    { "omx_libname", "OpenMAX library name", OFFSET(libname), AV_OPT_TYPE_STRING, { 0 }, 0, 0, VDE },
++    { "omx_libprefix", "OpenMAX library prefix", OFFSET(libprefix), AV_OPT_TYPE_STRING, { 0 }, 0, 0, VDE },
++    { "zerocopy", "Try to avoid copying input frames if possible", OFFSET(input_zerocopy), AV_OPT_TYPE_INT, { .i64 = CONFIG_OMX_RPI }, 0, 1, VE },
++    { "profile",  "Set the encoding profile", OFFSET(profile), AV_OPT_TYPE_INT,   { .i64 = FF_PROFILE_UNKNOWN },       FF_PROFILE_UNKNOWN, FF_PROFILE_H264_HIGH, VE, "profile" },
++    { "baseline", "",                         0,               AV_OPT_TYPE_CONST, { .i64 = FF_PROFILE_H264_BASELINE }, 0, 0, VE, "profile" },
++    { "main",     "",                         0,               AV_OPT_TYPE_CONST, { .i64 = FF_PROFILE_H264_MAIN },     0, 0, VE, "profile" },
++    { "high",     "",                         0,               AV_OPT_TYPE_CONST, { .i64 = FF_PROFILE_H264_HIGH },     0, 0, VE, "profile" },
++    { NULL }
++};
++
++
++static const AVClass omx_mpeg4dec_class = {
++    .class_name = "mpeg4_omx",
++    .item_name  = av_default_item_name,
++    .option     = options,
++    .version    = LIBAVUTIL_VERSION_INT,
++};
++AVCodec ff_mpeg4_omx_decoder = {
++    .name             = "mpeg4_omx",
++    .long_name        = NULL_IF_CONFIG_SMALL("OpenMAX IL MPEG-4 video decoder"),
++    .type             = AVMEDIA_TYPE_VIDEO,
++    .id               = AV_CODEC_ID_MPEG4,
++    .priv_data_size   = sizeof(OMXCodecContext),
++    .init             = omx_decode_init,
++    .decode           = omx_decode_frame,
++    .close            = omx_decode_end,
++    .capabilities     = AV_CODEC_CAP_DELAY,
++    .caps_internal    = FF_CODEC_CAP_INIT_THREADSAFE | FF_CODEC_CAP_INIT_CLEANUP,
++    .priv_class       = &omx_mpeg4dec_class,
++};
++
++static const AVClass omx_h264dec_class = {
++    .class_name = "h264_omx",
++    .item_name  = av_default_item_name,
++    .option     = options,
++    .version    = LIBAVUTIL_VERSION_INT,
++};
++AVCodec ff_h264_omx_decoder = {
++    .name             = "h264_omx",
++    .long_name        = NULL_IF_CONFIG_SMALL("OpenMAX IL H.264 video decoder"),
++    .type             = AVMEDIA_TYPE_VIDEO,
++    .id               = AV_CODEC_ID_H264,
++    .priv_data_size   = sizeof(OMXCodecContext),
++    .init             = omx_decode_init,
++    .decode           = omx_decode_frame,
++    .close            = omx_decode_end,
++    .capabilities     = AV_CODEC_CAP_DELAY,
++    .caps_internal    = FF_CODEC_CAP_INIT_THREADSAFE | FF_CODEC_CAP_INIT_CLEANUP,
++    .priv_class       = &omx_h264dec_class,
++};
+-- 
+2.17.1
+

+ 210 - 0
multimedia/patch/ffmpeg/0006-add-hevc-decoder-and-encoder-support.patch

@@ -0,0 +1,210 @@
+From 0adf789b7c3060e241b82e6afdfc2f7975f1ee39 Mon Sep 17 00:00:00 2001
+From: "sw.multimedia" <se.multimedia@starfivetech.com>
+Date: Wed, 25 May 2022 20:28:21 +0800
+Subject: [PATCH 2/8] add hevc decoder and encoder support
+
+Signed-off-by: sw.multimedia <se.multimedia@starfivetech.com>
+---
+ configure              |  2 ++
+ libavcodec/allcodecs.c |  2 ++
+ libavcodec/omx.c       | 35 +++++++++++++++++++++++++++++++++++
+ libavcodec/omxdec.c    | 35 ++++++++++++++++++++++++++++++++++-
+ libavformat/utils.c    |  7 +++++++
+ 5 files changed, 80 insertions(+), 1 deletion(-)
+
+diff --git a/configure b/configure
+index e3976df..80be074 100755
+--- a/configure
++++ b/configure
+@@ -3093,6 +3093,8 @@ h264_vaapi_encoder_select="cbs_h264 vaapi_encode"
+ h264_v4l2m2m_decoder_deps="v4l2_m2m h264_v4l2_m2m"
+ h264_v4l2m2m_decoder_select="h264_mp4toannexb_bsf"
+ h264_v4l2m2m_encoder_deps="v4l2_m2m h264_v4l2_m2m"
++hevc_omx_encoder_deps="omx"
++hevc_omx_decoder_deps="omx"
+ hevc_amf_encoder_deps="amf"
+ hevc_cuvid_decoder_deps="cuvid"
+ hevc_cuvid_decoder_select="hevc_mp4toannexb_bsf"
+diff --git a/libavcodec/allcodecs.c b/libavcodec/allcodecs.c
+index e42fa30..0c7a18d 100644
+--- a/libavcodec/allcodecs.c
++++ b/libavcodec/allcodecs.c
+@@ -793,6 +793,8 @@ extern AVCodec ff_h264_mf_encoder;
+ extern AVCodec ff_h264_nvenc_encoder;
+ extern AVCodec ff_h264_omx_encoder;
+ extern AVCodec ff_h264_omx_decoder;
++extern AVCodec ff_hevc_omx_encoder;
++extern AVCodec ff_hevc_omx_decoder;
+ extern AVCodec ff_h264_qsv_encoder;
+ extern AVCodec ff_h264_v4l2m2m_encoder;
+ extern AVCodec ff_h264_vaapi_encoder;
+diff --git a/libavcodec/omx.c b/libavcodec/omx.c
+index 0a6a308..86e32a8 100644
+--- a/libavcodec/omx.c
++++ b/libavcodec/omx.c
+@@ -43,6 +43,7 @@
+ #include "avcodec.h"
+ #include "h264.h"
+ #include "internal.h"
++#include "profiles.h"
+ 
+ #ifdef OMX_SKIP64BIT
+ static OMX_TICKS to_omx_ticks(int64_t value)
+@@ -501,6 +502,8 @@ static av_cold int omx_component_init(AVCodecContext *avctx, const char *role)
+         out_port_params.format.video.eCompressionFormat = OMX_VIDEO_CodingMPEG4;
+     else if (avctx->codec->id == AV_CODEC_ID_H264)
+         out_port_params.format.video.eCompressionFormat = OMX_VIDEO_CodingAVC;
++    else if (avctx->codec->id == AV_CODEC_ID_HEVC)
++        out_port_params.format.video.eCompressionFormat = OMX_VIDEO_CodingHEVC;
+ 
+     err = OMX_SetParameter(s->handle, OMX_IndexParamPortDefinition, &out_port_params);
+     CHECK(err);
+@@ -666,6 +669,9 @@ static av_cold int omx_encode_init(AVCodecContext *avctx)
+     case AV_CODEC_ID_H264:
+         role = "video_encoder.avc";
+         break;
++    case AV_CODEC_ID_HEVC:
++		role = "video_encoder.hevc";
++		break;
+     default:
+         return AVERROR(ENOSYS);
+     }
+@@ -941,6 +947,13 @@ static const AVOption options[] = {
+     { NULL }
+ };
+ 
++static const AVOption options_hevc[] = {
++    { "omx_libname", "OpenMAX library name", OFFSET(libname), AV_OPT_TYPE_STRING, { 0 }, 0, 0, VDE },
++    { "omx_libprefix", "OpenMAX library prefix", OFFSET(libprefix), AV_OPT_TYPE_STRING, { 0 }, 0, 0, VDE },
++    { "zerocopy", "Try to avoid copying input frames if possible", OFFSET(input_zerocopy), AV_OPT_TYPE_INT, { .i64 = CONFIG_OMX_RPI }, 0, 1, VE },
++	{ NULL },
++};
++
+ static const enum AVPixelFormat omx_encoder_pix_fmts[] = {
+     AV_PIX_FMT_YUV420P, AV_PIX_FMT_NONE
+ };
+@@ -986,3 +999,25 @@ AVCodec ff_h264_omx_encoder = {
+     .caps_internal    = FF_CODEC_CAP_INIT_THREADSAFE | FF_CODEC_CAP_INIT_CLEANUP,
+     .priv_class       = &omx_h264enc_class,
+ };
++
++static const AVClass omx_hevcenc_class = {
++	.class_name = "hevc_omx",
++	.item_name = av_default_item_name,
++	.option = options_hevc,
++	.version = LIBAVUTIL_VERSION_INT,
++};
++AVCodec ff_hevc_omx_encoder = {
++	.name 			  = "hevc_omx",
++	.long_name 		  = NULL_IF_CONFIG_SMALL("OpenMAX IL HEVC video encoder"),
++	.type 			  = AVMEDIA_TYPE_VIDEO,
++	.id 			  = AV_CODEC_ID_HEVC,
++	.priv_data_size   = sizeof(OMXCodecContext),
++	.init 			  = omx_encode_init,
++	.encode2 		  = omx_encode_frame,
++	.close 			  = omx_encode_end,
++	.pix_fmts 		  = omx_encoder_pix_fmts,
++	.profiles         = NULL_IF_CONFIG_SMALL(ff_hevc_profiles),
++	.capabilities 	  = AV_CODEC_CAP_DELAY,
++	.caps_internal 	  = FF_CODEC_CAP_INIT_THREADSAFE | FF_CODEC_CAP_INIT_CLEANUP,
++	.priv_class 	  = &omx_hevcenc_class,
++};
+diff --git a/libavcodec/omxdec.c b/libavcodec/omxdec.c
+index 7f76ec3..96a2829 100644
+--- a/libavcodec/omxdec.c
++++ b/libavcodec/omxdec.c
+@@ -44,6 +44,7 @@
+ #include "avcodec.h"
+ #include "h264.h"
+ #include "internal.h"
++#include "profiles.h"
+ 
+ #ifdef OMX_SKIP64BIT
+ static OMX_TICKS to_omx_ticks(int64_t value)
+@@ -494,6 +495,8 @@ static av_cold int omx_component_init(AVCodecContext *avctx, const char *role)
+         out_port_params.format.video.eCompressionFormat = OMX_VIDEO_CodingMPEG4;
+     else if (avctx->codec->id == AV_CODEC_ID_H264)
+         out_port_params.format.video.eCompressionFormat = OMX_VIDEO_CodingAVC;
++    else if (avctx->codec->id == AV_CODEC_ID_HEVC)
++        out_port_params.format.video.eCompressionFormat = OMX_VIDEO_CodingHEVC;
+ 
+     err = OMX_SetParameter(s->handle, OMX_IndexParamPortDefinition, &out_port_params);
+     CHECK(err);
+@@ -627,6 +630,9 @@ static av_cold int omx_decode_init(AVCodecContext *avctx)
+     case AV_CODEC_ID_H264:
+         role = "video_decoder.avc";
+         break;
++    case AV_CODEC_ID_HEVC:
++		role = "video_decoder.hevc";
++		break;
+     default:
+         return AVERROR(ENOSYS);
+     }
+@@ -733,7 +739,7 @@ static int omx_decode_frame(AVCodecContext *avctx, void *data,
+ 			buffer->nFilledLen = pkt->size;
+ 		}
+ 
+-		/* reduce memcpy. point it addr*/
++		/* avoid memcpy. point it addr*/
+ 		//buffer->pAppPrivate = pkt;
+ 		//buffer->pBuffer = pkt->data;
+ 		//buffer->nFilledLen = pkt->size;
+@@ -851,6 +857,12 @@ static const AVOption options[] = {
+     { NULL }
+ };
+ 
++static const AVOption options_hevc[] = {
++    { "omx_libname", "OpenMAX library name", OFFSET(libname), AV_OPT_TYPE_STRING, { 0 }, 0, 0, VDE },
++    { "omx_libprefix", "OpenMAX library prefix", OFFSET(libprefix), AV_OPT_TYPE_STRING, { 0 }, 0, 0, VDE },
++    { "zerocopy", "Try to avoid copying input frames if possible", OFFSET(input_zerocopy), AV_OPT_TYPE_INT, { .i64 = CONFIG_OMX_RPI }, 0, 1, VE },
++	{ NULL },
++};
+ 
+ static const AVClass omx_mpeg4dec_class = {
+     .class_name = "mpeg4_omx",
+@@ -891,3 +903,24 @@ AVCodec ff_h264_omx_decoder = {
+     .caps_internal    = FF_CODEC_CAP_INIT_THREADSAFE | FF_CODEC_CAP_INIT_CLEANUP,
+     .priv_class       = &omx_h264dec_class,
+ };
++
++static const AVClass omx_hevcdec_class = {
++    .class_name = "hevc_omx",
++    .item_name  = av_default_item_name,
++    .option     = options_hevc,
++    .version    = LIBAVUTIL_VERSION_INT,
++};
++AVCodec ff_hevc_omx_decoder = {
++    .name             = "hevc_omx",
++    .long_name        = NULL_IF_CONFIG_SMALL("OpenMAX IL HEVC video decoder"),
++    .type             = AVMEDIA_TYPE_VIDEO,
++    .id               = AV_CODEC_ID_HEVC,
++    .priv_data_size   = sizeof(OMXCodecContext),
++    .init             = omx_decode_init,
++    .decode           = omx_decode_frame,
++    .close            = omx_decode_end,
++    .profiles         = NULL_IF_CONFIG_SMALL(ff_hevc_profiles),
++    .capabilities     = AV_CODEC_CAP_DELAY,
++    .caps_internal    = FF_CODEC_CAP_INIT_THREADSAFE | FF_CODEC_CAP_INIT_CLEANUP,
++    .priv_class       = &omx_hevcdec_class,
++};
+diff --git a/libavformat/utils.c b/libavformat/utils.c
+index 75e5350..262ff5f 100644
+--- a/libavformat/utils.c
++++ b/libavformat/utils.c
+@@ -212,6 +212,13 @@ static const AVCodec *find_probe_decoder(AVFormatContext *s, const AVStream *st,
+         return avcodec_find_decoder_by_name("h264");
+ #endif
+ 
++#if CONFIG_HEVC_DECODER
++	/* Other parts of the code assume this decoder to be used for h265,
++	 * so force it if possible. */
++	if (codec_id == AV_CODEC_ID_HEVC)
++		return avcodec_find_decoder_by_name("hevc");
++#endif
++
+     codec = find_decoder(s, st, codec_id);
+     if (!codec)
+         return NULL;
+-- 
+2.17.1
+

+ 307 - 0
multimedia/patch/ffmpeg/0007-avcoder-fix-decoder-bug.patch

@@ -0,0 +1,307 @@
+From 9ca5512fde8d237977b102e240c957e7506a276a Mon Sep 17 00:00:00 2001
+From: "sw.multimedia" <se.multimedia@starfivetech.com>
+Date: Wed, 25 May 2022 20:52:37 +0800
+Subject: [PATCH 5/8] avcoder fix decoder bug
+
+Signed-off-by: sw.multimedia <se.multimedia@starfivetech.com>
+---
+ libavcodec/omxdec.c | 182 +++++++++++++++++++++++++++-----------------
+ 1 file changed, 114 insertions(+), 68 deletions(-)
+
+diff --git a/libavcodec/omxdec.c b/libavcodec/omxdec.c
+index 96a2829..9eb5dca 100644
+--- a/libavcodec/omxdec.c
++++ b/libavcodec/omxdec.c
+@@ -46,6 +46,11 @@
+ #include "internal.h"
+ #include "profiles.h"
+ 
++int evnet_bufferflag;
++int dec_out_height;
++int dec_out_width;
++int dec_pix_fmt;
++
+ #ifdef OMX_SKIP64BIT
+ static OMX_TICKS to_omx_ticks(int64_t value)
+ {
+@@ -223,7 +228,7 @@ typedef struct OMXCodecContext {
+ 
+     int mutex_cond_inited;
+ 
+-    int eos_sent, got_eos, first_get_outbuffer;
++    int eos_sent, got_eos, evnet_bufferflag;
+ 
+ 	int extradata_sent;
+ 
+@@ -274,6 +279,11 @@ static OMX_ERRORTYPE event_handler(OMX_HANDLETYPE component, OMX_PTR app_data, O
+     // This uses casts in the printfs, since OMX_U32 actually is a typedef for
+     // unsigned long in official header versions (but there are also modified
+     // versions where it is something else).
++    OMX_PARAM_PORTDEFINITIONTYPE out_port_params = { 0 };
++	OMX_PORT_PARAM_TYPE video_port_params = { 0 };
++	OMX_ERRORTYPE err;
++	int i;
++
+     switch (event) {
+     case OMX_EventError:
+         pthread_mutex_lock(&s->state_mutex);
+@@ -299,8 +309,36 @@ static OMX_ERRORTYPE event_handler(OMX_HANDLETYPE component, OMX_PTR app_data, O
+         }
+         break;
+     case OMX_EventPortSettingsChanged:
+-        av_log(s->avctx, AV_LOG_VERBOSE, "OMX port %"PRIu32" settings changed\n", (uint32_t) data1);
++        av_log(s->avctx, AV_LOG_ERROR, "OMX port %"PRIu32" settings changed\n", (uint32_t) data1);
++		INIT_STRUCT(video_port_params);
++		err = OMX_GetParameter(s->handle, OMX_IndexParamVideoInit, &video_port_params);
++		if(err != OMX_ErrorNone){
++            av_log(s->avctx, AV_LOG_ERROR, "err %d\n",err);          
++            return AVERROR_UNKNOWN;
++			}
++
++		for (i = 0; i < video_port_params.nPorts; i++) {
++	        int port = video_port_params.nStartPortNumber + i;
++	        OMX_PARAM_PORTDEFINITIONTYPE port_params = { 0 };
++	        INIT_STRUCT(port_params);
++	        port_params.nPortIndex = port;
++	        err = OMX_GetParameter(s->handle, OMX_IndexParamPortDefinition, &port_params);
++	        if (err != OMX_ErrorNone) {
++	            av_log(s->avctx, AV_LOG_WARNING, "port %d error %x\n", port, err);
++            return AVERROR_UNKNOWN;
++	        }
++	        if (port_params.eDir == OMX_DirOutput) {
++				out_port_params = port_params;
++				dec_out_width = out_port_params.format.video.nFrameWidth;
++				dec_out_height = out_port_params.format.video.nFrameHeight;
++				dec_pix_fmt = out_port_params.format.video.eColorFormat;
++	            
++	        } 
++	    }		
+         break;
++	case OMX_EventBufferFlag:
++		av_log(s->avctx, AV_LOG_VERBOSE, "OMX decoder competd set event_bufferflag\n");
++		evnet_bufferflag = 1;
+     default:
+         av_log(s->avctx, AV_LOG_VERBOSE, "OMX event %d %"PRIx32" %"PRIx32"\n",
+                                          event, (uint32_t) data1, (uint32_t) data2);
+@@ -453,7 +491,7 @@ static av_cold int omx_component_init(AVCodecContext *avctx, const char *role)
+ 
+     in_port_params.format.video.pNativeRender         = NULL;
+     in_port_params.format.video.bFlagErrorConcealment = OMX_FALSE;
+-    //in_port_params.format.video.eColorFormat          = s->color_format;
++    in_port_params.format.video.eColorFormat          = s->color_format;
+     s->stride     = avctx->width;
+     s->plane_size = avctx->height;
+     // If specific codecs need to manually override the stride/plane_size,
+@@ -473,11 +511,10 @@ static av_cold int omx_component_init(AVCodecContext *avctx, const char *role)
+     err = OMX_GetParameter(s->handle, OMX_IndexParamPortDefinition, &in_port_params);
+     CHECK(err);
+ 
+-#if 1
+     s->stride         = in_port_params.format.video.nStride;
+     s->plane_size     = in_port_params.format.video.nSliceHeight;
+     s->num_in_buffers = in_port_params.nBufferCountActual;
+-#endif
++
+     err = OMX_GetParameter(s->handle, OMX_IndexParamPortDefinition, &out_port_params);
+     out_port_params.bEnabled   = OMX_TRUE;
+     out_port_params.bPopulated = OMX_FALSE;
+@@ -645,51 +682,51 @@ static av_cold int omx_decode_init(AVCodecContext *avctx)
+     if ((ret = omx_component_init(avctx, role)) < 0)
+         goto fail;
+ 
+-#if 0
+-    if (avctx->flags & AV_CODEC_FLAG_GLOBAL_HEADER) {
+-        while (1) {
+-            buffer = get_buffer(&s->output_mutex, &s->output_cond,
+-                                &s->num_done_out_buffers, s->done_out_buffers, 1);
+-            if (buffer->nFlags & OMX_BUFFERFLAG_CODECCONFIG) {
+-                if ((ret = av_reallocp(&avctx->extradata, avctx->extradata_size + buffer->nFilledLen + AV_INPUT_BUFFER_PADDING_SIZE)) < 0) {
+-                    avctx->extradata_size = 0;
+-                    goto fail;
+-                }
+-                memcpy(avctx->extradata + avctx->extradata_size, buffer->pBuffer + buffer->nOffset, buffer->nFilledLen);
+-                avctx->extradata_size += buffer->nFilledLen;
+-                memset(avctx->extradata + avctx->extradata_size, 0, AV_INPUT_BUFFER_PADDING_SIZE);
+-            }
+-            err = OMX_FillThisBuffer(s->handle, buffer);
+-            if (err != OMX_ErrorNone) {
+-                append_buffer(&s->output_mutex, &s->output_cond,
+-                              &s->num_done_out_buffers, s->done_out_buffers, buffer);
+-                av_log(avctx, AV_LOG_ERROR, "OMX_FillThisBuffer failed: %x\n", err);
+-                ret = AVERROR_UNKNOWN;
+-                goto fail;
+-            }
+-            if (avctx->codec->id == AV_CODEC_ID_H264) {
+-                // For H.264, the extradata can be returned in two separate buffers
+-                // (the videocore encoder on raspberry pi does this);
+-                // therefore check that we have got both SPS and PPS before continuing.
+-                int nals[32] = { 0 };
+-                int i;
+-                for (i = 0; i + 4 < avctx->extradata_size; i++) {
+-                     if (!avctx->extradata[i + 0] &&
+-                         !avctx->extradata[i + 1] &&
+-                         !avctx->extradata[i + 2] &&
+-                         avctx->extradata[i + 3] == 1) {
+-                         nals[avctx->extradata[i + 4] & 0x1f]++;
+-                     }
+-                }
+-                if (nals[H264_NAL_SPS] && nals[H264_NAL_PPS])
+-                    break;
+-            } else {
+-                if (avctx->extradata_size > 0)
+-                    break;
+-            }
+-        }
+-    }
+-#endif
++// #if 0
++//     if (avctx->flags & AV_CODEC_FLAG_GLOBAL_HEADER) {
++//         while (1) {
++//             buffer = get_buffer(&s->output_mutex, &s->output_cond,
++//                                 &s->num_done_out_buffers, s->done_out_buffers, 1);
++//             if (buffer->nFlags & OMX_BUFFERFLAG_CODECCONFIG) {
++//                 if ((ret = av_reallocp(&avctx->extradata, avctx->extradata_size + buffer->nFilledLen + AV_INPUT_BUFFER_PADDING_SIZE)) < 0) {
++//                     avctx->extradata_size = 0;
++//                     goto fail;
++//                 }
++//                 memcpy(avctx->extradata + avctx->extradata_size, buffer->pBuffer + buffer->nOffset, buffer->nFilledLen);
++//                 avctx->extradata_size += buffer->nFilledLen;
++//                 memset(avctx->extradata + avctx->extradata_size, 0, AV_INPUT_BUFFER_PADDING_SIZE);
++//             }
++//             err = OMX_FillThisBuffer(s->handle, buffer);
++//             if (err != OMX_ErrorNone) {
++//                 append_buffer(&s->output_mutex, &s->output_cond,
++//                               &s->num_done_out_buffers, s->done_out_buffers, buffer);
++//                 av_log(avctx, AV_LOG_ERROR, "OMX_FillThisBuffer failed: %x\n", err);
++//                 ret = AVERROR_UNKNOWN;
++//                 goto fail;
++//             }
++//             if (avctx->codec->id == AV_CODEC_ID_H264) {
++//                 // For H.264, the extradata can be returned in two separate buffers
++//                 // (the videocore encoder on raspberry pi does this);
++//                 // therefore check that we have got both SPS and PPS before continuing.
++//                 int nals[32] = { 0 };
++//                 int i;
++//                 for (i = 0; i + 4 < avctx->extradata_size; i++) {
++//                      if (!avctx->extradata[i + 0] &&
++//                          !avctx->extradata[i + 1] &&
++//                          !avctx->extradata[i + 2] &&
++//                          avctx->extradata[i + 3] == 1) {
++//                          nals[avctx->extradata[i + 4] & 0x1f]++;
++//                      }
++//                 }
++//                 if (nals[H264_NAL_SPS] && nals[H264_NAL_PPS])
++//                     break;
++//             } else {
++//                 if (avctx->extradata_size > 0)
++//                     break;
++//             }
++//         }
++//     }
++// #endif
+ 	
+     return 0;
+ fail:
+@@ -712,10 +749,6 @@ static int omx_decode_frame(AVCodecContext *avctx, void *data,
+ 	int linesize[4];
+ 
+     if (pkt->size) {
+-
+-		//VPU init and fill buffer slow, so empty buf sleep to send before get vpu fill buf.
+-		if(!s->first_get_outbuffer)
+-			av_usleep(100000);
+ 	
+         buffer = get_buffer(&s->input_mutex, &s->input_cond,
+                             &s->num_free_in_buffers, s->free_in_buffers, 1);
+@@ -726,7 +759,7 @@ static int omx_decode_frame(AVCodecContext *avctx, void *data,
+         }
+ 
+ 		//cpy the extradata
+-		if(!s->extradata_sent) {
++		if(!s->extradata_sent && avctx->extradata ) {
+ 
+ 			memcpy(buffer->pBuffer + buffer->nOffset, avctx->extradata, avctx->extradata_size);	
+ 			memcpy(buffer->pBuffer + buffer->nOffset + avctx->extradata_size, pkt->data, pkt->size);
+@@ -746,6 +779,7 @@ static int omx_decode_frame(AVCodecContext *avctx, void *data,
+ 		
+ 		buffer->pOutputPortPrivate = NULL;
+ 		buffer->pAppPrivate = avctx->priv_data;
++        buffer->nFlags = OMX_BUFFERFLAG_ENDOFFRAME;
+ 
+ 	    err = OMX_EmptyThisBuffer(s->handle, buffer);
+ 	    if (err != OMX_ErrorNone) {
+@@ -754,16 +788,19 @@ static int omx_decode_frame(AVCodecContext *avctx, void *data,
+ 	        return AVERROR_UNKNOWN;
+ 	    }
+     } else if (!s->eos_sent) {
+-
+-		if(!s->first_get_outbuffer)
+-			av_usleep(1000000);
+ 		
+         buffer = get_buffer(&s->input_mutex, &s->input_cond,
+                             &s->num_free_in_buffers, s->free_in_buffers, 1);
+ 
++        if(!buffer) {
++        	av_log(avctx, AV_LOG_ERROR, "get_buffer NULL\n");
++            return AVERROR(ENOMEM);
++		}
++
+         buffer->nFilledLen = 0;
+         buffer->nFlags = OMX_BUFFERFLAG_EOS;
+         buffer->pAppPrivate = buffer->pOutputPortPrivate = NULL;
++        
+         err = OMX_EmptyThisBuffer(s->handle, buffer);
+         if (err != OMX_ErrorNone) {
+             append_buffer(&s->input_mutex, &s->input_cond, &s->num_free_in_buffers, s->free_in_buffers, buffer);
+@@ -782,20 +819,26 @@ static int omx_decode_frame(AVCodecContext *avctx, void *data,
+                             !pkt || had_partial);
+ 
+         if (!buffer) {
+-            /*eos is sent but fill buf still can't get then continue*/
+-            if(!s->first_get_outbuffer && s->eos_sent)
++            /*eos is sent wait for vpu evnet_bufferflag to get all frames*/
+++           if(s->eos_sent && !evnet_bufferflag){}
+ 				continue; 
+-			else
++            }
+ 				break;
+         }
+ 		//if (!buffer)
+            // break;
+ 
+-		if(!buffer->nFilledLen)
+-			goto end;
+-		
+-		if(!s->first_get_outbuffer)
+-			s->first_get_outbuffer = 1;
++		if(!buffer->nFilledLen){
++            av_log(avctx, AV_LOG_ERROR, "buffer->nFilledLen %d\n",(int)buffer->nFilledLen);
++            goto end;
++        }
++
++        avctx->width = dec_out_width;
++		avctx->height = dec_out_height;
++		avctx->pix_fmt = AV_PIX_FMT_YUV420P;
++		s->stride     = avctx->width;
++    	s->plane_size = avctx->height;
++		//avctx->pix_fmt = dec_pix_fmt;
+ 		
+         if (buffer->nFlags & OMX_BUFFERFLAG_EOS)
+             s->got_eos = 1;
+@@ -807,8 +850,11 @@ static int omx_decode_frame(AVCodecContext *avctx, void *data,
+ 		
+ 		ret = av_image_fill_arrays(dst, linesize, buffer->pBuffer,
+ 								   avctx->pix_fmt, s->stride, s->plane_size, 1);
+-		if (ret < 0)
+-			goto end;
++		if (ret < 0){
++            av_log(avctx, AV_LOG_ERROR, "av_image_fill_arrays ret:%d\n", ret);
++            goto end;
++        }
++
+ 		av_image_copy(avframe->data, avframe->linesize, (const uint8_t**)dst, linesize, 
+ 							avctx->pix_fmt, avctx->width, avctx->height);
+ 	
+-- 
+2.17.1
+

+ 83 - 0
multimedia/patch/ffmpeg/0008-fix-omx-decoder-setting-pix-fmt-bug.patch

@@ -0,0 +1,83 @@
+From 39a500044fe9216453960529ae9320fda2ef73e4 Mon Sep 17 00:00:00 2001
+From: "sw.multimedia" <se.multimedia@starfivetech.com>
+Date: Wed, 25 May 2022 20:58:57 +0800
+Subject: [PATCH 6/8] fix omx decoder setting pix-fmt bug
+
+Signed-off-by: sw.multimedia <se.multimedia@starfivetech.com>
+---
+ libavcodec/omxdec.c | 36 ++++++++++++++++++++++++++++++++++--
+ 1 file changed, 34 insertions(+), 2 deletions(-)
+
+diff --git a/libavcodec/omxdec.c b/libavcodec/omxdec.c
+index 9eb5dca..8d00b86 100644
+--- a/libavcodec/omxdec.c
++++ b/libavcodec/omxdec.c
+@@ -28,6 +28,7 @@
+ #include <dlfcn.h>
+ #include <OMX_Core.h>
+ #include <OMX_Component.h>
++#include <OMX_IVCommon.h>
+ #include <pthread.h>
+ #include <stdio.h>
+ #include <stdlib.h>
+@@ -80,6 +81,8 @@ static int64_t from_omx_ticks(OMX_TICKS value)
+         }                                                                 \
+     } while (0)
+ 
++#define FF_ARRAY_ELEMS(a) (sizeof(a) / sizeof((a)[0]))
++
+ typedef struct OMXContext {
+     void *lib;
+     void *lib2;
+@@ -93,6 +96,36 @@ typedef struct OMXContext {
+     void (*host_init)(void);
+ } OMXContext;
+ 
++static const struct {
++
++    int color_format;
++    enum AVPixelFormat pix_fmt;
++
++} color_formats[] = {
++
++    { OMX_COLOR_FormatYUV420Planar,                           AV_PIX_FMT_YUV420P },
++    { OMX_COLOR_FormatYUV420SemiPlanar,                       AV_PIX_FMT_NV12    },
++    { OMX_COLOR_FormatYUV420PackedSemiPlanar,                 AV_PIX_FMT_NV21    },
++    { 0 }
++};
++
++static enum AVPixelFormat omx_map_color_format(AVCodecContext *avctx, int color_format)
++{
++    int i;
++    enum AVPixelFormat ret = AV_PIX_FMT_NONE;
++
++    for (i = 0; i < FF_ARRAY_ELEMS(color_formats); i++) {
++        if (color_formats[i].color_format == color_format) {
++            return color_formats[i].pix_fmt;
++        }
++    }
++
++    av_log(avctx, AV_LOG_ERROR, "Output color format 0x%x (value=%d) is not supported\n",
++        color_format, color_format);
++
++    return ret;
++}
++
+ static av_cold void *dlsym_prefixed(void *handle, const char *symbol, const char *prefix)
+ {
+     char buf[50];
+@@ -835,10 +868,9 @@ static int omx_decode_frame(AVCodecContext *avctx, void *data,
+ 
+         avctx->width = dec_out_width;
+ 		avctx->height = dec_out_height;
+-		avctx->pix_fmt = AV_PIX_FMT_YUV420P;
++        avctx->pix_fmt = omx_map_color_format(avctx, dec_pix_fmt);
+ 		s->stride     = avctx->width;
+     	s->plane_size = avctx->height;
+-		//avctx->pix_fmt = dec_pix_fmt;
+ 		
+         if (buffer->nFlags & OMX_BUFFERFLAG_EOS)
+             s->got_eos = 1;
+-- 
+2.17.1
+

+ 222 - 0
multimedia/patch/ffmpeg/0009-ffmpeg-add-mjpeg-decoder-support-and-fix-some-bug.patch

@@ -0,0 +1,222 @@
+From fff5724489cd0a503d206c6f513b1d8572247484 Mon Sep 17 00:00:00 2001
+From: "sw.multimedia" <se.multimedia@starfivetech.com>
+Date: Wed, 25 May 2022 21:28:23 +0800
+Subject: [PATCH] ffmpeg: add mjpeg decoder support and fix some bug
+
+Signed-off-by: sw.multimedia <se.multimedia@starfivetech.com>
+---
+ configure              |  1 +
+ libavcodec/allcodecs.c |  1 +
+ libavcodec/omx.c       | 25 +++++++++++++++++++++---
+ libavcodec/omxdec.c    | 44 +++++++++++++++++++++++++++++++++++++-----
+ libavformat/utils.c    |  7 +++++++
+ 5 files changed, 70 insertions(+), 8 deletions(-)
+
+diff --git a/configure b/configure
+index 80be074..ca465a1 100755
+--- a/configure
++++ b/configure
+@@ -3095,6 +3095,7 @@ h264_v4l2m2m_decoder_select="h264_mp4toannexb_bsf"
+ h264_v4l2m2m_encoder_deps="v4l2_m2m h264_v4l2_m2m"
+ hevc_omx_encoder_deps="omx"
+ hevc_omx_decoder_deps="omx"
++mjpeg_omx_decoder_deps="omx"
+ hevc_amf_encoder_deps="amf"
+ hevc_cuvid_decoder_deps="cuvid"
+ hevc_cuvid_decoder_select="hevc_mp4toannexb_bsf"
+diff --git a/libavcodec/allcodecs.c b/libavcodec/allcodecs.c
+index 0c7a18d..69627df 100644
+--- a/libavcodec/allcodecs.c
++++ b/libavcodec/allcodecs.c
+@@ -817,6 +817,7 @@ extern AVCodec ff_libkvazaar_encoder;
+ extern AVCodec ff_mjpeg_cuvid_decoder;
+ extern AVCodec ff_mjpeg_qsv_encoder;
+ extern AVCodec ff_mjpeg_qsv_decoder;
++extern AVCodec ff_mjpeg_omx_decoder;
+ extern AVCodec ff_mjpeg_vaapi_encoder;
+ extern AVCodec ff_mp3_mf_encoder;
+ extern AVCodec ff_mpeg1_cuvid_decoder;
+diff --git a/libavcodec/omx.c b/libavcodec/omx.c
+index 86e32a8..023db9c 100644
+--- a/libavcodec/omx.c
++++ b/libavcodec/omx.c
+@@ -42,6 +42,7 @@
+ 
+ #include "avcodec.h"
+ #include "h264.h"
++#include "hevc.h"
+ #include "internal.h"
+ #include "profiles.h"
+ 
+@@ -474,9 +475,11 @@ static av_cold int omx_component_init(AVCodecContext *avctx, const char *role)
+     in_port_params.format.video.nFrameWidth  = avctx->width;
+     in_port_params.format.video.nFrameHeight = avctx->height;
+     if (avctx->framerate.den > 0 && avctx->framerate.num > 0)
+-        in_port_params.format.video.xFramerate = (1LL << 16) * avctx->framerate.num / avctx->framerate.den;
++        //in_port_params.format.video.xFramerate = (1LL << 16) * avctx->framerate.num / avctx->framerate.den;
++        in_port_params.format.video.xFramerate = avctx->framerate.num / avctx->framerate.den;
+     else
+-        in_port_params.format.video.xFramerate = (1LL << 16) * avctx->time_base.den / avctx->time_base.num;
++        //in_port_params.format.video.xFramerate = (1LL << 16) * avctx->time_base.den / avctx->time_base.num;
++        in_port_params.format.video.xFramerate = avctx->time_base.den / avctx->time_base.num;
+ 
+     err = OMX_SetParameter(s->handle, OMX_IndexParamPortDefinition, &in_port_params);
+     CHECK(err);
+@@ -721,7 +724,23 @@ static av_cold int omx_encode_init(AVCodecContext *avctx)
+                 }
+                 if (nals[H264_NAL_SPS] && nals[H264_NAL_PPS])
+                     break;
+-            } else {
++            } else if (avctx->codec->id == AV_CODEC_ID_HEVC) {
++                // For H.265, the extradata can be returned in two separate buffers
++                // (the videocore encoder on raspberry pi does this);
++                // therefore check that we have got both SPS and PPS before continuing.
++                int nals[128] = { 0 };
++                int i;
++                for (i = 0; i + 4 < avctx->extradata_size; i++) {
++                     if (!avctx->extradata[i + 0] &&
++                         !avctx->extradata[i + 1] &&
++                         !avctx->extradata[i + 2] &&
++                         avctx->extradata[i + 3] == 1) {
++                         nals[(avctx->extradata[i + 4] & 0x7E) >> 1]++;
++                     }
++                }
++                if (nals[HEVC_NAL_SPS] && nals[HEVC_NAL_PPS] && nals[HEVC_NAL_VPS])
++                    break;
++		} else {
+                 if (avctx->extradata_size > 0)
+                     break;
+             }
+diff --git a/libavcodec/omxdec.c b/libavcodec/omxdec.c
+index 8d00b86..32f39f3 100644
+--- a/libavcodec/omxdec.c
++++ b/libavcodec/omxdec.c
+@@ -106,6 +106,7 @@ static const struct {
+     { OMX_COLOR_FormatYUV420Planar,                           AV_PIX_FMT_YUV420P },
+     { OMX_COLOR_FormatYUV420SemiPlanar,                       AV_PIX_FMT_NV12    },
+     { OMX_COLOR_FormatYUV420PackedSemiPlanar,                 AV_PIX_FMT_NV21    },
++    { OMX_COLOR_FormatYUV444Interleaved,                      AV_PIX_FMT_YUV444P },
+     { 0 }
+ };
+ 
+@@ -366,6 +367,7 @@ static OMX_ERRORTYPE event_handler(OMX_HANDLETYPE component, OMX_PTR app_data, O
+ 				dec_out_height = out_port_params.format.video.nFrameHeight;
+ 				dec_pix_fmt = out_port_params.format.video.eColorFormat;
+ 	            
++                av_log(s->avctx, AV_LOG_VERBOSE, "w:%d, h:%d, fmt:%d\n", dec_out_width, dec_out_height, dec_pix_fmt); 
+ 	        } 
+ 	    }		
+         break;
+@@ -535,9 +537,11 @@ static av_cold int omx_component_init(AVCodecContext *avctx, const char *role)
+     in_port_params.format.video.nFrameHeight = avctx->height;
+ 
+     if (avctx->framerate.den > 0 && avctx->framerate.num > 0)
+-        in_port_params.format.video.xFramerate = (1LL << 16) * avctx->framerate.num / avctx->framerate.den;
++        //in_port_params.format.video.xFramerate = (1LL << 16) * avctx->framerate.num / avctx->framerate.den;
++        in_port_params.format.video.xFramerate = avctx->framerate.num / avctx->framerate.den;
+     else
+-        in_port_params.format.video.xFramerate = (1LL << 16) * avctx->time_base.den / avctx->time_base.num;
++        //in_port_params.format.video.xFramerate = (1LL << 16) * avctx->time_base.den / avctx->time_base.num;
++        in_port_params.format.video.xFramerate = avctx->time_base.den / avctx->time_base.num;
+ 
+     err = OMX_SetParameter(s->handle, OMX_IndexParamPortDefinition, &in_port_params);
+     CHECK(err);
+@@ -567,6 +571,8 @@ static av_cold int omx_component_init(AVCodecContext *avctx, const char *role)
+         out_port_params.format.video.eCompressionFormat = OMX_VIDEO_CodingAVC;
+     else if (avctx->codec->id == AV_CODEC_ID_HEVC)
+         out_port_params.format.video.eCompressionFormat = OMX_VIDEO_CodingHEVC;
++    else if (avctx->codec->id == AV_CODEC_ID_MJPEG)
++        out_port_params.format.video.eCompressionFormat = OMX_VIDEO_CodingMJPEG;
+ 
+     err = OMX_SetParameter(s->handle, OMX_IndexParamPortDefinition, &out_port_params);
+     CHECK(err);
+@@ -703,6 +709,9 @@ static av_cold int omx_decode_init(AVCodecContext *avctx)
+     case AV_CODEC_ID_HEVC:
+ 		role = "video_decoder.hevc";
+ 		break;
++    case AV_CODEC_ID_MJPEG:
++		role = "video_decoder.mjpeg";
++		break;
+     default:
+         return AVERROR(ENOSYS);
+     }
+@@ -852,11 +861,13 @@ static int omx_decode_frame(AVCodecContext *avctx, void *data,
+                             !pkt || had_partial);
+ 
+         if (!buffer) {
+-            /*eos is sent wait for vpu evnet_bufferflag to get all frames*/
+-+           if(s->eos_sent && !evnet_bufferflag){}
++            /*eos is sent wait for vpu evnet_bufferflag to get all frames
++			  mjpeg: sent a frame, then wait for a decoder frame 
++			*/
++            if((s->eos_sent && !evnet_bufferflag) || (avctx->codec_id == AV_CODEC_ID_MJPEG )) {
+ 				continue; 
+             }
+-				break;
++            break;
+         }
+ 		//if (!buffer)
+            // break;
+@@ -924,6 +935,7 @@ static av_cold int omx_decode_end(AVCodecContext *avctx)
+ #define OFFSET(x) offsetof(OMXCodecContext, x)
+ #define VDE AV_OPT_FLAG_VIDEO_PARAM | AV_OPT_FLAG_DECODING_PARAM | AV_OPT_FLAG_ENCODING_PARAM
+ #define VE  AV_OPT_FLAG_VIDEO_PARAM | AV_OPT_FLAG_ENCODING_PARAM
++#define VD  AV_OPT_FLAG_VIDEO_PARAM | AV_OPT_FLAG_DECODING_PARAM
+ static const AVOption options[] = {
+     { "omx_libname", "OpenMAX library name", OFFSET(libname), AV_OPT_TYPE_STRING, { 0 }, 0, 0, VDE },
+     { "omx_libprefix", "OpenMAX library prefix", OFFSET(libprefix), AV_OPT_TYPE_STRING, { 0 }, 0, 0, VDE },
+@@ -980,6 +992,7 @@ AVCodec ff_h264_omx_decoder = {
+     .capabilities     = AV_CODEC_CAP_DELAY,
+     .caps_internal    = FF_CODEC_CAP_INIT_THREADSAFE | FF_CODEC_CAP_INIT_CLEANUP,
+     .priv_class       = &omx_h264dec_class,
++    .bsfs             = "h264_mp4toannexb",
+ };
+ 
+ static const AVClass omx_hevcdec_class = {
+@@ -1001,4 +1014,25 @@ AVCodec ff_hevc_omx_decoder = {
+     .capabilities     = AV_CODEC_CAP_DELAY,
+     .caps_internal    = FF_CODEC_CAP_INIT_THREADSAFE | FF_CODEC_CAP_INIT_CLEANUP,
+     .priv_class       = &omx_hevcdec_class,
++    .bsfs             = "hevc_mp4toannexb",
++};
++
++static const AVClass omx_mjpegdec_class = {
++    .class_name = "mjpeg_omx",
++    .item_name  = av_default_item_name,
++    .version    = LIBAVUTIL_VERSION_INT,
++};
++AVCodec ff_mjpeg_omx_decoder = {
++    .name             = "mjpeg_omx",
++    .long_name        = NULL_IF_CONFIG_SMALL("OpenMAX IL mjpeg video decoder"),
++    .type             = AVMEDIA_TYPE_VIDEO,
++    .id               = AV_CODEC_ID_MJPEG,
++    .priv_data_size   = sizeof(OMXCodecContext),
++    .init             = omx_decode_init,
++    .decode           = omx_decode_frame,
++    .close            = omx_decode_end,
++	.capabilities	  = AV_CODEC_CAP_DR1,
++	.max_lowres 	  = 3,
++	.caps_internal	  = FF_CODEC_CAP_INIT_THREADSAFE,
++    .priv_class       = &omx_mjpegdec_class,
+ };
+diff --git a/libavformat/utils.c b/libavformat/utils.c
+index 262ff5f..4867810 100644
+--- a/libavformat/utils.c
++++ b/libavformat/utils.c
+@@ -219,6 +219,13 @@ static const AVCodec *find_probe_decoder(AVFormatContext *s, const AVStream *st,
+ 		return avcodec_find_decoder_by_name("hevc");
+ #endif
+ 
++#if CONFIG_MJPEG_DECODER
++		/* Other parts of the code assume this decoder to be used for mjpeg,
++		 * so force it if possible. */
++		if (codec_id == AV_CODEC_ID_MJPEG)
++			return avcodec_find_decoder_by_name("mjpeg");
++#endif
++
+     codec = find_decoder(s, st, codec_id);
+     if (!codec)
+         return NULL;
+-- 
+2.17.1
+

+ 48 - 0
multimedia/patch/ffmpeg/0010-ffmpeg-support-to-change-gop.patch

@@ -0,0 +1,48 @@
+From 8b976d184eb1ded7a0635582a4e1b1c619852ed8 Mon Sep 17 00:00:00 2001
+From: "arvin.zhu" <arvin.zhu@starfivetech.com>
+Date: Thu, 22 Sep 2022 18:56:28 +0800
+Subject: [PATCH] ffmpeg: support to change gop
+
+support to change gop of hevc for omc encoder
+
+Signed-off-by: arvin.zhu <arvin.zhu@starfivetech.com>
+---
+ libavcodec/omx.c | 12 ++++++++++++
+ 1 file changed, 12 insertions(+)
+ mode change 100644 => 100755 libavcodec/omx.c
+
+diff --git a/libavcodec/omx.c b/libavcodec/omx.c
+old mode 100644
+new mode 100755
+index 84b0835..1d121af
+--- a/libavcodec/omx.c
++++ b/libavcodec/omx.c
+@@ -28,6 +28,8 @@
+ #include <dlfcn.h>
+ #include <OMX_Core.h>
+ #include <OMX_Component.h>
++#include <OMX_IndexExt.h>
++#include <OMX_VideoExt.h>
+ #include <pthread.h>
+ #include <stdio.h>
+ #include <stdlib.h>
+@@ -545,6 +547,16 @@ static av_cold int omx_component_init(AVCodecContext *avctx, const char *role)
+         }
+         err = OMX_SetParameter(s->handle, OMX_IndexParamVideoAvc, &avc);
+         CHECK(err);
++    } else if (avctx->codec->id == AV_CODEC_ID_HEVC) {
++        OMX_VIDEO_PARAM_AVCTYPE hevc = { 0 };
++        INIT_STRUCT(hevc);
++        hevc.nPortIndex = s->out_port;
++        err = OMX_GetParameter(s->handle, OMX_IndexParamVideoHevc, &hevc);
++        CHECK(err);
++        hevc.nBFrames = 0;
++        hevc.nPFrames = avctx->gop_size;
++        err = OMX_SetParameter(s->handle, OMX_IndexParamVideoHevc, &hevc);
++        CHECK(err);
+     }
+ 
+     err = OMX_SendCommand(s->handle, OMX_CommandStateSet, OMX_StateIdle, NULL);
+-- 
+2.17.1
+

+ 59 - 0
multimedia/patch/ffmpeg/0011-ffmpeg-add-delay-for-decoding.patch

@@ -0,0 +1,59 @@
+From f7401faa595d8cd70392c73ce9578ed1e83ed81b Mon Sep 17 00:00:00 2001
+From: "arvin.zhu" <arvin.zhu@starfivetech.com>
+Date: Thu, 22 Sep 2022 19:06:50 +0800
+Subject: [PATCH] ffmpeg: add delay for decoding
+
+add delay for decoding
+
+Signed-off-by: arvin.zhu <arvin.zhu@starfivetech.com>
+---
+ libavcodec/omxdec.c | 9 ++++++++-
+ 1 file changed, 8 insertions(+), 1 deletion(-)
+ mode change 100644 => 100755 libavcodec/omxdec.c
+
+diff --git a/libavcodec/omxdec.c b/libavcodec/omxdec.c
+old mode 100644
+new mode 100755
+index 32f39f3..03a2bca
+--- a/libavcodec/omxdec.c
++++ b/libavcodec/omxdec.c
+@@ -262,7 +262,7 @@ typedef struct OMXCodecContext {
+ 
+     int mutex_cond_inited;
+ 
+-    int eos_sent, got_eos, evnet_bufferflag;
++    int eos_sent, got_eos, evnet_bufferflag, first_get_outbuffer;
+ 
+ 	int extradata_sent;
+ 
+@@ -792,6 +792,9 @@ static int omx_decode_frame(AVCodecContext *avctx, void *data,
+ 
+     if (pkt->size) {
+ 	
++        //VPU init and fill buffer slow, so empty buf sleep to send before get vpu fill buf.
++        if(!s->first_get_outbuffer)
++        av_usleep(100000);
+         buffer = get_buffer(&s->input_mutex, &s->input_cond,
+                             &s->num_free_in_buffers, s->free_in_buffers, 1);
+ 
+@@ -831,6 +834,8 @@ static int omx_decode_frame(AVCodecContext *avctx, void *data,
+ 	    }
+     } else if (!s->eos_sent) {
+ 		
++        if(!s->first_get_outbuffer)
++        av_usleep(1000000);
+         buffer = get_buffer(&s->input_mutex, &s->input_cond,
+                             &s->num_free_in_buffers, s->free_in_buffers, 1);
+ 
+@@ -871,6 +876,8 @@ static int omx_decode_frame(AVCodecContext *avctx, void *data,
+         }
+ 		//if (!buffer)
+            // break;
++        if(!s->first_get_outbuffer)
++            s->first_get_outbuffer = 1;
+ 
+ 		if(!buffer->nFilledLen){
+             av_log(avctx, AV_LOG_ERROR, "buffer->nFilledLen %d\n",(int)buffer->nFilledLen);
+-- 
+2.17.1
+

+ 120 - 0
multimedia/patch/ffmpeg/0012-ffmpeg-add-omx_pix_fmt.patch

@@ -0,0 +1,120 @@
+From c4fe39d5b2e35b7195ae3041adcb6ef786322dbb Mon Sep 17 00:00:00 2001
+From: "arvin.zhu" <arvin.zhu@starfivetech.com>
+Date: Fri, 23 Sep 2022 10:04:25 +0800
+Subject: [PATCH] ffmpeg: add omx_pix_fmt
+
+add omx_pix_fmt for mjpeg_omx decoder
+
+Signed-off-by: arvin.zhu <arvin.zhu@starfivetech.com>
+---
+ libavcodec/omxdec.c | 58 +++++++++++++++++++++++++++++++++++++++++----
+ 1 file changed, 54 insertions(+), 4 deletions(-)
+
+diff --git a/libavcodec/omxdec.c b/libavcodec/omxdec.c
+index 03a2bca..b489c09 100755
+--- a/libavcodec/omxdec.c
++++ b/libavcodec/omxdec.c
+@@ -105,8 +105,13 @@ static const struct {
+ 
+     { OMX_COLOR_FormatYUV420Planar,                           AV_PIX_FMT_YUV420P },
+     { OMX_COLOR_FormatYUV420SemiPlanar,                       AV_PIX_FMT_NV12    },
+-    { OMX_COLOR_FormatYUV420PackedSemiPlanar,                 AV_PIX_FMT_NV21    },
+-    { OMX_COLOR_FormatYUV444Interleaved,                      AV_PIX_FMT_YUV444P },
++    { OMX_COLOR_FormatYVU420SemiPlanar,                       AV_PIX_FMT_NV21    },
++    { OMX_COLOR_FormatYUV422SemiPlanar,                       AV_PIX_FMT_NV16    },
++    { OMX_COLOR_FormatYUV422Planar,                           AV_PIX_FMT_YUV422P },
++    { OMX_COLOR_FormatYCbYCr,                                 AV_PIX_FMT_YUYV422 },
++    { OMX_COLOR_FormatYCrYCb,                                 AV_PIX_FMT_YVYU422 },
++    { OMX_COLOR_FormatCbYCrY,                                 AV_PIX_FMT_UYVY422 },
++    { OMX_COLOR_FormatYUV444Planar,                           AV_PIX_FMT_YUV444P },
+     { 0 }
+ };
+ 
+@@ -271,6 +276,7 @@ typedef struct OMXCodecContext {
+ 
+     int input_zerocopy;
+     int profile;
++    char *pixel_format;     /**< Set by a private option. */
+ } OMXCodecContext;
+ 
+ static void append_buffer(pthread_mutex_t *mutex, pthread_cond_t *cond,
+@@ -571,9 +577,43 @@ static av_cold int omx_component_init(AVCodecContext *avctx, const char *role)
+         out_port_params.format.video.eCompressionFormat = OMX_VIDEO_CodingAVC;
+     else if (avctx->codec->id == AV_CODEC_ID_HEVC)
+         out_port_params.format.video.eCompressionFormat = OMX_VIDEO_CodingHEVC;
+-    else if (avctx->codec->id == AV_CODEC_ID_MJPEG)
++    else if (avctx->codec->id == AV_CODEC_ID_MJPEG){
+         out_port_params.format.video.eCompressionFormat = OMX_VIDEO_CodingMJPEG;
+ 
++        av_log(avctx, AV_LOG_VERBOSE, "OMX_pixel_format:%s\n", s->pixel_format);
++        switch (av_get_pix_fmt(s->pixel_format)) {
++        case AV_PIX_FMT_NV12:
++            out_port_params.format.video.eColorFormat = OMX_COLOR_FormatYUV420SemiPlanar;
++            break;
++        case AV_PIX_FMT_NV21:
++            out_port_params.format.video.eColorFormat = OMX_COLOR_FormatYVU420SemiPlanar;
++            break;
++        case AV_PIX_FMT_YUV420P:
++            out_port_params.format.video.eColorFormat = OMX_COLOR_FormatYUV420Planar;
++            break;
++        case AV_PIX_FMT_NV16:
++            out_port_params.format.video.eColorFormat = OMX_COLOR_FormatYUV422SemiPlanar;
++            break;
++        case AV_PIX_FMT_YUV422P:
++            out_port_params.format.video.eColorFormat = OMX_COLOR_FormatYUV422Planar;
++            break;
++        case AV_PIX_FMT_YUYV422:
++            out_port_params.format.video.eColorFormat = OMX_COLOR_FormatYCbYCr;
++            break;
++        case AV_PIX_FMT_YVYU422:
++            out_port_params.format.video.eColorFormat = OMX_COLOR_FormatYCrYCb;
++            break;
++        case AV_PIX_FMT_UYVY422:
++            out_port_params.format.video.eColorFormat = OMX_COLOR_FormatCbYCrY;
++            break;
++        case AV_PIX_FMT_YUV444P:
++            out_port_params.format.video.eColorFormat = OMX_COLOR_FormatYUV444Planar;
++            break;
++        default:
++            out_port_params.format.video.eColorFormat = OMX_COLOR_FormatYUV420Planar;
++        }
++    }
++
+     err = OMX_SetParameter(s->handle, OMX_IndexParamPortDefinition, &out_port_params);
+     CHECK(err);
+     err = OMX_GetParameter(s->handle, OMX_IndexParamPortDefinition, &out_port_params);
+@@ -961,6 +1001,15 @@ static const AVOption options_hevc[] = {
+ 	{ NULL },
+ };
+ 
++static const AVOption options_mjpeg[] = {
++    { "omx_libname", "OpenMAX library name", OFFSET(libname), AV_OPT_TYPE_STRING, { 0 }, 0, 0, VDE },
++    { "omx_libprefix", "OpenMAX library prefix", OFFSET(libprefix), AV_OPT_TYPE_STRING, { 0 }, 0, 0, VDE },
++    { "zerocopy", "Try to avoid copying input frames if possible", OFFSET(input_zerocopy), AV_OPT_TYPE_INT, { .i64 = CONFIG_OMX_RPI }, 0, 1, VD },
++    { "omx_pix_fmt", "Set the decoding pixel format for mjpeg_omx decoder. The following formats are supported: yuv420p, nv12, nv21, nv16, yuv422p, yuyv422, yvyu422, uyvy422, yuv444p.", OFFSET(pixel_format), AV_OPT_TYPE_STRING, { .str = "yuv420p" }, 0, 0, VD },
++
++    { NULL },
++};
++
+ static const AVClass omx_mpeg4dec_class = {
+     .class_name = "mpeg4_omx",
+     .item_name  = av_default_item_name,
+@@ -1027,6 +1076,7 @@ AVCodec ff_hevc_omx_decoder = {
+ static const AVClass omx_mjpegdec_class = {
+     .class_name = "mjpeg_omx",
+     .item_name  = av_default_item_name,
++    .option     = options_mjpeg,
+     .version    = LIBAVUTIL_VERSION_INT,
+ };
+ AVCodec ff_mjpeg_omx_decoder = {
+@@ -1040,6 +1090,6 @@ AVCodec ff_mjpeg_omx_decoder = {
+     .close            = omx_decode_end,
+ 	.capabilities	  = AV_CODEC_CAP_DR1,
+ 	.max_lowres 	  = 3,
+-	.caps_internal	  = FF_CODEC_CAP_INIT_THREADSAFE,
++	.caps_internal	  = FF_CODEC_CAP_INIT_THREADSAFE | FF_CODEC_CAP_INIT_CLEANUP,
+     .priv_class       = &omx_mjpegdec_class,
+ };
+-- 
+2.17.1
+

+ 433 - 0
multimedia/patch/ffmpeg/0013-ffmpeg-replace-the-indent-with-space.patch

@@ -0,0 +1,433 @@
+From 1a2947195001880a05af0e225b6e93c6f2349123 Mon Sep 17 00:00:00 2001
+From: "arvin.zhu" <arvin.zhu@starfivetech.com>
+Date: Fri, 23 Sep 2022 10:17:00 +0800
+Subject: [PATCH] ffmpeg: replace the indent with space
+
+replace the indent with space
+
+Signed-off-by: arvin.zhu <arvin.zhu@starfivetech.com>
+---
+ libavcodec/omx.c    |  46 +++++-----
+ libavcodec/omxdec.c | 212 ++++++++++++++++++++++----------------------
+ 2 files changed, 129 insertions(+), 129 deletions(-)
+
+diff --git a/libavcodec/omx.c b/libavcodec/omx.c
+index 1d121af..1c4179f 100755
+--- a/libavcodec/omx.c
++++ b/libavcodec/omx.c
+@@ -1,6 +1,6 @@
+ /*
+- * OMX Video encoder
+- * Copyright (C) 2011 Martin Storsjo
++ * OMX Video decoder
++ * Copyright (C) 2018-2022 Starfive Technology
+  *
+  * This file is part of FFmpeg.
+  *
+@@ -685,8 +685,8 @@ static av_cold int omx_encode_init(AVCodecContext *avctx)
+         role = "video_encoder.avc";
+         break;
+     case AV_CODEC_ID_HEVC:
+-		role = "video_encoder.hevc";
+-		break;
++        role = "video_encoder.hevc";
++        break;
+     default:
+         return AVERROR(ENOSYS);
+     }
+@@ -752,7 +752,7 @@ static av_cold int omx_encode_init(AVCodecContext *avctx)
+                 }
+                 if (nals[HEVC_NAL_SPS] && nals[HEVC_NAL_PPS] && nals[HEVC_NAL_VPS])
+                     break;
+-		} else {
++        } else {
+                 if (avctx->extradata_size > 0)
+                     break;
+             }
+@@ -982,7 +982,7 @@ static const AVOption options_hevc[] = {
+     { "omx_libname", "OpenMAX library name", OFFSET(libname), AV_OPT_TYPE_STRING, { 0 }, 0, 0, VDE },
+     { "omx_libprefix", "OpenMAX library prefix", OFFSET(libprefix), AV_OPT_TYPE_STRING, { 0 }, 0, 0, VDE },
+     { "zerocopy", "Try to avoid copying input frames if possible", OFFSET(input_zerocopy), AV_OPT_TYPE_INT, { .i64 = CONFIG_OMX_RPI }, 0, 1, VE },
+-	{ NULL },
++    { NULL },
+ };
+ 
+ static const enum AVPixelFormat omx_encoder_pix_fmts[] = {
+@@ -1032,23 +1032,23 @@ AVCodec ff_h264_omx_encoder = {
+ };
+ 
+ static const AVClass omx_hevcenc_class = {
+-	.class_name = "hevc_omx",
+-	.item_name = av_default_item_name,
+-	.option = options_hevc,
+-	.version = LIBAVUTIL_VERSION_INT,
++    .class_name = "hevc_omx",
++    .item_name = av_default_item_name,
++    .option = options_hevc,
++    .version = LIBAVUTIL_VERSION_INT,
+ };
+ AVCodec ff_hevc_omx_encoder = {
+-	.name 			  = "hevc_omx",
+-	.long_name 		  = NULL_IF_CONFIG_SMALL("OpenMAX IL HEVC video encoder"),
+-	.type 			  = AVMEDIA_TYPE_VIDEO,
+-	.id 			  = AV_CODEC_ID_HEVC,
+-	.priv_data_size   = sizeof(OMXCodecContext),
+-	.init 			  = omx_encode_init,
+-	.encode2 		  = omx_encode_frame,
+-	.close 			  = omx_encode_end,
+-	.pix_fmts 		  = omx_encoder_pix_fmts,
+-	.profiles         = NULL_IF_CONFIG_SMALL(ff_hevc_profiles),
+-	.capabilities 	  = AV_CODEC_CAP_DELAY,
+-	.caps_internal 	  = FF_CODEC_CAP_INIT_THREADSAFE | FF_CODEC_CAP_INIT_CLEANUP,
+-	.priv_class 	  = &omx_hevcenc_class,
++    .name             = "hevc_omx",
++    .long_name        = NULL_IF_CONFIG_SMALL("OpenMAX IL HEVC video encoder"),
++    .type             = AVMEDIA_TYPE_VIDEO,
++    .id               = AV_CODEC_ID_HEVC,
++    .priv_data_size   = sizeof(OMXCodecContext),
++    .init             = omx_encode_init,
++    .encode2          = omx_encode_frame,
++    .close            = omx_encode_end,
++    .pix_fmts         = omx_encoder_pix_fmts,
++    .profiles         = NULL_IF_CONFIG_SMALL(ff_hevc_profiles),
++    .capabilities     = AV_CODEC_CAP_DELAY,
++    .caps_internal    = FF_CODEC_CAP_INIT_THREADSAFE | FF_CODEC_CAP_INIT_CLEANUP,
++    .priv_class       = &omx_hevcenc_class,
+ };
+diff --git a/libavcodec/omxdec.c b/libavcodec/omxdec.c
+index b489c09..20298b8 100755
+--- a/libavcodec/omxdec.c
++++ b/libavcodec/omxdec.c
+@@ -269,7 +269,7 @@ typedef struct OMXCodecContext {
+ 
+     int eos_sent, got_eos, evnet_bufferflag, first_get_outbuffer;
+ 
+-	int extradata_sent;
++    int extradata_sent;
+ 
+     uint8_t *output_buf;
+     int output_buf_size;
+@@ -320,9 +320,9 @@ static OMX_ERRORTYPE event_handler(OMX_HANDLETYPE component, OMX_PTR app_data, O
+     // unsigned long in official header versions (but there are also modified
+     // versions where it is something else).
+     OMX_PARAM_PORTDEFINITIONTYPE out_port_params = { 0 };
+-	OMX_PORT_PARAM_TYPE video_port_params = { 0 };
+-	OMX_ERRORTYPE err;
+-	int i;
++    OMX_PORT_PARAM_TYPE video_port_params = { 0 };
++    OMX_ERRORTYPE err;
++    int i;
+ 
+     switch (event) {
+     case OMX_EventError:
+@@ -350,36 +350,36 @@ static OMX_ERRORTYPE event_handler(OMX_HANDLETYPE component, OMX_PTR app_data, O
+         break;
+     case OMX_EventPortSettingsChanged:
+         av_log(s->avctx, AV_LOG_ERROR, "OMX port %"PRIu32" settings changed\n", (uint32_t) data1);
+-		INIT_STRUCT(video_port_params);
+-		err = OMX_GetParameter(s->handle, OMX_IndexParamVideoInit, &video_port_params);
+-		if(err != OMX_ErrorNone){
++        INIT_STRUCT(video_port_params);
++        err = OMX_GetParameter(s->handle, OMX_IndexParamVideoInit, &video_port_params);
++        if(err != OMX_ErrorNone){
+             av_log(s->avctx, AV_LOG_ERROR, "err %d\n",err);          
+             return AVERROR_UNKNOWN;
+-			}
+-
+-		for (i = 0; i < video_port_params.nPorts; i++) {
+-	        int port = video_port_params.nStartPortNumber + i;
+-	        OMX_PARAM_PORTDEFINITIONTYPE port_params = { 0 };
+-	        INIT_STRUCT(port_params);
+-	        port_params.nPortIndex = port;
+-	        err = OMX_GetParameter(s->handle, OMX_IndexParamPortDefinition, &port_params);
+-	        if (err != OMX_ErrorNone) {
+-	            av_log(s->avctx, AV_LOG_WARNING, "port %d error %x\n", port, err);
++        }
++
++        for (i = 0; i < video_port_params.nPorts; i++) {
++            int port = video_port_params.nStartPortNumber + i;
++            OMX_PARAM_PORTDEFINITIONTYPE port_params = { 0 };
++            INIT_STRUCT(port_params);
++            port_params.nPortIndex = port;
++            err = OMX_GetParameter(s->handle, OMX_IndexParamPortDefinition, &port_params);
++            if (err != OMX_ErrorNone) {
++                av_log(s->avctx, AV_LOG_WARNING, "port %d error %x\n", port, err);
+             return AVERROR_UNKNOWN;
+-	        }
+-	        if (port_params.eDir == OMX_DirOutput) {
+-				out_port_params = port_params;
+-				dec_out_width = out_port_params.format.video.nFrameWidth;
+-				dec_out_height = out_port_params.format.video.nFrameHeight;
+-				dec_pix_fmt = out_port_params.format.video.eColorFormat;
+-	            
++            }
++            if (port_params.eDir == OMX_DirOutput) {
++                out_port_params = port_params;
++                dec_out_width = out_port_params.format.video.nFrameWidth;
++                dec_out_height = out_port_params.format.video.nFrameHeight;
++                dec_pix_fmt = out_port_params.format.video.eColorFormat;
++
+                 av_log(s->avctx, AV_LOG_VERBOSE, "w:%d, h:%d, fmt:%d\n", dec_out_width, dec_out_height, dec_pix_fmt); 
+-	        } 
+-	    }		
++            } 
++        }
+         break;
+-	case OMX_EventBufferFlag:
+-		av_log(s->avctx, AV_LOG_VERBOSE, "OMX decoder competd set event_bufferflag\n");
+-		evnet_bufferflag = 1;
++    case OMX_EventBufferFlag:
++        av_log(s->avctx, AV_LOG_VERBOSE, "OMX decoder competd set event_bufferflag\n");
++        evnet_bufferflag = 1;
+     default:
+         av_log(s->avctx, AV_LOG_VERBOSE, "OMX event %d %"PRIx32" %"PRIx32"\n",
+                                          event, (uint32_t) data1, (uint32_t) data2);
+@@ -563,7 +563,7 @@ static av_cold int omx_component_init(AVCodecContext *avctx, const char *role)
+     out_port_params.bPopulated = OMX_FALSE;
+     out_port_params.eDomain    = OMX_PortDomainVideo;
+     out_port_params.format.video.pNativeRender = NULL;
+-	out_port_params.format.video.bFlagErrorConcealment  = OMX_FALSE;
++    out_port_params.format.video.bFlagErrorConcealment  = OMX_FALSE;
+     out_port_params.format.video.nFrameWidth   = avctx->width;
+     out_port_params.format.video.nFrameHeight  = avctx->height;
+     out_port_params.format.video.nStride       = 0;
+@@ -747,11 +747,11 @@ static av_cold int omx_decode_init(AVCodecContext *avctx)
+         role = "video_decoder.avc";
+         break;
+     case AV_CODEC_ID_HEVC:
+-		role = "video_decoder.hevc";
+-		break;
++        role = "video_decoder.hevc";
++        break;
+     case AV_CODEC_ID_MJPEG:
+-		role = "video_decoder.mjpeg";
+-		break;
++        role = "video_decoder.mjpeg";
++        break;
+     default:
+         return AVERROR(ENOSYS);
+     }
+@@ -809,7 +809,7 @@ static av_cold int omx_decode_init(AVCodecContext *avctx)
+ //         }
+ //     }
+ // #endif
+-	
++
+     return 0;
+ fail:
+     return ret;
+@@ -825,64 +825,64 @@ static int omx_decode_frame(AVCodecContext *avctx, void *data,
+     OMX_ERRORTYPE err;
+     int had_partial = 0;
+ 
+-	AVFrame *avframe = data;
+-	
+-	uint8_t *dst[4];
+-	int linesize[4];
++    AVFrame *avframe = data;
++
++    uint8_t *dst[4];
++    int linesize[4];
+ 
+     if (pkt->size) {
+-	
++
+         //VPU init and fill buffer slow, so empty buf sleep to send before get vpu fill buf.
+         if(!s->first_get_outbuffer)
+         av_usleep(100000);
+         buffer = get_buffer(&s->input_mutex, &s->input_cond,
+                             &s->num_free_in_buffers, s->free_in_buffers, 1);
+ 
+-		if (!buffer) {
+-        	av_log(avctx, AV_LOG_ERROR, "get_buffer NULL\n");
++        if (!buffer) {
++            av_log(avctx, AV_LOG_ERROR, "get_buffer NULL\n");
+             return AVERROR(ENOMEM);
+         }
+ 
+-		//cpy the extradata
+-		if(!s->extradata_sent && avctx->extradata ) {
+-
+-			memcpy(buffer->pBuffer + buffer->nOffset, avctx->extradata, avctx->extradata_size);	
+-			memcpy(buffer->pBuffer + buffer->nOffset + avctx->extradata_size, pkt->data, pkt->size);
+-			buffer->nFilledLen = pkt->size + avctx->extradata_size;
+-			s->extradata_sent = 1;
+-		
+-		}
+-		else {
+-			memcpy(buffer->pBuffer + buffer->nOffset, pkt->data, pkt->size);
+-			buffer->nFilledLen = pkt->size;
+-		}
+-
+-		/* avoid memcpy. point it addr*/
+-		//buffer->pAppPrivate = pkt;
+-		//buffer->pBuffer = pkt->data;
+-		//buffer->nFilledLen = pkt->size;
+-		
+-		buffer->pOutputPortPrivate = NULL;
+-		buffer->pAppPrivate = avctx->priv_data;
++        //cpy the extradata
++        if(!s->extradata_sent && avctx->extradata ) {
++
++            memcpy(buffer->pBuffer + buffer->nOffset, avctx->extradata, avctx->extradata_size);    
++            memcpy(buffer->pBuffer + buffer->nOffset + avctx->extradata_size, pkt->data, pkt->size);
++            buffer->nFilledLen = pkt->size + avctx->extradata_size;
++            s->extradata_sent = 1;
++        
++        }
++        else {
++            memcpy(buffer->pBuffer + buffer->nOffset, pkt->data, pkt->size);
++            buffer->nFilledLen = pkt->size;
++        }
++
++        /* avoid memcpy. point it addr*/
++        //buffer->pAppPrivate = pkt;
++        //buffer->pBuffer = pkt->data;
++        //buffer->nFilledLen = pkt->size;
++        
++        buffer->pOutputPortPrivate = NULL;
++        buffer->pAppPrivate = avctx->priv_data;
+         buffer->nFlags = OMX_BUFFERFLAG_ENDOFFRAME;
+ 
+-	    err = OMX_EmptyThisBuffer(s->handle, buffer);
+-	    if (err != OMX_ErrorNone) {
+-	        append_buffer(&s->input_mutex, &s->input_cond, &s->num_free_in_buffers, s->free_in_buffers, buffer);
+-	        av_log(avctx, AV_LOG_ERROR, "OMX_EmptyThisBuffer failed: %x\n", err);
+-	        return AVERROR_UNKNOWN;
+-	    }
++        err = OMX_EmptyThisBuffer(s->handle, buffer);
++        if (err != OMX_ErrorNone) {
++            append_buffer(&s->input_mutex, &s->input_cond, &s->num_free_in_buffers, s->free_in_buffers, buffer);
++            av_log(avctx, AV_LOG_ERROR, "OMX_EmptyThisBuffer failed: %x\n", err);
++            return AVERROR_UNKNOWN;
++        }
+     } else if (!s->eos_sent) {
+-		
++
+         if(!s->first_get_outbuffer)
+         av_usleep(1000000);
+         buffer = get_buffer(&s->input_mutex, &s->input_cond,
+                             &s->num_free_in_buffers, s->free_in_buffers, 1);
+ 
+         if(!buffer) {
+-        	av_log(avctx, AV_LOG_ERROR, "get_buffer NULL\n");
++            av_log(avctx, AV_LOG_ERROR, "get_buffer NULL\n");
+             return AVERROR(ENOMEM);
+-		}
++        }
+ 
+         buffer->nFilledLen = 0;
+         buffer->nFlags = OMX_BUFFERFLAG_EOS;
+@@ -907,58 +907,58 @@ static int omx_decode_frame(AVCodecContext *avctx, void *data,
+ 
+         if (!buffer) {
+             /*eos is sent wait for vpu evnet_bufferflag to get all frames
+-			  mjpeg: sent a frame, then wait for a decoder frame 
+-			*/
++              mjpeg: sent a frame, then wait for a decoder frame 
++            */
+             if((s->eos_sent && !evnet_bufferflag) || (avctx->codec_id == AV_CODEC_ID_MJPEG )) {
+-				continue; 
++                continue; 
+             }
+             break;
+         }
+-		//if (!buffer)
++        //if (!buffer)
+            // break;
+         if(!s->first_get_outbuffer)
+             s->first_get_outbuffer = 1;
+ 
+-		if(!buffer->nFilledLen){
++        if(!buffer->nFilledLen){
+             av_log(avctx, AV_LOG_ERROR, "buffer->nFilledLen %d\n",(int)buffer->nFilledLen);
+             goto end;
+         }
+ 
+         avctx->width = dec_out_width;
+-		avctx->height = dec_out_height;
++        avctx->height = dec_out_height;
+         avctx->pix_fmt = omx_map_color_format(avctx, dec_pix_fmt);
+-		s->stride     = avctx->width;
+-    	s->plane_size = avctx->height;
+-		
++        s->stride     = avctx->width;
++        s->plane_size = avctx->height;
++        
+         if (buffer->nFlags & OMX_BUFFERFLAG_EOS)
+             s->got_eos = 1;
+-       
+-		if ((ret = ff_get_buffer(avctx, avframe, 0)) < 0) {
+-			av_log(avctx, AV_LOG_ERROR, "Unable to allocate buffer\n");
+-			goto end;
+-		}
+-		
+-		ret = av_image_fill_arrays(dst, linesize, buffer->pBuffer,
+-								   avctx->pix_fmt, s->stride, s->plane_size, 1);
+-		if (ret < 0){
++
++        if ((ret = ff_get_buffer(avctx, avframe, 0)) < 0) {
++            av_log(avctx, AV_LOG_ERROR, "Unable to allocate buffer\n");
++            goto end;
++        }
++
++        ret = av_image_fill_arrays(dst, linesize, buffer->pBuffer,
++                                   avctx->pix_fmt, s->stride, s->plane_size, 1);
++        if (ret < 0){
+             av_log(avctx, AV_LOG_ERROR, "av_image_fill_arrays ret:%d\n", ret);
+             goto end;
+         }
+ 
+-		av_image_copy(avframe->data, avframe->linesize, (const uint8_t**)dst, linesize, 
+-							avctx->pix_fmt, avctx->width, avctx->height);
+-	
+-		//avframe->pts	 = buffer->nTimeStamp;
+-		//avframe->pkt_dts = AV_NOPTS_VALUE;
+-		//avframe->pict_type= AV_PICTURE_TYPE_I;
+-    	//avframe->key_frame= 1;
++        av_image_copy(avframe->data, avframe->linesize, (const uint8_t**)dst, linesize, 
++                            avctx->pix_fmt, avctx->width, avctx->height);
+ 
+-		*got_packet = 1;
++        //avframe->pts     = buffer->nTimeStamp;
++        //avframe->pkt_dts = AV_NOPTS_VALUE;
++        //avframe->pict_type= AV_PICTURE_TYPE_I;
++        //avframe->key_frame= 1;
+ 
+-	/*
+-		if ((ret = av_frame_ref(data, avframe)) < 0)
+-         	goto end;
+-         	*/
++        *got_packet = 1;
++
++        /*
++        if ((ret = av_frame_ref(data, avframe)) < 0)
++            goto end;
++        */
+ 
+ end:     
+         err = OMX_FillThisBuffer(s->handle, buffer);
+@@ -998,7 +998,7 @@ static const AVOption options_hevc[] = {
+     { "omx_libname", "OpenMAX library name", OFFSET(libname), AV_OPT_TYPE_STRING, { 0 }, 0, 0, VDE },
+     { "omx_libprefix", "OpenMAX library prefix", OFFSET(libprefix), AV_OPT_TYPE_STRING, { 0 }, 0, 0, VDE },
+     { "zerocopy", "Try to avoid copying input frames if possible", OFFSET(input_zerocopy), AV_OPT_TYPE_INT, { .i64 = CONFIG_OMX_RPI }, 0, 1, VE },
+-	{ NULL },
++    { NULL },
+ };
+ 
+ static const AVOption options_mjpeg[] = {
+@@ -1088,8 +1088,8 @@ AVCodec ff_mjpeg_omx_decoder = {
+     .init             = omx_decode_init,
+     .decode           = omx_decode_frame,
+     .close            = omx_decode_end,
+-	.capabilities	  = AV_CODEC_CAP_DR1,
+-	.max_lowres 	  = 3,
+-	.caps_internal	  = FF_CODEC_CAP_INIT_THREADSAFE | FF_CODEC_CAP_INIT_CLEANUP,
++    .capabilities     = AV_CODEC_CAP_DR1,
++    .max_lowres       = 3,
++    .caps_internal    = FF_CODEC_CAP_INIT_THREADSAFE | FF_CODEC_CAP_INIT_CLEANUP,
+     .priv_class       = &omx_mjpegdec_class,
+ };
+-- 
+2.17.1
+

+ 159 - 0
multimedia/patch/ffmpeg/0014-ffmpeg-add-pixel-foramt.patch

@@ -0,0 +1,159 @@
+From 474232dff26df2188fe37287aeb7834f6d3853d1 Mon Sep 17 00:00:00 2001
+From: "arvin.zhu" <arvin.zhu@starfivetech.com>
+Date: Thu, 29 Sep 2022 16:05:09 +0800
+Subject: [PATCH] ffmpeg: add pixel foramt
+
+add pixel format option for codecs of VPU
+
+Signed-off-by: arvin.zhu <arvin.zhu@starfivetech.com>
+---
+ libavcodec/omx.c    | 19 ++++++++++++++++---
+ libavcodec/omxdec.c | 43 +++++++++++++++++++++++++++++++++++++------
+ 2 files changed, 53 insertions(+), 9 deletions(-)
+
+diff --git a/libavcodec/omx.c b/libavcodec/omx.c
+index 1c4179f..6976884 100755
+--- a/libavcodec/omx.c
++++ b/libavcodec/omx.c
+@@ -456,18 +456,31 @@ static av_cold int omx_component_init(AVCodecContext *avctx, const char *role)
+             break;
+         }
+     }
++
+     if (s->color_format == 0) {
+         av_log(avctx, AV_LOG_ERROR, "No supported pixel formats (%d formats available)\n", i);
+         return AVERROR_UNKNOWN;
+     }
+-
++    av_log(avctx, AV_LOG_VERBOSE, "OMX setting pixel_format:%s\n", av_get_pix_fmt_name(avctx->pix_fmt));
+     in_port_params.bEnabled   = OMX_TRUE;
+     in_port_params.bPopulated = OMX_FALSE;
+     in_port_params.eDomain    = OMX_PortDomainVideo;
+ 
+     in_port_params.format.video.pNativeRender         = NULL;
+     in_port_params.format.video.bFlagErrorConcealment = OMX_FALSE;
+-    in_port_params.format.video.eColorFormat          = s->color_format;
++    switch (avctx->pix_fmt) {
++        case AV_PIX_FMT_NV12:
++            in_port_params.format.video.eColorFormat = OMX_COLOR_FormatYUV420SemiPlanar;
++            break;
++        case AV_PIX_FMT_NV21:
++            in_port_params.format.video.eColorFormat = OMX_COLOR_FormatYVU420SemiPlanar;
++            break;
++        case AV_PIX_FMT_YUV420P:
++            out_port_params.format.video.eColorFormat = OMX_COLOR_FormatYUV420Planar;
++            break;
++        default:
++            return AVERROR_OPTION_NOT_FOUND;
++        }
+     s->stride     = avctx->width;
+     s->plane_size = avctx->height;
+     // If specific codecs need to manually override the stride/plane_size,
+@@ -986,7 +999,7 @@ static const AVOption options_hevc[] = {
+ };
+ 
+ static const enum AVPixelFormat omx_encoder_pix_fmts[] = {
+-    AV_PIX_FMT_YUV420P, AV_PIX_FMT_NONE
++    AV_PIX_FMT_YUV420P, AV_PIX_FMT_NV12, AV_PIX_FMT_NV21, AV_PIX_FMT_NONE
+ };
+ 
+ static const AVClass omx_mpeg4enc_class = {
+diff --git a/libavcodec/omxdec.c b/libavcodec/omxdec.c
+index 20298b8..074ff4a 100755
+--- a/libavcodec/omxdec.c
++++ b/libavcodec/omxdec.c
+@@ -481,7 +481,7 @@ static av_cold int omx_component_init(AVCodecContext *avctx, const char *role)
+     //OMX_VIDEO_PARAM_BITRATETYPE vid_param_bitrate = { 0 };
+     OMX_ERRORTYPE err;
+     int i;
+-
++av_log(avctx, AV_LOG_VERBOSE, "OMX avctx pixel_format:%s\n", av_get_pix_fmt_name(avctx->pix_fmt));
+     s->version.s.nVersionMajor = 1;
+     s->version.s.nVersionMinor = 1;
+     s->version.s.nRevision     = 2;
+@@ -571,16 +571,46 @@ static av_cold int omx_component_init(AVCodecContext *avctx, const char *role)
+     out_port_params.format.video.nBitrate      = avctx->bit_rate;
+     out_port_params.format.video.xFramerate    = in_port_params.format.video.xFramerate;
+     out_port_params.format.video.bFlagErrorConcealment  = OMX_FALSE;
++    av_log(avctx, AV_LOG_VERBOSE, "OMX setting pixel_format:%s\n", s->pixel_format);
+     if (avctx->codec->id == AV_CODEC_ID_MPEG4)
+         out_port_params.format.video.eCompressionFormat = OMX_VIDEO_CodingMPEG4;
+-    else if (avctx->codec->id == AV_CODEC_ID_H264)
++    else if (avctx->codec->id == AV_CODEC_ID_H264) {
+         out_port_params.format.video.eCompressionFormat = OMX_VIDEO_CodingAVC;
+-    else if (avctx->codec->id == AV_CODEC_ID_HEVC)
++
++        switch (av_get_pix_fmt(s->pixel_format)) {
++        case AV_PIX_FMT_NV12:
++            out_port_params.format.video.eColorFormat = OMX_COLOR_FormatYUV420SemiPlanar;
++            break;
++        case AV_PIX_FMT_NV21:
++            out_port_params.format.video.eColorFormat = OMX_COLOR_FormatYVU420SemiPlanar;
++            break;
++        case AV_PIX_FMT_YUV420P:
++            out_port_params.format.video.eColorFormat = OMX_COLOR_FormatYUV420Planar;
++            break;
++        default:
++            return AVERROR_OPTION_NOT_FOUND;
++        }
++    }
++    else if (avctx->codec->id == AV_CODEC_ID_HEVC) {
+         out_port_params.format.video.eCompressionFormat = OMX_VIDEO_CodingHEVC;
++
++        switch (av_get_pix_fmt(s->pixel_format)) {
++        case AV_PIX_FMT_NV12:
++            out_port_params.format.video.eColorFormat = OMX_COLOR_FormatYUV420SemiPlanar;
++            break;
++        case AV_PIX_FMT_NV21:
++            out_port_params.format.video.eColorFormat = OMX_COLOR_FormatYVU420SemiPlanar;
++            break;
++        case AV_PIX_FMT_YUV420P:
++            out_port_params.format.video.eColorFormat = OMX_COLOR_FormatYUV420Planar;
++            break;
++        default:
++            return AVERROR_OPTION_NOT_FOUND;
++        }
++    }
+     else if (avctx->codec->id == AV_CODEC_ID_MJPEG){
+         out_port_params.format.video.eCompressionFormat = OMX_VIDEO_CodingMJPEG;
+ 
+-        av_log(avctx, AV_LOG_VERBOSE, "OMX_pixel_format:%s\n", s->pixel_format);
+         switch (av_get_pix_fmt(s->pixel_format)) {
+         case AV_PIX_FMT_NV12:
+             out_port_params.format.video.eColorFormat = OMX_COLOR_FormatYUV420SemiPlanar;
+@@ -610,7 +640,7 @@ static av_cold int omx_component_init(AVCodecContext *avctx, const char *role)
+             out_port_params.format.video.eColorFormat = OMX_COLOR_FormatYUV444Planar;
+             break;
+         default:
+-            out_port_params.format.video.eColorFormat = OMX_COLOR_FormatYUV420Planar;
++            return AVERROR_OPTION_NOT_FOUND;
+         }
+     }
+ 
+@@ -991,6 +1021,7 @@ static const AVOption options[] = {
+     { "baseline", "",                         0,               AV_OPT_TYPE_CONST, { .i64 = FF_PROFILE_H264_BASELINE }, 0, 0, VE, "profile" },
+     { "main",     "",                         0,               AV_OPT_TYPE_CONST, { .i64 = FF_PROFILE_H264_MAIN },     0, 0, VE, "profile" },
+     { "high",     "",                         0,               AV_OPT_TYPE_CONST, { .i64 = FF_PROFILE_H264_HIGH },     0, 0, VE, "profile" },
++    { "omx_pix_fmt", "Set the decoding pixel format for h264_omx decoder. The following formats are supported: yuv420p, nv12, nv21.", OFFSET(pixel_format), AV_OPT_TYPE_STRING, { .str = "yuv420p" }, 0, 0, VD },
+     { NULL }
+ };
+ 
+@@ -998,6 +1029,7 @@ static const AVOption options_hevc[] = {
+     { "omx_libname", "OpenMAX library name", OFFSET(libname), AV_OPT_TYPE_STRING, { 0 }, 0, 0, VDE },
+     { "omx_libprefix", "OpenMAX library prefix", OFFSET(libprefix), AV_OPT_TYPE_STRING, { 0 }, 0, 0, VDE },
+     { "zerocopy", "Try to avoid copying input frames if possible", OFFSET(input_zerocopy), AV_OPT_TYPE_INT, { .i64 = CONFIG_OMX_RPI }, 0, 1, VE },
++    { "omx_pix_fmt", "Set the decoding pixel format for hevc_omx decoder. The following formats are supported: yuv420p, nv12, nv21.", OFFSET(pixel_format), AV_OPT_TYPE_STRING, { .str = "yuv420p" }, 0, 0, VD },
+     { NULL },
+ };
+ 
+@@ -1006,7 +1038,6 @@ static const AVOption options_mjpeg[] = {
+     { "omx_libprefix", "OpenMAX library prefix", OFFSET(libprefix), AV_OPT_TYPE_STRING, { 0 }, 0, 0, VDE },
+     { "zerocopy", "Try to avoid copying input frames if possible", OFFSET(input_zerocopy), AV_OPT_TYPE_INT, { .i64 = CONFIG_OMX_RPI }, 0, 1, VD },
+     { "omx_pix_fmt", "Set the decoding pixel format for mjpeg_omx decoder. The following formats are supported: yuv420p, nv12, nv21, nv16, yuv422p, yuyv422, yvyu422, uyvy422, yuv444p.", OFFSET(pixel_format), AV_OPT_TYPE_STRING, { .str = "yuv420p" }, 0, 0, VD },
+-
+     { NULL },
+ };
+ 
+-- 
+2.17.1
+

+ 52 - 0
multimedia/patch/ffmpeg/0015-ffmpeg-remove-delay.patch

@@ -0,0 +1,52 @@
+From 74c4c2a25bb185ede7162a39a2d43c75a6276573 Mon Sep 17 00:00:00 2001
+From: "arvin.zhu" <arvin.zhu@starfivetech.com>
+Date: Fri, 30 Sep 2022 12:01:06 +0800
+Subject: [PATCH] ffmpeg: remove delay
+
+remove delay for decoding in omx.c file
+
+Signed-off-by: arvin.zhu <arvin.zhu@starfivetech.com>
+---
+ libavcodec/omxdec.c | 12 ++++++------
+ 1 file changed, 6 insertions(+), 6 deletions(-)
+
+diff --git a/libavcodec/omxdec.c b/libavcodec/omxdec.c
+index 074ff4a..0957605 100755
+--- a/libavcodec/omxdec.c
++++ b/libavcodec/omxdec.c
+@@ -863,8 +863,8 @@ static int omx_decode_frame(AVCodecContext *avctx, void *data,
+     if (pkt->size) {
+ 
+         //VPU init and fill buffer slow, so empty buf sleep to send before get vpu fill buf.
+-        if(!s->first_get_outbuffer)
+-        av_usleep(100000);
++        // if(!s->first_get_outbuffer)
++        // av_usleep(100000);
+         buffer = get_buffer(&s->input_mutex, &s->input_cond,
+                             &s->num_free_in_buffers, s->free_in_buffers, 1);
+ 
+@@ -904,8 +904,8 @@ static int omx_decode_frame(AVCodecContext *avctx, void *data,
+         }
+     } else if (!s->eos_sent) {
+ 
+-        if(!s->first_get_outbuffer)
+-        av_usleep(1000000);
++        // if(!s->first_get_outbuffer)
++        // av_usleep(1000000);
+         buffer = get_buffer(&s->input_mutex, &s->input_cond,
+                             &s->num_free_in_buffers, s->free_in_buffers, 1);
+ 
+@@ -946,8 +946,8 @@ static int omx_decode_frame(AVCodecContext *avctx, void *data,
+         }
+         //if (!buffer)
+            // break;
+-        if(!s->first_get_outbuffer)
+-            s->first_get_outbuffer = 1;
++        // if(!s->first_get_outbuffer)
++        //     s->first_get_outbuffer = 1;
+ 
+         if(!buffer->nFilledLen){
+             av_log(avctx, AV_LOG_ERROR, "buffer->nFilledLen %d\n",(int)buffer->nFilledLen);
+-- 
+2.17.1
+

+ 209 - 0
multimedia/patch/ffmpeg/0016-FFmpeg-omx-add-scale-option-for-omx.patch

@@ -0,0 +1,209 @@
+From 246a5308151416e99dbe53ad52d415abdfb6df29 Mon Sep 17 00:00:00 2001
+From: "arvin.zhu" <arvin.zhu@starfivetech.com>
+Date: Fri, 14 Oct 2022 16:50:55 +0800
+Subject: [PATCH] FFmpeg:omx: add scale option for omx
+
+add scale option for omx decoder
+
+Signed-off-by: arvin.zhu <arvin.zhu@starfivetech.com>
+---
+ libavcodec/omx.c    |  4 +-
+ libavcodec/omxdec.c | 92 +++++++++++++++++++++++++++------------------
+ 2 files changed, 58 insertions(+), 38 deletions(-)
+
+diff --git a/libavcodec/omx.c b/libavcodec/omx.c
+index 6976884..77012ce 100755
+--- a/libavcodec/omx.c
++++ b/libavcodec/omx.c
+@@ -1,6 +1,6 @@
+ /*
+- * OMX Video decoder
+- * Copyright (C) 2018-2022 Starfive Technology
++ * OMX Video encoder
++ * Copyright (C) 2011 Martin Storsjo
+  *
+  * This file is part of FFmpeg.
+  *
+diff --git a/libavcodec/omxdec.c b/libavcodec/omxdec.c
+index 0957605..87e5a1a 100755
+--- a/libavcodec/omxdec.c
++++ b/libavcodec/omxdec.c
+@@ -1,6 +1,6 @@
+ /*
+- * OMX Video encoder
+- * Copyright (C) 2011 Martin Storsjo
++ * OMX Video decoder
++ * Copyright (C) 2018-2022 Starfive Technology
+  *
+  * This file is part of FFmpeg.
+  *
+@@ -274,6 +274,9 @@ typedef struct OMXCodecContext {
+     uint8_t *output_buf;
+     int output_buf_size;
+ 
++    OMX_U32 scale_width;
++    OMX_U32 scale_height;
++
+     int input_zerocopy;
+     int profile;
+     char *pixel_format;     /**< Set by a private option. */
+@@ -481,7 +484,6 @@ static av_cold int omx_component_init(AVCodecContext *avctx, const char *role)
+     //OMX_VIDEO_PARAM_BITRATETYPE vid_param_bitrate = { 0 };
+     OMX_ERRORTYPE err;
+     int i;
+-av_log(avctx, AV_LOG_VERBOSE, "OMX avctx pixel_format:%s\n", av_get_pix_fmt_name(avctx->pix_fmt));
+     s->version.s.nVersionMajor = 1;
+     s->version.s.nVersionMinor = 1;
+     s->version.s.nRevision     = 2;
+@@ -564,8 +566,6 @@ av_log(avctx, AV_LOG_VERBOSE, "OMX avctx pixel_format:%s\n", av_get_pix_fmt_name
+     out_port_params.eDomain    = OMX_PortDomainVideo;
+     out_port_params.format.video.pNativeRender = NULL;
+     out_port_params.format.video.bFlagErrorConcealment  = OMX_FALSE;
+-    out_port_params.format.video.nFrameWidth   = avctx->width;
+-    out_port_params.format.video.nFrameHeight  = avctx->height;
+     out_port_params.format.video.nStride       = 0;
+     out_port_params.format.video.nSliceHeight  = 0;
+     out_port_params.format.video.nBitrate      = avctx->bit_rate;
+@@ -576,41 +576,29 @@ av_log(avctx, AV_LOG_VERBOSE, "OMX avctx pixel_format:%s\n", av_get_pix_fmt_name
+         out_port_params.format.video.eCompressionFormat = OMX_VIDEO_CodingMPEG4;
+     else if (avctx->codec->id == AV_CODEC_ID_H264) {
+         out_port_params.format.video.eCompressionFormat = OMX_VIDEO_CodingAVC;
+-
+-        switch (av_get_pix_fmt(s->pixel_format)) {
+-        case AV_PIX_FMT_NV12:
+-            out_port_params.format.video.eColorFormat = OMX_COLOR_FormatYUV420SemiPlanar;
+-            break;
+-        case AV_PIX_FMT_NV21:
+-            out_port_params.format.video.eColorFormat = OMX_COLOR_FormatYVU420SemiPlanar;
+-            break;
+-        case AV_PIX_FMT_YUV420P:
+-            out_port_params.format.video.eColorFormat = OMX_COLOR_FormatYUV420Planar;
+-            break;
+-        default:
+-            return AVERROR_OPTION_NOT_FOUND;
+-        }
+     }
+     else if (avctx->codec->id == AV_CODEC_ID_HEVC) {
+         out_port_params.format.video.eCompressionFormat = OMX_VIDEO_CodingHEVC;
+-
+-        switch (av_get_pix_fmt(s->pixel_format)) {
+-        case AV_PIX_FMT_NV12:
+-            out_port_params.format.video.eColorFormat = OMX_COLOR_FormatYUV420SemiPlanar;
+-            break;
+-        case AV_PIX_FMT_NV21:
+-            out_port_params.format.video.eColorFormat = OMX_COLOR_FormatYVU420SemiPlanar;
+-            break;
+-        case AV_PIX_FMT_YUV420P:
+-            out_port_params.format.video.eColorFormat = OMX_COLOR_FormatYUV420Planar;
+-            break;
+-        default:
+-            return AVERROR_OPTION_NOT_FOUND;
+-        }
+     }
+     else if (avctx->codec->id == AV_CODEC_ID_MJPEG){
+         out_port_params.format.video.eCompressionFormat = OMX_VIDEO_CodingMJPEG;
+ 
++        /* Set Scale config setting*/
++        if (s->scale_width || s->scale_height) {
++            av_log(avctx, AV_LOG_TRACE, "mjpeg decoder: scaling width: %ld scaling height: %ld .\n", s->scale_width, s->scale_height);
++            OMX_CONFIG_SCALEFACTORTYPE ScaleConfig;
++            INIT_STRUCT(ScaleConfig);
++            ScaleConfig.nPortIndex = 1;
++            OMX_GetConfig(s->handle, OMX_IndexConfigCommonScale, &ScaleConfig);
++            /* in Q16 format */
++            ScaleConfig.xWidth = (1 << 16) >> (s->scale_width & 0x3);
++            ScaleConfig.xHeight = (1 << 16) >> (s->scale_height & 0x3);
++            OMX_SetConfig(s->handle, OMX_IndexConfigCommonScale, &ScaleConfig);
++        } else {
++            out_port_params.format.video.nFrameWidth   = avctx->width;
++            out_port_params.format.video.nFrameHeight  = avctx->height;
++        }
++
+         switch (av_get_pix_fmt(s->pixel_format)) {
+         case AV_PIX_FMT_NV12:
+             out_port_params.format.video.eColorFormat = OMX_COLOR_FormatYUV420SemiPlanar;
+@@ -643,6 +631,35 @@ av_log(avctx, AV_LOG_VERBOSE, "OMX avctx pixel_format:%s\n", av_get_pix_fmt_name
+             return AVERROR_OPTION_NOT_FOUND;
+         }
+     }
++    if (avctx->codec->id == AV_CODEC_ID_H264 || avctx->codec->id == AV_CODEC_ID_HEVC) {
++        /* Set Scale config setting*/
++        if ((s->scale_width != 0) && ((s->scale_width < avctx->width/8) ||
++             (s->scale_width > avctx->width) || (s->scale_width % 8 != 0))) {
++            av_log(avctx, AV_LOG_ERROR, "scale_width: Invalid scale parameter\n");
++            return AVERROR_INVALIDDATA;
++        }
++        if ((s->scale_height != 0) && ((s->scale_height < avctx->height/8) ||
++             (s->scale_height > avctx->height) || (s->scale_height % 8 != 0))) {
++            av_log(avctx, AV_LOG_ERROR, "scale_height: Invalid scale parameter\n");
++            return AVERROR_INVALIDDATA;
++        }
++        out_port_params.format.video.nFrameWidth   = s->scale_width ? s->scale_width : avctx->width;
++        out_port_params.format.video.nFrameHeight  = s->scale_height ? s->scale_height : avctx->height;
++
++        switch (av_get_pix_fmt(s->pixel_format)) {
++        case AV_PIX_FMT_NV12:
++            out_port_params.format.video.eColorFormat = OMX_COLOR_FormatYUV420SemiPlanar;
++            break;
++        case AV_PIX_FMT_NV21:
++            out_port_params.format.video.eColorFormat = OMX_COLOR_FormatYVU420SemiPlanar;
++            break;
++        case AV_PIX_FMT_YUV420P:
++            out_port_params.format.video.eColorFormat = OMX_COLOR_FormatYUV420Planar;
++            break;
++        default:
++            return AVERROR_OPTION_NOT_FOUND;
++        }
++    }
+ 
+     err = OMX_SetParameter(s->handle, OMX_IndexParamPortDefinition, &out_port_params);
+     CHECK(err);
+@@ -650,10 +667,8 @@ av_log(avctx, AV_LOG_VERBOSE, "OMX avctx pixel_format:%s\n", av_get_pix_fmt_name
+     CHECK(err);
+     s->num_out_buffers = out_port_params.nBufferCountActual;
+ 
+-
+     err = OMX_SendCommand(s->handle, OMX_CommandStateSet, OMX_StateIdle, NULL);
+     CHECK(err);
+-
+     s->in_buffer_headers  = av_mallocz(sizeof(OMX_BUFFERHEADERTYPE*) * s->num_in_buffers);
+     s->free_in_buffers    = av_mallocz(sizeof(OMX_BUFFERHEADERTYPE*) * s->num_in_buffers);
+     s->out_buffer_headers = av_mallocz(sizeof(OMX_BUFFERHEADERTYPE*) * s->num_out_buffers);
+@@ -674,7 +689,6 @@ av_log(avctx, AV_LOG_VERBOSE, "OMX avctx pixel_format:%s\n", av_get_pix_fmt_name
+         err = OMX_AllocateBuffer(s->handle, &s->out_buffer_headers[i], s->out_port, s, out_port_params.nBufferSize);
+     CHECK(err);
+     s->num_out_buffers = i;
+-
+     if (wait_for_state(s, OMX_StateIdle) < 0) {
+         av_log(avctx, AV_LOG_ERROR, "Didn't get OMX_StateIdle\n");
+         return AVERROR_UNKNOWN;
+@@ -1022,6 +1036,8 @@ static const AVOption options[] = {
+     { "main",     "",                         0,               AV_OPT_TYPE_CONST, { .i64 = FF_PROFILE_H264_MAIN },     0, 0, VE, "profile" },
+     { "high",     "",                         0,               AV_OPT_TYPE_CONST, { .i64 = FF_PROFILE_H264_HIGH },     0, 0, VE, "profile" },
+     { "omx_pix_fmt", "Set the decoding pixel format for h264_omx decoder. The following formats are supported: yuv420p, nv12, nv21.", OFFSET(pixel_format), AV_OPT_TYPE_STRING, { .str = "yuv420p" }, 0, 0, VD },
++    { "scale_width", "Set the scaling width for omx (Only zoom out is support, ceil 8(width/8) and minimum 1/8)", OFFSET(scale_width), AV_OPT_TYPE_INT, { .i64 = 0 }, 0, INT_MAX, VD },
++    { "scale_height", "Set the scaling height for omx (Only zoom out is support, ceil 8(height/8) and minimum 1/8)", OFFSET(scale_height), AV_OPT_TYPE_INT, { .i64 = 0 }, 0, INT_MAX, VD },
+     { NULL }
+ };
+ 
+@@ -1030,6 +1046,8 @@ static const AVOption options_hevc[] = {
+     { "omx_libprefix", "OpenMAX library prefix", OFFSET(libprefix), AV_OPT_TYPE_STRING, { 0 }, 0, 0, VDE },
+     { "zerocopy", "Try to avoid copying input frames if possible", OFFSET(input_zerocopy), AV_OPT_TYPE_INT, { .i64 = CONFIG_OMX_RPI }, 0, 1, VE },
+     { "omx_pix_fmt", "Set the decoding pixel format for hevc_omx decoder. The following formats are supported: yuv420p, nv12, nv21.", OFFSET(pixel_format), AV_OPT_TYPE_STRING, { .str = "yuv420p" }, 0, 0, VD },
++    { "scale_width", "Set the scaling width for omx (Only zoom out is support, ceil 8(width/8) and minimum 1/8)", OFFSET(scale_width), AV_OPT_TYPE_INT, { .i64 = 0 }, 0, INT_MAX, VD },
++    { "scale_height", "Set the scaling height for omx (Only zoom out is support, ceil 8(height/8) and minimum 1/8)", OFFSET(scale_height), AV_OPT_TYPE_INT, { .i64 = 0 }, 0, INT_MAX, VD },
+     { NULL },
+ };
+ 
+@@ -1038,6 +1056,8 @@ static const AVOption options_mjpeg[] = {
+     { "omx_libprefix", "OpenMAX library prefix", OFFSET(libprefix), AV_OPT_TYPE_STRING, { 0 }, 0, 0, VDE },
+     { "zerocopy", "Try to avoid copying input frames if possible", OFFSET(input_zerocopy), AV_OPT_TYPE_INT, { .i64 = CONFIG_OMX_RPI }, 0, 1, VD },
+     { "omx_pix_fmt", "Set the decoding pixel format for mjpeg_omx decoder. The following formats are supported: yuv420p, nv12, nv21, nv16, yuv422p, yuyv422, yvyu422, uyvy422, yuv444p.", OFFSET(pixel_format), AV_OPT_TYPE_STRING, { .str = "yuv420p" }, 0, 0, VD },
++    { "scale_width", "Set the scaling width for omx (Only zoom out is support, Horizontal downscale: 0(none), 1(1/2), 2(1/4), 3(1/8)) and minimum 1/8)", OFFSET(scale_width), AV_OPT_TYPE_INT, { .i64 = 0 }, 0, 3, VD },
++    { "scale_height", "Set the scaling height for omx (Only zoom out is support, Vertical downscale: 0(none), 1(1/2), 2(1/4), 3(1/8)) and minimum 1/8)", OFFSET(scale_height), AV_OPT_TYPE_INT, { .i64 = 0 }, 0, 3, VD },
+     { NULL },
+ };
+ 
+-- 
+2.17.1
+

+ 265 - 0
multimedia/patch/ffmpeg/0017-FFmpeg-omxdec-add-mirror-rotation-and-crop-option.patch

@@ -0,0 +1,265 @@
+From 0c7df7e92eb46143fabeefd42af3023e6b7fc53f Mon Sep 17 00:00:00 2001
+From: "arvin.zhu" <arvin.zhu@starfivetech.com>
+Date: Fri, 21 Oct 2022 10:39:31 +0800
+Subject: [PATCH] FFmpeg:omxdec: add mirror rotation and crop option
+
+add mirror rotation and crop option for mjpeg_omx decoder
+
+Signed-off-by: arvin.zhu <arvin.zhu@starfivetech.com>
+---
+ libavcodec/omxdec.c | 181 +++++++++++++++++++++++++++++++++++---------
+ 1 file changed, 146 insertions(+), 35 deletions(-)
+
+diff --git a/libavcodec/omxdec.c b/libavcodec/omxdec.c
+index 87e5a1a..b4a7c96 100755
+--- a/libavcodec/omxdec.c
++++ b/libavcodec/omxdec.c
+@@ -276,6 +276,15 @@ typedef struct OMXCodecContext {
+ 
+     OMX_U32 scale_width;
+     OMX_U32 scale_height;
++    OMX_U32 rotation;
++    OMX_U32 mirror;
++    char *crop_expr;
++    struct {
++        int x;
++        int y;
++        int w;
++        int h;
++    } crop;
+ 
+     int input_zerocopy;
+     int profile;
+@@ -359,7 +368,6 @@ static OMX_ERRORTYPE event_handler(OMX_HANDLETYPE component, OMX_PTR app_data, O
+             av_log(s->avctx, AV_LOG_ERROR, "err %d\n",err);          
+             return AVERROR_UNKNOWN;
+         }
+-
+         for (i = 0; i < video_port_params.nPorts; i++) {
+             int port = video_port_params.nStartPortNumber + i;
+             OMX_PARAM_PORTDEFINITIONTYPE port_params = { 0 };
+@@ -571,7 +579,7 @@ static av_cold int omx_component_init(AVCodecContext *avctx, const char *role)
+     out_port_params.format.video.nBitrate      = avctx->bit_rate;
+     out_port_params.format.video.xFramerate    = in_port_params.format.video.xFramerate;
+     out_port_params.format.video.bFlagErrorConcealment  = OMX_FALSE;
+-    av_log(avctx, AV_LOG_VERBOSE, "OMX setting pixel_format:%s\n", s->pixel_format);
++
+     if (avctx->codec->id == AV_CODEC_ID_MPEG4)
+         out_port_params.format.video.eCompressionFormat = OMX_VIDEO_CodingMPEG4;
+     else if (avctx->codec->id == AV_CODEC_ID_H264) {
+@@ -580,12 +588,12 @@ static av_cold int omx_component_init(AVCodecContext *avctx, const char *role)
+     else if (avctx->codec->id == AV_CODEC_ID_HEVC) {
+         out_port_params.format.video.eCompressionFormat = OMX_VIDEO_CodingHEVC;
+     }
+-    else if (avctx->codec->id == AV_CODEC_ID_MJPEG){
++    else if (avctx->codec->id == AV_CODEC_ID_MJPEG) {
+         out_port_params.format.video.eCompressionFormat = OMX_VIDEO_CodingMJPEG;
+ 
+         /* Set Scale config setting*/
+         if (s->scale_width || s->scale_height) {
+-            av_log(avctx, AV_LOG_TRACE, "mjpeg decoder: scaling width: %ld scaling height: %ld .\n", s->scale_width, s->scale_height);
++            av_log(avctx, AV_LOG_TRACE, "mjpeg decoder: scaling width: %d scaling height: %d .\n", s->scale_width, s->scale_height);
+             OMX_CONFIG_SCALEFACTORTYPE ScaleConfig;
+             INIT_STRUCT(ScaleConfig);
+             ScaleConfig.nPortIndex = 1;
+@@ -599,38 +607,138 @@ static av_cold int omx_component_init(AVCodecContext *avctx, const char *role)
+             out_port_params.format.video.nFrameHeight  = avctx->height;
+         }
+ 
+-        switch (av_get_pix_fmt(s->pixel_format)) {
+-        case AV_PIX_FMT_NV12:
+-            out_port_params.format.video.eColorFormat = OMX_COLOR_FormatYUV420SemiPlanar;
+-            break;
+-        case AV_PIX_FMT_NV21:
+-            out_port_params.format.video.eColorFormat = OMX_COLOR_FormatYVU420SemiPlanar;
+-            break;
+-        case AV_PIX_FMT_YUV420P:
+-            out_port_params.format.video.eColorFormat = OMX_COLOR_FormatYUV420Planar;
+-            break;
+-        case AV_PIX_FMT_NV16:
+-            out_port_params.format.video.eColorFormat = OMX_COLOR_FormatYUV422SemiPlanar;
+-            break;
+-        case AV_PIX_FMT_YUV422P:
+-            out_port_params.format.video.eColorFormat = OMX_COLOR_FormatYUV422Planar;
+-            break;
+-        case AV_PIX_FMT_YUYV422:
+-            out_port_params.format.video.eColorFormat = OMX_COLOR_FormatYCbYCr;
+-            break;
+-        case AV_PIX_FMT_YVYU422:
+-            out_port_params.format.video.eColorFormat = OMX_COLOR_FormatYCrYCb;
+-            break;
+-        case AV_PIX_FMT_UYVY422:
+-            out_port_params.format.video.eColorFormat = OMX_COLOR_FormatCbYCrY;
+-            break;
+-        case AV_PIX_FMT_YUV444P:
+-            out_port_params.format.video.eColorFormat = OMX_COLOR_FormatYUV444Planar;
+-            break;
+-        default:
++        /* Set pixel format to decoder output*/
++        if (s->pixel_format) {
++            switch (av_get_pix_fmt(s->pixel_format)) {
++            case AV_PIX_FMT_NV12:
++                out_port_params.format.video.eColorFormat = OMX_COLOR_FormatYUV420SemiPlanar;
++                break;
++            case AV_PIX_FMT_NV21:
++                out_port_params.format.video.eColorFormat = OMX_COLOR_FormatYVU420SemiPlanar;
++                break;
++            case AV_PIX_FMT_YUV420P:
++                out_port_params.format.video.eColorFormat = OMX_COLOR_FormatYUV420Planar;
++                break;
++            case AV_PIX_FMT_NV16:
++                out_port_params.format.video.eColorFormat = OMX_COLOR_FormatYUV422SemiPlanar;
++                break;
++            case AV_PIX_FMT_YUV422P:
++                out_port_params.format.video.eColorFormat = OMX_COLOR_FormatYUV422Planar;
++                break;
++            case AV_PIX_FMT_YUYV422:
++                out_port_params.format.video.eColorFormat = OMX_COLOR_FormatYCbYCr;
++                break;
++            case AV_PIX_FMT_YVYU422:
++                out_port_params.format.video.eColorFormat = OMX_COLOR_FormatYCrYCb;
++                break;
++            case AV_PIX_FMT_UYVY422:
++                out_port_params.format.video.eColorFormat = OMX_COLOR_FormatCbYCrY;
++                break;
++            case AV_PIX_FMT_YUV444P:
++                out_port_params.format.video.eColorFormat = OMX_COLOR_FormatYUV444Planar;
++                break;
++            default:
++                return AVERROR_OPTION_NOT_FOUND;
++            }
++            av_log(avctx, AV_LOG_VERBOSE, "OMX setting pixel_format:%s\n", s->pixel_format);
++        } else {
++            switch (avctx->pix_fmt) {
++            case AV_PIX_FMT_YUVJ420P:
++                out_port_params.format.video.eColorFormat = OMX_COLOR_FormatYUV420Planar;
++                break;
++            case AV_PIX_FMT_YUVJ422P:
++                out_port_params.format.video.eColorFormat = OMX_COLOR_FormatYUV422Planar;
++                break;
++            case AV_PIX_FMT_YUVJ444P:
++                out_port_params.format.video.eColorFormat = OMX_COLOR_FormatYUV444Planar;
++                break;
++            default:
++                out_port_params.format.video.eColorFormat = OMX_COLOR_FormatYUV420Planar;
++                break;
++            }
++        }
++
++        /* Set Mirror config setting*/
++        if (s->mirror) {
++            if (s->pixel_format || s->scale_width || s->scale_height || s->rotation) {
++                av_log(avctx, AV_LOG_ERROR, "Mirror cannot work with other option together.\n");
++                return AVERROR_INVALIDDATA;
++            }
++            av_log(avctx, AV_LOG_TRACE, "mjpeg decoder: mirror\n");
++            OMX_CONFIG_MIRRORTYPE MirrorConfig;
++            INIT_STRUCT(MirrorConfig);
++            MirrorConfig.nPortIndex = 1;
++            OMX_GetConfig(s->handle, OMX_IndexConfigCommonMirror, &MirrorConfig);
++            MirrorConfig.eMirror = s->mirror;
++            OMX_SetConfig(s->handle, OMX_IndexConfigCommonMirror, &MirrorConfig);
++        }
++
++        /* Set Rotation config setting*/
++        if (s->rotation) {
++            if (s->pixel_format || s->scale_width || s->scale_height || s->mirror) {
++                av_log(avctx, AV_LOG_ERROR, "Rotation cannot work with other option together.\n");
++                return AVERROR_INVALIDDATA;
++            }
++            av_log(avctx, AV_LOG_TRACE, "mjpeg decoder: rotation\n");
++            OMX_CONFIG_ROTATIONTYPE RotatConfig;
++            INIT_STRUCT(RotatConfig);
++            RotatConfig.nPortIndex = 1;
++            OMX_GetConfig(s->handle, OMX_IndexConfigCommonRotate, &RotatConfig);
++            switch (s->rotation) {
++            case 1:
++                RotatConfig.nRotation = 90;
++                break;
++            case 2:
++                RotatConfig.nRotation = 180;
++                break;
++            case 3:
++                RotatConfig.nRotation = 270;
++                break;
++            default:
++                RotatConfig.nRotation = 0;
++                break;
++            }
++            OMX_SetConfig(s->handle, OMX_IndexConfigCommonRotate, &RotatConfig);
++        }
++
++        /* Set Roi config setting*/
++        if (s->crop_expr && sscanf(s->crop_expr, "%d,%d,%d,%d",
++                                &s->crop.x, &s->crop.y,
++                                &s->crop.w, &s->crop.h) != 4) {
++            av_log(avctx, AV_LOG_ERROR, "Invalid cropping expressions.\n");
+             return AVERROR_OPTION_NOT_FOUND;
+         }
++
++        if (s->crop.w && s->crop.h) {
++            if (s->pixel_format || s->scale_width || s->scale_height || s->mirror || s->rotation) {
++                av_log(avctx, AV_LOG_ERROR, "Crop cannot work with other option together.\n");
++                return AVERROR_INVALIDDATA;
++            }
++            if ((s->crop.x < 0) || (s->crop.x > (avctx->width - s->crop.w)) ||
++                 (s->crop.y < 0) || (s->crop.y > (avctx->height - s->crop.h)) ||
++                 (s->crop.w < 16) || (s->crop.w > (avctx->width - s->crop.x)) ||
++                 (s->crop.h < 16) || (s->crop.h > (avctx->height - s->crop.y))) {
++                av_log(avctx, AV_LOG_ERROR, "Invalid cropping range.\n");
++                return AVERROR_INVALIDDATA;
++            }
++            if (s->crop.w % 16 != 0 || s->crop.h % 16 != 0) {
++                av_log(avctx, AV_LOG_ERROR, "The width/height must be an integer multiple of 16\n");
++                return AVERROR_INVALIDDATA;
++            }
++
++            OMX_CONFIG_RECTTYPE RectConfig; 
++            INIT_STRUCT(RectConfig);
++            RectConfig.nPortIndex = 1;
++            OMX_GetConfig(s->handle, OMX_IndexConfigCommonOutputCrop, &RectConfig);
++            RectConfig.nLeft = s->crop.x;
++            RectConfig.nTop = s->crop.y;
++            RectConfig.nWidth = s->crop.w;
++            RectConfig.nHeight = s->crop.h;
++            OMX_SetConfig(s->handle, OMX_IndexConfigCommonOutputCrop, &RectConfig);
++            av_log(avctx, AV_LOG_ERROR, "mjpeg decoder:roi x: %d y: %d w: %d h: %d .\n", s->crop.x, s->crop.y, s->crop.w, s->crop.h);
++        }
+     }
++
+     if (avctx->codec->id == AV_CODEC_ID_H264 || avctx->codec->id == AV_CODEC_ID_HEVC) {
+         /* Set Scale config setting*/
+         if ((s->scale_width != 0) && ((s->scale_width < avctx->width/8) ||
+@@ -675,6 +783,7 @@ static av_cold int omx_component_init(AVCodecContext *avctx, const char *role)
+     s->done_out_buffers   = av_mallocz(sizeof(OMX_BUFFERHEADERTYPE*) * s->num_out_buffers);
+     if (!s->in_buffer_headers || !s->free_in_buffers || !s->out_buffer_headers || !s->done_out_buffers)
+         return AVERROR(ENOMEM);
++
+     for (i = 0; i < s->num_in_buffers && err == OMX_ErrorNone; i++) {
+         if (s->input_zerocopy)
+             err = OMX_UseBuffer(s->handle, &s->in_buffer_headers[i], s->in_port, s, in_port_params.nBufferSize, NULL);
+@@ -699,7 +808,6 @@ static av_cold int omx_component_init(AVCodecContext *avctx, const char *role)
+         av_log(avctx, AV_LOG_ERROR, "Didn't get OMX_StateExecuting\n");
+         return AVERROR_UNKNOWN;
+     }
+-
+     for (i = 0; i < s->num_out_buffers && err == OMX_ErrorNone; i++)
+         err = OMX_FillThisBuffer(s->handle, s->out_buffer_headers[i]);
+     if (err != OMX_ErrorNone) {
+@@ -1055,9 +1163,12 @@ static const AVOption options_mjpeg[] = {
+     { "omx_libname", "OpenMAX library name", OFFSET(libname), AV_OPT_TYPE_STRING, { 0 }, 0, 0, VDE },
+     { "omx_libprefix", "OpenMAX library prefix", OFFSET(libprefix), AV_OPT_TYPE_STRING, { 0 }, 0, 0, VDE },
+     { "zerocopy", "Try to avoid copying input frames if possible", OFFSET(input_zerocopy), AV_OPT_TYPE_INT, { .i64 = CONFIG_OMX_RPI }, 0, 1, VD },
+-    { "omx_pix_fmt", "Set the decoding pixel format for mjpeg_omx decoder. The following formats are supported: yuv420p, nv12, nv21, nv16, yuv422p, yuyv422, yvyu422, uyvy422, yuv444p.", OFFSET(pixel_format), AV_OPT_TYPE_STRING, { .str = "yuv420p" }, 0, 0, VD },
++    { "omx_pix_fmt", "Set the decoding pixel format for mjpeg_omx decoder. The following formats are supported: yuv420p, nv12, nv21, nv16, yuv422p, yuyv422, yvyu422, uyvy422, yuv444p.", OFFSET(pixel_format), AV_OPT_TYPE_STRING, { .str = NULL }, 0, 0, VD },
+     { "scale_width", "Set the scaling width for omx (Only zoom out is support, Horizontal downscale: 0(none), 1(1/2), 2(1/4), 3(1/8)) and minimum 1/8)", OFFSET(scale_width), AV_OPT_TYPE_INT, { .i64 = 0 }, 0, 3, VD },
+     { "scale_height", "Set the scaling height for omx (Only zoom out is support, Vertical downscale: 0(none), 1(1/2), 2(1/4), 3(1/8)) and minimum 1/8)", OFFSET(scale_height), AV_OPT_TYPE_INT, { .i64 = 0 }, 0, 3, VD },
++    { "mirror", "mirror 0(none), 1(V), 2(H) or 3(VH), cannot be set at the same time as pixel format conversion, rotation and crop", OFFSET(mirror), AV_OPT_TYPE_INT, { .i64 = 0 }, 0, 3, VD },
++    { "rotation", "rotation 0(0), 1(90), 2(180) or 3(270), cannot be set at the same time as pixel format conversion, mirror and crop", OFFSET(rotation), AV_OPT_TYPE_INT, { .i64 = 0 }, 0, 3, VD },
++    { "crop", "crop <x>,<y>,<w>,<h>: crop coord and width/height(from left/top, must be an integer multiple of 16), cannot be set at the same time as pixel format conversion, mirror and rotation", OFFSET(crop_expr), AV_OPT_TYPE_STRING, { .str = NULL }, 0, 0, VD },
+     { NULL },
+ };
+ 
+-- 
+2.17.1
+

+ 35 - 0
multimedia/patch/ffmpeg/0018-FFmpeg-omx-gop-parameter-of-hevc-encoder.patch

@@ -0,0 +1,35 @@
+From 67b9e00e3abaa9179de91387bab9d00aa994d250 Mon Sep 17 00:00:00 2001
+From: "arvin.zhu" <arvin.zhu@starfivetech.com>
+Date: Wed, 2 Nov 2022 12:32:56 +0800
+Subject: [PATCH] FFmpeg:omx: gop parameter of hevc encoder
+
+repair gop parameter of hevc encoder not take effect
+
+Signed-off-by: arvin.zhu <arvin.zhu@starfivetech.com>
+---
+ libavcodec/omx.c | 5 ++---
+ 1 file changed, 2 insertions(+), 3 deletions(-)
+
+diff --git a/libavcodec/omx.c b/libavcodec/omx.c
+index 77012ce..d101fa4 100755
+--- a/libavcodec/omx.c
++++ b/libavcodec/omx.c
+@@ -561,13 +561,12 @@ static av_cold int omx_component_init(AVCodecContext *avctx, const char *role)
+         err = OMX_SetParameter(s->handle, OMX_IndexParamVideoAvc, &avc);
+         CHECK(err);
+     } else if (avctx->codec->id == AV_CODEC_ID_HEVC) {
+-        OMX_VIDEO_PARAM_AVCTYPE hevc = { 0 };
++        OMX_VIDEO_PARAM_HEVCTYPE hevc = { 0 };
+         INIT_STRUCT(hevc);
+         hevc.nPortIndex = s->out_port;
+         err = OMX_GetParameter(s->handle, OMX_IndexParamVideoHevc, &hevc);
+         CHECK(err);
+-        hevc.nBFrames = 0;
+-        hevc.nPFrames = avctx->gop_size;
++        hevc.nKeyFrameInterval = avctx->gop_size;
+         err = OMX_SetParameter(s->handle, OMX_IndexParamVideoHevc, &hevc);
+         CHECK(err);
+     }
+-- 
+2.17.1
+

+ 34 - 0
multimedia/patch/ffmpeg/0019-FFmpeg-omcdec-fix-mjpeg_omx-decoder-scale-option-bug.patch

@@ -0,0 +1,34 @@
+From bc391d931fa56ab788ae9cee7c249f91a9263d5b Mon Sep 17 00:00:00 2001
+From: "arvin.zhu" <arvin.zhu@starfivetech.com>
+Date: Wed, 2 Nov 2022 17:21:01 +0800
+Subject: [PATCH] FFmpeg:omcdec: fix mjpeg_omx decoder scale option bug
+
+fix mjpeg_omx decoder scale option bug
+
+Signed-off-by: arvin.zhu <arvin.zhu@starfivetech.com>
+---
+ libavcodec/omxdec.c | 6 +++---
+ 1 file changed, 3 insertions(+), 3 deletions(-)
+
+diff --git a/libavcodec/omxdec.c b/libavcodec/omxdec.c
+index b4a7c96..019db2e 100755
+--- a/libavcodec/omxdec.c
++++ b/libavcodec/omxdec.c
+@@ -602,11 +602,11 @@ static av_cold int omx_component_init(AVCodecContext *avctx, const char *role)
+             ScaleConfig.xWidth = (1 << 16) >> (s->scale_width & 0x3);
+             ScaleConfig.xHeight = (1 << 16) >> (s->scale_height & 0x3);
+             OMX_SetConfig(s->handle, OMX_IndexConfigCommonScale, &ScaleConfig);
+-        } else {
+-            out_port_params.format.video.nFrameWidth   = avctx->width;
+-            out_port_params.format.video.nFrameHeight  = avctx->height;
+         }
+ 
++        out_port_params.format.video.nFrameWidth   = avctx->width;
++        out_port_params.format.video.nFrameHeight  = avctx->height;
++
+         /* Set pixel format to decoder output*/
+         if (s->pixel_format) {
+             switch (av_get_pix_fmt(s->pixel_format)) {
+-- 
+2.17.1
+

+ 72 - 0
multimedia/patch/gstreamer1/gst1-plugins-bad/0001-add-I420.patch

@@ -0,0 +1,72 @@
+--- a/sys/fbdev/gstfbdevsink.c	2021-07-06 16:02:46.304928800 +0800
++++ b/sys/fbdev/gstfbdevsink.c	2021-07-06 16:02:11.593693100 +0800
+@@ -73,7 +73,7 @@
+ static GstStateChangeReturn gst_fbdevsink_change_state (GstElement * element,
+     GstStateChange transition);
+ 
+-#define VIDEO_CAPS "{ RGB, BGR, BGRx, xBGR, RGB, RGBx, xRGB, RGB15, RGB16 }"
++#define VIDEO_CAPS "{ RGB, BGR, BGRx, xBGR, RGB, RGBx, xRGB, RGB15, RGB16, I420, NV12, NV21 }"
+ 
+ static GstStaticPadTemplate sink_template = GST_STATIC_PAD_TEMPLATE ("sink",
+     GST_PAD_SINK,
+@@ -133,7 +133,10 @@
+   /* FIXME: locking */
+   if (!fbdevsink->framebuffer)
+     goto done;
+-
++  if (fbdevsink->varinfo.grayscale == 1) {
++    format = GST_VIDEO_FORMAT_NV12;
++    goto common;
++  }
+   bpp = fbdevsink->varinfo.bits_per_pixel;
+ 
+   rmask = ((1 << fbdevsink->varinfo.red.length) - 1)
+@@ -182,6 +185,8 @@
+   if (format == GST_VIDEO_FORMAT_UNKNOWN)
+     goto unknown_format;
+ 
++common:
++
+   caps = gst_caps_make_writable (caps);
+   gst_caps_set_simple (caps, "format", G_TYPE_STRING,
+       gst_video_format_to_string (format), NULL);
+@@ -272,12 +277,33 @@
+   if (!gst_buffer_map (buf, &map, GST_MAP_READ))
+     return GST_FLOW_ERROR;
+ 
+-  for (i = 0; i < fbdevsink->lines; i++) {
+-    memcpy (fbdevsink->framebuffer
+-        + (i + fbdevsink->cy) * fbdevsink->fixinfo.line_length
+-        + fbdevsink->cx * fbdevsink->bytespp,
+-        map.data + i * fbdevsink->width * fbdevsink->bytespp,
+-        fbdevsink->linelen);
++  if (fbdevsink->varinfo.grayscale == 1) {
++    int screen_offset = fbdevsink->varinfo.xres * fbdevsink->varinfo.yres;
++    int buffer_offset = fbdevsink->height * fbdevsink->width;
++
++    for (i = 0; i < fbdevsink->lines; i++) {
++      memcpy (fbdevsink->framebuffer
++          + (i + fbdevsink->cy) * fbdevsink->varinfo.xres
++          + fbdevsink->cx * 1,
++          map.data + i * fbdevsink->width * 1,
++          fbdevsink->width);
++    }
++
++    for (i = 0; i < fbdevsink->lines/2; i++) {
++      memcpy (fbdevsink->framebuffer + screen_offset
++          + (i + fbdevsink->cy/2) * fbdevsink->varinfo.xres
++          + fbdevsink->cx * 1,
++          map.data + buffer_offset + i * fbdevsink->width * 1,
++          fbdevsink->width);
++    }  
++  } else {
++    for (i = 0; i < fbdevsink->lines; i++) {
++      memcpy (fbdevsink->framebuffer
++          + (i + fbdevsink->cy) * fbdevsink->fixinfo.line_length
++          + fbdevsink->cx * fbdevsink->bytespp,
++          map.data + i * fbdevsink->width * fbdevsink->bytespp,
++          fbdevsink->linelen);
++    } 
+   }
+ 
+   gst_buffer_unmap (buf, &map);

+ 1027 - 0
multimedia/patch/gstreamer1/gst1-plugins-bad/0002-reset-drm-in-atomic-api.patch

@@ -0,0 +1,1027 @@
+Reset DRM to initial state on each playing.
+
+Signed-off-by: Windsome Zeng <windsome.zeng@starfivetech.com>
+
+diff -purN a/sys/kms/gstkmsallocator.c b/sys/kms/gstkmsallocator.c
+--- a/sys/kms/gstkmsallocator.c	2022-09-06 11:07:01.464180909 +0800
++++ b/sys/kms/gstkmsallocator.c	2022-09-06 11:03:57.761729431 +0800
+@@ -384,6 +384,9 @@ gst_kms_memory_map (GstMemory * mem, gsi
+   }
+   kmsmem->bo->ptr = out;
+ 
++  /* clear the framebuffer to 0 */
++  memset(out, 0, kmsmem->bo->size);
++
+ out:
+   g_atomic_int_inc (&kmsmem->bo->refs);
+   return kmsmem->bo->ptr;
+diff -purN a/sys/kms/gstkmssink.c b/sys/kms/gstkmssink.c
+--- a/sys/kms/gstkmssink.c	2022-09-06 11:07:01.464180909 +0800
++++ b/sys/kms/gstkmssink.c	2022-09-06 11:03:57.761729431 +0800
+@@ -74,6 +74,8 @@ static GstFlowReturn gst_kms_sink_show_f
+ static void gst_kms_sink_video_overlay_init (GstVideoOverlayInterface * iface);
+ static void gst_kms_sink_drain (GstKMSSink * self);
+ 
++int gst_kms_sink_reset_drm(GstKMSSink * self);
++
+ #define parent_class gst_kms_sink_parent_class
+ G_DEFINE_TYPE_WITH_CODE (GstKMSSink, gst_kms_sink, GST_TYPE_VIDEO_SINK,
+     GST_DEBUG_CATEGORY_INIT (GST_CAT_DEFAULT, GST_PLUGIN_NAME, 0,
+@@ -818,6 +820,9 @@ retry_find_plane:
+   g_object_notify_by_pspec (G_OBJECT (self), g_properties[PROP_DISPLAY_WIDTH]);
+   g_object_notify_by_pspec (G_OBJECT (self), g_properties[PROP_DISPLAY_HEIGHT]);
+ 
++  GST_INFO_OBJECT (self, "Reset DRM to initial state using atomic API.");
++  gst_kms_sink_reset_drm (self);
++
+   gst_kms_sink_update_connector_properties (self);
+   gst_kms_sink_update_plane_properties (self);
+ 
+diff -purN a/sys/kms/meson.build b/sys/kms/meson.build
+--- a/sys/kms/meson.build	2022-09-06 11:07:01.464180909 +0800
++++ b/sys/kms/meson.build	2022-09-06 11:03:57.761729431 +0800
+@@ -3,6 +3,7 @@ kmssink_sources = [
+   'gstkmsbufferpool.c',
+   'gstkmssink.c',
+   'gstkmsutils.c',
++  'mymodetest.c',
+ ]
+ 
+ if host_system != 'linux'
+diff -purN a/sys/kms/mymodetest.c b/sys/kms/mymodetest.c
+--- a/sys/kms/mymodetest.c	1970-01-01 08:00:00.000000000 +0800
++++ b/sys/kms/mymodetest.c	2022-09-06 11:03:57.761729431 +0800
+@@ -0,0 +1,973 @@
++/*
++ * SPDX-License-Identifier: GPL-2.0
++ *
++ * Copyright (C) 2022 StarFive Technology Co., Ltd.
++ *
++ * Description: Reset drm to initial state in atomic API.
++ * This code is modified from libdrm/modetest. Instead of command line, all 
++ * its info comes from GstKMSSink, such as crtc/plane/connector etc.
++ *   Windsome Zeng <windsome.zeng@starfivetech.com>
++ */
++
++/*
++ * DRM based mode setting test program
++ * Copyright 2008 Tungsten Graphics
++ *   Jakob Bornecrantz <jakob@tungstengraphics.com>
++ * Copyright 2008 Intel Corporation
++ *   Jesse Barnes <jesse.barnes@intel.com>
++ *
++ * Permission is hereby granted, free of charge, to any person obtaining a
++ * copy of this software and associated documentation files (the "Software"),
++ * to deal in the Software without restriction, including without limitation
++ * the rights to use, copy, modify, merge, publish, distribute, sublicense,
++ * and/or sell copies of the Software, and to permit persons to whom the
++ * Software is furnished to do so, subject to the following conditions:
++ *
++ * The above copyright notice and this permission notice shall be included in
++ * all copies or substantial portions of the Software.
++ *
++ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
++ * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
++ * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
++ * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
++ * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
++ * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
++ * IN THE SOFTWARE.
++ */
++
++/*
++ * This fairly simple test program dumps output in a similar format to the
++ * "xrandr" tool everyone knows & loves.  It's necessarily slightly different
++ * since the kernel separates outputs into encoder and connector structures,
++ * each with their own unique ID.  The program also allows test testing of the
++ * memory management and mode setting APIs by allowing the user to specify a
++ * connector and mode to use for mode setting.  If all works as expected, a
++ * blue background should be painted on the monitor attached to the specified
++ * connector after the selected mode is set.
++ *
++ * TODO: use cairo to write the mode info on the selected output once
++ *       the mode has been programmed, along with possible test patterns.
++ */
++
++#include <assert.h>
++#include <ctype.h>
++#include <stdbool.h>
++#include <stdio.h>
++#include <stdlib.h>
++#include <stdint.h>
++#include <inttypes.h>
++#include <unistd.h>
++#include <string.h>
++#include <strings.h>
++#include <errno.h>
++#include <poll.h>
++#include <sys/time.h>
++#if HAVE_SYS_SELECT_H
++#include <sys/select.h>
++#endif
++#include <math.h>
++
++#include "xf86drm.h"
++#include "xf86drmMode.h"
++#include "drm_fourcc.h"
++
++#include <gst/video/video.h>
++#include "gstkmssink.h"
++
++int gst_kms_sink_reset_drm(GstKMSSink * self);
++
++struct crtc {
++    drmModeCrtc *crtc;
++    drmModeObjectProperties *props;
++    drmModePropertyRes **props_info;
++    drmModeModeInfo *mode;
++};
++
++struct encoder {
++    drmModeEncoder *encoder;
++};
++
++struct connector {
++    drmModeConnector *connector;
++    drmModeObjectProperties *props;
++    drmModePropertyRes **props_info;
++    char *name;
++};
++
++struct fb {
++    drmModeFB *fb;
++};
++
++struct plane {
++    drmModePlane *plane;
++    drmModeObjectProperties *props;
++    drmModePropertyRes **props_info;
++};
++
++struct resources {
++    struct crtc *crtcs;
++    int count_crtcs;
++    struct encoder *encoders;
++    int count_encoders;
++    struct connector *connectors;
++    int count_connectors;
++    struct fb *fbs;
++    int count_fbs;
++    struct plane *planes;
++    uint32_t count_planes;
++};
++
++struct device {
++    int fd;
++
++    struct resources *resources;
++
++    struct {
++        unsigned int width;
++        unsigned int height;
++
++        unsigned int fb_id;
++        struct bo *bo;
++        struct bo *cursor_bo;
++    } mode;
++
++    int use_atomic;
++    drmModeAtomicReq *req;
++};
++
++static inline int64_t U642I64(uint64_t val)
++{
++    return (int64_t)*((int64_t *)&val);
++}
++
++static float mode_vrefresh(drmModeModeInfo *mode)
++{
++    return  mode->clock * 1000.00
++            / (mode->htotal * mode->vtotal);
++}
++
++static void free_resources(struct resources *res)
++{
++    int i;
++
++    if (!res)
++        return;
++
++#define free_resource(_res, type, Type)                    \
++    do {                                    \
++        if (!(_res)->type##s)                        \
++            break;                            \
++        for (i = 0; i < (int)(_res)->count_##type##s; ++i) {    \
++            if (!(_res)->type##s[i].type)                \
++                break;                        \
++            drmModeFree##Type((_res)->type##s[i].type);        \
++        }                                \
++        free((_res)->type##s);                        \
++    } while (0)
++
++#define free_properties(_res, type)                    \
++    do {                                    \
++        for (i = 0; i < (int)(_res)->count_##type##s; ++i) {    \
++            unsigned int j;                                        \
++            for (j = 0; j < res->type##s[i].props->count_props; ++j)\
++                drmModeFreeProperty(res->type##s[i].props_info[j]);\
++            free(res->type##s[i].props_info);            \
++            drmModeFreeObjectProperties(res->type##s[i].props);    \
++        }                                \
++    } while (0)
++
++    free_properties(res, plane);
++    free_resource(res, plane, Plane);
++
++    free_properties(res, connector);
++    free_properties(res, crtc);
++
++    for (i = 0; i < res->count_connectors; i++)
++        free(res->connectors[i].name);
++
++    free_resource(res, fb, FB);
++    free_resource(res, connector, Connector);
++    free_resource(res, encoder, Encoder);
++    free_resource(res, crtc, Crtc);
++
++    free(res);
++}
++
++struct type_name {
++    unsigned int type;
++    const char *name;
++};
++
++#define ARRAY_SIZE(arr) (sizeof(arr) / sizeof((arr)[0]))
++
++static const char *util_lookup_type_name(unsigned int type,
++                     const struct type_name *table,
++                     unsigned int count)
++{
++    unsigned int i;
++
++    for (i = 0; i < count; i++)
++        if (table[i].type == type)
++            return table[i].name;
++
++    return NULL;
++}
++
++static const struct type_name connector_type_names[] = {
++    { DRM_MODE_CONNECTOR_Unknown, "unknown" },
++    { DRM_MODE_CONNECTOR_VGA, "VGA" },
++    { DRM_MODE_CONNECTOR_DVII, "DVI-I" },
++    { DRM_MODE_CONNECTOR_DVID, "DVI-D" },
++    { DRM_MODE_CONNECTOR_DVIA, "DVI-A" },
++    { DRM_MODE_CONNECTOR_Composite, "composite" },
++    { DRM_MODE_CONNECTOR_SVIDEO, "s-video" },
++    { DRM_MODE_CONNECTOR_LVDS, "LVDS" },
++    { DRM_MODE_CONNECTOR_Component, "component" },
++    { DRM_MODE_CONNECTOR_9PinDIN, "9-pin DIN" },
++    { DRM_MODE_CONNECTOR_DisplayPort, "DP" },
++    { DRM_MODE_CONNECTOR_HDMIA, "HDMI-A" },
++    { DRM_MODE_CONNECTOR_HDMIB, "HDMI-B" },
++    { DRM_MODE_CONNECTOR_TV, "TV" },
++    { DRM_MODE_CONNECTOR_eDP, "eDP" },
++    { DRM_MODE_CONNECTOR_VIRTUAL, "Virtual" },
++    { DRM_MODE_CONNECTOR_DSI, "DSI" },
++    { DRM_MODE_CONNECTOR_DPI, "DPI" },
++};
++
++static const char *util_lookup_connector_type_name(unsigned int type)
++{
++    return util_lookup_type_name(type, connector_type_names,
++                     ARRAY_SIZE(connector_type_names));
++}
++
++static struct resources *get_resources(struct device *dev)
++{
++    drmModeRes *_res;
++    drmModePlaneRes *plane_res;
++    struct resources *res;
++    int i;
++
++    res = calloc(1, sizeof(*res));
++    if (res == 0)
++        return NULL;
++
++    drmSetClientCap(dev->fd, DRM_CLIENT_CAP_UNIVERSAL_PLANES, 1);
++
++    _res = drmModeGetResources(dev->fd);
++    if (!_res) {
++        fprintf(stderr, "drmModeGetResources failed: %s\n",
++            strerror(errno));
++        free(res);
++        return NULL;
++    }
++
++    res->count_crtcs = _res->count_crtcs;
++    res->count_encoders = _res->count_encoders;
++    res->count_connectors = _res->count_connectors;
++    res->count_fbs = _res->count_fbs;
++
++    res->crtcs = calloc(res->count_crtcs, sizeof(*res->crtcs));
++    res->encoders = calloc(res->count_encoders, sizeof(*res->encoders));
++    res->connectors = calloc(res->count_connectors, sizeof(*res->connectors));
++    res->fbs = calloc(res->count_fbs, sizeof(*res->fbs));
++
++    if (!res->crtcs || !res->encoders || !res->connectors || !res->fbs) {
++        drmModeFreeResources(_res);
++        goto error;
++    }
++
++#define get_resource(_res, __res, type, Type)                    \
++    do {                                    \
++        for (i = 0; i < (int)(_res)->count_##type##s; ++i) {    \
++            uint32_t type##id = (__res)->type##s[i];            \
++            (_res)->type##s[i].type =                            \
++                drmModeGet##Type(dev->fd, type##id);            \
++            if (!(_res)->type##s[i].type)                        \
++                fprintf(stderr, "could not get %s %i: %s\n",    \
++                    #type, type##id,                            \
++                    strerror(errno));            \
++        }                                \
++    } while (0)
++
++    get_resource(res, _res, crtc, Crtc);
++    get_resource(res, _res, encoder, Encoder);
++    get_resource(res, _res, connector, Connector);
++    get_resource(res, _res, fb, FB);
++
++    drmModeFreeResources(_res);
++
++    /* Set the name of all connectors based on the type name and the per-type ID. */
++    for (i = 0; i < res->count_connectors; i++) {
++        struct connector *connector = &res->connectors[i];
++        drmModeConnector *conn = connector->connector;
++        int num;
++
++        num = asprintf(&connector->name, "%s-%u",
++             util_lookup_connector_type_name(conn->connector_type),
++             conn->connector_type_id);
++        if (num < 0)
++            goto error;
++    }
++
++#define get_properties(_res, type, Type)                    \
++    do {                                    \
++        for (i = 0; i < (int)(_res)->count_##type##s; ++i) {    \
++            struct type *obj = &res->type##s[i];            \
++            unsigned int j;                        \
++            obj->props =                        \
++                drmModeObjectGetProperties(dev->fd, obj->type->type##_id, \
++                               DRM_MODE_OBJECT_##Type); \
++            if (!obj->props) {                    \
++                fprintf(stderr,                    \
++                    "could not get %s %i properties: %s\n", \
++                    #type, obj->type->type##_id,        \
++                    strerror(errno));            \
++                continue;                    \
++            }                            \
++            obj->props_info = calloc(obj->props->count_props,    \
++                         sizeof(*obj->props_info));    \
++            if (!obj->props_info)                    \
++                continue;                    \
++            for (j = 0; j < obj->props->count_props; ++j)        \
++                obj->props_info[j] =                \
++                    drmModeGetProperty(dev->fd, obj->props->props[j]); \
++        }                                \
++    } while (0)
++
++    get_properties(res, crtc, CRTC);
++    get_properties(res, connector, CONNECTOR);
++
++    for (i = 0; i < res->count_crtcs; ++i)
++        res->crtcs[i].mode = &res->crtcs[i].crtc->mode;
++
++    plane_res = drmModeGetPlaneResources(dev->fd);
++    if (!plane_res) {
++        fprintf(stderr, "drmModeGetPlaneResources failed: %s\n",
++            strerror(errno));
++        return res;
++    }
++
++    res->count_planes = plane_res->count_planes;
++
++    res->planes = calloc(res->count_planes, sizeof(*res->planes));
++    if (!res->planes) {
++        drmModeFreePlaneResources(plane_res);
++        goto error;
++    }
++
++    get_resource(res, plane_res, plane, Plane);
++    drmModeFreePlaneResources(plane_res);
++    get_properties(res, plane, PLANE);
++
++    return res;
++
++error:
++    free_resources(res);
++    return NULL;
++}
++
++static struct crtc *get_crtc_by_id(struct device *dev, uint32_t id)
++{
++    int i;
++
++    for (i = 0; i < dev->resources->count_crtcs; ++i) {
++        drmModeCrtc *crtc = dev->resources->crtcs[i].crtc;
++        if (crtc && crtc->crtc_id == id)
++            return &dev->resources->crtcs[i];
++    }
++
++    return NULL;
++}
++
++static uint32_t get_crtc_mask(struct device *dev, struct crtc *crtc)
++{
++    unsigned int i;
++
++    for (i = 0; i < (unsigned int)dev->resources->count_crtcs; i++) {
++        if (crtc->crtc->crtc_id == dev->resources->crtcs[i].crtc->crtc_id)
++            return 1 << i;
++    }
++    /* Unreachable: crtc->crtc is one of resources->crtcs[] */
++    /* Don't return zero or static analysers will complain */
++    abort();
++    return 0;
++}
++
++static drmModeConnector *get_connector_by_id(struct device *dev, uint32_t id)
++{
++    drmModeConnector *connector;
++    int i;
++
++    for (i = 0; i < dev->resources->count_connectors; i++) {
++        connector = dev->resources->connectors[i].connector;
++        if (connector && connector->connector_id == id)
++            return connector;
++    }
++
++    return NULL;
++}
++
++static drmModeEncoder *get_encoder_by_id(struct device *dev, uint32_t id)
++{
++    drmModeEncoder *encoder;
++    int i;
++
++    for (i = 0; i < dev->resources->count_encoders; i++) {
++        encoder = dev->resources->encoders[i].encoder;
++        if (encoder && encoder->encoder_id == id)
++            return encoder;
++    }
++
++    return NULL;
++}
++
++/* -----------------------------------------------------------------------------
++ * Pipes and planes
++ */
++
++/*
++ * Mode setting with the kernel interfaces is a bit of a chore.
++ * First you have to find the connector in question and make sure the
++ * requested mode is available.
++ * Then you need to find the encoder attached to that connector so you
++ * can bind it with a free crtc.
++ */
++struct pipe_arg {
++    const char **cons;
++    uint32_t *con_ids;
++    unsigned int num_cons;
++    uint32_t crtc_id;
++    char mode_str[64];
++    char format_str[5];
++    float vrefresh;
++    unsigned int fourcc;
++    drmModeModeInfo *mode;
++    struct crtc *crtc;
++    unsigned int fb_id[2], current_fb_id;
++    struct timeval start;
++
++    int swap_count;
++};
++
++struct plane_arg {
++    uint32_t plane_id;  /* the id of plane to use */
++    uint32_t crtc_id;  /* the id of CRTC to bind to */
++    bool has_position;
++    int32_t x, y;
++    uint32_t w, h;
++    double scale;
++    unsigned int fb_id;
++    unsigned int old_fb_id;
++    struct bo *bo;
++    struct bo *old_bo;
++    char format_str[5]; /* need to leave room for terminating \0 */
++    unsigned int fourcc;
++};
++
++static drmModeModeInfo *
++connector_find_mode(struct device *dev, uint32_t con_id, const char *mode_str,
++    const float vrefresh)
++{
++    drmModeConnector *connector;
++    drmModeModeInfo *mode;
++    int i;
++
++    connector = get_connector_by_id(dev, con_id);
++    if (!connector || !connector->count_modes)
++        return NULL;
++
++    /* Pick by Index */
++    if (mode_str[0] == '#') {
++        int index = atoi(mode_str + 1);
++
++        if (index >= connector->count_modes || index < 0)
++            return NULL;
++        return &connector->modes[index];
++    }
++
++    /* Pick by Name */
++    for (i = 0; i < connector->count_modes; i++) {
++        mode = &connector->modes[i];
++        if (!strcmp(mode->name, mode_str)) {
++            /* If the vertical refresh frequency is not specified
++             * then return the first mode that match with the name.
++             * Else, return the mode that match the name and
++             * the specified vertical refresh frequency.
++             */
++            if (vrefresh == 0)
++                return mode;
++            else if (fabs(mode_vrefresh(mode) - vrefresh) < 0.005)
++                return mode;
++        }
++    }
++
++    return NULL;
++}
++
++static struct crtc *pipe_find_crtc(struct device *dev, struct pipe_arg *pipe)
++{
++    uint32_t possible_crtcs = ~0;
++    uint32_t active_crtcs = 0;
++    unsigned int crtc_idx;
++    unsigned int i;
++    int j;
++
++    for (i = 0; i < pipe->num_cons; ++i) {
++        uint32_t crtcs_for_connector = 0;
++        drmModeConnector *connector;
++        drmModeEncoder *encoder;
++        struct crtc *crtc;
++
++        connector = get_connector_by_id(dev, pipe->con_ids[i]);
++        if (!connector)
++            return NULL;
++
++        for (j = 0; j < connector->count_encoders; ++j) {
++            encoder = get_encoder_by_id(dev, connector->encoders[j]);
++            if (!encoder)
++                continue;
++
++            crtcs_for_connector |= encoder->possible_crtcs;
++            crtc = get_crtc_by_id(dev, encoder->crtc_id);
++            if (!crtc)
++                continue;
++            active_crtcs |= get_crtc_mask(dev, crtc);
++        }
++
++        possible_crtcs &= crtcs_for_connector;
++    }
++
++    if (!possible_crtcs)
++        return NULL;
++
++    /* Return the first possible and active CRTC if one exists, or the first
++     * possible CRTC otherwise.
++     */
++    if (possible_crtcs & active_crtcs)
++        crtc_idx = ffs(possible_crtcs & active_crtcs);
++    else
++        crtc_idx = ffs(possible_crtcs);
++
++    return &dev->resources->crtcs[crtc_idx - 1];
++}
++
++static int pipe_find_crtc_and_mode(struct device *dev, struct pipe_arg *pipe)
++{
++    drmModeModeInfo *mode = NULL;
++    int i;
++
++    pipe->mode = NULL;
++
++    for (i = 0; i < (int)pipe->num_cons; i++) {
++        mode = connector_find_mode(dev, pipe->con_ids[i],
++                       pipe->mode_str, pipe->vrefresh);
++        if (mode == NULL) {
++            if (pipe->vrefresh)
++                fprintf(stderr,
++                "failed to find mode "
++                "\"%s-%.2fHz\" for connector %d\n",
++                pipe->mode_str, pipe->vrefresh, pipe->con_ids[i]);
++            else
++                fprintf(stderr,
++                "failed to find mode \"%s\" for connector %d\n",
++                pipe->mode_str, pipe->con_ids[i]);
++            return -EINVAL;
++        }
++    }
++
++    /* If the CRTC ID was specified, get the corresponding CRTC. Otherwise
++     * locate a CRTC that can be attached to all the connectors.
++     */
++    if (pipe->crtc_id != (uint32_t)-1) {
++        pipe->crtc = get_crtc_by_id(dev, pipe->crtc_id);
++    } else {
++        pipe->crtc = pipe_find_crtc(dev, pipe);
++        pipe->crtc_id = pipe->crtc->crtc->crtc_id;
++    }
++
++    if (!pipe->crtc) {
++        fprintf(stderr, "failed to find CRTC for pipe\n");
++        return -EINVAL;
++    }
++
++    pipe->mode = mode;
++    pipe->crtc->mode = mode;
++
++    return 0;
++}
++
++/* -----------------------------------------------------------------------------
++ * Properties
++ */
++
++struct property_arg {
++    uint32_t obj_id;
++    uint32_t obj_type;
++    char name[DRM_PROP_NAME_LEN+1];
++    uint32_t prop_id;
++    uint64_t value;
++    bool optional;
++};
++
++static bool set_property(struct device *dev, struct property_arg *p)
++{
++    drmModeObjectProperties *props = NULL;
++    drmModePropertyRes **props_info = NULL;
++    const char *obj_type;
++    int ret;
++    int i;
++
++    p->obj_type = 0;
++    p->prop_id = 0;
++
++#define find_object(_res, type, Type)                    \
++    do {                                    \
++        for (i = 0; i < (int)(_res)->count_##type##s; ++i) {    \
++            struct type *obj = &(_res)->type##s[i];            \
++            if (obj->type->type##_id != p->obj_id)            \
++                continue;                    \
++            p->obj_type = DRM_MODE_OBJECT_##Type;            \
++            obj_type = #Type;                    \
++            props = obj->props;                    \
++            props_info = obj->props_info;                \
++        }                                \
++    } while(0)                                \
++
++    find_object(dev->resources, crtc, CRTC);
++    if (p->obj_type == 0)
++        find_object(dev->resources, connector, CONNECTOR);
++    if (p->obj_type == 0)
++        find_object(dev->resources, plane, PLANE);
++    if (p->obj_type == 0) {
++        fprintf(stderr, "Object %i not found, can't set property\n",
++            p->obj_id);
++        return false;
++    }
++
++    if (!props) {
++        fprintf(stderr, "%s %i has no properties\n",
++            obj_type, p->obj_id);
++        return false;
++    }
++
++    for (i = 0; i < (int)props->count_props; ++i) {
++        if (!props_info[i])
++            continue;
++        if (strcmp(props_info[i]->name, p->name) == 0)
++            break;
++    }
++
++    if (i == (int)props->count_props) {
++        if (!p->optional)
++            fprintf(stderr, "%s %i has no %s property\n",
++                obj_type, p->obj_id, p->name);
++        return false;
++    }
++
++    p->prop_id = props->props[i];
++
++    if (!dev->use_atomic)
++        ret = drmModeObjectSetProperty(dev->fd, p->obj_id, p->obj_type,
++                                       p->prop_id, p->value);
++    else
++        ret = drmModeAtomicAddProperty(dev->req, p->obj_id, p->prop_id, p->value);
++
++    if (ret < 0)
++        fprintf(stderr, "failed to set %s %i property %s to %" PRIu64 ": %s\n",
++            obj_type, p->obj_id, p->name, p->value, strerror(errno));
++
++    return true;
++}
++
++/* -------------------------------------------------------------------------- */
++
++static void add_property(struct device *dev, uint32_t obj_id,
++                   const char *name, uint64_t value)
++{
++    struct property_arg p;
++
++    p.obj_id = obj_id;
++    strcpy(p.name, name);
++    p.value = value;
++
++    set_property(dev, &p);
++}
++
++static bool add_property_optional(struct device *dev, uint32_t obj_id,
++                  const char *name, uint64_t value)
++{
++    struct property_arg p;
++
++    p.obj_id = obj_id;
++    strcpy(p.name, name);
++    p.value = value;
++    p.optional = true;
++
++    return set_property(dev, &p);
++}
++
++static void util_smpte_c8_gamma(unsigned size, struct drm_color_lut *lut)
++{
++    if (size < 7 + 7 + 8) {
++        printf("Error: gamma too small: %d < %d\n", size, 7 + 7 + 8);
++        return;
++    }
++    memset(lut, 0, size * sizeof(struct drm_color_lut));
++
++#define FILL_COLOR(idx, r, g, b) \
++    lut[idx].red = (r) << 8; \
++    lut[idx].green = (g) << 8; \
++    lut[idx].blue = (b) << 8
++
++    FILL_COLOR( 0, 192, 192, 192);    /* grey */
++    FILL_COLOR( 1, 192, 192, 0  );    /* yellow */
++    FILL_COLOR( 2, 0,   192, 192);    /* cyan */
++    FILL_COLOR( 3, 0,   192, 0  );    /* green */
++    FILL_COLOR( 4, 192, 0,   192);    /* magenta */
++    FILL_COLOR( 5, 192, 0,   0  );    /* red */
++    FILL_COLOR( 6, 0,   0,   192);    /* blue */
++
++    FILL_COLOR( 7, 0,   0,   192);    /* blue */
++    FILL_COLOR( 8, 19,  19,  19 );    /* black */
++    FILL_COLOR( 9, 192, 0,   192);    /* magenta */
++    FILL_COLOR(10, 19,  19,  19 );    /* black */
++    FILL_COLOR(11, 0,   192, 192);    /* cyan */
++    FILL_COLOR(12, 19,  19,  19 );    /* black */
++    FILL_COLOR(13, 192, 192, 192);    /* grey */
++
++    FILL_COLOR(14, 0,   33,  76);    /* in-phase */
++    FILL_COLOR(15, 255, 255, 255);    /* super white */
++    FILL_COLOR(16, 50,  0,   106);    /* quadrature */
++    FILL_COLOR(17, 19,  19,  19);    /* black */
++    FILL_COLOR(18, 9,   9,   9);    /* 3.5% */
++    FILL_COLOR(19, 19,  19,  19);    /* 7.5% */
++    FILL_COLOR(20, 29,  29,  29);    /* 11.5% */
++    FILL_COLOR(21, 19,  19,  19);    /* black */
++
++#undef FILL_COLOR
++}
++
++static void set_gamma(struct device *dev, unsigned crtc_id, unsigned fourcc)
++{
++    unsigned blob_id = 0;
++    /* TODO: support 1024-sized LUTs, when the use-case arises */
++    struct drm_color_lut gamma_lut[256];
++    int i, ret;
++
++    if (fourcc == DRM_FORMAT_C8) {
++        /* TODO: Add C8 support for more patterns */
++        util_smpte_c8_gamma(256, gamma_lut);
++        drmModeCreatePropertyBlob(dev->fd, gamma_lut, sizeof(gamma_lut), &blob_id);
++    } else {
++        for (i = 0; i < 256; i++) {
++            gamma_lut[i].red =
++            gamma_lut[i].green =
++            gamma_lut[i].blue = i << 8;
++        }
++    }
++
++    add_property_optional(dev, crtc_id, "DEGAMMA_LUT", 0);
++    add_property_optional(dev, crtc_id, "CTM", 0);
++    if (!add_property_optional(dev, crtc_id, "GAMMA_LUT", blob_id)) {
++        uint16_t r[256], g[256], b[256];
++
++        for (i = 0; i < 256; i++) {
++            r[i] = gamma_lut[i].red;
++            g[i] = gamma_lut[i].green;
++            b[i] = gamma_lut[i].blue;
++        }
++
++        ret = drmModeCrtcSetGamma(dev->fd, crtc_id, 256, r, g, b);
++        if (ret)
++            fprintf(stderr, "failed to set gamma: %s\n", strerror(errno));
++    }
++}
++
++static void atomic_set_planes(struct device *dev, struct plane_arg *p,
++                  unsigned int count)
++{
++    if (p && (count > 0))
++        set_gamma(dev, p[0].crtc_id, p[0].fourcc);
++}
++
++static void atomic_clear_planes(struct device *dev, struct plane_arg *p, unsigned int count)
++{
++    unsigned int i;
++
++    for (i = 0; i < count; i++) {
++        add_property(dev, p[i].plane_id, "FB_ID", 0);
++        add_property(dev, p[i].plane_id, "CRTC_ID", 0);
++        add_property(dev, p[i].plane_id, "SRC_X", 0);
++        add_property(dev, p[i].plane_id, "SRC_Y", 0);
++        add_property(dev, p[i].plane_id, "SRC_W", 0);
++        add_property(dev, p[i].plane_id, "SRC_H", 0);
++        add_property(dev, p[i].plane_id, "CRTC_X", 0);
++        add_property(dev, p[i].plane_id, "CRTC_Y", 0);
++        add_property(dev, p[i].plane_id, "CRTC_W", 0);
++        add_property(dev, p[i].plane_id, "CRTC_H", 0);
++    }
++}
++
++static void atomic_clear_FB(struct device *dev, struct plane_arg *p, unsigned int count)
++{
++    unsigned int i;
++
++    for (i = 0; i < count; i++) {
++        if (p[i].fb_id) {
++            drmModeRmFB(dev->fd, p[i].fb_id);
++            p[i].fb_id = 0;
++        }
++        if (p[i].old_fb_id) {
++            drmModeRmFB(dev->fd, p[i].old_fb_id);
++            p[i].old_fb_id = 0;
++        }
++    }
++}
++
++static void set_mode(struct device *dev, struct pipe_arg *pipes, unsigned int count)
++{
++    unsigned int i, j;
++    int ret;
++
++    for (i = 0; i < count; i++) {
++        struct pipe_arg *pipe = &pipes[i];
++        ret = pipe_find_crtc_and_mode(dev, pipe);
++        if (ret < 0)
++            continue;
++    }
++
++    for (i = 0; i < count; i++) {
++        struct pipe_arg *pipe = &pipes[i];
++        uint32_t blob_id;
++
++        if (pipe->mode == NULL)
++            continue;
++
++        for (j = 0; j < pipe->num_cons; ++j) {
++            add_property(dev, pipe->con_ids[j], "CRTC_ID", pipe->crtc_id);
++        }
++
++        drmModeCreatePropertyBlob(dev->fd, pipe->mode, sizeof(*pipe->mode), &blob_id);
++        add_property(dev, pipe->crtc_id, "MODE_ID", blob_id);
++        add_property(dev, pipe->crtc_id, "ACTIVE", 1);
++    }
++}
++
++static void atomic_clear_mode(struct device *dev, struct pipe_arg *pipes, unsigned int count)
++{
++    unsigned int i;
++    unsigned int j;
++
++    for (i = 0; i < count; i++) {
++        struct pipe_arg *pipe = &pipes[i];
++
++        if (pipe->mode == NULL)
++            continue;
++
++        for (j = 0; j < pipe->num_cons; ++j)
++            add_property(dev, pipe->con_ids[j], "CRTC_ID",0);
++
++        add_property(dev, pipe->crtc_id, "MODE_ID", 0);
++        add_property(dev, pipe->crtc_id, "ACTIVE", 0);
++    }
++}
++
++int
++gst_kms_sink_reset_drm(GstKMSSink * self)
++{
++    int ret = -1;
++    struct device dev;
++    uint64_t cap = 0;
++
++    const unsigned int connector_count = 1, plane_count = 1;
++    struct pipe_arg pipe_args[1];
++    struct plane_arg plane_args[1];
++    uint32_t connectors[1];
++    GValueArray *formats = NULL;
++    GstStructure *st = NULL;
++
++    if (!self || (self->fd < 0))
++        return ret;
++
++    memset (&dev, 0, sizeof(dev));
++    memset (&plane_args, 0, sizeof(*plane_args));
++    memset (&pipe_args, 0, sizeof(*pipe_args));
++
++    dev.fd = self->fd;
++    ret = drmSetClientCap(dev.fd, DRM_CLIENT_CAP_ATOMIC, 1);
++    if (ret) {
++        fprintf(stderr, "no atomic modesetting support: %s\n", strerror(errno));
++        goto out;
++    }
++
++    ret = drmGetCap(dev.fd, DRM_CAP_DUMB_BUFFER, &cap);
++    if (ret || cap == 0) {
++        fprintf(stderr, "driver doesn't support the dumb buffer API\n");
++        goto out;
++    }
++
++    dev.use_atomic = 1;
++    dev.resources = get_resources(&dev);
++    if (!dev.resources)
++        goto out;
++
++    plane_args[0].plane_id = self->plane_id;  /* the id of plane to use */
++    plane_args[0].crtc_id = self->crtc_id;  /* the id of CRTC to bind to */
++    plane_args[0].w = self->render_rect.w;
++    plane_args[0].h = self->render_rect.h;
++    plane_args[0].scale = 1.0;
++    strncpy(plane_args[0].format_str, "NV12", 4);
++
++    /* Find the first supported format and also has correct FOURCC value. */
++    st = gst_caps_get_structure(self->allowed_caps, 0);
++    if (st && gst_structure_get_list (st, "format", &formats)) {
++        for (int i = 0; formats && (i < formats->n_values); ++i) {
++            const gchar *format = g_value_get_string(g_value_array_get_nth(formats, i));
++            if (format && (gst_video_format_to_fourcc(gst_video_format_from_string(format)) != 0)) {
++                strncpy(plane_args[0].format_str, format, 4);
++                break;
++            }
++        }
++
++        g_value_array_free(formats);
++        formats = NULL;
++    }
++
++    plane_args[0].fourcc = gst_video_format_to_fourcc(gst_video_format_from_string(plane_args[0].format_str));
++
++    connectors[0] = self->conn_id;
++    pipe_args[0].con_ids = connectors;
++    pipe_args[0].num_cons = connector_count;
++    pipe_args[0].crtc_id = self->crtc_id;
++    strncpy(pipe_args[0].format_str, plane_args[0].format_str, 4);
++    pipe_args[0].fourcc = plane_args[0].fourcc;
++    snprintf(pipe_args[0].mode_str, sizeof(pipe_args[0].mode_str), "%dx%d", plane_args[0].w, plane_args[0].h);
++
++    dev.req = drmModeAtomicAlloc();
++    set_mode(&dev, pipe_args, connector_count);
++    atomic_set_planes(&dev, plane_args, plane_count);
++    ret = drmModeAtomicCommit(dev.fd, dev.req, DRM_MODE_ATOMIC_ALLOW_MODESET, NULL);
++    if (ret) {
++        fprintf(stderr, "Atomic Commit failed [1]\n");
++        goto out;
++    }
++
++    // clear
++    atomic_clear_planes(&dev, plane_args, plane_count);
++    atomic_clear_mode(&dev, pipe_args, connector_count);
++    ret = drmModeAtomicCommit(dev.fd, dev.req, DRM_MODE_ATOMIC_ALLOW_MODESET, NULL);
++    if (ret)
++        fprintf(stderr, "Atomic Commit failed\n");
++    atomic_clear_FB(&dev, plane_args, plane_count);
++    drmModeAtomicFree(dev.req);
++    ret = 0;
++
++out:
++    free_resources(dev.resources);
++
++    ret = drmSetClientCap(dev.fd, DRM_CLIENT_CAP_ATOMIC, 0);
++    if (ret)
++        fprintf(stderr, "no atomic modesetting support: %s\n", strerror(errno));
++
++    return ret;
++}

+ 25 - 0
multimedia/patch/gstreamer1/gst1-plugins-bad/0003-add-default-param-for-kmssink.patch

@@ -0,0 +1,25 @@
+Add default parameters for kmssink so that it can be used by other plug
+
+Signed-off-by: Leo Lu <leo.lu@starfivetech.com>
+
+diff -purN a/sys/kms/gstkmssink.c b/sys/kms/gstkmssink.c
+--- a/sys/kms/gstkmssink.c	2022-10-31 17:48:04.390225610 +0800
++++ b/sys/kms/gstkmssink.c	2022-10-31 17:45:50.462455826 +0800
+@@ -175,7 +175,7 @@ gst_kms_sink_video_overlay_init (GstVide
+ static int
+ kms_open (gchar ** driver)
+ {
+-  static const char *drivers[] = { "i915", "radeon", "nouveau", "vmwgfx",
++  static const char *drivers[] = { "starfive", "i915", "radeon", "nouveau", "vmwgfx",
+     "exynos", "amdgpu", "imx-drm", "rockchip", "atmel-hlcdc", "msm",
+     "xlnx", "vc4", "meson", "sun4i-drm", "mxsfb-drm",
+     "xilinx_drm",               /* DEPRECATED. Replaced by xlnx */
+@@ -2005,7 +2005,7 @@ gst_kms_sink_class_init (GstKMSSinkClass
+    */
+   g_properties[PROP_FORCE_MODESETTING] =
+       g_param_spec_boolean ("force-modesetting", "Force modesetting",
+-      "When enabled, the sink try to configure the display mode", FALSE,
++      "When enabled, the sink try to configure the display mode", TRUE,
+       G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS | G_PARAM_CONSTRUCT);
+ 
+   /**

+ 45 - 0
multimedia/patch/gstreamer1/gst1-plugins-base/0001-gst-libs-gst-video-gstvideoaggregator.c-fix-build-wi.patch

@@ -0,0 +1,45 @@
+From 012ca1bcd234ee02f576dd58b1f09baba18b89b0 Mon Sep 17 00:00:00 2001
+From: Fabrice Fontaine <fontaine.fabrice@gmail.com>
+Date: Mon, 14 Dec 2020 07:42:55 +0100
+Subject: [PATCH] gst-libs/gst/video/gstvideoaggregator.c: fix build with gcc
+ 4.8
+
+Fix the following build failure with gcc 4.8 which has been added with
+https://gitlab.freedesktop.org/gstreamer/gst-plugins-base/-/commit/d268c193ad39fb970351ed62898be806ebd0a71e:
+
+../gst-libs/gst/video/gstvideoaggregator.c: In function 'gst_video_aggregator_init':
+../gst-libs/gst/video/gstvideoaggregator.c:2762:3: error: 'for' loop initial declarations are only allowed in C99 mode
+   for (gint i = 0; i < gst_caps_get_size (src_template); i++) {
+   ^
+
+Signed-off-by: Fabrice Fontaine <fontaine.fabrice@gmail.com>
+[Upstream status:
+https://gitlab.freedesktop.org/gstreamer/gst-plugins-base/-/merge_requests/974]
+---
+ gst-libs/gst/video/gstvideoaggregator.c | 3 ++-
+ 1 file changed, 2 insertions(+), 1 deletion(-)
+
+diff --git a/gst-libs/gst/video/gstvideoaggregator.c b/gst-libs/gst/video/gstvideoaggregator.c
+index 2e3d813be..04dad6563 100644
+--- a/gst-libs/gst/video/gstvideoaggregator.c
++++ b/gst-libs/gst/video/gstvideoaggregator.c
+@@ -2786,6 +2786,7 @@ gst_video_aggregator_init (GstVideoAggregator * vagg,
+ {
+   GstCaps *src_template;
+   GstPadTemplate *pad_template;
++  gint i;
+ 
+   vagg->priv = gst_video_aggregator_get_instance_private (vagg);
+   vagg->priv->current_caps = NULL;
+@@ -2800,7 +2801,7 @@ gst_video_aggregator_init (GstVideoAggregator * vagg,
+   pad_template =
+       gst_element_class_get_pad_template (GST_ELEMENT_CLASS (klass), "src");
+   src_template = gst_pad_template_get_caps (pad_template);
+-  for (gint i = 0; i < gst_caps_get_size (src_template); i++) {
++  for (i = 0; i < gst_caps_get_size (src_template); i++) {
+     const GValue *v =
+         gst_structure_get_value (gst_caps_get_structure (src_template, i),
+         "format");
+-- 
+2.29.2
+

+ 106 - 0
multimedia/patch/gstreamer1/gst1-plugins-base/0002-quit-when-pipeline-interrupted.patch

@@ -0,0 +1,106 @@
+diff -Naur a/tools/gst-play.c b/tools/gst-play.c
+--- a/tools/gst-play.c	2020-10-26 19:10:32.459882500 +0800
++++ b/tools/gst-play.c	2022-04-07 14:13:15.126395339 +0800
+@@ -19,7 +19,6 @@
+  * Free Software Foundation, Inc., 51 Franklin St, Fifth Floor,
+  * Boston, MA 02110-1301, USA.
+  */
+-
+ #ifdef HAVE_CONFIG_H
+ #include "config.h"
+ #endif
+@@ -40,6 +39,9 @@
+ #include <glib/gprintf.h>
+
+ #include "gst-play-kb.h"
++#ifdef G_OS_UNIX
++#include <glib-unix.h>
++#endif
+
+ #define VOLUME_STEPS 20
+
+@@ -104,6 +106,31 @@
+   gdouble rate;
+ } GstPlay;
+ 
++#if defined(G_OS_UNIX)
++static guint signal_watch_intr_id;
++#endif
++
++#if defined(G_OS_UNIX)
++/* As the interrupt handler is dispatched from GMainContext as a GSourceFunc
++ * handler, we can react to this by posting a message. */
++static gboolean
++intr_handler (gpointer user_data)
++{
++  GstPlay *play =  (GstPlay *) user_data;
++  gst_print ("handling interrupt.\n");
++
++  /* post an application specific message */
++  gst_element_post_message (GST_ELEMENT (play->playbin),
++      gst_message_new_application (GST_OBJECT (play->playbin),
++          gst_structure_new ("GstLaunchInterrupt",
++              "message", G_TYPE_STRING, "Pipeline interrupted", NULL)));
++
++  /* remove signal handler */
++  signal_watch_intr_id = 0;
++  return G_SOURCE_REMOVE;
++}
++#endif /* G_OS_UNIX */
++
+ static gboolean quiet = FALSE;
+ static gboolean instant_rate_changes = FALSE;
+ 
+@@ -594,6 +621,24 @@
+       }
+       break;
+     }
++    case GST_MESSAGE_APPLICATION:{
++        const GstStructure *s;
++
++        s = gst_message_get_structure (msg);
++
++        if (gst_structure_has_name (s, "GstLaunchInterrupt")) {
++          /* this application message is posted when we caught an interrupt and
++           * we need to stop the pipeline. */
++
++            gst_print (_("Interrupt: GST_MESSAGE_APPLICATION ...\n"));
++            if (play->desired_state == GST_STATE_PLAYING){
++               gst_print (_("Interrupt: Stopping pipeline ...\n"));
++               g_main_loop_quit (play->loop);  
++            }
++
++        }
++        break;
++      }
+     default:
+       if (gst_is_missing_plugin_message (msg)) {
+         gchar *desc;
+@@ -1622,6 +1667,12 @@
+     }
+   }
+ 
++#ifdef G_OS_UNIX
++    gst_print("g_source_remove signal_watch_intr_id \n");
++    signal_watch_intr_id =
++        g_unix_signal_add (SIGINT, (GSourceFunc) intr_handler, play);
++#endif
++
+   /* play */
+   do_play (play);
+ 
+@@ -1632,6 +1683,14 @@
+   g_free (video_sink);
+ 
+   gst_print ("\n");
++
++#ifdef G_OS_UNIX
++    if (signal_watch_intr_id > 0){
++      gst_print("g_source_remove signal_watch_intr_id \n");
++      g_source_remove (signal_watch_intr_id);
++    }
++#endif
++
+   gst_deinit ();
+   return 0;
+ }

+ 69 - 0
multimedia/patch/gstreamer1/sf-gst-omx/0001-add-starfive-support.patch

@@ -0,0 +1,69 @@
+diff --git a/config/meson.build b/config/meson.build
+index 1068c6d..7f8abe9 100755
+--- a/config/meson.build
++++ b/config/meson.build
+@@ -6,6 +6,8 @@ elif omx_target == 'zynqultrascaleplus'
+   sub = 'zynqultrascaleplus'
+ elif omx_target == 'tizonia'
+   sub = 'tizonia'
++ elif omx_target == 'stf'
++  sub = 'stf'
+ else
+   # No config file defined for the 'generic' target
+   sub = ''
+diff --git a/config/stf/gstomx.conf b/config/stf/gstomx.conf
+new file mode 100644
+index 0000000..b4cdebb
+--- /dev/null
++++ b/config/stf/gstomx.conf
+@@ -0,0 +1,17 @@
++[omxh265dec]
++type-name=GstOMXH265Dec
++core-name=/usr/lib/libsf-omx-il.so
++component-name=sf.dec.decoder.h265
++in-port-index=0
++out-port-index=1
++rank=0
++hacks=pass-profile-to-decoder;pass-color-format-to-decoder;ensure-buffer-count-actual
++
++[omxh264dec]
++type-name=GstOMXH264Dec
++core-name=/usr/lib/libsf-omx-il.so
++component-name=sf.dec.decoder.h264
++in-port-index=0
++out-port-index=1
++rank=1
++hacks=pass-profile-to-decoder;pass-color-format-to-decoder;ensure-buffer-count-actual
+diff --git a/config/stf/meson.build b/config/stf/meson.build
+new file mode 100644
+index 0000000..dc99c08
+--- /dev/null
++++ b/config/stf/meson.build
+@@ -0,0 +1 @@
++install_data (['gstomx.conf'], install_dir : omx_conf_dir)
+diff --git a/meson.build b/meson.build
+index 32c7fa7..aeda56a 100755
+--- a/meson.build
++++ b/meson.build
+@@ -216,6 +216,8 @@ elif omx_target == 'tizonia'
+   tizil_includedir = tizil_dep.get_pkgconfig_variable('includedir')
+   gst_omx_args += ['-I' + tizil_includedir + '/tizonia']
+   omx_inc = []
++elif omx_target == 'stf'
++  warning('stf selected')
+ else
+   error ('Unsupported omx target specified. Use the -Dtarget option')
+ endif
+diff --git a/meson_options.txt b/meson_options.txt
+index e18beb2..4fc6139 100755
+--- a/meson_options.txt
++++ b/meson_options.txt
+@@ -1,7 +1,7 @@
+ option('header_path', type : 'string', value : '',
+     description : 'An extra include directory to find the OpenMax headers')
+ option('target', type : 'combo',
+-    choices : ['none', 'generic', 'rpi', 'bellagio', 'tizonia', 'zynqultrascaleplus'], value : 'none',
++    choices : ['none', 'generic', 'rpi', 'bellagio', 'tizonia', 'zynqultrascaleplus', 'stf'], value : 'none',
+     description : 'The OMX platform to target')
+ option('struct_packing', type : 'combo',
+     choices : ['0', '1', '2', '4', '8'], value : '0',

+ 70 - 0
multimedia/patch/gstreamer1/sf-gst-omx/0002-Fix-gst-omx-Enable-the-gst-omx-VPU-decoding-and-enco.patch

@@ -0,0 +1,70 @@
+From 810800c303965e5cb0cf135210667c003f050ee0 Mon Sep 17 00:00:00 2001
+From: "andy.hu" <andy.hu@starfive.com>
+Date: Tue, 10 Aug 2021 19:18:00 +0800
+Subject: [PATCH] Fix(gst-omx): Enable the gst-omx VPU decoding and encoding
+ support h264 High L5.2
+
+---
+
+diff --git a/meson.build b/meson.build
+index aeda56a..48121ab 100644
+--- a/meson.build
++++ b/meson.build
+@@ -217,6 +217,7 @@ elif omx_target == 'tizonia'
+   gst_omx_args += ['-I' + tizil_includedir + '/tizonia']
+   omx_inc = []
+ elif omx_target == 'stf'
++  cdata.set('USE_OMX_TARGET_STARFIVE', 1)
+   warning('stf selected')
+ else
+   error ('Unsupported omx target specified. Use the -Dtarget option')
+diff --git a/omx/gstomxh264enc.c b/omx/gstomxh264enc.c
+index 51d84a8..875f974 100644
+--- a/omx/gstomxh264enc.c
++++ b/omx/gstomxh264enc.c
+@@ -824,6 +824,11 @@ gst_omx_h264_enc_get_caps (GstOMXVideoEnc * enc, GstOMXPort * port,
+       case OMX_VIDEO_AVCLevel51:
+         level = "5.1";
+         break;
++#ifdef USE_OMX_TARGET_STARFIVE
++      case OMX_VIDEO_AVCLevel52:
++        level = "5.2";
++        break;
++#else
+ #ifdef USE_OMX_TARGET_ZYNQ_USCALE_PLUS
+       case OMX_ALG_VIDEO_AVCLevel52:
+         level = "5.2";
+@@ -837,6 +842,7 @@ gst_omx_h264_enc_get_caps (GstOMXVideoEnc * enc, GstOMXPort * port,
+       case OMX_ALG_VIDEO_AVCLevel62:
+         level = "6.2";
+         break;
++#endif
+ #endif
+       default:
+         g_assert_not_reached ();
+diff --git a/omx/gstomxh264utils.c b/omx/gstomxh264utils.c
+index ba29211..a6e95e3 100644
+--- a/omx/gstomxh264utils.c
++++ b/omx/gstomxh264utils.c
+@@ -115,6 +115,10 @@ gst_omx_h264_utils_get_level_from_str (const gchar * level)
+     return OMX_VIDEO_AVCLevel5;
+   } else if (g_str_equal (level, "5.1")) {
+     return OMX_VIDEO_AVCLevel51;
++#ifdef USE_OMX_TARGET_STARFIVE
++  } else if (g_str_equal (level, "5.2")) {
++    return OMX_VIDEO_AVCLevel52;
++#else
+ #ifdef USE_OMX_TARGET_ZYNQ_USCALE_PLUS
+   } else if (g_str_equal (level, "5.2")) {
+     return (OMX_VIDEO_AVCLEVELTYPE) OMX_ALG_VIDEO_AVCLevel52;
+@@ -124,6 +128,7 @@ gst_omx_h264_utils_get_level_from_str (const gchar * level)
+     return (OMX_VIDEO_AVCLEVELTYPE) OMX_ALG_VIDEO_AVCLevel61;
+   } else if (g_str_equal (level, "6.2")) {
+     return (OMX_VIDEO_AVCLEVELTYPE) OMX_ALG_VIDEO_AVCLevel62;
++#endif
+ #endif
+   }
+
+-- 
+2.17.1
+

+ 50 - 0
multimedia/patch/gstreamer1/sf-gst-omx/0003-add-video-scale-support.patch

@@ -0,0 +1,50 @@
+--- a/omx/gstomxvideodec.c	2020-10-26 19:17:03.000000000 +0800
++++ b/omx/gstomxvideodec.c	2021-08-16 16:23:08.082113380 +0800
+@@ -2185,6 +2185,7 @@
+ gst_omx_video_dec_negotiate (GstOMXVideoDec * self)
+ {
+   OMX_VIDEO_PARAM_PORTFORMATTYPE param;
++  OMX_PARAM_PORTDEFINITIONTYPE port_def;
+   OMX_ERRORTYPE err;
+   GstCaps *comp_supported_caps;
+   GList *negotiation_map = NULL, *l;
+@@ -2192,6 +2193,7 @@
+   GstVideoFormat format;
+   GstStructure *s;
+   const gchar *format_str;
++  gint width, height;
+ 
+   GST_DEBUG_OBJECT (self, "Trying to negotiate a video format with downstream");
+ 
+@@ -2281,6 +2283,31 @@
+         gst_omx_error_to_string (err), err);
+   }
+ 
++  if (gst_structure_get_int(s, "width", &width) && gst_structure_get_int(s, "height", &height))
++  {
++    GST_OMX_INIT_STRUCT (&port_def);
++    port_def.nPortIndex = self->dec_out_port->index;
++    err = gst_omx_component_get_parameter (self->dec,
++        OMX_IndexParamPortDefinition, &port_def);
++    if (err != OMX_ErrorNone) {
++      GST_ERROR_OBJECT (self, "Failed to get video port definition: %s (0x%08x)",
++          gst_omx_error_to_string (err), err);
++      return FALSE;
++    }
++
++    port_def.format.video.nFrameHeight = height;
++    port_def.format.video.nFrameWidth = width;
++
++    err =
++      gst_omx_component_set_parameter (self->dec,
++      OMX_IndexParamPortDefinition, &port_def);
++    if (err != OMX_ErrorNone) {
++      GST_ERROR_OBJECT (self, "Failed to set video port definition: %s (0x%08x)",
++          gst_omx_error_to_string (err), err);
++      return FALSE;
++    }
++  }
++
+   gst_caps_unref (intersection);
+   return (err == OMX_ErrorNone);
+ }

+ 24 - 0
multimedia/patch/gstreamer1/sf-gst-omx/0004-add-encoder-support.patch

@@ -0,0 +1,24 @@
+--- a/config/stf/gstomx.conf	2021-08-18 14:51:51.384176799 +0800
++++ b/config/stf/gstomx.conf	2021-08-18 14:50:58.972186447 +0800
+@@ -15,3 +15,21 @@
+ out-port-index=1
+ rank=1
+ hacks=pass-profile-to-decoder;pass-color-format-to-decoder;ensure-buffer-count-actual
++
++[omxh265enc]
++type-name=GstOMXH265Enc
++core-name=/usr/lib/libsf-omx-il.so
++component-name=sf.enc.encoder.h265
++in-port-index=0
++out-port-index=1
++rank=1
++hacks=pass-profile-to-decoder;pass-color-format-to-decoder;ensure-buffer-count-actual;video-framerate-integer
++
++[omxh264enc]
++type-name=GstOMXH264Enc
++core-name=/usr/lib/libsf-omx-il.so
++component-name=sf.enc.encoder.h264
++in-port-index=0
++out-port-index=1
++rank=1
++hacks=pass-profile-to-decoder;pass-color-format-to-decoder;ensure-buffer-count-actual;video-framerate-integer

+ 19 - 0
multimedia/patch/gstreamer1/sf-gst-omx/0005-rank-257-for-sf-codecs.patch

@@ -0,0 +1,19 @@
+diff -Naur a/config/stf/gstomx.conf b/config/stf/gstomx.conf
+--- a/config/stf/gstomx.conf	2021-08-23 16:04:48.928905742 +0800
++++ b/config/stf/gstomx.conf	2021-08-23 16:03:38.590578856 +0800
+@@ -4,7 +4,7 @@
+ component-name=sf.dec.decoder.h265
+ in-port-index=0
+ out-port-index=1
+-rank=0
++rank=257
+ hacks=pass-profile-to-decoder;pass-color-format-to-decoder;ensure-buffer-count-actual
+ 
+ [omxh264dec]
+@@ -13,5 +13,5 @@
+ component-name=sf.dec.decoder.h264
+ in-port-index=0
+ out-port-index=1
+-rank=1
++rank=257
+ hacks=pass-profile-to-decoder;pass-color-format-to-decoder;ensure-buffer-count-actual

+ 19 - 0
multimedia/patch/gstreamer1/sf-gst-omx/0006-dont-invoke-USE_BUFFER-if-no-dmabuffer.patch

@@ -0,0 +1,19 @@
+--- a/omx/gstomxvideodec.c
++++ b/omx/gstomxvideodec.c
+@@ -718,6 +718,15 @@
+     goto out;
+   }
+ 
++#ifdef USE_OMX_TARGET_STARFIVE
++  if ( !gst_is_dmabuf_memory (mem)) {
++    GST_INFO_OBJECT (self,
++        " %d-th buffer doesn't contain dmabuf, go to out. port->port_def.nBufferSize: %lu",
++        i, port->port_def.nBufferSize);
++    goto out;
++  } 
++#endif
++
+   *frame = g_slice_new0 (GstVideoFrame);
+ 
+   is_mapped = gst_video_frame_map (*frame, v_info, buffer, flags);
+

+ 16 - 0
multimedia/patch/gstreamer1/sf-gst-omx/0007-add-omxmjpegdec-support.patch

@@ -0,0 +1,16 @@
+--- a/config/stf/gstomx.conf
++++ b/config/stf/gstomx.conf
+@@ -33,3 +33,12 @@
+ out-port-index=1
+ rank=1
+ hacks=pass-profile-to-decoder;pass-color-format-to-decoder;ensure-buffer-count-actual;video-framerate-integer
++
++[omxmjpegdec]
++type-name=GstOMXMJPEGDec
++core-name=/usr/lib/libsf-omx-il.so
++component-name=sf.dec.decoder.mjpeg
++in-port-index=0
++out-port-index=1
++rank=1
++hacks=pass-profile-to-decoder;pass-color-format-to-decoder;ensure-buffer-count-actual
+

+ 100 - 0
multimedia/patch/gstreamer1/sf-gst-omx/0008-support-nv21-i422-y444-for-omxmjpegdec.patch

@@ -0,0 +1,100 @@
+--- a/config/stf/gstomx.conf
++++ b/config/stf/gstomx.conf
+@@ -42,3 +42,4 @@
+ out-port-index=1
+ rank=1
+ hacks=pass-profile-to-decoder;pass-color-format-to-decoder;ensure-buffer-count-actual
++src-template-caps=video/x-raw, format = (string) { NV16_10LE32, NV12_10LE32, NV16, YUY2, YVYU, UYVY, NV12, NV21, I420, RGB16, BGR16, ABGR, ARGB, GRAY8, Y42B, Y444 }, width = (int) [ 1, max ], height = (int) [ 1, max ], framerate = (fraction) [ 0, max ]
+--- a/omx/gstomxbufferpool.c
++++ b/omx/gstomxbufferpool.c
+@@ -370,9 +370,29 @@
+       case GST_VIDEO_FORMAT_NV12_10LE32:
+       case GST_VIDEO_FORMAT_NV16:
+       case GST_VIDEO_FORMAT_NV16_10LE32:
++#ifdef USE_OMX_TARGET_STARFIVE
++      case GST_VIDEO_FORMAT_NV21:
++#endif
+         stride[1] = nstride;
+         offset[1] = offset[0] + stride[0] * nslice;
+         break;
++#ifdef USE_OMX_TARGET_STARFIVE
++      case GST_VIDEO_FORMAT_Y42B:
++        stride[0] = GST_ROUND_UP_4 (nstride);
++        stride[1] = GST_ROUND_UP_8 (nstride) / 2;
++        stride[2] = stride[1];
++
++        offset[1] = stride[0] * nslice;
++        offset[2] = offset[1] + stride[1] * nslice;
++        break;
++
++      case GST_VIDEO_FORMAT_Y444:
++        stride[1] = nstride;
++        stride[2] = nstride;
++        offset[1] = offset[0] + stride[0] * nslice;
++        offset[2] = offset[1] + stride[1] * nslice;
++        break;
++#endif
+       default:
+         g_assert_not_reached ();
+         break;
+--- a/omx/gstomxvideo.c
++++ b/omx/gstomxvideo.c
+@@ -93,6 +93,17 @@
+       format = GST_VIDEO_FORMAT_NV16_10LE32;
+       break;
+ #pragma GCC diagnostic pop
++#endif
++#ifdef USE_OMX_TARGET_STARFIVE
++    case OMX_COLOR_FormatYUV422Planar:
++      format = GST_VIDEO_FORMAT_Y42B;
++      break;
++    case OMX_COLOR_FormatYUV444Interleaved:
++      format = GST_VIDEO_FORMAT_Y444;
++      break;
++    case OMX_COLOR_FormatYVU420SemiPlanar:
++      format = GST_VIDEO_FORMAT_NV21;
++      break;
+ #endif
+     default:
+       format = GST_VIDEO_FORMAT_UNKNOWN;
+--- a/omx/gstomxvideodec.c
++++ b/omx/gstomxvideodec.c
+@@ -592,6 +592,9 @@
+       case GST_VIDEO_FORMAT_YUY2:
+       case GST_VIDEO_FORMAT_UYVY:
+       case GST_VIDEO_FORMAT_YVYU:
++#ifdef USE_OMX_TARGET_STARFIVE
++      case GST_VIDEO_FORMAT_Y42B:
++#endif
+         dst_width[0] = GST_VIDEO_INFO_WIDTH (vinfo) * 2;
+         break;
+       case GST_VIDEO_FORMAT_GRAY8:
+@@ -609,6 +612,9 @@
+         dst_height[2] = GST_VIDEO_INFO_FIELD_HEIGHT (vinfo) / 2;
+         break;
+       case GST_VIDEO_FORMAT_NV12:
++#ifdef USE_OMX_TARGET_STARFIVE
++      case GST_VIDEO_FORMAT_NV21:
++#endif
+         dst_width[0] = GST_VIDEO_INFO_WIDTH (vinfo);
+         src_stride[1] = nstride;
+         src_size[1] = src_stride[1] * nslice / 2;
+@@ -638,6 +644,18 @@
+         src_size[1] = src_stride[1] * nslice;
+         dst_height[1] = GST_VIDEO_INFO_FIELD_HEIGHT (vinfo);
+         break;
++#ifdef USE_OMX_TARGET_STARFIVE
++      case GST_VIDEO_FORMAT_Y444:
++        dst_width[0] = GST_VIDEO_INFO_WIDTH (vinfo);
++        dst_width[1] = GST_VIDEO_INFO_WIDTH (vinfo);
++        dst_width[2] = GST_VIDEO_INFO_WIDTH (vinfo);
++        src_size[1] = src_size[0];
++        src_size[2] = src_size[0];
++        src_stride[1] = nstride;
++        dst_height[1] = dst_height[0];
++        dst_height[2] = dst_height[0];
++        break;
++#endif
+       default:
+         g_assert_not_reached ();
+         break;

+ 247 - 0
multimedia/patch/gstreamer1/sf-gst-omx/0009-suport-usebuffer-mode-for-encoding.patch

@@ -0,0 +1,247 @@
+diff -Naur a/omx/gstomx.h b/omx/gstomx.h
+--- a/omx/gstomx.h	2020-10-26 19:17:03.488452200 +0800
++++ b/omx/gstomx.h	2022-04-24 18:50:01.353690234 +0800
+@@ -332,6 +332,7 @@
+    */
+   gint settings_cookie;
+   gint configured_settings_cookie;
++  GstBuffer * c_buf;
+ };
+
+ struct _GstOMXComponent {
+diff -Naur a/omx/gstomxvideoenc.c b/omx/gstomxvideoenc.c
+--- a/omx/gstomxvideoenc.c	2020-10-26 19:17:03.492452100 +0800
++++ b/omx/gstomxvideoenc.c	2022-04-24 18:53:42.211595096 +0800
+@@ -2102,6 +2102,143 @@
+   return TRUE;
+ }
+
++#ifdef USE_OMX_TARGET_STARFIVE
++static GstBuffer *
++gst_omx_try_importing_buffer (GstOMXVideoEnc * self, GstBufferPool * pool,
++    GstOMXPort * port, GstVideoInfo * v_info, guint i, GstVideoFrame ** frame)
++{
++  GstBufferPoolAcquireParams params = { 0, };
++  GstBuffer *buffer = NULL;
++  GstMemory *mem;
++  GstMapFlags flags = GST_MAP_WRITE | GST_VIDEO_FRAME_MAP_FLAG_NO_REF;
++  gboolean is_mapped = FALSE;
++
++  *frame = NULL;
++
++  if (gst_buffer_pool_acquire_buffer (pool, &buffer, &params) != GST_FLOW_OK) {
++    GST_INFO_OBJECT (self, "Failed to acquire %d-th buffer", i);
++    return NULL;
++  }
++
++  if (gst_buffer_n_memory (buffer) != 1) {
++    GST_INFO_OBJECT (self, "%d-th buffer has more than one memory (%d)", i,
++        gst_buffer_n_memory (buffer));
++    goto out;
++  }
++
++  mem = gst_buffer_peek_memory (buffer, 0);
++  if (!mem) {
++    GST_INFO_OBJECT (self, "Failed to acquire memory of %d-th buffer", i);
++    goto out;
++  }
++
++  if ( !gst_is_dmabuf_memory (mem)) {
++    GST_INFO_OBJECT (self,
++        " %d-th buffer doesn't contain dmabuf, go to out. port->port_def.nBufferSize: %lu",
++        i, port->port_def.nBufferSize);
++    goto out;
++  }
++
++  *frame = g_slice_new0 (GstVideoFrame);
++
++  is_mapped = gst_video_frame_map (*frame, v_info, buffer, flags);
++  GST_INFO_OBJECT (self, " *frame = %p, mem: %p", *frame, mem);
++
++  if (!is_mapped) {
++    GST_INFO_OBJECT (self, "Failed to map %d-th buffer", i);
++    goto out;
++  }
++
++  if (GST_VIDEO_FRAME_SIZE (*frame) < port->port_def.nBufferSize) {
++    GST_INFO_OBJECT (self,
++        "Frame size of %d-th buffer (%" G_GSIZE_FORMAT
++        ") is too small for port buffer size (%d)", i,
++        GST_VIDEO_FRAME_SIZE (*frame), (guint32) port->port_def.nBufferSize);
++    goto out;
++  }
++
++  return buffer;
++
++out:
++  if (*frame) {
++    if (is_mapped)
++      gst_video_frame_unmap (*frame);
++    g_slice_free (GstVideoFrame, *frame);
++    *frame = NULL;
++  }
++  gst_buffer_unref (buffer);
++  return NULL;
++}
++#endif
++
++#ifdef USE_OMX_TARGET_STARFIVE
++static gboolean
++gst_omx_video_enc_use_buffers (GstOMXVideoEnc * self)
++{
++  GstOMXPort * port = self->enc_in_port;
++  GstBufferPool *pool = port->c_buf->pool;
++  GstVideoCodecState *input_state = self->input_state;
++  GstVideoInfo v_info = input_state->info;
++  guint n = port->port_def.nBufferCountActual;
++  GList *images = NULL;
++  GList *buffers = NULL;
++  GList *frames = NULL;
++  OMX_ERRORTYPE err = OMX_ErrorNone;
++  gboolean is_mapped = FALSE ;
++  GST_DEBUG(" nBufferCountActual=%d, nBufferCountMin= %lu", n, port->port_def.nBufferCountMin);
++
++  if ( pool != NULL ) {
++    guint i;
++    for (i = 0; i < n; i++) {
++      GstBuffer * buf = NULL;
++      GstVideoFrame *frame = NULL;
++      buf = gst_omx_try_importing_buffer (self, pool, port, &v_info, i, &frame);
++      if (!buf) {
++        GST_DEBUG_OBJECT (self, "Failed to import %d-th buffer", i);
++        g_list_free (images);
++        g_list_free_full (frames, (GDestroyNotify) gst_video_frame_unmap);
++        g_list_free_full (buffers, (GDestroyNotify) gst_buffer_unref);
++        break;
++      } else {
++          buffers = g_list_append (buffers, buf);
++          frames = g_list_append (frames, frame);
++          images = g_list_append (images, GST_VIDEO_FRAME_PLANE_DATA (frame, 0));
++          GST_DEBUG_OBJECT (self, " pBuffer: %p", GST_VIDEO_FRAME_PLANE_DATA (frame, 0));
++      }
++    }
++  }
++
++  if (images) {
++    is_mapped = TRUE;
++
++    err = gst_omx_port_use_buffers (port, images);
++    g_list_free (images);
++    g_list_free_full (frames, (GDestroyNotify) gst_video_frame_unmap);
++
++    if (err == OMX_ErrorNone) {
++      GST_DEBUG_OBJECT (self, "Using %d buffers", n);
++    } else {
++      GST_INFO_OBJECT (self,
++          "Failed to OMX_UseBuffer on port: %s (0x%08x)",
++          gst_omx_error_to_string (err), err);
++      g_list_free_full (buffers, (GDestroyNotify) gst_buffer_unref);
++      return FALSE;
++    }
++  }
++
++  if ( !is_mapped ) {
++    GST_WARNING_OBJECT (self, " failed to call UseBuffer, trying to use allocation");
++    self->input_dmabuf = FALSE;
++    self->input_allocation = GST_OMX_BUFFER_ALLOCATION_ALLOCATE_BUFFER;
++    if (gst_omx_port_allocate_buffers (self->enc_in_port) != OMX_ErrorNone){
++      return FALSE;
++    }
++  }
++
++  return TRUE;
++}
++#endif
++
+ static gboolean
+ gst_omx_video_enc_allocate_in_buffers (GstOMXVideoEnc * self)
+ {
+@@ -2115,6 +2252,9 @@
+         return FALSE;
+       break;
+     case GST_OMX_BUFFER_ALLOCATION_USE_BUFFER:
++#ifdef USE_OMX_TARGET_STARFIVE
++      return gst_omx_video_enc_use_buffers(self);
++#endif
+     default:
+       /* Not supported */
+       g_return_val_if_reached (FALSE);
+@@ -2179,8 +2319,18 @@
+ gst_omx_video_enc_pick_input_allocation_mode (GstOMXVideoEnc * self,
+     GstBuffer * inbuf)
+ {
++#ifdef USE_OMX_TARGET_STARFIVE
++  if (!gst_omx_is_dynamic_allocation_supported ()){
++    if (gst_is_dmabuf_memory (gst_buffer_peek_memory (inbuf, 0))){
++       GST_DEBUG_OBJECT (self, "select GST_OMX_BUFFER_ALLOCATION_USE_BUFFER");
++       return GST_OMX_BUFFER_ALLOCATION_USE_BUFFER;
++    }
++    return GST_OMX_BUFFER_ALLOCATION_ALLOCATE_BUFFER;
++  }
++#else
+   if (!gst_omx_is_dynamic_allocation_supported ())
+     return GST_OMX_BUFFER_ALLOCATION_ALLOCATE_BUFFER;
++#endif
+
+   if (can_use_dynamic_buffer_mode (self, inbuf)) {
+     GST_DEBUG_OBJECT (self,
+@@ -2272,6 +2422,15 @@
+         input);
+     self->input_dmabuf = FALSE;
+
++#ifdef USE_OMX_TARGET_STARFIVE
++    if (self->input_allocation ==
++          GST_OMX_BUFFER_ALLOCATION_USE_BUFFER) {
++      self->input_dmabuf = TRUE;
++      self->enc_in_port->c_buf = input;
++      GST_DEBUG_OBJECT (self, "input_dmabuf is true");
++    }
++#endif
++
+ #ifdef USE_OMX_TARGET_ZYNQ_USCALE_PLUS
+     if (gst_is_dmabuf_memory (gst_buffer_peek_memory (input, 0))) {
+       if (self->input_allocation ==
+@@ -2786,7 +2945,11 @@
+   }
+
+   if (self->enc_in_port->allocation ==
+-      GST_OMX_BUFFER_ALLOCATION_USE_BUFFER_DYNAMIC) {
++      GST_OMX_BUFFER_ALLOCATION_USE_BUFFER_DYNAMIC
++#ifdef USE_OMX_TARGET_STARFIVE
++      || self->enc_in_port->allocation == GST_OMX_BUFFER_ALLOCATION_USE_BUFFER
++#endif
++     ) {
+     if (gst_buffer_n_memory (inbuf) > 1) {
+       GST_ELEMENT_ERROR (self, STREAM, FORMAT, (NULL),
+           ("input buffer now has more than one memory, can't use dynamic allocation any more"));
+@@ -2812,12 +2975,29 @@
+           gst_buffer_get_size (inbuf));
+     } else {
+       /* dmabuf input */
++#ifdef USE_OMX_TARGET_STARFIVE
++      if (self->enc_in_port->allocation == GST_OMX_BUFFER_ALLOCATION_USE_BUFFER) {
++        GstMemory *mem = gst_buffer_peek_memory (inbuf, 0);
++        GST_LOG_OBJECT (self, " mem: %p, inbuf: %p, outbuf: %p", mem, inbuf, outbuf);
++        if (!gst_omx_buffer_map_memory (outbuf, mem)) {
++          GST_ELEMENT_ERROR (self, STREAM, FORMAT, (NULL),("failed to import"));
++          GST_ERROR_OBJECT (self, "failed to import dmabuf %p, inbuf: %p",mem, inbuf);
++          return FALSE;
++        }
++      } else {
++        if (!gst_omx_buffer_import_fd (outbuf, inbuf)) {
++          GST_ELEMENT_ERROR (self, STREAM, FORMAT, (NULL),
++              ("failed to import dmabuf"));
++          return FALSE;
++        }
++      }
++#else
+       if (!gst_omx_buffer_import_fd (outbuf, inbuf)) {
+         GST_ELEMENT_ERROR (self, STREAM, FORMAT, (NULL),
+             ("failed to import dmabuf"));
+         return FALSE;
+       }
+-
++#endif
+       GST_LOG_OBJECT (self, "Import dmabuf of %" G_GSIZE_FORMAT " bytes",
+           gst_buffer_get_size (inbuf));
+     }

+ 144 - 0
multimedia/patch/gstreamer1/sf-gst-omx/0010-add-property-framerate.patch

@@ -0,0 +1,144 @@
+--- a/omx/gstomxmjpegdec.h
++++ b/omx/gstomxmjpegdec.h
+@@ -45,6 +45,16 @@
+ struct _GstOMXMJPEGDec
+ {
+   GstOMXVideoDec parent;
++#ifdef USE_OMX_TARGET_STARFIVE
++  //If container shows fps as public api, it won't work
++  //as GstBuffer containing pts/dts/duration and it will calculate fps in upstream.
++  //i.e.
++  //not works:
++  //   gst-launch-1.0 -v filesrc location=xxx.mp4 ! qtdemux name=demux demux.video_0 ! jpegparse ! omxmjpegdec framerate=xxx ! kmssink driver-name=starfive force-modesetting=1
++  //works:
++  //   gst-launch-1.0 -v filesrc location=xxx.mjpeg ! jpegparse ! omxmjpegdec framerate=xxx ! kmssink driver-name=starfive force-modesetting=1
++  gint framerate;
++#endif
+ };
+
+ struct _GstOMXMJPEGDecClass
+
+--- a/omx/gstomxmjpegdec.c
++++ b/omx/gstomxmjpegdec.c
+@@ -28,16 +28,28 @@
+
+ GST_DEBUG_CATEGORY_STATIC (gst_omx_mjpeg_dec_debug_category);
+ #define GST_CAT_DEFAULT gst_omx_mjpeg_dec_debug_category
+-
++#ifdef USE_OMX_TARGET_STARFIVE
++#define DEFAULT_FRAMERATE                0
++#endif
+ /* prototypes */
+ static gboolean gst_omx_mjpeg_dec_is_format_change (GstOMXVideoDec * dec,
+     GstOMXPort * port, GstVideoCodecState * state);
+ static gboolean gst_omx_mjpeg_dec_set_format (GstOMXVideoDec * dec,
+     GstOMXPort * port, GstVideoCodecState * state);
+-
++#ifdef USE_OMX_TARGET_STARFIVE
++static void gst_omx_mjpeg_dec_set_property (GObject * object, guint prop_id,
++    const GValue * value, GParamSpec * pspec);
++static void gst_omx_mjpeg_dec_get_property (GObject * object, guint prop_id,
++    GValue * value, GParamSpec * pspec);
++#endif
+ enum
+ {
++#ifdef USE_OMX_TARGET_STARFIVE
++  PROP_0,
++  PROP_FRAMERATE
++#else
+   PROP_0
++#endif
+ };
+
+ /* class initialization */
+@@ -54,11 +66,16 @@
+ {
+   GstOMXVideoDecClass *videodec_class = GST_OMX_VIDEO_DEC_CLASS (klass);
+   GstElementClass *element_class = GST_ELEMENT_CLASS (klass);
+-
++#ifdef USE_OMX_TARGET_STARFIVE
++  GObjectClass *gobject_class = G_OBJECT_CLASS (videodec_class);
++#endif
+   videodec_class->is_format_change =
+       GST_DEBUG_FUNCPTR (gst_omx_mjpeg_dec_is_format_change);
+   videodec_class->set_format = GST_DEBUG_FUNCPTR (gst_omx_mjpeg_dec_set_format);
+-
++#ifdef USE_OMX_TARGET_STARFIVE
++  gobject_class->set_property = GST_DEBUG_FUNCPTR (gst_omx_mjpeg_dec_set_property);
++  gobject_class->get_property = GST_DEBUG_FUNCPTR (gst_omx_mjpeg_dec_get_property);
++#endif
+   videodec_class->cdata.default_sink_template_caps = "image/jpeg, "
+       "width=(int) [1,MAX], " "height=(int) [1,MAX]";
+
+@@ -67,7 +84,16 @@
+       "Codec/Decoder/Video/Hardware",
+       "Decode MJPEG video streams",
+       "Sebastian Dröge <sebastian.droege@collabora.co.uk>");
+-
++#ifdef USE_OMX_TARGET_STARFIVE
++  g_object_class_install_property (gobject_class, PROP_FRAMERATE,
++      g_param_spec_int ("framerate",
++        "Framerate",
++        "Frames per second in raw stream.\n                \
++         Only workable for the container not exposing fps.\n\
++                         '0' means disabled.",
++         0, G_MAXINT,
++         DEFAULT_FRAMERATE, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
++#endif
+   gst_omx_set_default_role (&videodec_class->cdata, "video_decoder.mjpeg");
+ }
+
+@@ -89,10 +115,53 @@
+ {
+   gboolean ret;
+   OMX_PARAM_PORTDEFINITIONTYPE port_def;
++#ifdef USE_OMX_TARGET_STARFIVE
++  GstVideoInfo *info = &state->info;
++  GstOMXMJPEGDec * mjpeg_dec = GST_OMX_MJPEG_DEC (dec);
+
++  if (mjpeg_dec->framerate != DEFAULT_FRAMERATE) {
++    GST_DEBUG_OBJECT (dec, " change fps from %d/%d to %d/1" GST_PTR_FORMAT, info->fps_n, info->fps_d, mjpeg_dec->framerate);
++    info->fps_n = mjpeg_dec->framerate;
++    info->fps_d = 1;
++    if (state->caps) {
++      gst_caps_set_simple (state->caps, "framerate", GST_TYPE_FRACTION, info->fps_n, info->fps_d, NULL);
++    }
++  }
++#endif
+   gst_omx_port_get_port_definition (port, &port_def);
+   port_def.format.video.eCompressionFormat = OMX_VIDEO_CodingMJPEG;
+   ret = gst_omx_port_update_port_definition (port, &port_def) == OMX_ErrorNone;
+
+   return ret;
+ }
++#ifdef USE_OMX_TARGET_STARFIVE
++static void
++gst_omx_mjpeg_dec_set_property (GObject * object, guint prop_id,
++    const GValue * value, GParamSpec * pspec)
++{
++  GstOMXMJPEGDec * mjpeg_dec = GST_OMX_MJPEG_DEC (object);
++  switch (prop_id) {
++    case PROP_FRAMERATE:
++      mjpeg_dec->framerate = g_value_get_int (value);
++      break;
++    default:
++      G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
++      break;
++  }
++}
++
++static void
++gst_omx_mjpeg_dec_get_property (GObject * object, guint prop_id, GValue * value,
++    GParamSpec * pspec)
++{
++  GstOMXMJPEGDec * mjpeg_dec = GST_OMX_MJPEG_DEC (object);
++  switch (prop_id) {
++    case PROP_FRAMERATE:
++      g_value_set_int (value, mjpeg_dec->framerate);
++      break;
++    default:
++      G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
++      break;
++  }
++}
++#endif

+ 38 - 0
multimedia/patch/gstreamer1/sf-gst-omx/0011-hanle-some-extra-profile-for-avc.patch

@@ -0,0 +1,38 @@
+--- a/omx/gstomxh264utils.c
++++ b/omx/gstomxh264utils.c
+@@ -54,6 +54,19 @@
+ #endif
+ };
+ 
++#ifdef USE_OMX_TARGET_STARFIVE
++static const H264ProfileMapping h264_ext_profiles[] = {
++  {"progressive-high",
++      (OMX_VIDEO_AVCPROFILETYPE) OMX_VIDEO_AVCProfileHigh},
++  {"constrained-high",
++      (OMX_VIDEO_AVCPROFILETYPE) OMX_VIDEO_AVCProfileHigh},
++  {"high-10-intra",
++      (OMX_VIDEO_AVCPROFILETYPE) OMX_VIDEO_AVCProfileHigh10},
++  {"high-4:2:2-intra",
++      (OMX_VIDEO_AVCPROFILETYPE) OMX_VIDEO_AVCProfileHigh422},
++};
++#endif
++
+ OMX_VIDEO_AVCPROFILETYPE
+ gst_omx_h264_utils_get_profile_from_str (const gchar * profile)
+ {
+@@ -63,6 +76,15 @@
+     if (g_str_equal (profile, h264_profiles[i].profile))
+       return h264_profiles[i].e;
+   }
++
++#ifdef USE_OMX_TARGET_STARFIVE
++  for (i = 0; i < G_N_ELEMENTS (h264_ext_profiles); i++) {
++    if (g_str_equal (profile, h264_ext_profiles[i].profile)) {
++      GST_INFO("extra profile: %s ==> avc profie type: %d", profile, h264_ext_profiles[i].e);
++      return h264_ext_profiles[i].e;
++    }
++  }
++#endif
+ 
+   return OMX_VIDEO_AVCProfileMax;
+ }

+ 67 - 0
multimedia/patch/gstreamer1/sf-gst-omx/0012-combine-sps-pps-header-to-idr.patch

@@ -0,0 +1,67 @@
+Combine SPS/PPS header to IDR frame.
+
+Signed-off-by: Windsome Zeng <windsome.zeng@starfivetech.com>
+
+diff -purN a/meson.build b/meson.build
+--- a/meson.build	2022-05-13 11:31:02.725614796 +0800
++++ b/meson.build	2022-05-13 11:32:26.213975419 +0800
+@@ -218,6 +218,7 @@ elif omx_target == 'tizonia'
+   omx_inc = []
+ elif omx_target == 'stf'
+   cdata.set('USE_OMX_TARGET_STARFIVE', 1)
++  gst_omx_args += ['-DCOMBINE_SPS_PPS_HEADERS']
+   warning('stf selected')
+ else
+   error ('Unsupported omx target specified. Use the -Dtarget option')
+diff -purN a/omx/gstomx.c b/omx/gstomx.c
+--- a/omx/gstomx.c	2022-05-13 11:31:02.725614796 +0800
++++ b/omx/gstomx.c	2022-05-13 11:33:15.461950978 +0800
+@@ -70,6 +70,11 @@ static GHashTable *core_handles;
+ G_LOCK_DEFINE_STATIC (buffer_flags_str);
+ static GHashTable *buffer_flags_str;
+ 
++#ifdef COMBINE_SPS_PPS_HEADERS
++static OMX_U8 *sps_pps_header = NULL;
++static OMX_U32 sps_pps_header_size = 0;
++#endif
++
+ GstOMXCore *
+ gst_omx_core_acquire (const gchar * filename)
+ {
+@@ -903,6 +908,36 @@ FillBufferDone (OMX_HANDLETYPE hComponen
+ 
+   g_assert (buf->omx_buf == pBuffer);
+ 
++#ifdef COMBINE_SPS_PPS_HEADERS
++  if (pBuffer->nFlags == 0x80) {
++    g_assert (sps_pps_header == NULL);
++    if (sps_pps_header)
++      g_free (sps_pps_header);
++
++    sps_pps_header = g_new (OMX_U8, pBuffer->nFilledLen);
++    if (sps_pps_header) {
++      sps_pps_header_size = pBuffer->nFilledLen;
++      memcpy(sps_pps_header, pBuffer->pBuffer + pBuffer->nOffset, pBuffer->nFilledLen);
++    }
++    return OMX_ErrorNone;
++  }
++  
++  if (sps_pps_header) {
++    g_assert (sps_pps_header_size + pBuffer->nFilledLen <= pBuffer->nAllocLen);
++    if (sps_pps_header_size + pBuffer->nFilledLen <= pBuffer->nAllocLen) {
++      OMX_U8 *buf;
++      buf = pBuffer->pBuffer + pBuffer->nOffset;
++      memmove (buf + sps_pps_header_size, buf, pBuffer->nFilledLen);
++      memcpy (buf, sps_pps_header, sps_pps_header_size);
++      pBuffer->nFilledLen += sps_pps_header_size;
++
++      g_free (sps_pps_header);
++      sps_pps_header = NULL;
++      sps_pps_header_size = 0;
++    }
++  }
++#endif /* COMBINE_SPS_PPS_HEADERS */
++
+   if (buf->port->tunneled) {
+     GST_ERROR ("FillBufferDone on tunneled port");
+     return OMX_ErrorBadParameter;

+ 62 - 0
multimedia/patch/gstreamer1/sf-gst-omx/0013-Modify-sf-component-name-to-in-std-format.patch

@@ -0,0 +1,62 @@
+From f207f1287cf8732459e928940dd6e762fb546bff Mon Sep 17 00:00:00 2001
+From: Som Qin <som.qin@starfivetech.com>
+Date: Mon, 25 Jul 2022 09:45:44 +0800
+Subject: [PATCH] Modify sf component name to adapt omx
+
+Signed-off-by: Som Qin <som.qin@starfivetech.com>
+---
+ config/stf/gstomx.conf | 10 +++++-----
+ 1 file changed, 5 insertions(+), 5 deletions(-)
+
+diff --git a/config/stf/gstomx.conf b/config/stf/gstomx.conf
+index 398a62c..4eae679 100644
+--- a/config/stf/gstomx.conf
++++ b/config/stf/gstomx.conf
+@@ -1,7 +1,7 @@
+ [omxh265dec]
+ type-name=GstOMXH265Dec
+ core-name=/usr/lib/libsf-omx-il.so
+-component-name=sf.dec.decoder.h265
++component-name=OMX.sf.video_decoder.hevc.internal
+ in-port-index=0
+ out-port-index=1
+ rank=257
+@@ -10,7 +10,7 @@ hacks=pass-profile-to-decoder;pass-color-format-to-decoder;ensure-buffer-count-a
+ [omxh264dec]
+ type-name=GstOMXH264Dec
+ core-name=/usr/lib/libsf-omx-il.so
+-component-name=sf.dec.decoder.h264
++component-name=OMX.sf.video_decoder.avc.internal
+ in-port-index=0
+ out-port-index=1
+ rank=257
+@@ -19,7 +19,7 @@ hacks=pass-profile-to-decoder;pass-color-format-to-decoder;ensure-buffer-count-a
+ [omxh265enc]
+ type-name=GstOMXH265Enc
+ core-name=/usr/lib/libsf-omx-il.so
+-component-name=sf.enc.encoder.h265
++component-name=OMX.sf.video_encoder.hevc
+ in-port-index=0
+ out-port-index=1
+ rank=1
+@@ -28,7 +28,7 @@ hacks=pass-profile-to-decoder;pass-color-format-to-decoder;ensure-buffer-count-a
+ [omxh264enc]
+ type-name=GstOMXH264Enc
+ core-name=/usr/lib/libsf-omx-il.so
+-component-name=sf.enc.encoder.h264
++component-name=OMX.sf.video_encoder.avc
+ in-port-index=0
+ out-port-index=1
+ rank=1
+@@ -37,7 +37,7 @@ hacks=pass-profile-to-decoder;pass-color-format-to-decoder;ensure-buffer-count-a
+ [omxmjpegdec]
+ type-name=GstOMXMJPEGDec
+ core-name=/usr/lib/libsf-omx-il.so
+-component-name=sf.dec.decoder.mjpeg
++component-name=OMX.sf.video_decoder.mjpeg.internal
+ in-port-index=0
+ out-port-index=1
+ rank=1
+-- 
+2.25.1
+

+ 16 - 0
multimedia/patch/gstreamer1/sf-gst-omx/0014-support-nv21-for-omxh264_5dec.patch

@@ -0,0 +1,16 @@
+Add NV21 for omxh264dec omxh265dec 
+
+Signed-off-by: Leo Lu <leo.lu@starfivetech.com>
+
+diff -purN a/omx/gstomxvideo.h b/omx/gstomxvideo.h
+--- a/omx/gstomxvideo.h	2021-09-09 03:05:28.042623000 +0800
++++ b/omx/gstomxvideo.h	2022-08-23 13:45:37.042353044 +0800
+@@ -36,7 +36,7 @@ G_BEGIN_DECLS
+ 
+ /* Keep synced with gst_omx_video_get_format_from_omx(). Sort by decreasing quality */
+ #define GST_OMX_VIDEO_DEC_SUPPORTED_FORMATS "{ NV16_10LE32, NV12_10LE32, " \
+-  "NV16, YUY2, YVYU, UYVY, NV12, I420, RGB16, BGR16, ABGR, ARGB, GRAY8 }"
++  "NV16, YUY2, YVYU, UYVY, NV12, NV21, I420, RGB16, BGR16, ABGR, ARGB, GRAY8 }"
+ 
+ #define GST_OMX_VIDEO_ENC_SUPPORTED_FORMATS "{ NV16_10LE32, NV12_10LE32, " \
+   "NV16, NV12, I420, GRAY8 }"

+ 72 - 0
multimedia/patch/gstreamer1/sf-gst-omx/0015_Add_NV21_for_gstomxvideoenc_class.patch

@@ -0,0 +1,72 @@
+Add NV21 for omxvideoenc class
+
+Signed-off-by: Leo Lu <leo.lu@starfivetech.com>
+
+diff -purN a/omx/gstomxvideoenc.c b/omx/gstomxvideoenc.c
+--- a/omx/gstomxvideoenc.c	2022-08-26 18:07:10.171979169 +0800
++++ b/omx/gstomxvideoenc.c	2022-08-29 09:54:54.979229076 +0800
+@@ -1391,6 +1391,7 @@ get_chroma_info_from_input (GstOMXVideoE
+       break;
+     case GST_VIDEO_FORMAT_I420:
+     case GST_VIDEO_FORMAT_NV12:
++    case GST_VIDEO_FORMAT_NV21:
+       *chroma_format = "4:2:0";
+       *bit_depth_luma = *bit_depth_chroma = 8;
+       break;
+@@ -2045,6 +2046,7 @@ gst_omx_video_enc_configure_input_buffer
+ 
+     case OMX_COLOR_FormatYUV420PackedSemiPlanar:
+     case OMX_COLOR_FormatYUV420SemiPlanar:
++    case OMX_COLOR_FormatYVU420SemiPlanar:
+       port_def.nBufferSize =
+           (port_def.format.video.nStride * port_def.format.video.nFrameHeight) +
+           (port_def.format.video.nStride *
+@@ -2693,6 +2695,9 @@ gst_omx_video_enc_set_format (GstVideoEn
+       case GST_VIDEO_FORMAT_ARGB:
+         port_def.format.video.eColorFormat = OMX_COLOR_Format32bitBGRA8888;
+         break;
++      case GST_VIDEO_FORMAT_NV21:
++        port_def.format.video.eColorFormat = OMX_COLOR_FormatYVU420SemiPlanar;
++        break;
+       default:
+         GST_ERROR_OBJECT (self, "Unsupported format %s",
+             gst_video_format_to_string (info->finfo->format));
+@@ -3094,6 +3099,7 @@ gst_omx_video_enc_fill_buffer (GstOMXVid
+       break;
+     }
+     case GST_VIDEO_FORMAT_NV12:
++    case GST_VIDEO_FORMAT_NV21:
+     case GST_VIDEO_FORMAT_NV16:
+     case GST_VIDEO_FORMAT_NV12_10LE32:
+     case GST_VIDEO_FORMAT_NV16_10LE32:
+@@ -3728,6 +3734,7 @@ filter_supported_formats (GList * negoti
+       case GST_VIDEO_FORMAT_NV16:
+       case GST_VIDEO_FORMAT_NV16_10LE32:
+       case GST_VIDEO_FORMAT_GRAY8:
++      case GST_VIDEO_FORMAT_NV21:
+         cur = g_list_next (cur);
+         continue;
+       default:
+diff -purN a/omx/gstomxvideo.h b/omx/gstomxvideo.h
+--- a/omx/gstomxvideo.h	2022-08-26 18:07:10.219979290 +0800
++++ b/omx/gstomxvideo.h	2022-08-29 09:55:41.307514974 +0800
+@@ -39,7 +39,7 @@ G_BEGIN_DECLS
+   "NV16, YUY2, YVYU, UYVY, NV12, NV21, I420, RGB16, BGR16, ABGR, ARGB, GRAY8 }"
+ 
+ #define GST_OMX_VIDEO_ENC_SUPPORTED_FORMATS "{ NV16_10LE32, NV12_10LE32, " \
+-  "NV16, NV12, I420, GRAY8 }"
++  "NV16, NV12, NV21, I420, GRAY8 }"
+ 
+ typedef struct
+ {
+diff -purN a/omx/openmax/OMX_IVCommon.h b/omx/openmax/OMX_IVCommon.h
+--- a/omx/openmax/OMX_IVCommon.h	2021-09-09 03:05:28.046623200 +0800
++++ b/omx/openmax/OMX_IVCommon.h	2022-08-29 09:56:49.415927528 +0800
+@@ -130,6 +130,7 @@ typedef enum OMX_COLOR_FORMATTYPE {
+     OMX_COLOR_Format18BitBGR666,
+     OMX_COLOR_Format24BitARGB6666,
+     OMX_COLOR_Format24BitABGR6666,
++    OMX_COLOR_FormatYVU420SemiPlanar,   /*  NV21  */
+     OMX_COLOR_FormatKhronosExtensions = 0x6F000000, /**< Reserved region for introducing Khronos Standard Extensions */ 
+     OMX_COLOR_FormatVendorStartUnused = 0x7F000000, /**< Reserved region for introducing Vendor Extensions */
+     OMX_COLOR_FormatMax = 0x7FFFFFFF

+ 86 - 0
multimedia/patch/gstreamer1/sf-gst-omx/0016-Modify-gstomxmjpegdec-format.patch

@@ -0,0 +1,86 @@
+Changed Y444 for correct format
+Add VYUY v308 NV61 format for omxmjpegdec
+
+Signed-off-by: Leo Lu <leo.lu@starfivetech.com>
+
+diff -purN a/omx/gstomxbufferpool.c b/omx/gstomxbufferpool.c
+--- a/omx/gstomxbufferpool.c	2022-09-23 11:48:33.011477728 +0800
++++ b/omx/gstomxbufferpool.c	2022-09-23 15:01:05.053735591 +0800
+@@ -355,10 +355,13 @@ gst_omx_buffer_pool_alloc_buffer (GstBuf
+       case GST_VIDEO_FORMAT_ARGB:
+       case GST_VIDEO_FORMAT_RGB16:
+       case GST_VIDEO_FORMAT_BGR16:
++      case GST_VIDEO_FORMAT_GRAY8:
++        break;
+       case GST_VIDEO_FORMAT_YUY2:
+       case GST_VIDEO_FORMAT_UYVY:
+       case GST_VIDEO_FORMAT_YVYU:
+-      case GST_VIDEO_FORMAT_GRAY8:
++      case GST_VIDEO_FORMAT_VYUY:
++        stride[0] = GST_ROUND_UP_4(nstride * 2);
+         break;
+       case GST_VIDEO_FORMAT_I420:
+         stride[1] = nstride / 2;
+@@ -370,6 +373,7 @@ gst_omx_buffer_pool_alloc_buffer (GstBuf
+       case GST_VIDEO_FORMAT_NV12_10LE32:
+       case GST_VIDEO_FORMAT_NV16:
+       case GST_VIDEO_FORMAT_NV16_10LE32:
++      case GST_VIDEO_FORMAT_NV61:
+ #ifdef USE_OMX_TARGET_STARFIVE
+       case GST_VIDEO_FORMAT_NV21:
+ #endif
+@@ -392,6 +396,9 @@ gst_omx_buffer_pool_alloc_buffer (GstBuf
+         offset[1] = offset[0] + stride[0] * nslice;
+         offset[2] = offset[1] + stride[1] * nslice;
+         break;
++      case GST_VIDEO_FORMAT_v308:
++        stride[0] = GST_ROUND_UP_4(nstride * 3);
++        break;
+ #endif
+       default:
+         g_assert_not_reached ();
+diff -purN a/omx/gstomxvideo.c b/omx/gstomxvideo.c
+--- a/omx/gstomxvideo.c	2022-09-23 11:48:33.011477728 +0800
++++ b/omx/gstomxvideo.c	2022-09-23 15:00:40.937734140 +0800
+@@ -99,11 +99,20 @@ gst_omx_video_get_format_from_omx (OMX_C
+       format = GST_VIDEO_FORMAT_Y42B;
+       break;
+     case OMX_COLOR_FormatYUV444Interleaved:
+-      format = GST_VIDEO_FORMAT_Y444;
++      format = GST_VIDEO_FORMAT_v308;
+       break;
+     case OMX_COLOR_FormatYVU420SemiPlanar:
+       format = GST_VIDEO_FORMAT_NV21;
+       break;
++    case OMX_COLOR_FormatCrYCbY:
++      format = GST_VIDEO_FORMAT_VYUY;
++      break;
++    case OMX_COLOR_FormatYUV444Planar:
++      format = GST_VIDEO_FORMAT_Y444;
++      break;
++    case OMX_COLOR_FormatYVU422SemiPlanar:
++      format = GST_VIDEO_FORMAT_NV61;
++      break;
+ #endif
+     default:
+       format = GST_VIDEO_FORMAT_UNKNOWN;
+diff -purN a/omx/gstomxvideodec.c b/omx/gstomxvideodec.c
+--- a/omx/gstomxvideodec.c	2022-09-23 11:48:33.011477728 +0800
++++ b/omx/gstomxvideodec.c	2022-09-23 11:42:45.251075196 +0800
+@@ -646,6 +646,7 @@ gst_omx_video_dec_fill_buffer (GstOMXVid
+         break;
+ #ifdef USE_OMX_TARGET_STARFIVE
+       case GST_VIDEO_FORMAT_Y444:
++      case GST_VIDEO_FORMAT_v308:
+         dst_width[0] = GST_VIDEO_INFO_WIDTH (vinfo);
+         dst_width[1] = GST_VIDEO_INFO_WIDTH (vinfo);
+         dst_width[2] = GST_VIDEO_INFO_WIDTH (vinfo);
+diff -purN a/config/stf/gstomx.conf b/config/stf/gstomx.conf
+--- a/config/stf/gstomx.conf	2022-09-23 11:48:33.079477828 +0800
++++ b/config/stf/gstomx.conf	2022-09-23 15:04:52.249741473 +0800
+@@ -42,4 +42,4 @@ in-port-index=0
+ out-port-index=1
+ rank=1
+ hacks=pass-profile-to-decoder;pass-color-format-to-decoder;ensure-buffer-count-actual
+-src-template-caps=video/x-raw, format = (string) { NV16_10LE32, NV12_10LE32, NV16, YUY2, YVYU, UYVY, NV12, NV21, I420, RGB16, BGR16, ABGR, ARGB, GRAY8, Y42B, Y444 }, width = (int) [ 1, max ], height = (int) [ 1, max ], framerate = (fraction) [ 0, max ]
++src-template-caps=video/x-raw, format = (string) { NV16_10LE32, NV12_10LE32, NV16, NV61, YUY2, YVYU, UYVY, NV12, NV21, I420, RGB16, BGR16, ABGR, ARGB, GRAY8, Y42B, Y444, VYUY, v308 }, width = (int) [ 1, max ], height = (int) [ 1, max ], framerate = (fraction) [ 0, max ]

+ 388 - 0
multimedia/patch/gstreamer1/sf-gst-omx/0017-support-mirror-rotation-scale-for-gstomxmjpegdec.patch

@@ -0,0 +1,388 @@
+Support mirror rotation scale for omxmjpegdec
+
+Signed-off-by: Leo Lu <leo.lu@starfivetech.com>
+
+diff -purN a/omx/gstomxmjpegdec.c b/omx/gstomxmjpegdec.c
+--- a/omx/gstomxmjpegdec.c	2022-10-14 13:42:12.749600862 +0800
++++ b/omx/gstomxmjpegdec.c	2022-10-14 13:36:08.488195374 +0800
+@@ -30,8 +30,19 @@ GST_DEBUG_CATEGORY_STATIC (gst_omx_mjpeg
+ #define GST_CAT_DEFAULT gst_omx_mjpeg_dec_debug_category
+ #ifdef USE_OMX_TARGET_STARFIVE
+ #define DEFAULT_FRAMERATE                0
++#define DEFAULT_MIRROR                   0
++#define DEFAULT_ROTATION                 0
++#define DEFAULT_SCALEH                   0
++#define DEFAULT_SCALEV                   0
+ #endif
+ /* prototypes */
++
++static gboolean
++gst_omx_jpegdec_set_scale (GstOMXVideoDec * self);
++static gboolean
++gst_omx_jpegdec_set_rotation (GstOMXVideoDec * self);
++static gboolean
++gst_omx_jpegdec_set_mirror (GstOMXVideoDec * self);
+ static gboolean gst_omx_mjpeg_dec_is_format_change (GstOMXVideoDec * dec,
+     GstOMXPort * port, GstVideoCodecState * state);
+ static gboolean gst_omx_mjpeg_dec_set_format (GstOMXVideoDec * dec,
+@@ -46,10 +57,14 @@ enum
+ {
+ #ifdef USE_OMX_TARGET_STARFIVE
+   PROP_0,
+-  PROP_FRAMERATE
++  PROP_FRAMERATE,
+ #else
+-  PROP_0
++  PROP_0,
+ #endif
++  PROP_MIRROR,
++  PROP_SCALEH,
++  PROP_SCALEV,
++  PROP_ROTATION,
+ };
+ 
+ /* class initialization */
+@@ -61,6 +76,67 @@ enum
+ G_DEFINE_TYPE_WITH_CODE (GstOMXMJPEGDec, gst_omx_mjpeg_dec,
+     GST_TYPE_OMX_VIDEO_DEC, DEBUG_INIT);
+ 
++#define GST_OMX_MJPEG_DEC_MIRROR_MODE (gst_omx_mjpeg_dec_mirror_mode_get_type ())
++
++static GType
++gst_omx_mjpeg_dec_mirror_mode_get_type (void)
++{
++  static GType qtype = 0;
++  if (qtype == 0) {
++    static const GEnumValue values[] = {
++      { OMX_MirrorNone, "None", "none"},
++      { OMX_MirrorVertical, "Mirror Vertical", "V"},
++      { OMX_MirrorHorizontal, "Mirror Horizontal", "H"},
++      { OMX_MirrorBoth, "Mirror Both",  "VH"},
++      { 0, NULL, NULL}
++      };
++
++    qtype = g_enum_register_static("GstOMXMJPEGDecMirrorMode",values);
++  }
++  return qtype;
++}
++
++#define GST_OMX_MJPEG_DEC_ROTATION_MODE (gst_omx_mjpeg_dec_rotation_mode_get_type ())
++
++static GType
++gst_omx_mjpeg_dec_rotation_mode_get_type (void)
++{
++  static GType qtype = 0;
++  if (qtype == 0) {
++    static const GEnumValue values[] = {
++      { ROTATION_0, "None", "none"},
++      { ROTATION_90, "Rotate 90", "M1"},
++      { ROTATION_180, "Rotate 180", "M2"},
++      { ROTATION_270, "Rotate 270",  "M3"},
++      { 0, NULL, NULL}
++      };
++      
++    qtype = g_enum_register_static ("GstOMXMJPEGDecRotationMode", values);
++  }
++  return qtype;
++}
++
++#define GST_OMX_MJPEG_DEC_SCALE_MODE (gst_omx_mjpeg_dec_scale_mode_get_type ())
++
++static GType 
++gst_omx_mjpeg_dec_scale_mode_get_type (void)
++{
++  static GType qtype = 0;
++  if (qtype == 0) {
++    static const GEnumValue values[] = {
++      { SCALEMODE_0, "None", "none"},
++      { SCALEMODE1_2, "Scale half", "1/2"},
++      { SCALEMODE1_4, "Scale quarter", "1/4"},
++      { SCALEMODE1_8, "Scale eighth",  "1/8"},
++      { 0, NULL, NULL}
++      };
++      
++    qtype = g_enum_register_static ("GstOMXMJPEGDecScaleMode", values);
++  }
++  return qtype;
++}
++
++
+ static void
+ gst_omx_mjpeg_dec_class_init (GstOMXMJPEGDecClass * klass)
+ {
+@@ -94,12 +170,40 @@ gst_omx_mjpeg_dec_class_init (GstOMXMJPE
+          0, G_MAXINT,
+          DEFAULT_FRAMERATE, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
+ #endif
++  g_object_class_install_property (gobject_class, PROP_MIRROR,
++      g_param_spec_enum ("mirror", "MIRROR",
++        "Mirror 0(none), 1(V), 2(H), 3(VH).",
++        GST_OMX_MJPEG_DEC_MIRROR_MODE,
++        DEFAULT_MIRROR, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
++
++  g_object_class_install_property (gobject_class, PROP_ROTATION,
++      g_param_spec_enum ("rotation", "ROTATION",
++        "Rotation 0(none), 1(90), 2(180), 3(270).",
++        GST_OMX_MJPEG_DEC_ROTATION_MODE,
++        DEFAULT_ROTATION, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
++
++  g_object_class_install_property (gobject_class, PROP_SCALEH,
++      g_param_spec_enum ("scaleH", "ScaleH",
++        "Horizontal downscale: 0(none), 1(1/2), 2(1/4), 3(1/8).",
++        GST_OMX_MJPEG_DEC_SCALE_MODE, DEFAULT_SCALEH,
++        G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
++
++  g_object_class_install_property (gobject_class, PROP_SCALEV,
++      g_param_spec_enum ("scaleV", "ScaleV",
++        "Vertical downscale: 0(none), 1(1/2), 2(1/4), 3(1/8).",
++        GST_OMX_MJPEG_DEC_SCALE_MODE, DEFAULT_SCALEV,
++        G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
++  
+   gst_omx_set_default_role (&videodec_class->cdata, "video_decoder.mjpeg");
+ }
+ 
+ static void
+ gst_omx_mjpeg_dec_init (GstOMXMJPEGDec * self)
+ {
++  self->mirror = DEFAULT_MIRROR;
++  self->rotation = DEFAULT_ROTATION;
++  self->scaleH = DEFAULT_SCALEH;
++  self->scaleV = DEFAULT_SCALEV;
+ }
+ 
+ static gboolean
+@@ -128,6 +232,15 @@ gst_omx_mjpeg_dec_set_format (GstOMXVide
+     }
+   }
+ #endif
++  if (!gst_omx_jpegdec_set_mirror (dec))
++    return FALSE;
++    
++  if (!gst_omx_jpegdec_set_rotation (dec))
++    return FALSE;
++
++  if (!gst_omx_jpegdec_set_scale (dec))
++    return FALSE;
++
+   gst_omx_port_get_port_definition (port, &port_def);
+   port_def.format.video.eCompressionFormat = OMX_VIDEO_CodingMJPEG;
+   ret = gst_omx_port_update_port_definition (port, &port_def) == OMX_ErrorNone;
+@@ -144,6 +257,18 @@ gst_omx_mjpeg_dec_set_property (GObject
+     case PROP_FRAMERATE:
+       mjpeg_dec->framerate = g_value_get_int (value);
+       break;
++    case PROP_MIRROR:
++      mjpeg_dec->mirror = g_value_get_enum (value);
++      break;
++    case PROP_ROTATION:
++      mjpeg_dec->rotation = g_value_get_enum (value);
++      break;
++    case PROP_SCALEH:
++      mjpeg_dec->scaleH = g_value_get_enum (value);
++      break;
++    case PROP_SCALEV:
++      mjpeg_dec->scaleV = g_value_get_enum (value);
++      break;
+     default:
+       G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
+       break;
+@@ -159,9 +284,164 @@ gst_omx_mjpeg_dec_get_property (GObject
+     case PROP_FRAMERATE:
+       g_value_set_int (value, mjpeg_dec->framerate);
+       break;
++    case PROP_MIRROR:
++      g_value_set_enum (value, mjpeg_dec->mirror);
++      break;
++    case PROP_ROTATION:
++      g_value_set_enum (value, mjpeg_dec->rotation);
++      break;
++    case PROP_SCALEH:
++      g_value_set_enum (value, mjpeg_dec->scaleH);
++      break;
++    case PROP_SCALEV:
++      g_value_set_enum (value, mjpeg_dec->scaleV);
++      break;
+     default:
+       G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
+       break;
+   }
+ }
+ #endif
++
++static gboolean
++gst_omx_jpegdec_set_rotation (GstOMXVideoDec * self)
++{
++  OMX_ERRORTYPE err;
++  OMX_CONFIG_ROTATIONTYPE rotation_config;
++  gboolean result = TRUE;
++  GstOMXMJPEGDec * mjpeg_dec = GST_OMX_MJPEG_DEC (self);
++
++  GST_OBJECT_LOCK (self);
++
++  GST_OMX_INIT_STRUCT (&rotation_config);
++  rotation_config.nPortIndex = self->dec_out_port->index;
++
++  err = 
++      gst_omx_component_get_config (self->dec,
++      OMX_IndexConfigCommonRotate, &rotation_config);
++  
++  if (err == OMX_ErrorNone) {
++    if (mjpeg_dec->rotation >= 0 && mjpeg_dec->rotation <= 270) 
++      rotation_config.nRotation = mjpeg_dec->rotation;
++
++    err = 
++        gst_omx_component_set_config (self->dec,
++        OMX_IndexConfigCommonRotate, &rotation_config);
++    if (err == OMX_ErrorUnsupportedIndex) {
++       GST_WARNING_OBJECT (self,
++           "Setting a rotation not supported by the component");
++    } else if (err == OMX_ErrorUnsupportedSetting) {
++      GST_WARNING_OBJECT (self,
++          "Setting rotation settings %u not supported by the component",
++          mjpeg_dec->rotation);
++    } else if (err != OMX_ErrorNone) {
++      GST_ERROR_OBJECT (self,
++          "Failed to set rotation parameters: %s (0x%08x)",
++          gst_omx_error_to_string (err), err);
++      result = FALSE;
++    }
++  } else {
++    GST_ERROR_OBJECT (self, "Failed to get rotation parameters: %s (0x%08x)",
++        gst_omx_error_to_string (err), err);
++  }
++  
++  GST_OBJECT_UNLOCK (self);
++  return result;
++}
++
++
++
++static gboolean
++gst_omx_jpegdec_set_mirror (GstOMXVideoDec * self)
++{
++  OMX_ERRORTYPE err;
++  OMX_CONFIG_MIRRORTYPE mirror_config;
++  gboolean result = TRUE;
++  GstOMXMJPEGDec * mjpeg_dec = GST_OMX_MJPEG_DEC (self);
++
++  GST_OBJECT_LOCK (self);
++
++  GST_OMX_INIT_STRUCT (&mirror_config);
++  mirror_config.nPortIndex = self->dec_out_port->index;
++
++  err = 
++      gst_omx_component_get_config (self->dec,
++      OMX_IndexConfigCommonMirror, &mirror_config);
++  
++  if (err == OMX_ErrorNone) {
++    if (mjpeg_dec->mirror >= 0 && mjpeg_dec->mirror <= OMX_MirrorBoth)
++      mirror_config.eMirror = mjpeg_dec->mirror;
++    
++    err = 
++        gst_omx_component_set_config (self->dec,
++        OMX_IndexConfigCommonMirror, &mirror_config);
++    if (err == OMX_ErrorUnsupportedIndex) {
++      GST_WARNING_OBJECT (self,
++          "Setting a mirror not supported by the component");
++    } else if (err == OMX_ErrorUnsupportedSetting) {
++      GST_WARNING_OBJECT (self,
++          "Setting mirror settings %u not supported by the component",
++           mjpeg_dec->mirror);
++    } else if (err != OMX_ErrorNone) {
++      GST_ERROR_OBJECT (self,
++          "Failed to set mirror parameters: %s (0x%08x)",
++          gst_omx_error_to_string (err), err);
++      result = FALSE;
++    }
++  } else {
++    GST_ERROR_OBJECT (self, "Failed to get mirror parameters: %s (0x%08x)",
++        gst_omx_error_to_string (err), err);
++  }
++  
++  GST_OBJECT_UNLOCK (self);
++  return result;
++}
++
++static gboolean
++gst_omx_jpegdec_set_scale (GstOMXVideoDec * self)
++{
++  OMX_ERRORTYPE err;
++  OMX_CONFIG_SCALEFACTORTYPE scale_config;
++  gboolean result = TRUE;
++  GstOMXMJPEGDec * mjpeg_dec = GST_OMX_MJPEG_DEC (self);
++
++  GST_OBJECT_LOCK (self);
++
++  GST_OMX_INIT_STRUCT (&scale_config);
++  scale_config.nPortIndex = self->dec_out_port->index;
++
++  err = 
++      gst_omx_component_get_config (self->dec,
++      OMX_IndexConfigCommonScale, &scale_config);
++  
++  if (err == OMX_ErrorNone) {
++    if ((mjpeg_dec->scaleV >= 0 && mjpeg_dec->scaleV <= 3 ) ||
++        (mjpeg_dec->scaleH >= 0 && mjpeg_dec->scaleH <= 3)) {    
++      scale_config.xWidth = (1 << 16) >> (mjpeg_dec->scaleH & 0x3);
++      scale_config.xHeight = (1 << 16) >> (mjpeg_dec->scaleV & 0x3);
++    }
++
++    err = 
++        gst_omx_component_set_config (self->dec,
++        OMX_IndexConfigCommonScale, &scale_config);
++    if (err == OMX_ErrorUnsupportedIndex) {
++      GST_WARNING_OBJECT (self,
++          "Setting a scale not supported by the component");
++    } else if (err == OMX_ErrorUnsupportedSetting) {
++      GST_WARNING_OBJECT (self,
++          "Setting scale settings %u %u  not supported by the component",
++           mjpeg_dec->scaleV, mjpeg_dec->scaleH);
++    } else if (err != OMX_ErrorNone) {
++      GST_ERROR_OBJECT (self,
++          "Failed to set scale parameters: %s (0x%08x)",
++          gst_omx_error_to_string (err), err);
++      result = FALSE;
++    }
++  } else {
++    GST_ERROR_OBJECT (self, "Failed to get scale parameters: %s (0x%08x)",
++        gst_omx_error_to_string (err), err);
++  }
++  
++  GST_OBJECT_UNLOCK (self);
++  return result;
++}
+\ No newline at end of file
+diff -purN a/omx/gstomxmjpegdec.h b/omx/gstomxmjpegdec.h
+--- a/omx/gstomxmjpegdec.h	2022-10-14 13:42:12.749600862 +0800
++++ b/omx/gstomxmjpegdec.h	2022-10-14 13:29:24.849638506 +0800
+@@ -55,6 +55,10 @@ struct _GstOMXMJPEGDec
+   //   gst-launch-1.0 -v filesrc location=xxx.mjpeg ! jpegparse ! omxmjpegdec framerate=xxx ! kmssink driver-name=starfive force-modesetting=1
+   gint framerate;
+ #endif
++  guint32 mirror;
++  guint32 scaleH;
++  guint32 scaleV;
++  guint32 rotation;
+ };
+ 
+ struct _GstOMXMJPEGDecClass
+@@ -68,3 +72,19 @@ G_END_DECLS
+ 
+ #endif /* __GST_OMX_MJPEG_DEC_H__ */
+ 
++typedef enum ROTATIONTYPE 
++{
++  ROTATION_0 = 0,
++  ROTATION_90 = 90,
++  ROTATION_180 = 180,
++  ROTATION_270 = 270,   
++} ROTATIONTYPE;
++
++typedef enum SCALETYPE 
++{
++  SCALEMODE_0 = 0,
++  SCALEMODE1_2,
++  SCALEMODE1_4,
++  SCALEMODE1_8,   
++} SCALETYPE;
++

+ 194 - 0
multimedia/patch/gstreamer1/sf-gst-omx/0018-support-cut-for-gstomxmjpegdec.patch

@@ -0,0 +1,194 @@
+Support cut property for omxmjpegdec
+
+Signed-off-by: Leo Lu <leo.lu@starfivetech.com>
+
+diff -purN a/omx/gstomxmjpegdec.c b/omx/gstomxmjpegdec.c
+--- a/omx/gstomxmjpegdec.c	2022-10-14 14:48:24.207804442 +0800
++++ b/omx/gstomxmjpegdec.c	2022-10-14 14:46:42.979857812 +0800
+@@ -34,6 +34,8 @@ GST_DEBUG_CATEGORY_STATIC (gst_omx_mjpeg
+ #define DEFAULT_ROTATION                 0
+ #define DEFAULT_SCALEH                   0
+ #define DEFAULT_SCALEV                   0
++#define DEFAULT_ROI                     ""
++
+ #endif
+ /* prototypes */
+ 
+@@ -43,6 +45,8 @@ static gboolean
+ gst_omx_jpegdec_set_rotation (GstOMXVideoDec * self);
+ static gboolean
+ gst_omx_jpegdec_set_mirror (GstOMXVideoDec * self);
++static gboolean
++gst_omx_jpegdec_set_roi (GstOMXVideoDec * self);
+ static gboolean gst_omx_mjpeg_dec_is_format_change (GstOMXVideoDec * dec,
+     GstOMXPort * port, GstVideoCodecState * state);
+ static gboolean gst_omx_mjpeg_dec_set_format (GstOMXVideoDec * dec,
+@@ -65,6 +69,7 @@ enum
+   PROP_SCALEH,
+   PROP_SCALEV,
+   PROP_ROTATION,
++  PROP_ROI,
+ };
+ 
+ /* class initialization */
+@@ -194,6 +199,17 @@ gst_omx_mjpeg_dec_class_init (GstOMXMJPE
+         GST_OMX_MJPEG_DEC_SCALE_MODE, DEFAULT_SCALEV,
+         G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
+   
++  g_object_class_install_property (gobject_class, PROP_ROI,
++      g_param_spec_string ("roi", "ROI",
++        "Cutting image roi=<x>,<y>,<w>,<h> .\n                 \
++        roi coord and width/height(from left top).\n                 \
++        <x>   X Coordinate of the top left corner of the rectangle.\n                 \
++        <y>   Y Coordinate of the top left corner of the rectangle.\n                 \
++        <w>   Width of the rectangle.\n                 \
++        <h>   Height of the rectangle.\n                 \
++        EX:omxmjpegdec roi=0,0,800,480",
++        DEFAULT_ROI, G_PARAM_READWRITE | G_PARAM_STATIC_STRINGS));
++
+   gst_omx_set_default_role (&videodec_class->cdata, "video_decoder.mjpeg");
+ }
+ 
+@@ -204,6 +220,7 @@ gst_omx_mjpeg_dec_init (GstOMXMJPEGDec *
+   self->rotation = DEFAULT_ROTATION;
+   self->scaleH = DEFAULT_SCALEH;
+   self->scaleV = DEFAULT_SCALEV;
++  memset (self->roi, 0, ROI_ARRAY_SIZE);
+ }
+ 
+ static gboolean
+@@ -240,6 +257,9 @@ gst_omx_mjpeg_dec_set_format (GstOMXVide
+ 
+   if (!gst_omx_jpegdec_set_scale (dec))
+     return FALSE;
++  
++  if (!gst_omx_jpegdec_set_roi (dec))
++    return FALSE;
+ 
+   gst_omx_port_get_port_definition (port, &port_def);
+   port_def.format.video.eCompressionFormat = OMX_VIDEO_CodingMJPEG;
+@@ -269,6 +289,9 @@ gst_omx_mjpeg_dec_set_property (GObject
+     case PROP_SCALEV:
+       mjpeg_dec->scaleV = g_value_get_enum (value);
+       break;
++    case PROP_ROI:
++      strncpy (mjpeg_dec->roi, g_value_get_string (value), ROI_ARRAY_SIZE);
++      break;
+     default:
+       G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
+       break;
+@@ -296,6 +319,9 @@ gst_omx_mjpeg_dec_get_property (GObject
+     case PROP_SCALEV:
+       g_value_set_enum (value, mjpeg_dec->scaleV);
+       break;
++    case PROP_ROI:
++      g_value_set_string (value, mjpeg_dec->roi);
++      break;
+     default:
+       G_OBJECT_WARN_INVALID_PROPERTY_ID (object, prop_id, pspec);
+       break;
+@@ -444,4 +470,81 @@ gst_omx_jpegdec_set_scale (GstOMXVideoDe
+   
+   GST_OBJECT_UNLOCK (self);
+   return result;
+-}
+\ No newline at end of file
++}
++
++static gboolean
++gst_omx_jpegdec_set_roi (GstOMXVideoDec *self)
++{
++  OMX_ERRORTYPE err;
++  OMX_CONFIG_RECTTYPE RectConfig;
++  gboolean result = TRUE;
++  GstOMXMJPEGDec * mjpeg_dec = GST_OMX_MJPEG_DEC (self);
++  gchar* val;
++  gchar roi_tmp[ROI_ARRAY_SIZE];
++  strncpy (roi_tmp, mjpeg_dec->roi, ROI_ARRAY_SIZE);
++
++  GST_OBJECT_LOCK (self);
++  GST_OMX_INIT_STRUCT (&RectConfig);
++  RectConfig.nPortIndex = self->dec_out_port->index;
++
++  err = 
++      gst_omx_component_get_config (self->dec,
++      OMX_IndexConfigCommonScale, &RectConfig);
++  if (err == OMX_ErrorNone) {
++    if (strlen (mjpeg_dec->roi) > 0) {    
++      val = strtok (roi_tmp, ",");
++      if (val == NULL) {
++        goto ROI_PARAM_ERROR;
++      } else {
++        RectConfig.nLeft = atoi (val);
++      }
++      val = strtok (NULL, ",");
++      if (val == NULL) {
++        goto ROI_PARAM_ERROR;
++      } else {
++        RectConfig.nTop = atoi (val);
++      }
++      val = strtok (NULL, ",");
++      if (val == NULL) {
++        goto ROI_PARAM_ERROR;
++      } else {
++        RectConfig.nWidth = atoi (val);
++      }
++      val = strtok (NULL, ",");
++      if (val == NULL) {
++        goto ROI_PARAM_ERROR;
++      } else {
++        RectConfig.nHeight = atoi (val);
++      }
++    } else {
++      GST_OBJECT_UNLOCK (self);
++      return TRUE;
++    }
++
++    err = 
++        gst_omx_component_set_config (self->dec,
++        OMX_IndexConfigCommonOutputCrop, &RectConfig);
++    if (err == OMX_ErrorUnsupportedIndex) {
++      GST_WARNING_OBJECT (self,
++          "Setting a ROI not supported by the component");
++    } else if (err == OMX_ErrorUnsupportedSetting) {
++      GST_WARNING_OBJECT (self,
++          "Setting ROI settings %s  not supported by the component", mjpeg_dec->roi);
++    } else if (err != OMX_ErrorNone) {
++      GST_ERROR_OBJECT (self,
++          "Failed to set ROI parameters: %s (0x%08x)", gst_omx_error_to_string (err), err);
++      result = FALSE;
++    }
++  } else {
++    GST_ERROR_OBJECT (self, "Failed to get ROI parameters: %s (0x%08x)",
++        gst_omx_error_to_string (err), err);
++  }
++
++  GST_OBJECT_UNLOCK (self);
++  return result;
++
++ROI_PARAM_ERROR:
++  GST_ERROR_OBJECT (self, "Invalid ROI parameter: %s", mjpeg_dec->roi);
++  GST_OBJECT_UNLOCK (self);
++  return FALSE;  
++}
+diff -purN a/omx/gstomxmjpegdec.h b/omx/gstomxmjpegdec.h
+--- a/omx/gstomxmjpegdec.h	2022-10-14 14:48:24.207804442 +0800
++++ b/omx/gstomxmjpegdec.h	2022-10-14 14:42:09.876591323 +0800
+@@ -42,6 +42,8 @@ G_BEGIN_DECLS
+ typedef struct _GstOMXMJPEGDec GstOMXMJPEGDec;
+ typedef struct _GstOMXMJPEGDecClass GstOMXMJPEGDecClass;
+ 
++#define ROI_ARRAY_SIZE 128
++
+ struct _GstOMXMJPEGDec
+ {
+   GstOMXVideoDec parent;
+@@ -59,6 +61,8 @@ struct _GstOMXMJPEGDec
+   guint32 scaleH;
+   guint32 scaleV;
+   guint32 rotation;
++  gchar roi[ROI_ARRAY_SIZE];
++
+ };
+ 
+ struct _GstOMXMJPEGDecClass

+ 125 - 0
multimedia/patch/mm_libs/0001-wave511-fix-building-issue.patch

@@ -0,0 +1,125 @@
+From 0e365b325c7ddd78087bb48c5c4d695f505a7f6b Mon Sep 17 00:00:00 2001
+From: Andy Hu <andy.hu@starfivetech.com>
+Date: Tue, 16 Aug 2022 19:52:20 +0800
+Subject: [PATCH] wave511: fix building issue
+
+---
+ wave511/wave511/WaveDecode_buildroot.mak     | 46 +++++++++++++++++++-
+ wave511/wave511/sample_v2/helper/vpuhelper.c | 19 +++++---
+ 2 files changed, 59 insertions(+), 6 deletions(-)
+
+diff --git a/wave511/wave511/WaveDecode_buildroot.mak b/wave511/wave511/WaveDecode_buildroot.mak
+index e8b9f06..d2d05f5 100644
+--- a/wave511/wave511/WaveDecode_buildroot.mak
++++ b/wave511/wave511/WaveDecode_buildroot.mak
+@@ -11,6 +11,10 @@ PRODUCT := WAVE511
+ 
+ $(shell cp sample_v2/component_list_decoder.h sample_v2/component/component_list.h)
+ 
++TARGET_DIR=../../target
++STAGING_DIR=$(TARGET_DIR)
++INSTALL=install
++
+ USE_PTHREAD = yes
+ USE_RTL_SIM = no
+ LINT_HOME   = etc/lint
+@@ -57,7 +61,8 @@ ifeq ("$(BUILD_CONFIGURATION)", "EmbeddedLinux")
+ endif
+ 
+ ifeq ("$(BUILD_CONFIGURATION)", "EmbeddedRiscvLinux")
+-    CROSS_CC_PREFIX = riscv64-buildroot-linux-gnu-
++#    CROSS_CC_PREFIX = riscv64-buildroot-linux-gnu-
++    CROSS_CC_PREFIX =
+     PLATFORM        = riscvlinux
+ endif
+ CC  = $(CROSS_CC_PREFIX)gcc
+@@ -166,3 +171,42 @@ obj/%.o: %.c $(MAKEFILE)
+ lint:
+ 	"$(LINT_HOME)/flint" -i"$(LINT_HOME)" $(DEFINES) $(INCLUDES) $(VPATH2) linux_std.lnt $(HAPS_RULE) $(NONOS_RULE)  $(SOURCES_COMMON)
+ 
++install:
++	@echo install ...
++	$(INSTALL) -D -m 0777 vdi/linux/driver/load.sh $(TARGET_DIR)/root/wave511/vdec_load.sh
++	$(INSTALL) -D -m 0777 vdi/linux/driver/unload.sh $(TARGET_DIR)/root/wave511/vdec_unload.sh
++	$(INSTALL) -D -m 0644 libsfdec.so $(TARGET_DIR)/usr/lib/libsfdec.so
++	$(INSTALL) -D -m 0644 ../firmware/chagall.bin $(TARGET_DIR)/lib/firmware/chagall.bin
++	@echo "install STAGING_DIR ..."
++	$(INSTALL) -D -m 0644 sample_v2/component_list_decoder.h                           $(STAGING_DIR)/usr/include/wave511/sample_v2/component_list_decoder.h
++	$(INSTALL) -D -m 0644 sample_v2/component_list_encoder.h                           $(STAGING_DIR)/usr/include/wave511/sample_v2/component_list_encoder.h
++	$(INSTALL) -D -m 0644 sample_v2/helper/misc/pbu.h                                  $(STAGING_DIR)/usr/include/wave511/sample_v2/helper/misc/pbu.h
++	$(INSTALL) -D -m 0644 sample_v2/helper/misc/header_struct.h                        $(STAGING_DIR)/usr/include/wave511/sample_v2/helper/misc/header_struct.h
++	$(INSTALL) -D -m 0644 sample_v2/helper/misc/json_output.h                          $(STAGING_DIR)/usr/include/wave511/sample_v2/helper/misc/json_output.h
++	$(INSTALL) -D -m 0644 sample_v2/helper/misc/debug.h                                $(STAGING_DIR)/usr/include/wave511/sample_v2/helper/misc/debug.h
++	$(INSTALL) -D -m 0644 sample_v2/helper/misc/bw_monitor.h                           $(STAGING_DIR)/usr/include/wave511/sample_v2/helper/misc/bw_monitor.h
++	$(INSTALL) -D -m 0644 sample_v2/helper/main_helper.h                               $(STAGING_DIR)/usr/include/wave511/sample_v2/helper/main_helper.h
++	$(INSTALL) -D -m 0644 sample_v2/component/component.h                              $(STAGING_DIR)/usr/include/wave511/sample_v2/component/component.h
++	$(INSTALL) -D -m 0644 sample_v2/component/cnm_app_internal.h                       $(STAGING_DIR)/usr/include/wave511/sample_v2/component/cnm_app_internal.h
++	$(INSTALL) -D -m 0644 sample_v2/component/cnm_app.h                                $(STAGING_DIR)/usr/include/wave511/sample_v2/component/cnm_app.h
++	$(INSTALL) -D -m 0644 sample_v2/component/component_list.h                         $(STAGING_DIR)/usr/include/wave511/sample_v2/component/component_list.h
++	$(INSTALL) -D -m 0644 sample_v2/component_list_all.h                               $(STAGING_DIR)/usr/include/wave511/sample_v2/component_list_all.h
++	$(INSTALL) -D -m 0644 sample_v2/component_encoder/encoder_listener.h               $(STAGING_DIR)/usr/include/wave511/sample_v2/component_encoder/encoder_listener.h
++	$(INSTALL) -D -m 0644 sample_v2/component_decoder/decoder_listener.h               $(STAGING_DIR)/usr/include/wave511/sample_v2/component_decoder/decoder_listener.h
++	$(INSTALL) -D -m 0644 vdi/vdi_osal.h                                               $(STAGING_DIR)/usr/include/wave511/vdi/vdi_osal.h
++	$(INSTALL) -D -m 0644 vdi/linux/driver/vmm.h                                       $(STAGING_DIR)/usr/include/wave511/vdi/linux/driver/vmm.h
++	$(INSTALL) -D -m 0644 vdi/linux/driver/vpu.h                                       $(STAGING_DIR)/usr/include/wave511/vdi/linux/driver/vpu.h
++	$(INSTALL) -D -m 0644 vdi/mm.h                                                     $(STAGING_DIR)/usr/include/wave511/vdi/mm.h
++	$(INSTALL) -D -m 0644 vdi/vdi.h                                                    $(STAGING_DIR)/usr/include/wave511/vdi/vdi.h
++	$(INSTALL) -D -m 0644 config.h                                                     $(STAGING_DIR)/usr/include/wave511/config.h
++	$(INSTALL) -D -m 0644 vpuapi/product.h                                             $(STAGING_DIR)/usr/include/wave511/vpuapi/product.h
++	$(INSTALL) -D -m 0644 vpuapi/coda9/coda9_vpuconfig.h                               $(STAGING_DIR)/usr/include/wave511/vpuapi/coda9/coda9_vpuconfig.h
++	$(INSTALL) -D -m 0644 vpuapi/coda9/coda9.h                                         $(STAGING_DIR)/usr/include/wave511/vpuapi/coda9/coda9.h
++	$(INSTALL) -D -m 0644 vpuapi/coda9/coda9_regdefine.h                               $(STAGING_DIR)/usr/include/wave511/vpuapi/coda9/coda9_regdefine.h
++	$(INSTALL) -D -m 0644 vpuapi/vpuapi.h                                              $(STAGING_DIR)/usr/include/wave511/vpuapi/vpuapi.h
++	$(INSTALL) -D -m 0644 vpuapi/vpuconfig.h                                           $(STAGING_DIR)/usr/include/wave511/vpuapi/vpuconfig.h
++	$(INSTALL) -D -m 0644 vpuapi/wave/wave5.h                                          $(STAGING_DIR)/usr/include/wave511/vpuapi/wave/wave5.h
++	$(INSTALL) -D -m 0644 vpuapi/wave/wave5_regdefine.h                                $(STAGING_DIR)/usr/include/wave511/vpuapi/wave/wave5_regdefine.h
++	$(INSTALL) -D -m 0644 vpuapi/vpuerror.h                                            $(STAGING_DIR)/usr/include/wave511/vpuapi/vpuerror.h
++	$(INSTALL) -D -m 0644 vpuapi/vputypes.h                                            $(STAGING_DIR)/usr/include/wave511/vpuapi/vputypes.h
++	$(INSTALL) -D -m 0644 vpuapi/vpuapifunc.h                                          $(STAGING_DIR)/usr/include/wave511/vpuapi/vpuapifunc.h
+diff --git a/wave511/wave511/sample_v2/helper/vpuhelper.c b/wave511/wave511/sample_v2/helper/vpuhelper.c
+index d655562..8c8e888 100644
+--- a/wave511/wave511/sample_v2/helper/vpuhelper.c
++++ b/wave511/wave511/sample_v2/helper/vpuhelper.c
+@@ -1418,9 +1418,12 @@ Int32 GetEncOpenParam(EncOpenParam *pEncOP, TestEncConfig *pEncConfig, ENC_CFG *
+                 return 0;
+             if (pEncCfg)
+                 strcpy(pEncConfig->yuvFileName, pCfg->SrcFileName);
+-            else
+-                sprintf(pEncConfig->yuvFileName,  "%s%s", yuvDir, pCfg->SrcFileName);
+-
++            else {
++                if (snprintf(pEncConfig->yuvFileName, sizeof(pEncConfig->yuvFileName), "%s%s", yuvDir, pCfg->SrcFileName) >= 
++                        sizeof(pEncConfig->yuvFileName)) {
++                    fprintf(stderr, "yuvFileName overflow -- string is truncated\n");
++                }
++            }
+             if (pCfg->waveCfg.roiEnable) {
+                 strcpy(pEncConfig->roi_file_name, pCfg->waveCfg.roiFileName);
+                 if (!strcmp(pCfg->waveCfg.roiQpMapFile, "0") || pCfg->waveCfg.roiQpMapFile[0] == 0) {
+@@ -1487,7 +1490,10 @@ Int32 GetEncOpenParam(EncOpenParam *pEncOP, TestEncConfig *pEncConfig, ENC_CFG *
+             if (parseAvcCfgFile(pCfg, pEncConfig->cfgFileName) == 0)
+                 return 0;
+             pEncConfig->picQpY = pCfg->PicQpY;	
+-            sprintf(pEncConfig->yuvFileName,  "%s/%s", yuvDir, pCfg->SrcFileName);
++            if (snprintf(pEncConfig->yuvFileName, sizeof(pEncConfig->yuvFileName), "%s/%s", yuvDir, pCfg->SrcFileName) >= 
++                    sizeof(pEncConfig->yuvFileName)) {
++                fprintf(stderr, "yuvFileName overflow -- string is truncated\n");
++            }
+ 
+             if(pCfg->frameCropLeft || pCfg->frameCropRight || pCfg->frameCropTop || pCfg->frameCropBottom)
+                 pCfg->frameCroppingFlag = 1;
+@@ -1497,7 +1503,10 @@ Int32 GetEncOpenParam(EncOpenParam *pEncOP, TestEncConfig *pEncConfig, ENC_CFG *
+             if (parseMp4CfgFile(pCfg, pEncConfig->cfgFileName) == 0)
+                 return 0;
+             pEncConfig->picQpY = pCfg->VopQuant;
+-            sprintf(pEncConfig->yuvFileName,  "%s/%s", yuvDir, pCfg->SrcFileName);
++            if (snprintf(pEncConfig->yuvFileName, sizeof(pEncConfig->yuvFileName), "%s/%s", yuvDir, pCfg->SrcFileName) >= 
++                    sizeof(pEncConfig->yuvFileName)) {
++                fprintf(stderr, "yuvFileName overflow -- string is truncated\n");
++            }
+             if (pCfg->ShortVideoHeader == 1) {
+                 pEncOP->bitstreamFormat = STD_H263;
+                 bitFormat = STD_H263;
+-- 
+2.36.0
+

+ 124 - 0
multimedia/patch/mm_libs/0002-wave420l-fix-building-issue.patch

@@ -0,0 +1,124 @@
+From 0131ee14620181c35ce8ad775fb289fcb4cf0a38 Mon Sep 17 00:00:00 2001
+From: Andy Hu <andy.hu@starfivetech.com>
+Date: Tue, 16 Aug 2022 21:52:55 +0800
+Subject: [PATCH] wave420l: fix building issue
+
+---
+ wave420l/code/WaveEncoder_buildroot.mak | 47 ++++++++++++++++++++++++-
+ wave420l/code/sample/helper/vpuhelper.c | 24 +++++++++----
+ 2 files changed, 64 insertions(+), 7 deletions(-)
+
+diff --git a/wave420l/code/WaveEncoder_buildroot.mak b/wave420l/code/WaveEncoder_buildroot.mak
+index a6ea84d..1f6b938 100755
+--- a/wave420l/code/WaveEncoder_buildroot.mak
++++ b/wave420l/code/WaveEncoder_buildroot.mak
+@@ -43,7 +43,8 @@ ifeq ("$(BUILD_CONFIGURATION)", "EmbeddedLinux")
+ endif
+ 
+ ifeq ("$(BUILD_CONFIGURATION)", "RiscvLinux")
+-    CROSS_CC_PREFIX = riscv64-buildroot-linux-gnu-
++#    CROSS_CC_PREFIX = riscv64-buildroot-linux-gnu-
++    CROSS_CC_PREFIX =
+     PLATFORM        = riscvlinux
+     USE_FFMPEG  	= no
+     USE_PTHREAD 	= yes
+@@ -273,3 +274,47 @@ endif
+ force_dependency : 
+ 	true
+ 
++TARGET_DIR=../../target
++STAGING_DIR=$(TARGET_DIR)
++INSTALL=install
++
++install:
++	@echo install ...
++	$(INSTALL) -D -m 0777 vdi/linux/driver/load.sh $(TARGET_DIR)/root/wave420l/venc_load.sh
++	$(INSTALL) -D -m 0777 vdi/linux/driver/unload.sh $(TARGET_DIR)/root/wave420l/venc_unload.sh
++	$(INSTALL) -D -m 0644 libsfenc.so $(TARGET_DIR)/usr/lib/libsfenc.so
++	$(INSTALL) -D -m 0644 ../firmware/monet.bin $(TARGET_DIR)/lib/firmware/monet.bin
++	$(INSTALL) -D -m 0644 cfg/encoder_defconfig.cfg $(TARGET_DIR)/lib/firmware/encoder_defconfig.cfg
++	@echo "install STAGING_DIR ..."
++	$(INSTALL) -D -m 0644 vpuapi/vpuconfig.h                                           $(STAGING_DIR)/usr/include/wave420l/vpuapi/vpuconfig.h
++	$(INSTALL) -D -m 0644 vpuapi/product.h                                             $(STAGING_DIR)/usr/include/wave420l/vpuapi/product.h
++	$(INSTALL) -D -m 0644 vpuapi/vputypes.h                                            $(STAGING_DIR)/usr/include/wave420l/vpuapi/vputypes.h
++	$(INSTALL) -D -m 0644 vpuapi/vpuapi.h                                              $(STAGING_DIR)/usr/include/wave420l/vpuapi/vpuapi.h
++	$(INSTALL) -D -m 0644 vpuapi/vpuapifunc.h                                          $(STAGING_DIR)/usr/include/wave420l/vpuapi/vpuapifunc.h
++	$(INSTALL) -D -m 0644 vpuapi/coda9/coda9_vpuconfig.h                               $(STAGING_DIR)/usr/include/wave420l/vpuapi/coda9/coda9_vpuconfig.h
++	$(INSTALL) -D -m 0644 vpuapi/coda9/coda9.h                                         $(STAGING_DIR)/usr/include/wave420l/vpuapi/coda9/coda9.h
++	$(INSTALL) -D -m 0644 vpuapi/coda9/coda9_regdefine.h                               $(STAGING_DIR)/usr/include/wave420l/vpuapi/coda9/coda9_regdefine.h
++	$(INSTALL) -D -m 0644 vpuapi/wave/coda7q/coda7q_regdefine.h                        $(STAGING_DIR)/usr/include/wave420l/vpuapi/wave/coda7q/coda7q_regdefine.h
++	$(INSTALL) -D -m 0644 vpuapi/wave/coda7q/coda7q.h                                  $(STAGING_DIR)/usr/include/wave420l/vpuapi/wave/coda7q/coda7q.h
++	$(INSTALL) -D -m 0644 vpuapi/wave/wave4/wave4.h                                    $(STAGING_DIR)/usr/include/wave420l/vpuapi/wave/wave4/wave4.h
++	$(INSTALL) -D -m 0644 vpuapi/wave/wave4/wave4_regdefine.h                          $(STAGING_DIR)/usr/include/wave420l/vpuapi/wave/wave4/wave4_regdefine.h
++	$(INSTALL) -D -m 0644 vpuapi/wave/wave5/wave5.h                                    $(STAGING_DIR)/usr/include/wave420l/vpuapi/wave/wave5/wave5.h
++	$(INSTALL) -D -m 0644 vpuapi/wave/wave5/wave5_regdefine.h                          $(STAGING_DIR)/usr/include/wave420l/vpuapi/wave/wave5/wave5_regdefine.h
++	$(INSTALL) -D -m 0644 vpuapi/wave/common/common.h                                  $(STAGING_DIR)/usr/include/wave420l/vpuapi/wave/common/common.h
++	$(INSTALL) -D -m 0644 vpuapi/wave/common/common_vpuconfig.h                        $(STAGING_DIR)/usr/include/wave420l/vpuapi/wave/common/common_vpuconfig.h
++	$(INSTALL) -D -m 0644 vpuapi/wave/common/common_regdefine.h                        $(STAGING_DIR)/usr/include/wave420l/vpuapi/wave/common/common_regdefine.h
++	$(INSTALL) -D -m 0644 vpuapi/vpuerror.h                                            $(STAGING_DIR)/usr/include/wave420l/vpuapi/vpuerror.h
++	$(INSTALL) -D -m 0644 sample/helper/misc/pbu.h                                     $(STAGING_DIR)/usr/include/wave420l/sample/helper/misc/pbu.h
++	$(INSTALL) -D -m 0644 sample/helper/misc/skip.h                                    $(STAGING_DIR)/usr/include/wave420l/sample/helper/misc/skip.h
++	$(INSTALL) -D -m 0644 sample/helper/misc/getopt.h                                  $(STAGING_DIR)/usr/include/wave420l/sample/helper/misc/getopt.h
++	$(INSTALL) -D -m 0644 sample/helper/misc/header_struct.h                           $(STAGING_DIR)/usr/include/wave420l/sample/helper/misc/header_struct.h
++	$(INSTALL) -D -m 0644 sample/helper/misc/debug.h                                   $(STAGING_DIR)/usr/include/wave420l/sample/helper/misc/debug.h
++	$(INSTALL) -D -m 0644 sample/helper/msvc/inttypes.h                                $(STAGING_DIR)/usr/include/wave420l/sample/helper/msvc/inttypes.h
++	$(INSTALL) -D -m 0644 sample/helper/msvc/stdint.h                                  $(STAGING_DIR)/usr/include/wave420l/sample/helper/msvc/stdint.h
++	$(INSTALL) -D -m 0644 sample/helper/main_helper.h                                  $(STAGING_DIR)/usr/include/wave420l/sample/helper/main_helper.h
++	$(INSTALL) -D -m 0644 vdi/mm.h                                                     $(STAGING_DIR)/usr/include/wave420l/vdi/mm.h
++	$(INSTALL) -D -m 0644 vdi/linux/driver/vmm.h                                       $(STAGING_DIR)/usr/include/wave420l/vdi/linux/driver/vmm.h
++	$(INSTALL) -D -m 0644 vdi/linux/driver/vpu.h                                       $(STAGING_DIR)/usr/include/wave420l/vdi/linux/driver/vpu.h
++	$(INSTALL) -D -m 0644 vdi/vdi.h                                                    $(STAGING_DIR)/usr/include/wave420l/vdi/vdi.h
++	$(INSTALL) -D -m 0644 vdi/vdi_osal.h                                               $(STAGING_DIR)/usr/include/wave420l/vdi/vdi_osal.h
++	$(INSTALL) -D -m 0644 config.h                                                     $(STAGING_DIR)/usr/include/wave420l/config.h
+diff --git a/wave420l/code/sample/helper/vpuhelper.c b/wave420l/code/sample/helper/vpuhelper.c
+index 711375b..688dbf6 100644
+--- a/wave420l/code/sample/helper/vpuhelper.c
++++ b/wave420l/code/sample/helper/vpuhelper.c
+@@ -1674,8 +1674,12 @@ Int32 GetEncOpenParam(EncOpenParam *pEncOP, TestEncConfig *pEncConfig, ENC_CFG *
+         pEncConfig->picQpY = pCfg->PicQpY;	
+         if (pEncCfg)
+             strcpy(pEncConfig->yuvFileName, pCfg->SrcFileName);			
+-        else
+-            sprintf(pEncConfig->yuvFileName,  "%s%s", yuvDir, pCfg->SrcFileName);		
++        else {
++            if (snprintf(pEncConfig->yuvFileName, sizeof(pEncConfig->yuvFileName), "%s%s", yuvDir, pCfg->SrcFileName) >= 
++                    sizeof(pEncConfig->yuvFileName)) {
++                fprintf(stderr, "yuvFileName overflow -- string is truncated\n");
++            }
++        }
+ 
+ #ifdef SUPPORT_980_RC_LIB
+         if(pEncOP->EncStdParam.avcParam.svcExtension == TRUE)
+@@ -1699,8 +1703,12 @@ Int32 GetEncOpenParam(EncOpenParam *pEncOP, TestEncConfig *pEncConfig, ENC_CFG *
+         pEncConfig->picQpY = pCfg->VopQuant;
+         if (pEncCfg)
+             strcpy(pEncConfig->yuvFileName, pCfg->SrcFileName);			
+-        else
+-            sprintf(pEncConfig->yuvFileName,  "%s%s", yuvDir, pCfg->SrcFileName);
++        else {
++            if (snprintf(pEncConfig->yuvFileName, sizeof(pEncConfig->yuvFileName), "%s%s", yuvDir, pCfg->SrcFileName) >= 
++                    sizeof(pEncConfig->yuvFileName)) {
++                fprintf(stderr, "yuvFileName overflow -- string is truncated\n");
++            }
++        }
+         if (pCfg->ShortVideoHeader == 1) {
+             pEncOP->bitstreamFormat = STD_H263;
+             bitFormat = STD_H263;
+@@ -1711,8 +1719,12 @@ Int32 GetEncOpenParam(EncOpenParam *pEncOP, TestEncConfig *pEncConfig, ENC_CFG *
+             return 0;
+         if (pEncCfg)
+             strcpy(pEncConfig->yuvFileName, pCfg->SrcFileName);			
+-        else
+-            sprintf(pEncConfig->yuvFileName,  "%s%s", yuvDir, pCfg->SrcFileName);
++        else {
++            if (snprintf(pEncConfig->yuvFileName, sizeof(pEncConfig->yuvFileName), "%s%s", yuvDir, pCfg->SrcFileName) >= 
++                    sizeof(pEncConfig->yuvFileName)) {
++                fprintf(stderr, "yuvFileName overflow -- string is truncated\n");
++            }
++        }
+         if (pEncConfig->bitstreamFileName[0] == 0 && pCfg->BitStreamFileName[0] != 0)
+             sprintf(pEncConfig->bitstreamFileName, "%s", pCfg->BitStreamFileName);
+ 
+-- 
+2.36.0
+

+ 58 - 0
multimedia/patch/mm_libs/0003-codaj12-fix-building-issue.patch

@@ -0,0 +1,58 @@
+From 103b02bd32ed65174b16af02215807d8d0e133a6 Mon Sep 17 00:00:00 2001
+From: Andy Hu <andy.hu@starfivetech.com>
+Date: Tue, 16 Aug 2022 21:58:00 +0800
+Subject: [PATCH] codaj12: fix building issue
+
+---
+ codaj12/codaj12_buildroot.mak | 31 ++++++++++++++++++++++++++++++-
+ 1 file changed, 30 insertions(+), 1 deletion(-)
+
+diff --git a/codaj12/codaj12_buildroot.mak b/codaj12/codaj12_buildroot.mak
+index bffe02d..6833f09 100644
+--- a/codaj12/codaj12_buildroot.mak
++++ b/codaj12/codaj12_buildroot.mak
+@@ -53,7 +53,8 @@ ifeq ("$(BUILD_CONFIGURATION)", "EmbeddedLinux")
+ endif
+ 
+ ifeq ("$(BUILD_CONFIGURATION)", "EmbeddedRiscvLinux")
+-    CROSS_CC_PREFIX = riscv64-buildroot-linux-gnu-
++#    CROSS_CC_PREFIX = riscv64-buildroot-linux-gnu-
++    CROSS_CC_PREFIX =
+     PLATFORM        = riscvlinux
+ endif
+ CC  = $(CROSS_CC_PREFIX)gcc
+@@ -146,3 +147,31 @@ obj/%.o: %.c $(MAKEFILE)
+ lint:
+ 	"$(LINT_HOME)/flint" -i"$(LINT_HOME)" $(DEFINES) $(INCLUDES) $(VPATH2) linux_std.lnt $(HAPS_RULE) $(NONOS_RULE)  $(SOURCES_COMMON)
+ 
++TARGET_DIR=../target
++STAGING_DIR=$(TARGET_DIR)
++INSTALL=install
++
++install:
++	@echo install ...
++	$(INSTALL) -D -m 0777 jdi/linux/driver/load.sh $(TARGET_DIR)/root/codaj12/load.sh
++	$(INSTALL) -D -m 0777 jdi/linux/driver/unload.sh $(TARGET_DIR)/root/codaj12/unload.sh
++	$(INSTALL) -D -m 0644 libcodadec.so $(TARGET_DIR)/usr/lib/libcodadec.so
++	@echo "install STAGING_DIR ..."
++	$(INSTALL) -D -m 0644 jpuapi/jpuapi.h                  $(STAGING_DIR)/usr/include/codaj12/jpuapi/jpuapi.h
++	$(INSTALL) -D -m 0644 jpuapi/jpuapifunc.h              $(STAGING_DIR)/usr/include/codaj12/jpuapi/jpuapifunc.h
++	$(INSTALL) -D -m 0644 jpuapi/regdefine.h               $(STAGING_DIR)/usr/include/codaj12/jpuapi/regdefine.h
++	$(INSTALL) -D -m 0644 jpuapi/jpuconfig.h               $(STAGING_DIR)/usr/include/codaj12/jpuapi/jpuconfig.h
++	$(INSTALL) -D -m 0644 jpuapi/jputypes.h                $(STAGING_DIR)/usr/include/codaj12/jpuapi/jputypes.h
++	$(INSTALL) -D -m 0644 jpuapi/jputable.h                $(STAGING_DIR)/usr/include/codaj12/jpuapi/jputable.h
++	$(INSTALL) -D -m 0644 sample/helper/cnm_fpga.h         $(STAGING_DIR)/usr/include/codaj12/sample/helper/cnm_fpga.h
++	$(INSTALL) -D -m 0644 sample/helper/platform.h         $(STAGING_DIR)/usr/include/codaj12/sample/helper/platform.h
++	$(INSTALL) -D -m 0644 sample/helper/yuv_feeder.h       $(STAGING_DIR)/usr/include/codaj12/sample/helper/yuv_feeder.h
++	$(INSTALL) -D -m 0644 sample/helper/datastructure.h    $(STAGING_DIR)/usr/include/codaj12/sample/helper/datastructure.h
++	$(INSTALL) -D -m 0644 sample/helper/jpulog.h           $(STAGING_DIR)/usr/include/codaj12/sample/helper/jpulog.h
++	$(INSTALL) -D -m 0644 sample/main_helper.h             $(STAGING_DIR)/usr/include/codaj12/sample/main_helper.h
++	$(INSTALL) -D -m 0644 jdi/linux/driver/jpu.h           $(STAGING_DIR)/usr/include/codaj12/jdi/linux/driver/jpu.h
++	$(INSTALL) -D -m 0644 jdi/linux/driver/jmm.h           $(STAGING_DIR)/usr/include/codaj12/jdi/linux/driver/jmm.h
++	$(INSTALL) -D -m 0644 jdi/jdi.h                        $(STAGING_DIR)/usr/include/codaj12/jdi/jdi.h
++	$(INSTALL) -D -m 0644 jdi/mm.h                         $(STAGING_DIR)/usr/include/codaj12/jdi/mm.h
++	$(INSTALL) -D -m 0644 config.h                         $(STAGING_DIR)/usr/include/codaj12/config.h
++
+-- 
+2.36.0
+

+ 88 - 0
multimedia/patch/mm_libs/0004-omx-il-fix-building-issue.patch

@@ -0,0 +1,88 @@
+From 14143fcf2c4561905c11cae22314bb5dfdbe171c Mon Sep 17 00:00:00 2001
+From: Andy Hu <andy.hu@starfivetech.com>
+Date: Wed, 17 Aug 2022 11:14:38 +0800
+Subject: [PATCH] omx-il: fix building issue
+
+---
+ omx-il/Makefile | 29 ++++++++++++++++++++++-------
+ 1 file changed, 22 insertions(+), 7 deletions(-)
+
+diff --git a/omx-il/Makefile b/omx-il/Makefile
+index 150c4d5..a51de48 100644
+--- a/omx-il/Makefile
++++ b/omx-il/Makefile
+@@ -10,7 +10,8 @@ BUILD_CONFIGURATION := EmbeddedRiscvLinux
+ PRODUCT := OMX_IL
+ 
+ ifeq ("$(BUILD_CONFIGURATION)", "EmbeddedRiscvLinux")
+-    CROSS_CC_PREFIX = riscv64-buildroot-linux-gnu-
++#    CROSS_CC_PREFIX = riscv64-buildroot-linux-gnu-
++    CROSS_CC_PREFIX =
+     PLATFORM        = riscvlinux
+ endif
+ CC  = $(CROSS_CC_PREFIX)gcc
+@@ -18,9 +19,13 @@ CXX = $(CROSS_CC_PREFIX)g++
+ LINKER=$(CC)
+ AR  = $(CROSS_CC_PREFIX)ar
+ 
++TARGET_DIR=../target
++STAGING_DIR=$(TARGET_DIR)
++INSTALL=install
++
+ DEFINES = -DUSE_FEEDING_METHOD_BUFFER
+ INCLUDES = -I./include/khronos -I./core -I./component/video/wave5/common -I./component/video/wave4 -I./component/helper
+-$(warning "the value of LOCAL_PATH is$(STAGING_DIR)")
++$(warning "the value of LOCAL_PATH is $(STAGING_DIR)")
+ INCLUDES += -I$(STAGING_DIR)/usr/include/
+ INCLUDES += -I$(STAGING_DIR)/usr/include/wave420l/
+ INCLUDES += -I$(STAGING_DIR)/usr/include/wave420l/vpuapi/
+@@ -29,12 +34,13 @@ INCLUDES += -I$(STAGING_DIR)/usr/include/wave420l/sample/helper/
+ 
+ #mjpeg
+ INCLUDES += -I./component/image/common
++INCLUDES += -I../out/include
+ 
+ CFLAGS  += -g -I. $(INCLUDES) $(DEFINES) $(PLATFORM_FLAGS)
+ CFLAGS  += -fpic
+ ARFLAGS += cru
+-LDFLAGS = -ldl -lpthread -Wl,--fatal-warning
+-LDFLAGS_FFMPEG = -lavformat -lavcodec -lavutil -lswresample
++LDFLAGS = -L./  -ldl -lpthread -Wl,--fatal-warning
++LDFLAGS_FFMPEG = -L../out/lib -lavformat -lavcodec -lavutil -lswresample 
+ 
+ OBJDIR=obj
+ ALLOBJS=*.o
+@@ -86,18 +92,27 @@ omx-il: CREATE_DIR $(OBJECTPATHS_COMMON)
+ test: video-dec-test video-enc-test mjpeg-dec-test
+ 
+ video-dec-test: CREATE_DIR $(OBJECTPATHS_DEC_TEST)
+-	$(CC) -o video_dec_test $(LDFLAGS) $(LDFLAGS_FFMPEG) -lsf-omx-il $(OBJECTPATHS_DEC_TEST) -L./
++	$(CC) -o video_dec_test $(OBJECTPATHS_DEC_TEST) $(LDFLAGS) $(LDFLAGS_FFMPEG) -lsf-omx-il 
+ 
+ video-enc-test: CREATE_DIR $(OBJECTPATHS_ENC_TEST)
+-	$(CC) -o video_enc_test $(LDFLAGS) $(LDFLAGS_FFMPEG) -lsf-omx-il $(OBJECTPATHS_ENC_TEST) -L./
++	$(CC) -o video_enc_test $(OBJECTPATHS_ENC_TEST) -L../out/lib -L./ -lsf-omx-il $(LDFLAGS_FFMPEG) $(LDFLAGS)
+ 
+ mjpeg-dec-test: CREATE_DIR $(OBJECTPATHS_MJPEG_DEC_TEST)
+-	$(CC) -g -o mjpeg_dec_test $(LDFLAGS) $(LDFLAGS_FFMPEG) -lsf-omx-il $(OBJECTPATHS_MJPEG_DEC_TEST) -L./
++	$(CC) -g -o mjpeg_dec_test $(OBJECTPATHS_MJPEG_DEC_TEST) $(LDFLAGS) $(LDFLAGS_FFMPEG) -lsf-omx-il -L./
+ 
+ clean:
++	rm -rf obj/
++	rm -rf libsf-omx-il.so video-dec-test video-enc-test mjpeg-dec-test
+ 
+ CREATE_DIR:
+ 	-mkdir -p $(OBJDIR)
+ 
+ obj/%.o: %.c $(MAKEFILE)
+ 	$(CC) $(CFLAGS) -Wall -Werror -c $< -o $@ -MD -MF $(@:.o=.dep)
++
++install:
++	@echo "install omx-il to STAGING_DIR ..."
++	$(INSTALL) -m 0644 libsf-omx-il.so $(STAGING_DIR)/usr/lib/libsf-omx-il.so
++	ln -sf libsf-omx-il.so $(STAGING_DIR)/usr/lib/libOMX_Core.so
++	$(INSTALL) -d include/khronos $(STAGING_DIR)/usr/include/omx-il
++	$(INSTALL) -m 0644 include/khronos/* $(STAGING_DIR)/usr/include/omx-il
+-- 
+2.36.0
+

+ 41 - 0
multimedia/run_env.sh

@@ -0,0 +1,41 @@
+#!/bin/bash
+##################################################################
+##                                                              ##
+##      SPDX-License-Identifier: GPL-2.0-or-later               ##
+##                                                              ##
+##      Copyright (C) 2018-2022   Starfive Technology           ##
+##                                                              ##
+##################################################################
+
+CURDIR=$(cd `dirname $0`; pwd)
+export LD_LIBRARY_PATH=$CURDIR/target/usr/lib/
+
+## load wave511 vdec.ko
+pushd $CURDIR/target/root/wave511
+./vdec_load.sh
+popd
+
+## load wave420l venc.ko
+pushd $CURDIR/target/root/wave420l
+./venc_load.sh
+popd
+
+## load codaj12 jpu.ko
+pushd $CURDIR/target/root/codaj12
+./load.sh
+popd
+
+## e.g.: ffmpeg use h264_omx to hard decode 264 file and output to yuv420p file, use 7yuv tool to check
+#$CURDIR/target/usr/bin/ffmpeg  -vcodec h264_omx -i Sintel_720_10s_10MB.mp4 -pix_fmt yuv420p 720p_i420.yuv
+
+## e.g. ffplay use h264_omx to hard decode and play video on SDL2.
+## Note: this not good now, will fix it asap
+#$CURDIR/target/usr/bin/ffplay  -vcodec h264_omx -i Sintel_720_10s_10MB.mp4
+
+## e.g. gstreamer decode + display
+# gst-launch-1.0 filesrc location=Audio_Video_Sync_Test_1_1920x1080.mp4 ! qtdemux name=demux demux.video_0 ! queue ! h264parse ! omxh264dec ! xvimagesink
+# gst-launch-1.0 filesrc location=4K_30FPS_AVC_MainL5_2.h265 ! h265parse ! omxh265dec ! videoconvert ! videoscale ! xvimagesink
+# gst-launch-1.0 filesrc location=youtube001_1080p_h264_aac.mp4 ! qtdemux name=demux demux.video_0 ! queue ! h264parse ! omxh264dec ! xvimagesink
+# gst-launch-1.0 filesrc location=2k.18fps.mp4 ! qtdemux name=demux demux.video_0 ! queue ! h265parse ! omxh265dec ! videoconvert ! videoscale ! xvimagesink
+# gst-launch-1.0 filesrc location=youtube002_1080p_h264_aac.mp4 ! qtdemux name=demux demux.video_0 ! queue ! h264parse ! omxh264dec ! xvimagesink
+# gst-launch-1.0 filesrc location=file_example_MP4_640_3MG.mp4 ! qtdemux name=demux demux.video_0 ! queue ! h264parse ! omxh264dec ! xvimagesink

BIN
multimedia/target.tar.xz