Browse Source

libcamera: add libcamera library

1. update libcamera version
2. add starfive pipeline handler
3. add starfive ipa
4. add list sensor controls and properties
5. add config media pipeline through yaml file, path is /data/sensors_pipeline.yaml

Signed-off-by: sw.multimedia <sw.multimedia@starfivetech.com>
Signed-off-by: andy.hu <andy.hu@starfivetech.com>
andy.hu 2 years ago
parent
commit
45de93d856

+ 1109 - 0
package/libcamera/0001-add-starfive-pipeline.patch

@@ -0,0 +1,1109 @@
+From 0a7af884c40080461ac59cf6eda86b8d1da035fd Mon Sep 17 00:00:00 2001
+From: sw.multimedia <sw.multimedia@starfivetech.com>
+Date: Sat, 9 Oct 2021 18:09:44 +0800
+Subject: [PATCH] add starfive pipeline
+
+---
+ include/libcamera/internal/v4l2_subdevice.h  |   1 +
+ include/linux/v4l2-subdev.h                  |   9 +
+ meson_options.txt                            |   2 +-
+ src/gstreamer/gstlibcamera-utils.cpp         |   7 +
+ src/gstreamer/gstlibcamerasrc.cpp            |  16 +-
+ src/libcamera/pipeline/starfive/meson.build  |   5 +
+ src/libcamera/pipeline/starfive/starfive.cpp | 940 +++++++++++++++++++
+ src/libcamera/v4l2_subdevice.cpp             |  12 +
+ 8 files changed, 990 insertions(+), 2 deletions(-)
+ create mode 100644 src/libcamera/pipeline/starfive/meson.build
+ create mode 100644 src/libcamera/pipeline/starfive/starfive.cpp
+
+diff --git a/include/libcamera/internal/v4l2_subdevice.h b/include/libcamera/internal/v4l2_subdevice.h
+index 97b89fb9..fe8835bd 100644
+--- a/include/libcamera/internal/v4l2_subdevice.h
++++ b/include/libcamera/internal/v4l2_subdevice.h
+@@ -63,6 +63,7 @@ public:
+ 
+ 	static std::unique_ptr<V4L2Subdevice>
+ 	fromEntityName(const MediaDevice *media, const std::string &entity);
++	int ioctlPrivate(unsigned long request, void *argp);
+ 
+ protected:
+ 	std::string logPrefix() const override;
+diff --git a/include/linux/v4l2-subdev.h b/include/linux/v4l2-subdev.h
+index a38454d9..5f127523 100644
+--- a/include/linux/v4l2-subdev.h
++++ b/include/linux/v4l2-subdev.h
+@@ -206,4 +206,13 @@ struct v4l2_subdev_capability {
+ #define VIDIOC_SUBDEV_QUERY_DV_TIMINGS		_IOR('V', 99, struct v4l2_dv_timings)
+ #define VIDIOC_SUBDEV_DV_TIMINGS_CAP		_IOWR('V', 100, struct v4l2_dv_timings_cap)
+ 
++#define STF_ISPFW_FILENAME_MAX_LEN  30
++
++struct stfisp_fw_info {
++        char filename[STF_ISPFW_FILENAME_MAX_LEN];
++};
++
++#define VIDIOC_STFISP_LOAD_FW \
++        _IOW('V', BASE_VIDIOC_PRIVATE + 1, struct stfisp_fw_info)
++
+ #endif
+diff --git a/meson_options.txt b/meson_options.txt
+index 2c80ad8b..14baa7ef 100644
+--- a/meson_options.txt
++++ b/meson_options.txt
+@@ -37,7 +37,7 @@ option('lc-compliance',
+ 
+ option('pipelines',
+         type : 'array',
+-        choices : ['ipu3', 'raspberrypi', 'rkisp1', 'simple', 'uvcvideo', 'vimc'],
++        choices : ['ipu3', 'raspberrypi', 'rkisp1', 'simple', 'uvcvideo', 'vimc', 'starfive'],
+         description : 'Select which pipeline handlers to include')
+ 
+ option('qcam',
+diff --git a/src/gstreamer/gstlibcamera-utils.cpp b/src/gstreamer/gstlibcamera-utils.cpp
+index 0af91c1a..b40b78ae 100644
+--- a/src/gstreamer/gstlibcamera-utils.cpp
++++ b/src/gstreamer/gstlibcamera-utils.cpp
+@@ -126,6 +126,12 @@ gst_libcamera_stream_configuration_to_caps(const StreamConfiguration &stream_cfg
+ 			  "width", G_TYPE_INT, stream_cfg.size.width,
+ 			  "height", G_TYPE_INT, stream_cfg.size.height,
+ 			  nullptr);
++
++	// Add framerate negotiation support
++	// the range will be [ 0/1, 2147483647/1 ] as there is not any args 
++	// required from driver for the time being.
++	gst_structure_set(s, "framerate", GST_TYPE_FRACTION_RANGE, 0, 1, G_MAXINT, 1, NULL);
++
+ 	gst_caps_append_structure(caps, s);
+ 
+ 	return caps;
+@@ -217,6 +223,7 @@ gst_libcamera_resume_task(GstTask *task)
+ 	/* We only want to resume the task if it's paused. */
+ 	GLibLocker lock(GST_OBJECT(task));
+ 	if (GST_TASK_STATE(task) == GST_TASK_PAUSED) {
++		GST_DEBUG("gst_libcamera_resume_task");
+ 		GST_TASK_STATE(task) = GST_TASK_STARTED;
+ 		GST_TASK_SIGNAL(task);
+ 	}
+diff --git a/src/gstreamer/gstlibcamerasrc.cpp b/src/gstreamer/gstlibcamerasrc.cpp
+index 2be44edd..fab2aa2a 100644
+--- a/src/gstreamer/gstlibcamerasrc.cpp
++++ b/src/gstreamer/gstlibcamerasrc.cpp
+@@ -46,6 +46,7 @@ using namespace libcamera;
+ 
+ GST_DEBUG_CATEGORY_STATIC(source_debug);
+ #define GST_CAT_DEFAULT source_debug
++#define TASK_PAUSE_COUNT_MAX 2
+ 
+ struct RequestWrap {
+ 	RequestWrap(std::unique_ptr<Request> request);
+@@ -125,6 +126,7 @@ struct _GstLibcameraSrc {
+ 	GstLibcameraSrcState *state;
+ 	GstLibcameraAllocator *allocator;
+ 	GstFlowCombiner *flow_combiner;
++	guint task_pause_count;
+ };
+ 
+ enum {
+@@ -340,8 +342,20 @@ gst_libcamera_src_task_run(gpointer user_data)
+ 			}
+ 		}
+ 
+-		if (do_pause)
++		if (do_pause){
++			self->task_pause_count ++;
++	        GST_DEBUG_OBJECT(self, "task_pause_count: %d'", self->task_pause_count);
++		} else {
++			self->task_pause_count = 0;
++	        GST_DEBUG_OBJECT(self, "reset task_pause_count ");
++		}
++
++		if ( self->task_pause_count > TASK_PAUSE_COUNT_MAX)
++		{
++	        GST_DEBUG_OBJECT(self, "gst_task_pause");
+ 			gst_task_pause(self->task);
++		}
++
+ 	}
+ }
+ 
+diff --git a/src/libcamera/pipeline/starfive/meson.build b/src/libcamera/pipeline/starfive/meson.build
+new file mode 100644
+index 00000000..2f2d2a75
+--- /dev/null
++++ b/src/libcamera/pipeline/starfive/meson.build
+@@ -0,0 +1,5 @@
++# SPDX-License-Identifier: CC0-1.0
++
++libcamera_sources += files([
++    'starfive.cpp',
++])
+diff --git a/src/libcamera/pipeline/starfive/starfive.cpp b/src/libcamera/pipeline/starfive/starfive.cpp
+new file mode 100644
+index 00000000..3517679d
+--- /dev/null
++++ b/src/libcamera/pipeline/starfive/starfive.cpp
+@@ -0,0 +1,940 @@
++/* SPDX-License-Identifier: LGPL-2.1-or-later */
++/*
++ * starfive.cpp - Pipeline handler for starfive devices
++ */
++#include <algorithm>
++#include <assert.h>
++#include <fcntl.h>
++#include <mutex>
++#include <queue>
++#include <sys/mman.h>
++#include <math.h>
++
++#include <libcamera/camera.h>
++#include <libcamera/control_ids.h>
++#include <libcamera/file_descriptor.h>
++#include <libcamera/formats.h>
++#include <libcamera/logging.h>
++#include <libcamera/property_ids.h>
++#include <libcamera/request.h>
++#include <libcamera/stream.h>
++#include <linux/media-bus-format.h>
++
++#include <linux/videodev2.h>
++#include <linux/v4l2-subdev.h>
++
++#include "libcamera/internal/camera.h"
++#include "libcamera/internal/camera_sensor.h"
++#include "libcamera/internal/device_enumerator.h"
++#include "libcamera/internal/ipa_manager.h"
++#include "libcamera/internal/media_device.h"
++#include "libcamera/internal/pipeline_handler.h"
++// #include "libcamera/internal/utils.h"
++// #include "libcamera/internal/v4l2_controls.h"
++#include "libcamera/internal/v4l2_videodevice.h"
++
++#define STF_MAX_CAMERAS   3
++
++namespace libcamera {
++
++LOG_DEFINE_CATEGORY(STARFIVE)
++
++static constexpr unsigned int BUFFER_COUNT = 4;
++static constexpr unsigned int MAX_STREAMS = 2;
++static const Size OUTPUT_MIN_SIZE = { 8, 8 };
++static const Size OUTPUT_MAX_SIZE = { 8192, 8192 };
++
++namespace {
++
++typedef enum {
++	DVP_YUV = 0,
++	MIPICSI0_YUV,
++	MIPICSI1_YUV,
++	DVP_ISP0,    // ISP0
++	MIPICSI0_ISP0,
++	MIPICSI1_ISP0,
++	DVP_ISP1,    // ISP1
++	MIPICSI0_ISP1,
++	MIPICSI1_ISP1,
++	SENSORTYPE_MAX
++} SensorType;
++
++typedef struct {
++	std::string sensorEntityName_;
++	std::string sensorFwImageName_;
++	SensorType sensorType_;
++} SensorConfig;
++
++const std::vector<SensorConfig> sensorConfigs = {
++	{ "imx219 0-0010", "stf_isp0_fw_dump.bin", MIPICSI0_ISP0 },
++	{ "imx219 2-0010", "stf_isp0_fw_dump.bin", MIPICSI1_ISP1 },
++};
++
++typedef struct {
++	std::string source;
++	std::string link;
++} PipelineConfigLink;
++
++const std::vector<PipelineConfigLink> dvpyuvConfig = {
++	{"stf_dvp0", "stf_vin0_wr"},
++};
++
++const std::vector<PipelineConfigLink> mipicsi0yuvConfig = {
++	{"stf_csiphy0", "stf_csi0"},
++	{"stf_csi0", "stf_vin0_wr"}
++};
++
++const std::vector<PipelineConfigLink> mipicsi1yuvConfig = {
++	{"stf_csiphy1", "stf_csi1"},
++	{"stf_csi1", "stf_vin0_wr"}
++};
++
++const std::vector<PipelineConfigLink> dvpraw0Config = {
++	{"stf_dvp0", "stf_isp0"},
++	{"stf_isp0", "stf_vin0_isp0"}
++};
++
++const std::vector<PipelineConfigLink> mipicsi0raw0Config = {
++	{"stf_csiphy0", "stf_csi0"},
++	{"stf_csi0", "stf_isp0"},
++	{"stf_isp0", "stf_vin0_isp0"}
++};
++
++const std::vector<PipelineConfigLink> mipicsi1raw0Config = {
++	{"stf_csiphy1", "stf_csi1"},
++	{"stf_csi1", "stf_isp0"},
++	{"stf_isp0", "stf_vin0_isp0"}
++};
++
++const std::vector<PipelineConfigLink> dvpraw1Config = {
++	{"stf_dvp0", "stf_isp1"},
++	{"stf_isp1", "stf_vin0_isp1"}
++};
++
++const std::vector<PipelineConfigLink> mipicsi0raw1Config = {
++	{"stf_csiphy0", "stf_csi0"},
++	{"stf_csi0", "stf_isp1"},
++	{"stf_isp1", "stf_vin0_isp1"}
++};
++
++const std::vector<PipelineConfigLink> mipicsi1raw1Config = {
++	{"stf_csiphy1", "stf_csi1"},
++	{"stf_csi1", "stf_isp1"},
++	{"stf_isp1", "stf_vin0_isp1"}
++};
++
++const std::vector<PipelineConfigLink> pipelineConfigs[SENSORTYPE_MAX] = {
++	dvpyuvConfig,
++	mipicsi0yuvConfig,
++	mipicsi1yuvConfig,
++	dvpraw0Config,
++	mipicsi0raw0Config,
++	mipicsi1raw0Config,
++	dvpraw1Config,
++	mipicsi0raw1Config,
++	mipicsi1raw1Config,
++};
++
++const std::map<uint32_t, PixelFormat> mbusCodesToPixelFormat = {
++	{ MEDIA_BUS_FMT_SBGGR10_1X10, formats::SBGGR12 },
++	{ MEDIA_BUS_FMT_SGBRG10_1X10, formats::SGBRG12 },
++	{ MEDIA_BUS_FMT_SGRBG10_1X10, formats::SGRBG12 },
++	{ MEDIA_BUS_FMT_SRGGB10_1X10, formats::SRGGB12 },
++};
++
++} /* namespace */
++
++class StarFiveCameraData : public Camera::Private
++{
++public:
++	StarFiveCameraData(PipelineHandler *pipe, MediaDevice *media,
++			std::string entityName,
++			std::string sensorEntityName)
++		: Camera::Private(pipe), media_(media)
++	{
++		LOG(STARFIVE, Debug) << __func__;
++		sensorEntityName_ = sensorEntityName;
++		videoEntityName_ = entityName;
++		if ( videoEntityName_ == "stf_vin0_isp0_video1")
++			ispEntityName_ = "stf_isp0";
++		else if (videoEntityName_ == "stf_vin0_isp1_video2")
++			ispEntityName_ = "stf_isp1";
++		else
++			ispEntityName_ = "unknow";
++
++		video_ = nullptr;
++		raw_ = nullptr;
++		sensor_ = nullptr;
++		ispSubDev_ = nullptr;
++		haveRaw_ = false;
++		rawActive_= false;
++	}
++
++	~StarFiveCameraData()
++	{
++		LOG(STARFIVE, Debug) << __func__;
++		delete sensor_;
++		delete video_;
++		if (raw_ != nullptr)
++			delete raw_;
++		if (ispSubDev_ != nullptr)
++			delete ispSubDev_;
++	}
++
++	int init();
++	void bufferReady(FrameBuffer *buffer);
++	bool haveRaw() const { return haveRaw_; }
++	bool rawActive() const { return rawActive_; }
++	void setRawActive(bool val) { rawActive_ = val; }
++	std::vector<SizeRange> sensorSizes() const;
++	std::vector<PixelFormat> sensorFormats() const;
++	std::vector<PixelFormat> videoFormats() const;
++
++	MediaDevice *media_;
++	V4L2VideoDevice *video_;
++	V4L2VideoDevice *raw_;
++	V4L2Subdevice *ispSubDev_;
++	CameraSensor *sensor_;
++	Stream outStream_;
++	Stream rawStream_;
++
++private:
++	bool haveRaw_;
++	bool rawActive_;
++	std::string videoEntityName_;
++	std::string sensorEntityName_;
++	std::string ispEntityName_;
++	std::string getRawVideoEntityName()
++	{
++		LOG(STARFIVE, Debug) << __func__;
++		if ( videoEntityName_ == "stf_vin0_isp0_video1")
++			return "stf_vin0_isp0_raw_video3";
++		else if (videoEntityName_ == "stf_vin0_isp1_video2")
++			return "stf_vin0_isp1_raw_video4";
++		else
++			return "unknow";
++	}
++	int ispLoadFW(const char *filename);
++};
++
++std::vector<PixelFormat> StarFiveCameraData::videoFormats() const
++{
++	if (!video_)
++		return {};
++
++	std::vector<PixelFormat> formats;
++	for (auto it : video_->formats()) {
++		formats.push_back(it.first.toPixelFormat());
++	}
++
++	return formats;
++}
++
++std::vector<PixelFormat> StarFiveCameraData::sensorFormats() const
++{
++	if (!sensor_)
++		return {};
++
++	std::vector<PixelFormat> formats;
++	for (unsigned int code : sensor_->mbusCodes()) {
++		auto it = mbusCodesToPixelFormat.find(code);
++		if (it != mbusCodesToPixelFormat.end())
++			formats.push_back(it->second);
++	}
++
++	return formats;
++}
++
++std::vector<SizeRange> StarFiveCameraData::sensorSizes() const
++{
++	if (!sensor_)
++		return {};
++
++	std::vector<SizeRange> sizes;
++	for (const Size &size : sensor_->sizes(sensor_->mbusCodes().at(0)))
++		sizes.emplace_back(size, size);
++
++	return sizes;
++}
++
++int StarFiveCameraData::ispLoadFW(const char *filename)
++{
++	struct stfisp_fw_info fw_info = {0};
++
++	if (!ispSubDev_)
++		return -ENODEV;
++
++	if (filename && (strlen(filename) < STF_ISPFW_FILENAME_MAX_LEN))
++		memcpy(fw_info.filename, filename, strlen(filename) + 1);
++	else
++		return -EINVAL;
++
++	LOG(STARFIVE, Debug)
++		<< "VIDIOC_STFISP_LOAD_FW: " << VIDIOC_STFISP_LOAD_FW
++		<< " filename: " << filename
++		<< " struct size: " << sizeof(struct stfisp_fw_info);
++
++	if (ispSubDev_->ioctlPrivate(VIDIOC_STFISP_LOAD_FW, &fw_info) < 0)
++		LOG(STARFIVE, Error) << "Load ISP fw failed" ;
++
++	return 0;
++}
++
++class StarFiveCameraConfiguration : public CameraConfiguration
++{
++public:
++	StarFiveCameraConfiguration(StarFiveCameraData *data);
++
++	Status validate() override;
++
++private:
++	StarFiveCameraData *data_;
++};
++
++int StarFiveCameraData::init()
++{
++	int ret;
++
++	LOG(STARFIVE, Debug) << __func__;
++	if (sensorEntityName_ != "unknow") {
++		sensor_ =
++		  new CameraSensor(media_->getEntityByName(sensorEntityName_));
++		ret = sensor_->init();
++		if (ret)
++			return ret;
++		LOG(STARFIVE, Debug) << "sensor id: " << sensor_->id();
++	} else {
++		LOG(STARFIVE, Debug) << " Can't find sensorEntityName!";
++		return -ENODEV;
++	}
++
++	if (ispEntityName_ != "unknow") {
++		ispSubDev_ =
++			new V4L2Subdevice(media_->getEntityByName(ispEntityName_));
++		LOG(STARFIVE, Debug) << "ispEntityName: " << ispEntityName_;
++		if (ispSubDev_->open())
++			return -ENODEV;
++
++		for (SensorConfig it : sensorConfigs) {
++			if (it.sensorEntityName_ == sensorEntityName_) {
++				ispLoadFW(it.sensorFwImageName_.c_str());
++				break;
++			}
++		}
++	}
++
++	video_ = new V4L2VideoDevice(media_->getEntityByName(videoEntityName_));
++	LOG(STARFIVE, Debug) << "videoEntityName: " << videoEntityName_;
++	if (video_->open())
++		return -ENODEV;
++
++	video_->bufferReady.connect(this, &StarFiveCameraData::bufferReady);
++	LOG(STARFIVE, Debug) << "driverName: " << video_->driverName();
++
++	std::string rawVideoEntityName = getRawVideoEntityName();
++	if (rawVideoEntityName != "unknow") {
++		raw_ =
++		  new V4L2VideoDevice(media_->getEntityByName(rawVideoEntityName));
++		LOG(STARFIVE, Debug)
++			<< "rawEntityName: " << rawVideoEntityName;
++		if (raw_->open()) {
++			LOG(STARFIVE, Debug) << "No raw data capture!!!";
++			haveRaw_ = false;
++		} else {
++			haveRaw_ = true;
++			raw_->bufferReady.connect(this,
++					&StarFiveCameraData::bufferReady);
++		}
++	}
++	
++	return 0;
++}
++
++class PipelineHandlerStarFive : public PipelineHandler
++{
++public:
++	PipelineHandlerStarFive(CameraManager *manager);
++	
++	CameraConfiguration *generateConfiguration(Camera *camera,
++	const StreamRoles &roles) override;
++	int configure(Camera *camera, CameraConfiguration *config) override;
++	
++	int exportFrameBuffers(Camera *camera, Stream *stream,
++	std::vector<std::unique_ptr<FrameBuffer>> *buffers) override;
++	
++	int start(Camera *camera, const ControlList *controls) override;
++	void stop(Camera *camera) override;
++	
++	int queueRequestDevice(Camera *camera, Request *request) override;
++	
++	bool match(DeviceEnumerator *enumerator) override;
++
++private:
++	int processControls(StarFiveCameraData *data, Request *request);
++
++	StarFiveCameraData *cameraData(Camera *camera)
++	{
++		return static_cast<StarFiveCameraData *>(camera->_d());
++	}
++
++	int registerCameras();
++	std::string getVideoEntityNameById(unsigned int id);
++	std::string findSensorEntityName(std::string entityName);
++	int enableLinks(std::vector<PipelineConfigLink> config);
++
++	MediaDevice *starFiveMediaDev_;
++};
++
++StarFiveCameraConfiguration::StarFiveCameraConfiguration(StarFiveCameraData *data)
++	: CameraConfiguration(), data_(data)
++{
++	LOG(STARFIVE, Debug) << __func__;
++}
++
++CameraConfiguration::Status StarFiveCameraConfiguration::validate()
++{
++	LOG(STARFIVE, Debug) << __func__;
++	Status status = Valid;
++
++	if (config_.empty())
++		return Invalid;
++
++	LOG(STARFIVE, Debug) << "config_.size " << config_.size();
++	/* Cap the number of entries to the available streams. */
++	if (config_.size() > MAX_STREAMS) {
++		config_.resize(MAX_STREAMS);
++		status = Adjusted;
++	}
++
++	for (unsigned int i = 0; i < config_.size(); ++i) {
++		const PixelFormatInfo &info =
++			PixelFormatInfo::info(config_[i].pixelFormat);
++		StreamConfiguration &cfg = config_[i];
++
++		LOG(STARFIVE, Debug)
++			<< "Validating stream: " << config_[i].toString();
++
++		const Size size = cfg.size;
++
++		cfg.size.width = std::max(OUTPUT_MIN_SIZE.width,
++				std::min(OUTPUT_MAX_SIZE.width, cfg.size.width));
++		cfg.size.height = std::max(OUTPUT_MIN_SIZE.height,
++				std::min(OUTPUT_MAX_SIZE.height, cfg.size.height));
++
++		if (cfg.size != size) {
++			LOG(STARFIVE, Debug)
++				<< "Adjusting size to " << cfg.size.toString();
++			status = Adjusted;
++		}
++
++		cfg.bufferCount = BUFFER_COUNT;
++
++		if (info.colourEncoding == PixelFormatInfo::ColourEncodingRAW
++				&& data_->haveRaw()) {
++			V4L2DeviceFormat format = {};
++			format.fourcc =
++				data_->raw_->toV4L2PixelFormat(cfg.pixelFormat);
++			format.size = cfg.size;
++
++			int ret = data_->raw_->tryFormat(&format);
++			if (ret)
++				return Invalid;
++
++			cfg.stride = format.planes[0].bpl;
++			cfg.frameSize = format.planes[0].size;
++
++			cfg.setStream(&data_->rawStream_);
++		} else {
++			V4L2DeviceFormat format = {};
++			format.fourcc =
++				data_->video_->toV4L2PixelFormat(cfg.pixelFormat);
++			format.size = cfg.size;
++
++			int ret = data_->video_->tryFormat(&format);
++			if (ret)
++				return Invalid;
++
++			cfg.stride = format.planes[0].bpl;
++			cfg.frameSize = format.planes[0].size;
++
++			cfg.setStream(&data_->outStream_);
++		}
++	}
++
++	return status;
++}
++
++PipelineHandlerStarFive::PipelineHandlerStarFive(CameraManager *manager)
++	: PipelineHandler(manager)
++{
++	LOG(STARFIVE, Debug) << __func__;
++}
++
++CameraConfiguration *
++PipelineHandlerStarFive::generateConfiguration(Camera *camera,
++					const StreamRoles &roles)
++{
++	LOG(STARFIVE, Debug) << __func__;
++	StarFiveCameraData *data = cameraData(camera);
++	StarFiveCameraConfiguration *config =
++		new StarFiveCameraConfiguration(data);
++
++	if (roles.empty())
++		return config;
++
++	for (const StreamRole role : roles) {
++		std::map<PixelFormat, std::vector<SizeRange>> streamFormats;
++		unsigned int bufferCount;
++		PixelFormat pixelFormat;
++		Size size;
++
++		LOG(STARFIVE, Debug) << "role: " << role;
++		size = data->sensor_->resolution();
++		switch (role) {
++		case StreamRole::StillCapture:
++		case StreamRole::Viewfinder:
++		case StreamRole::VideoRecording:
++			for (const auto &pixelformat : data->videoFormats()) {
++				streamFormats[pixelformat] = data->sensorSizes();
++			}
++			pixelFormat = data->videoFormats().at(0);
++			bufferCount = BUFFER_COUNT;
++
++			break;
++
++		case StreamRole::Raw: {
++			std::vector<unsigned int> mbusCodes =
++				utils::map_keys(mbusCodesToPixelFormat);
++
++			V4L2SubdeviceFormat sensorFormat =
++				data->sensor_->getFormat(mbusCodes, size);
++			if (!sensorFormat.mbus_code) {
++				LOG(STARFIVE, Error)
++					<< "Sensor does not support mbus code";
++				break;
++			}
++
++			pixelFormat =
++				mbusCodesToPixelFormat.at(sensorFormat.mbus_code);
++			size = sensorFormat.size;
++			bufferCount = BUFFER_COUNT;
++
++			streamFormats[pixelFormat] = data->sensorSizes();
++
++			break;
++		}
++		default:
++			LOG(STARFIVE, Error)
++				<< "Requested stream role not supported: "
++				<< role;
++			break;
++		}
++
++		StreamFormats formats(streamFormats);
++		StreamConfiguration cfg(formats);
++		cfg.size = size;
++		cfg.pixelFormat = pixelFormat;
++		cfg.bufferCount = bufferCount;
++		config->addConfiguration(cfg);
++	}
++
++	if (config->validate() == CameraConfiguration::Invalid)
++		return {};
++
++	return config;
++}
++
++int PipelineHandlerStarFive::configure(Camera *camera, CameraConfiguration *c)
++{
++	StarFiveCameraConfiguration *config =
++		static_cast<StarFiveCameraConfiguration *>(c);
++	StarFiveCameraData *data = cameraData(camera);
++	int ret;
++
++	LOG(STARFIVE, Debug) << __func__
++		<< " config->size: " << config->size();
++
++	for (unsigned int i = 0; i < config->size(); ++i) {
++		StreamConfiguration &cfg = (*config)[i];
++		Stream *stream = cfg.stream();
++
++		LOG(STARFIVE, Debug)
++			<< "config stream: " << cfg.toString();
++
++		if (stream == &data->rawStream_) {
++			V4L2DeviceFormat format = {};
++			format.fourcc =
++				data->raw_->toV4L2PixelFormat(cfg.pixelFormat);
++			format.size = cfg.size;
++
++			ret = data->raw_->setFormat(&format);
++			if (ret)
++				return ret;
++
++			if (format.size != cfg.size ||
++			    format.fourcc !=
++			    data->raw_->toV4L2PixelFormat(cfg.pixelFormat))
++				return -EINVAL;
++
++		} else if (stream == &data->outStream_) {
++			V4L2DeviceFormat format = {};
++			format.fourcc =
++				data->video_->toV4L2PixelFormat(cfg.pixelFormat);
++			format.size = cfg.size;
++
++			ret = data->video_->setFormat(&format);
++			if (ret)
++				return ret;
++
++			if (format.size != cfg.size ||
++			    format.fourcc !=
++			    data->video_->toV4L2PixelFormat(cfg.pixelFormat))
++				return -EINVAL;
++		}
++	}
++
++	return 0;
++}
++
++int PipelineHandlerStarFive::exportFrameBuffers(Camera *camera, Stream *stream,
++			std::vector<std::unique_ptr<FrameBuffer>> *buffers)
++{
++	StarFiveCameraData *data = cameraData(camera);
++	unsigned int count = stream->configuration().bufferCount;
++
++	LOG(STARFIVE, Debug) << __func__ << " bufferCount: " << count;
++
++	if (stream == &data->outStream_)
++		return data->video_->exportBuffers(count, buffers);
++	else if (stream == &data->rawStream_)
++		return data->raw_->exportBuffers(count, buffers);
++
++	return -EINVAL;
++}
++
++int PipelineHandlerStarFive::start(Camera *camera, const ControlList *controls)
++{
++	StarFiveCameraData *data = cameraData(camera);
++	unsigned int count = data->outStream_.configuration().bufferCount;
++	int ret = -EINVAL;
++	
++	LOG(STARFIVE, Debug) << __func__ << " bufferCount: " << count;
++
++	ret = data->video_->importBuffers(count);
++	if (ret < 0)
++		return ret;
++
++	ret = data->video_->streamOn();
++	if (ret < 0)
++		goto error;
++
++	if (data->haveRaw()) {
++		count = data->rawStream_.configuration().bufferCount;
++
++		LOG(STARFIVE, Debug) << "rawbufferCount: " << count;
++		if (count) {
++			ret = data->raw_->importBuffers(count);
++			if (ret < 0) {
++				data->setRawActive(false);
++				LOG(STARFIVE, Debug)
++					<< "raw video importBuffers failed!";
++				goto error;
++			}
++			ret = data->raw_->streamOn();
++			if (ret < 0) {
++				data->setRawActive(false);
++				data->raw_->releaseBuffers();
++				LOG(STARFIVE, Debug)
++					<< "raw video streamOn failed!";
++				goto error;
++			}
++			data->setRawActive(true);
++		}
++	}
++
++	return ret;
++error:
++	data->video_->releaseBuffers();
++	return ret;
++}
++
++void PipelineHandlerStarFive::stop(Camera *camera)
++{
++	LOG(STARFIVE, Debug) << __func__;
++
++	StarFiveCameraData *data = cameraData(camera);
++	data->video_->streamOff();
++	data->video_->releaseBuffers();
++	if (data->rawActive()) {
++		data->raw_->streamOff();
++		data->raw_->releaseBuffers();
++		data->setRawActive(false);
++	}
++}
++
++int PipelineHandlerStarFive::processControls(StarFiveCameraData *data, Request *request)
++{
++	LOG(STARFIVE, Debug) << __func__;
++
++	ControlList controls(data->sensor_->controls());
++
++	for (auto it : request->controls()) {
++		unsigned int id = it.first;
++		unsigned int offset;
++		uint32_t cid;
++
++		if (id == controls::Brightness) {
++			cid = V4L2_CID_BRIGHTNESS;
++			offset = 128;
++		} else if (id == controls::Contrast) {
++			cid = V4L2_CID_CONTRAST;
++			offset = 0;
++		} else if (id == controls::Saturation) {
++			cid = V4L2_CID_SATURATION;
++			offset = 0;
++		} else {
++			continue;
++		}
++
++		int32_t value = lroundf(it.second.get<float>() * 128 + offset);
++		controls.set(cid, std::clamp(value, 0, 255));
++	}
++
++	for (const auto &ctrl : controls)
++		LOG(STARFIVE, Debug)
++			<< "Setting control " << utils::hex(ctrl.first)
++			<< " to " << ctrl.second.toString();
++
++	int ret = data->sensor_->setControls(&controls);
++	if (ret) {
++		LOG(STARFIVE, Debug)
++			<< "Failed to set controls: " << ret;
++		return ret < 0 ? ret : -EINVAL;
++	}
++
++	return ret;
++}
++
++int PipelineHandlerStarFive::queueRequestDevice(Camera *camera, Request *request)
++{
++	StarFiveCameraData *data = cameraData(camera);
++	int error = 0;
++
++	LOG(STARFIVE, Debug) << __func__;
++	int ret = processControls(data, request);
++	if (ret < 0)
++		return ret;
++
++	for (auto it : request->buffers()) {
++		const Stream *stream = it.first;
++		FrameBuffer *buffer = it.second;
++		int ret;
++
++		LOG(STARFIVE, Debug)
++			<< "stream queueBuffer : " << stream->configuration().toString();
++
++		if (stream == &data->outStream_)
++			ret = data->video_->queueBuffer(buffer);
++		else if (stream == &data->rawStream_)
++			ret = data->raw_->queueBuffer(buffer);
++		else
++			continue;
++
++		if (ret < 0)
++			error = ret;
++	}
++
++	return error;
++}
++
++std::string PipelineHandlerStarFive::getVideoEntityNameById(unsigned int id)
++{
++	LOG(STARFIVE, Debug) << __func__;
++	switch (id) {
++	case 0:
++		return "stf_vin0_wr_video0";
++	case 1:
++		return "stf_vin0_isp0_video1";
++	case 2:
++		return "stf_vin0_isp1_video2";
++	case 3:
++		return "stf_vin0_isp0_raw_video3";
++	case 4:
++		return "stf_vin0_isp1_raw_video4";
++	default:
++		return "unknow";
++	}
++
++	return "unknow";
++}
++
++std::string PipelineHandlerStarFive::findSensorEntityName(std::string entityName)
++{
++	std::string sensorEntityName;
++	bool found = false;
++	MediaEntity *sensorEntity = starFiveMediaDev_->getEntityByName(entityName);
++
++	while (1) {
++		LOG(STARFIVE, Debug) << "findSensorEntityName: " << sensorEntity->name();
++		const std::vector<MediaPad *> &pads = sensorEntity->pads();
++
++		if (pads.empty())
++			break;
++
++		MediaPad *sink = pads[0];
++		MediaLink *link;
++		bool found_enable_link = false;
++
++		for (MediaLink *it : sink->links()) {
++			if (it->flags() & MEDIA_LNK_FL_ENABLED) {
++				found_enable_link = true;
++				link = it;
++				break;
++			}
++		}
++
++		if (!found_enable_link)
++			break;
++
++		sensorEntity = link->source()->entity();
++		if (sensorEntity->function() == MEDIA_ENT_F_CAM_SENSOR) {
++			found = true;
++			break;
++		}
++	}
++
++	if (found)
++		sensorEntityName = sensorEntity->name();
++	else
++		sensorEntityName = "unknow";
++
++	LOG(STARFIVE, Debug) << "sensorEntityName: " << sensorEntityName;
++	return sensorEntityName;
++}
++
++int PipelineHandlerStarFive::registerCameras()
++{
++	unsigned int numCameras = 0;
++
++	LOG(STARFIVE, Debug) << __func__;
++	for (unsigned int id = 0;
++			id < STF_MAX_CAMERAS 
++			&& numCameras < STF_MAX_CAMERAS; ++id) {
++		std::string videoEntiryName;
++		videoEntiryName = getVideoEntityNameById(id);
++		if (videoEntiryName == "unknow")
++			continue;
++
++		std::string sensorEntityName;
++		sensorEntityName = findSensorEntityName(videoEntiryName);
++		if (sensorEntityName == "unknow")
++			continue;
++
++		std::unique_ptr<StarFiveCameraData> data =
++			std::make_unique<StarFiveCameraData>(this, starFiveMediaDev_,
++					videoEntiryName, sensorEntityName);
++
++		/* Locate and open the capture video node. */
++		if (data->init())
++			continue;
++
++		/* Create and register the camera. */
++		LOG(STARFIVE, Debug) << "register deviceName: "
++				     << videoEntiryName;
++		if (data->haveRaw()) {
++			std::set<Stream *> streams{ &data->outStream_,
++						&data->rawStream_ };
++			std::shared_ptr<Camera> camera =
++				Camera::create(std::move(data), videoEntiryName, streams);
++			registerCamera(std::move(camera));
++		} else {
++			std::set<Stream *> streams{ &data->outStream_ };
++			std::shared_ptr<Camera> camera =
++				Camera::create(std::move(data), videoEntiryName, streams);
++			registerCamera(std::move(camera));
++		}
++		numCameras++;
++	}
++
++	return numCameras;
++}
++
++int PipelineHandlerStarFive::enableLinks(std::vector<PipelineConfigLink> config)
++{
++	int ret = 0;
++
++	LOG(STARFIVE, Debug) << __func__;
++	for (PipelineConfigLink it : config) {
++		MediaLink *link = starFiveMediaDev_->link(it.source, 1, it.link, 0);
++		if (!link)
++			return -ENODEV;
++
++		ret = link->setEnabled(true);
++		if (ret < 0)
++			return ret;
++	}
++
++	return ret;
++}
++
++bool PipelineHandlerStarFive::match(DeviceEnumerator *enumerator)
++{
++	int numCameras = 0;
++
++	DeviceMatch dm("stf-vin");
++	dm.add("stf_vin0_wr_video0");
++	dm.add("stf_vin0_isp0_video1");
++	dm.add("stf_vin0_isp1_video2");
++
++	LOG(STARFIVE, Debug) << __func__;
++	starFiveMediaDev_ = acquireMediaDevice(enumerator, dm);
++	if (!starFiveMediaDev_)
++		return false;
++
++	if (starFiveMediaDev_->disableLinks())
++		return false;
++
++	for (SensorConfig it : sensorConfigs) {
++		MediaEntity *sensorEntity =
++			starFiveMediaDev_->getEntityByName(it.sensorEntityName_);
++		int ret;
++
++		if (sensorEntity != nullptr) {
++			if (it.sensorType_ < DVP_YUV
++				|| it.sensorType_ >= SENSORTYPE_MAX)
++				continue;
++			ret = enableLinks(pipelineConfigs[it.sensorType_]);
++			if (ret < 0) {
++				LOG(STARFIVE, Error)
++					<< it.sensorEntityName_
++					<< " enableLinks failed!";
++				continue;
++			}
++		}
++	}
++
++	numCameras = registerCameras();
++	if (numCameras)
++		LOG(STARFIVE, Debug)
++			<< "StarFive " << numCameras
++			<< " Device Identified";
++
++	return numCameras != 0;
++}
++
++void StarFiveCameraData::bufferReady(FrameBuffer *buffer)
++{
++	LOG(STARFIVE, Debug) << __func__;
++	PipelineHandlerStarFive *pipe =
++		static_cast<PipelineHandlerStarFive *>(this->pipe());
++	Request *request = buffer->request();
++
++	if (!pipe->completeBuffer(request, buffer))
++		return;
++
++	pipe->completeRequest(request);
++}
++
++REGISTER_PIPELINE_HANDLER(PipelineHandlerStarFive)
++
++} /* namespace libcamera */
+diff --git a/src/libcamera/v4l2_subdevice.cpp b/src/libcamera/v4l2_subdevice.cpp
+index 8fe5e45b..0bdcf440 100644
+--- a/src/libcamera/v4l2_subdevice.cpp
++++ b/src/libcamera/v4l2_subdevice.cpp
+@@ -524,4 +524,16 @@ std::vector<SizeRange> V4L2Subdevice::enumPadSizes(unsigned int pad,
+ 	return sizes;
+ }
+ 
++int V4L2Subdevice::ioctlPrivate(unsigned long request, void *argp)
++{
++	/*
++	 * Printing out an error message is usually better performed
++	 * in the caller, which can provide more context.
++	 */
++	if (V4L2Device::ioctl(request, argp) < 0)
++		return -errno;
++
++	return 0;
++}
++
+ } /* namespace libcamera */
+-- 
+2.17.1
+

+ 182 - 0
package/libcamera/0002-add-sensor-ctrls.patch

@@ -0,0 +1,182 @@
+From 0dafff3d38dd2b44a0bf602fcd1c25e55cdaaec6 Mon Sep 17 00:00:00 2001
+From: sw.multimedia <sw.multimedia@starfivetech.com>
+Date: Tue, 26 Oct 2021 12:29:30 +0800
+Subject: [PATCH] add sensor ctrls
+
+---
+ src/libcamera/pipeline/starfive/starfive.cpp | 118 ++++++++++++++++---
+ 1 file changed, 99 insertions(+), 19 deletions(-)
+
+diff --git a/src/libcamera/pipeline/starfive/starfive.cpp b/src/libcamera/pipeline/starfive/starfive.cpp
+index 8797bdcb..f63af910 100644
+--- a/src/libcamera/pipeline/starfive/starfive.cpp
++++ b/src/libcamera/pipeline/starfive/starfive.cpp
+@@ -370,7 +370,7 @@ public:
+ 	bool match(DeviceEnumerator *enumerator) override;
+ 
+ private:
+-	int processControls(StarFiveCameraData *data, Request *request);
++	int processControls(StarFiveCameraData *data, const ControlList &controls);
+ 
+ 	StarFiveCameraData *cameraData(Camera *camera)
+ 	{
+@@ -620,6 +620,13 @@ int PipelineHandlerStarFive::start(Camera *camera, const ControlList *controls)
+ 	
+ 	LOG(STARFIVE, Debug) << __func__ << " bufferCount: " << count;
+ 
++	if (controls != nullptr) {
++		ret = processControls(data, *controls);
++		if (ret < 0)
++			return ret;
++	} else
++		LOG(STARFIVE, Debug) << __func__ << " start no controls";
++
+ 	ret = data->video_->importBuffers(count);
+ 	if (ret < 0)
+ 		return ret;
+@@ -672,43 +679,116 @@ void PipelineHandlerStarFive::stop(Camera *camera)
+ 	}
+ }
+ 
+-int PipelineHandlerStarFive::processControls(StarFiveCameraData *data, Request *request)
++int PipelineHandlerStarFive::processControls(StarFiveCameraData *data, const ControlList &controls)
+ {
+-	LOG(STARFIVE, Debug) << __func__;
++	LOG(STARFIVE, Debug) << __func__ << ": test1 controls";
++	const ControlInfoMap &SensorControlsInfoMap = data->sensor_->controls();
++	ControlList SensorControls(SensorControlsInfoMap);
+ 
+-	ControlList controls(data->sensor_->controls());
+-
+-	for (auto it : request->controls()) {
++	for (auto it : controls) {
+ 		unsigned int id = it.first;
+-		unsigned int offset;
+-		uint32_t cid;
++		unsigned int offset = 0;
++		uint32_t cid = 0;
++		int32_t value = 0;
+ 
+-		if (id == controls::Brightness) {
++		switch (id) {
++		case controls::BRIGHTNESS:
+ 			cid = V4L2_CID_BRIGHTNESS;
+ 			offset = 128;
+-		} else if (id == controls::Contrast) {
++			value = lroundf(it.second.get<float>() * 128 + offset);
++			LOG(STARFIVE, Debug) << "Brightness controls: " << id
++				<< ", value: " << value;
++			break;
++		case controls::CONTRAST:
+ 			cid = V4L2_CID_CONTRAST;
+ 			offset = 0;
+-		} else if (id == controls::Saturation) {
++			value = lroundf(it.second.get<float>() * 128 + offset);
++			LOG(STARFIVE, Debug) << "Contrast controls: " << id
++				<< ", value: " << value;
++			break;
++		case controls::SATURATION:
+ 			cid = V4L2_CID_SATURATION;
+ 			offset = 0;
+-		} else {
+-			continue;
++			value = lroundf(it.second.get<float>() * 128 + offset);
++			LOG(STARFIVE, Debug) << "Saturation controls: " << id
++				<< ", value: " << value;
++			break;
++		case controls::SHARPNESS:
++			cid = V4L2_CID_SHARPNESS;
++			offset = 0;
++			value = lroundf(it.second.get<float>() * 128 + offset);
++			LOG(STARFIVE, Debug) << "Sharpness controls: " << id
++				<< ", value: " << value;
++			break;
++		case controls::COLOUR_GAINS:
++			cid = V4L2_CID_RED_BALANCE;
++			offset = 0;
++			value = lroundf(it.second.get<Span<const float>>()[0]);
++			LOG(STARFIVE, Debug) << "ColourGains controls: " << id
++				<< ", COLOUR_GAINS RED value: " << value;
++			if (SensorControlsInfoMap.find(cid) != SensorControlsInfoMap.end())
++				SensorControls.set(cid, value);
++			else
++				LOG(STARFIVE, Debug)
++					<< "SensorControls not supported controls: " << id;
++			cid = V4L2_CID_BLUE_BALANCE;
++			value = lroundf(it.second.get<Span<const float>>()[1]);
++			LOG(STARFIVE, Debug) << "COLOUR_GAINS BLUE value: " << value;
++			break;
++		case controls::AWB_ENABLE:
++			cid = V4L2_CID_AUTO_WHITE_BALANCE;
++			value = it.second.get<bool>();
++			LOG(STARFIVE, Debug) << "AwbMode controls: " << id
++				<< ", value: " << value;
++			break;
++		case controls::EXPOSURE_VALUE:
++			cid = V4L2_CID_EXPOSURE;
++			offset = 0;
++			value = lroundf(it.second.get<float>());
++			LOG(STARFIVE, Debug) << "ExposureValue controls: " << id
++				<< ", value: " << value;
++			break;
++		case controls::AE_ENABLE:
++			cid = V4L2_CID_EXPOSURE_AUTO;
++			value = it.second.get<bool>();
++			LOG(STARFIVE, Debug) << "AeExposureMode controls: " << id
++				<< ", value: " << value;
++			break;
++		case controls::ANALOGUE_GAIN:
++			cid = V4L2_CID_ANALOGUE_GAIN;
++			offset = 0;
++			value = lroundf(it.second.get<float>());
++			LOG(STARFIVE, Debug) << "AnalogueGain controls: " << id
++				<< ", value: " << value;
++			break;
++		case controls::DIGITAL_GAIN:
++			cid = V4L2_CID_GAIN;
++			offset = 0;
++			value = lroundf(it.second.get<float>());
++			LOG(STARFIVE, Debug) << "AnalogueGain controls: " << id
++				<< ", value: " << value;
++			break;
++		default:
++			LOG(STARFIVE, Debug) << "default controls: " << id;
++			break;
+ 		}
+ 
+-		int32_t value = lroundf(it.second.get<float>() * 128 + offset);
+-		controls.set(cid, std::clamp(value, 0, 255));
++		if (SensorControlsInfoMap.find(cid) != SensorControlsInfoMap.end())
++			SensorControls.set(cid, value);
++		else
++			LOG(STARFIVE, Debug) << "SensorControls not supported controls: " << id;
++
+ 	}
+ 
+-	for (const auto &ctrl : controls)
++	for (const auto &ctrl : SensorControls)
+ 		LOG(STARFIVE, Debug)
+ 			<< "Setting control " << utils::hex(ctrl.first)
+ 			<< " to " << ctrl.second.toString();
+ 
+-	int ret = data->sensor_->setControls(&controls);
++	int ret = data->sensor_->setControls(&SensorControls);
+ 	if (ret) {
+ 		LOG(STARFIVE, Debug)
+-			<< "Failed to set controls: " << ret;
++			<< "Failed to set sensor controls: " << ret;
+ 		return ret < 0 ? ret : -EINVAL;
+ 	}
+ 
+@@ -721,7 +801,7 @@ int PipelineHandlerStarFive::queueRequestDevice(Camera *camera, Request *request
+ 	int error = 0;
+ 
+ 	LOG(STARFIVE, Debug) << __func__;
+-	int ret = processControls(data, request);
++	int ret = processControls(data, request->controls());
+ 	if (ret < 0)
+ 		return ret;
+ 
+-- 
+2.17.1
+

+ 496 - 0
package/libcamera/0003-add-Meta-and-starfive-ipa-dummy.patch

@@ -0,0 +1,496 @@
+diff -Naur libcamera-06e53199c2563105030bda4c72752b853da7edc8/include/libcamera/ipa/meson.build libcamera-06e53199c2563105030bda4c72752b853da7edc8_after/include/libcamera/ipa/meson.build
+--- libcamera-06e53199c2563105030bda4c72752b853da7edc8/include/libcamera/ipa/meson.build	2021-09-07 07:11:15.000000000 +0800
++++ libcamera-06e53199c2563105030bda4c72752b853da7edc8_after/include/libcamera/ipa/meson.build	2021-10-26 10:35:08.196358414 +0800
+@@ -63,6 +63,7 @@
+     'raspberrypi.mojom',
+     'rkisp1.mojom',
+     'vimc.mojom',
++	'starfive.mojom',
+ ]
+ 
+ ipa_mojoms = []
+diff -Naur libcamera-06e53199c2563105030bda4c72752b853da7edc8/include/libcamera/ipa/starfive.mojom libcamera-06e53199c2563105030bda4c72752b853da7edc8_after/include/libcamera/ipa/starfive.mojom
+--- libcamera-06e53199c2563105030bda4c72752b853da7edc8/include/libcamera/ipa/starfive.mojom	1970-01-01 08:00:00.000000000 +0800
++++ libcamera-06e53199c2563105030bda4c72752b853da7edc8_after/include/libcamera/ipa/starfive.mojom	2021-10-26 10:35:08.196358414 +0800
+@@ -0,0 +1,45 @@
++/* SPDX-License-Identifier: LGPL-2.1-or-later */
++
++/*
++ * \todo Document the interface and remove the related EXCLUDE_PATTERNS entry.
++ */
++
++module ipa.starfive;
++
++import "include/libcamera/ipa/core.mojom";
++
++const string StarfiveIPAFIFOPath = "/tmp/libcamera_ipa_starfive_fifo";
++
++enum IPAOperationCode {
++	IPAOperationNone,
++	IPAOperationInit,
++	IPAOperationStart,
++	IPAOperationStop,
++};
++
++interface IPAStarfiveInterface {
++	init(libcamera.IPASettings settings) => (int32 ret);
++
++	configure(libcamera.IPACameraSensorInfo sensorInfo,
++		  map<uint32, libcamera.IPAStream> streamConfig,
++		  map<uint32, libcamera.ControlInfoMap> entityControls) => (int32 ret);
++
++	start() => (int32 ret);
++	stop();
++
++	mapBuffers(array<libcamera.IPABuffer> buffers);
++	unmapBuffers(array<uint32> ids);
++
++	/*
++	 * The starfive driver doesn't use parameters buffers. To maximize coverage
++	 * of unit tests that rely on the starfive pipeline handler, we still define
++	 * interface functions that mimick how other pipeline handlers typically
++	 * handle parameters at runtime.
++	 */
++	[async] fillParams(uint32 frame, uint32 bufferId);
++	[async] processControls(uint32 frame, libcamera.ControlList controls);
++};
++
++interface IPAStarfiveEventInterface {
++	paramsFilled(uint32 bufferId);
++};
+diff -Naur libcamera-06e53199c2563105030bda4c72752b853da7edc8/meson_options.txt libcamera-06e53199c2563105030bda4c72752b853da7edc8_after/meson_options.txt
+--- libcamera-06e53199c2563105030bda4c72752b853da7edc8/meson_options.txt	2021-10-26 10:40:32.171834666 +0800
++++ libcamera-06e53199c2563105030bda4c72752b853da7edc8_after/meson_options.txt	2021-10-26 10:35:08.196358414 +0800
+@@ -27,7 +27,7 @@
+ 
+ option('ipas',
+         type : 'array',
+-        choices : ['ipu3', 'raspberrypi', 'rkisp1', 'vimc'],
++        choices : ['ipu3', 'raspberrypi', 'rkisp1', 'vimc' ,'starfive'],
+         description : 'Select which IPA modules to build')
+ 
+ option('lc-compliance',
+diff -Naur libcamera-06e53199c2563105030bda4c72752b853da7edc8/src/gstreamer/gstlibcamerasrc.cpp libcamera-06e53199c2563105030bda4c72752b853da7edc8_after/src/gstreamer/gstlibcamerasrc.cpp
+--- libcamera-06e53199c2563105030bda4c72752b853da7edc8/src/gstreamer/gstlibcamerasrc.cpp	2021-10-26 10:40:32.171834666 +0800
++++ libcamera-06e53199c2563105030bda4c72752b853da7edc8_after/src/gstreamer/gstlibcamerasrc.cpp	2021-10-26 10:36:19.850655340 +0800
+@@ -140,9 +140,19 @@
+ 
+ #define TEMPLATE_CAPS GST_STATIC_CAPS("video/x-raw; image/jpeg")
+ 
++#define TEMPLATE_CAPS_SRC GST_STATIC_CAPS("video/x-raw, "\
++      "format = { (string)BG24,(string)RG24,(string)BA24, \
++      	          (string)NV12, (string)NV21, (string)NV16, \
++      	          (string)NV61, (string)NV24, (string)UYVY, \
++      	          (string)VYUY, (string)YUYV, (string)YVYU}, "\
++      "width = " GST_VIDEO_SIZE_RANGE ", "\
++      "height = " GST_VIDEO_SIZE_RANGE ", "\
++      "framerate = " GST_VIDEO_FPS_RANGE ";"\
++)
++
+ /* For the simple case, we have a src pad that is always present. */
+ GstStaticPadTemplate src_template = {
+-	"src", GST_PAD_SRC, GST_PAD_ALWAYS, TEMPLATE_CAPS
++	"src", GST_PAD_SRC, GST_PAD_ALWAYS, TEMPLATE_CAPS_SRC
+ };
+ 
+ /* More pads can be requested in state < PAUSED */
+diff -Naur libcamera-06e53199c2563105030bda4c72752b853da7edc8/src/ipa/starfive/data/meson.build libcamera-06e53199c2563105030bda4c72752b853da7edc8_after/src/ipa/starfive/data/meson.build
+--- libcamera-06e53199c2563105030bda4c72752b853da7edc8/src/ipa/starfive/data/meson.build	1970-01-01 08:00:00.000000000 +0800
++++ libcamera-06e53199c2563105030bda4c72752b853da7edc8_after/src/ipa/starfive/data/meson.build	2021-10-26 10:35:08.196358414 +0800
+@@ -0,0 +1,8 @@
++# SPDX-License-Identifier: CC0-1.0
++
++conf_files = files([
++    'starfive.conf',
++])
++
++install_data(conf_files,
++             install_dir : ipa_data_dir / 'starfive')
+diff -Naur libcamera-06e53199c2563105030bda4c72752b853da7edc8/src/ipa/starfive/data/starfive.conf libcamera-06e53199c2563105030bda4c72752b853da7edc8_after/src/ipa/starfive/data/starfive.conf
+--- libcamera-06e53199c2563105030bda4c72752b853da7edc8/src/ipa/starfive/data/starfive.conf	1970-01-01 08:00:00.000000000 +0800
++++ libcamera-06e53199c2563105030bda4c72752b853da7edc8_after/src/ipa/starfive/data/starfive.conf	2021-10-26 10:35:08.196358414 +0800
+@@ -0,0 +1,3 @@
++# SPDX-License-Identifier: LGPL-2.1-or-later
++#
++# Dummy configuration file for the starfive IPA.
+diff -Naur libcamera-06e53199c2563105030bda4c72752b853da7edc8/src/ipa/starfive/meson.build libcamera-06e53199c2563105030bda4c72752b853da7edc8_after/src/ipa/starfive/meson.build
+--- libcamera-06e53199c2563105030bda4c72752b853da7edc8/src/ipa/starfive/meson.build	1970-01-01 08:00:00.000000000 +0800
++++ libcamera-06e53199c2563105030bda4c72752b853da7edc8_after/src/ipa/starfive/meson.build	2021-10-26 10:35:08.196358414 +0800
+@@ -0,0 +1,23 @@
++# SPDX-License-Identifier: CC0-1.0
++
++ipa_name = 'ipa_starfive'
++
++mod = shared_module(ipa_name,
++                    ['starfive.cpp', libcamera_generated_ipa_headers],
++                    name_prefix : '',
++                    include_directories : [ipa_includes, libipa_includes],
++                    dependencies : libcamera_private,
++                    link_with : libipa,
++                    install : true,
++                    install_dir : ipa_install_dir)
++
++if ipa_sign_module
++    custom_target(ipa_name + '.so.sign',
++                  input : mod,
++                  output : ipa_name + '.so.sign',
++                  command : [ipa_sign, ipa_priv_key, '@INPUT@', '@OUTPUT@'],
++                  install : false,
++                  build_by_default : true)
++endif
++
++subdir('data')
+diff -Naur libcamera-06e53199c2563105030bda4c72752b853da7edc8/src/ipa/starfive/starfive.cpp libcamera-06e53199c2563105030bda4c72752b853da7edc8_after/src/ipa/starfive/starfive.cpp
+--- libcamera-06e53199c2563105030bda4c72752b853da7edc8/src/ipa/starfive/starfive.cpp	1970-01-01 08:00:00.000000000 +0800
++++ libcamera-06e53199c2563105030bda4c72752b853da7edc8_after/src/ipa/starfive/starfive.cpp	2021-10-26 10:35:08.196358414 +0800
+@@ -0,0 +1,200 @@
++/* SPDX-License-Identifier: LGPL-2.1-or-later */
++/*
++ * Copyright (C) 2019, Google Inc.
++ *
++ * ipa_starfive.cpp - Starfive Image Processing Algorithm module
++ */
++#include <libcamera/ipa/starfive_ipa_interface.h>
++
++#include <fcntl.h>
++#include <string.h>
++#include <sys/stat.h>
++#include <unistd.h>
++
++#include <iostream>
++
++#include <libcamera/base/file.h>
++#include <libcamera/base/log.h>
++
++#include <libcamera/ipa/ipa_interface.h>
++#include <libcamera/ipa/ipa_module_info.h>
++
++#include "libcamera/internal/mapped_framebuffer.h"
++
++namespace libcamera {
++
++LOG_DEFINE_CATEGORY(IPAStarfive)
++
++class IPAStarfive : public ipa::starfive::IPAStarfiveInterface
++{
++public:
++	IPAStarfive();
++	~IPAStarfive();
++
++	int init(const IPASettings &settings) override;
++
++	int start() override;
++	void stop() override;
++
++	int configure(const IPACameraSensorInfo &sensorInfo,
++		      const std::map<unsigned int, IPAStream> &streamConfig,
++		      const std::map<unsigned int, ControlInfoMap> &entityControls) override;
++
++	void mapBuffers(const std::vector<IPABuffer> &buffers) override;
++	void unmapBuffers(const std::vector<unsigned int> &ids) override;
++
++	void fillParams(uint32_t frame, uint32_t bufferId) override;
++	void processControls(uint32_t frame, const ControlList &controls) override;
++
++private:
++	void initTrace();
++	void trace(enum ipa::starfive::IPAOperationCode operation);
++
++	int fd_;
++	std::map<unsigned int, MappedFrameBuffer> buffers_;
++};
++
++IPAStarfive::IPAStarfive()
++	: fd_(-1)
++{
++	initTrace();
++}
++
++IPAStarfive::~IPAStarfive()
++{
++	if (fd_ != -1)
++		::close(fd_);
++}
++
++int IPAStarfive::init(const IPASettings &settings)
++{
++	trace(ipa::starfive::IPAOperationInit);
++
++	LOG(IPAStarfive, Debug)
++		<< "initializing starfive IPA with configuration file "
++		<< settings.configurationFile;
++
++	File conf(settings.configurationFile);
++	if (!conf.open(File::OpenModeFlag::ReadOnly)) {
++		LOG(IPAStarfive, Error) << "Failed to open configuration file";
++		return -EINVAL;
++	}
++
++	return 0;
++}
++
++int IPAStarfive::start()
++{
++	trace(ipa::starfive::IPAOperationStart);
++
++	LOG(IPAStarfive, Debug) << "start starfive IPA!";
++
++	return 0;
++}
++
++void IPAStarfive::stop()
++{
++	trace(ipa::starfive::IPAOperationStop);
++
++	LOG(IPAStarfive, Debug) << "stop starfive IPA!";
++}
++
++int IPAStarfive::configure([[maybe_unused]] const IPACameraSensorInfo &sensorInfo,
++			[[maybe_unused]] const std::map<unsigned int, IPAStream> &streamConfig,
++			[[maybe_unused]] const std::map<unsigned int, ControlInfoMap> &entityControls)
++{
++	LOG(IPAStarfive, Debug) << "configure()";
++
++	return 0;
++}
++
++void IPAStarfive::mapBuffers(const std::vector<IPABuffer> &buffers)
++{
++	LOG(IPAStarfive, Debug) << "mapBuffers()";
++	for (const IPABuffer &buffer : buffers) {
++		const FrameBuffer fb(buffer.planes);
++		buffers_.emplace(std::piecewise_construct,
++				 std::forward_as_tuple(buffer.id),
++				 std::forward_as_tuple(&fb, MappedFrameBuffer::MapFlag::Read));
++	}
++}
++
++void IPAStarfive::unmapBuffers(const std::vector<unsigned int> &ids)
++{
++	LOG(IPAStarfive, Debug) << "unmapBuffers()";
++	for (unsigned int id : ids) {
++		auto it = buffers_.find(id);
++		if (it == buffers_.end())
++			continue;
++
++		buffers_.erase(it);
++	}
++}
++
++void IPAStarfive::fillParams([[maybe_unused]] uint32_t frame, uint32_t bufferId)
++{
++	auto it = buffers_.find(bufferId);
++	if (it == buffers_.end()) {
++		LOG(IPAStarfive, Error) << "Could not find parameter buffer";
++		return;
++	}
++
++	paramsFilled.emit(bufferId);
++}
++
++void IPAStarfive::processControls([[maybe_unused]] uint32_t frame,
++			      [[maybe_unused]] const ControlList &controls)
++{
++	LOG(IPAStarfive,Debug) << "IPA ProcessControl";
++}
++
++void IPAStarfive::initTrace()
++{
++	struct stat fifoStat;
++	int ret = stat(ipa::starfive::StarfiveIPAFIFOPath.c_str(), &fifoStat);
++	if (ret)
++		return;
++
++	ret = ::open(ipa::starfive::StarfiveIPAFIFOPath.c_str(), O_WRONLY);
++	if (ret < 0) {
++		ret = errno;
++		LOG(IPAStarfive, Error) << "Failed to open starfive IPA test FIFO: "
++				    << strerror(ret);
++		return;
++	}
++
++	fd_ = ret;
++}
++
++void IPAStarfive::trace(enum ipa::starfive::IPAOperationCode operation)
++{
++	if (fd_ < 0)
++		return;
++
++	int ret = ::write(fd_, &operation, sizeof(operation));
++	if (ret < 0) {
++		ret = errno;
++		LOG(IPAStarfive, Error) << "Failed to write to starfive IPA test FIFO: "
++				    << strerror(ret);
++	}
++}
++
++/*
++ * External IPA module interface
++ */
++
++extern "C" {
++const struct IPAModuleInfo ipaModuleInfo = {
++	IPA_MODULE_API_VERSION,
++	0,
++	"PipelineHandlerStarFive",
++	"starfive",
++};
++
++IPAInterface *ipaCreate()
++{
++	return new IPAStarfive();
++}
++}
++
++} /* namespace libcamera */
+diff -Naur libcamera-06e53199c2563105030bda4c72752b853da7edc8/src/libcamera/ipa_manager.cpp libcamera-06e53199c2563105030bda4c72752b853da7edc8_after/src/libcamera/ipa_manager.cpp
+--- libcamera-06e53199c2563105030bda4c72752b853da7edc8/src/libcamera/ipa_manager.cpp	2021-09-07 07:11:15.000000000 +0800
++++ libcamera-06e53199c2563105030bda4c72752b853da7edc8_after/src/libcamera/ipa_manager.cpp	2021-10-26 10:35:08.196358414 +0800
+@@ -276,7 +276,9 @@
+ 
+ bool IPAManager::isSignatureValid([[maybe_unused]] IPAModule *ipa) const
+ {
+-#if HAVE_IPA_PUBKEY
++#if SKIP_SIGN_VERIFY
++	return true;
++#elif HAVE_IPA_PUBKEY
+ 	char *force = utils::secure_getenv("LIBCAMERA_IPA_FORCE_ISOLATION");
+ 	if (force && force[0] != '\0') {
+ 		LOG(IPAManager, Debug)
+diff -Naur libcamera-06e53199c2563105030bda4c72752b853da7edc8/src/libcamera/pipeline/starfive/starfive.cpp libcamera-06e53199c2563105030bda4c72752b853da7edc8_after/src/libcamera/pipeline/starfive/starfive.cpp
+--- libcamera-06e53199c2563105030bda4c72752b853da7edc8/src/libcamera/pipeline/starfive/starfive.cpp	2021-10-26 10:40:32.175834819 +0800
++++ libcamera-06e53199c2563105030bda4c72752b853da7edc8_after/src/libcamera/pipeline/starfive/starfive.cpp	2021-10-26 10:35:08.196358414 +0800
+@@ -23,6 +23,11 @@
+ #include <linux/videodev2.h>
+ #include <linux/v4l2-subdev.h>
+ 
++#include <libcamera/ipa/ipa_interface.h>
++#include <libcamera/ipa/ipa_module_info.h>
++#include <libcamera/ipa/starfive_ipa_interface.h>
++#include <libcamera/ipa/starfive_ipa_proxy.h>
++
+ #include "libcamera/internal/camera.h"
+ #include "libcamera/internal/camera_sensor.h"
+ #include "libcamera/internal/device_enumerator.h"
+@@ -189,6 +194,7 @@
+ 	std::vector<SizeRange> sensorSizes() const;
+ 	std::vector<PixelFormat> sensorFormats() const;
+ 	std::vector<PixelFormat> videoFormats() const;
++	void paramsFilled(unsigned int id){}
+ 
+ 	MediaDevice *media_;
+ 	V4L2VideoDevice *video_;
+@@ -197,6 +203,8 @@
+ 	CameraSensor *sensor_;
+ 	Stream outStream_;
+ 	Stream rawStream_;
++	std::unique_ptr<ipa::starfive::IPAProxyStarfive> ipa_;
++	std::vector<std::unique_ptr<FrameBuffer>> mockIPABufs_;
+ 
+ private:
+ 	bool haveRaw_;
+@@ -591,6 +599,22 @@
+ 			    data->video_->toV4L2PixelFormat(cfg.pixelFormat))
+ 				return -EINVAL;
+ 		}
++
++		if (data->ipa_) {
++		/* Inform IPA of stream configuration and sensor controls. */
++		std::map<unsigned int, IPAStream> streamConfig;
++		streamConfig.emplace(std::piecewise_construct,
++				     std::forward_as_tuple(0),
++				     std::forward_as_tuple(cfg.pixelFormat, cfg.size));
++
++		std::map<unsigned int, ControlInfoMap> entityControls;
++		entityControls.emplace(0, data->sensor_->controls());
++
++		IPACameraSensorInfo sensorInfo;
++		data->sensor_->sensorInfo(&sensorInfo);
++
++		data->ipa_->configure(sensorInfo, streamConfig, entityControls);
++		}
+ 	}
+ 
+ 	return 0;
+@@ -627,6 +651,17 @@
+ 	ret = data->video_->importBuffers(count);
+ 	if (ret < 0)
+ 		return ret;
++	
++	/* Map the mock IPA buffers to VIMC IPA to exercise IPC code paths. */
++	std::vector<IPABuffer> ipaBuffers;
++	for (auto [i, buffer] : utils::enumerate(data->mockIPABufs_)) {
++		buffer->setCookie(i + 1);
++		ipaBuffers.emplace_back(buffer->cookie(), buffer->planes());
++	}
++	data->ipa_->mapBuffers(ipaBuffers);
++	ret = data->ipa_->start();
++	if(ret < 0)
++		goto error;
+ 
+ 	ret = data->video_->streamOn();
+ 	if (ret < 0)
+@@ -658,6 +693,7 @@
+ 
+ 	return ret;
+ error:
++	data->ipa_->stop();
+ 	data->video_->releaseBuffers();
+ 	return ret;
+ }
+@@ -668,6 +704,13 @@
+ 
+ 	StarFiveCameraData *data = cameraData(camera);
+ 	data->video_->streamOff();
++
++	std::vector<unsigned int> ids;
++	for (const std::unique_ptr<FrameBuffer> &buffer : data->mockIPABufs_)
++		ids.push_back(buffer->cookie());
++	data->ipa_->unmapBuffers(ids);
++	data->ipa_->stop();
++
+ 	data->video_->releaseBuffers();
+ 	if (data->rawActive()) {
+ 		data->raw_->streamOff();
+@@ -810,8 +853,11 @@
+ 		LOG(STARFIVE, Debug)
+ 			<< "stream queueBuffer : " << stream->configuration().toString();
+ 
+-		if (stream == &data->outStream_)
++		if (stream == &data->outStream_){
+ 			ret = data->video_->queueBuffer(buffer);
++			data->ipa_->processControls(request->sequence(), request->controls());
++		}
++			
+ 		else if (stream == &data->rawStream_)
+ 			ret = data->raw_->queueBuffer(buffer);
+ 		else
+@@ -915,6 +961,16 @@
+ 		if (data->init())
+ 			continue;
+ 
++		/*create ipa module*/
++		data->ipa_ = IPAManager::createIPA<ipa::starfive::IPAProxyStarfive>(this, 0, 0);
++		if (!data->ipa_) {
++			LOG(STARFIVE, Error) << "no matching IPA found";
++			return false;
++		}
++		data->ipa_->paramsFilled.connect(data.get(), &StarFiveCameraData::paramsFilled);
++		std::string conf = data->ipa_->configurationFile("starfive.conf");
++		data->ipa_->init(IPASettings{ conf, data->sensor_->model() });
++
+ 		/* Create and register the camera. */
+ 		LOG(STARFIVE, Debug) << "register deviceName: "
+ 				     << videoEntiryName;
+diff -Naur libcamera-06e53199c2563105030bda4c72752b853da7edc8/src/meson.build libcamera-06e53199c2563105030bda4c72752b853da7edc8_after/src/meson.build
+--- libcamera-06e53199c2563105030bda4c72752b853da7edc8/src/meson.build	2021-09-07 07:11:15.000000000 +0800
++++ libcamera-06e53199c2563105030bda4c72752b853da7edc8_after/src/meson.build	2021-10-26 10:35:08.196358414 +0800
+@@ -20,6 +20,7 @@
+                                  output : ['ipa-priv-key.pem'],
+                                  command : [gen_ipa_priv_key, '@OUTPUT@'])
+     config_h.set('HAVE_IPA_PUBKEY', 1)
++    config_h.set('SKIP_SIGN_VERIFY',1)
+     ipa_sign_module = true
+ else
+     ipa_sign_module = false

+ 74 - 0
package/libcamera/0004-update-toV4L2PixelFormat.patch

@@ -0,0 +1,74 @@
+From 62bce91263c747e42d39bcaa6a15f03458c38e70 Mon Sep 17 00:00:00 2001
+From: sw.multimedia <sw.multimedia@starfivetech.com>
+Date: Tue, 16 Nov 2021 17:23:17 +0800
+Subject: [PATCH] update toV4L2PixelFormat
+
+---
+ src/libcamera/pipeline/starfive/starfive.cpp | 18 ++++++++++++------
+ 1 file changed, 12 insertions(+), 6 deletions(-)
+
+diff --git a/src/libcamera/pipeline/starfive/starfive.cpp b/src/libcamera/pipeline/starfive/starfive.cpp
+index 36ef3b7b..159ead04 100644
+--- a/src/libcamera/pipeline/starfive/starfive.cpp
++++ b/src/libcamera/pipeline/starfive/starfive.cpp
+@@ -441,7 +441,8 @@ CameraConfiguration::Status StarFiveCameraConfiguration::validate()
+ 				&& data_->haveRaw()) {
+ 			V4L2DeviceFormat format = {};
+ 			format.fourcc =
+-				data_->raw_->toV4L2PixelFormat(cfg.pixelFormat);
++				V4L2PixelFormat::fromPixelFormat(cfg.pixelFormat,
++						data_->raw_->caps().isMultiplanar());
+ 			format.size = cfg.size;
+ 
+ 			int ret = data_->raw_->tryFormat(&format);
+@@ -455,7 +456,8 @@ CameraConfiguration::Status StarFiveCameraConfiguration::validate()
+ 		} else {
+ 			V4L2DeviceFormat format = {};
+ 			format.fourcc =
+-				data_->video_->toV4L2PixelFormat(cfg.pixelFormat);
++				V4L2PixelFormat::fromPixelFormat(cfg.pixelFormat,
++						data_->video_->caps().isMultiplanar());
+ 			format.size = cfg.size;
+ 
+ 			int ret = data_->video_->tryFormat(&format);
+@@ -572,7 +574,8 @@ int PipelineHandlerStarFive::configure(Camera *camera, CameraConfiguration *c)
+ 		if (stream == &data->rawStream_) {
+ 			V4L2DeviceFormat format = {};
+ 			format.fourcc =
+-				data->raw_->toV4L2PixelFormat(cfg.pixelFormat);
++				V4L2PixelFormat::fromPixelFormat(cfg.pixelFormat,
++						data->raw_->caps().isMultiplanar());
+ 			format.size = cfg.size;
+ 
+ 			ret = data->raw_->setFormat(&format);
+@@ -581,13 +584,15 @@ int PipelineHandlerStarFive::configure(Camera *camera, CameraConfiguration *c)
+ 
+ 			if (format.size != cfg.size ||
+ 			    format.fourcc !=
+-			    data->raw_->toV4L2PixelFormat(cfg.pixelFormat))
++			    V4L2PixelFormat::fromPixelFormat(cfg.pixelFormat,
++				    data->raw_->caps().isMultiplanar()))
+ 				return -EINVAL;
+ 
+ 		} else if (stream == &data->outStream_) {
+ 			V4L2DeviceFormat format = {};
+ 			format.fourcc =
+-				data->video_->toV4L2PixelFormat(cfg.pixelFormat);
++				V4L2PixelFormat::fromPixelFormat(cfg.pixelFormat,
++						data->video_->caps().isMultiplanar());
+ 			format.size = cfg.size;
+ 
+ 			ret = data->video_->setFormat(&format);
+@@ -596,7 +601,8 @@ int PipelineHandlerStarFive::configure(Camera *camera, CameraConfiguration *c)
+ 
+ 			if (format.size != cfg.size ||
+ 			    format.fourcc !=
+-			    data->video_->toV4L2PixelFormat(cfg.pixelFormat))
++			    V4L2PixelFormat::fromPixelFormat(cfg.pixelFormat,
++				    data->video_->caps().isMultiplanar()))
+ 				return -EINVAL;
+ 		}
+ 
+-- 
+2.17.1
+

+ 30 - 0
package/libcamera/0005-fix-build-gen-version-error.patch

@@ -0,0 +1,30 @@
+From 482311f6d06fa536c113dc0896829f74b172421c Mon Sep 17 00:00:00 2001
+From: sw.multimedia <sw.multimedia@starfivetech.com>
+Date: Wed, 17 Nov 2021 10:07:35 +0800
+Subject: [PATCH] fix build gen version error
+
+---
+ utils/gen-version.sh | 3 +++
+ 1 file changed, 3 insertions(+)
+
+diff --git a/utils/gen-version.sh b/utils/gen-version.sh
+index eb7c7268..ef548100 100755
+--- a/utils/gen-version.sh
++++ b/utils/gen-version.sh
+@@ -3,6 +3,8 @@
+ # SPDX-License-Identifier: GPL-2.0-or-later
+ # Generate a version string using git describe
+ 
++workdir=$PWD
++cd ../../../../buildroot/dl/libcamera/git/
+ build_dir="$1"
+ src_dir="$2"
+ 
+@@ -48,3 +50,4 @@ git diff-index --quiet HEAD || version="$version-dirty ($(date --iso-8601=second
+ version=$(echo "$version" | sed -e 's/-/+/' | sed -e 's/-g/-/' | cut -c 2-)
+ 
+ echo "$version"
++cd $workdir
+-- 
+2.17.1
+

+ 18 - 0
package/libcamera/0006-fix-yuy2-format-unsupported.patch

@@ -0,0 +1,18 @@
+--- a/src/gstreamer/gstlibcamerasrc.cpp
++++ b/src/gstreamer/gstlibcamerasrc.cpp
+@@ -152,12 +152,12 @@
+ 
+ /* For the simple case, we have a src pad that is always present. */
+ GstStaticPadTemplate src_template = {
+-	"src", GST_PAD_SRC, GST_PAD_ALWAYS, TEMPLATE_CAPS_SRC
++	"src", GST_PAD_SRC, GST_PAD_ALWAYS, TEMPLATE_CAPS
+ };
+ 
+ /* More pads can be requested in state < PAUSED */
+ GstStaticPadTemplate request_src_template = {
+-	"src_%u", GST_PAD_SRC, GST_PAD_REQUEST, TEMPLATE_CAPS
++	"src_%u", GST_PAD_SRC, GST_PAD_REQUEST, TEMPLATE_CAPS_SRC
+ };
+ 
+ void
+

+ 318 - 0
package/libcamera/0007-add-list-controls-and-properties.patch

@@ -0,0 +1,318 @@
+From 9eb2fc23e4d453acaef7f1f45874aca381d2fefd Mon Sep 17 00:00:00 2001
+From: sw.multimedia <sw.multimedia@starfivetech.com>
+Date: Tue, 23 Nov 2021 10:44:04 +0800
+Subject: [PATCH] add list controls and properties
+
+---
+ src/libcamera/pipeline/starfive/starfive.cpp | 219 ++++++++++++++-----
+ 1 file changed, 165 insertions(+), 54 deletions(-)
+
+diff --git a/src/libcamera/pipeline/starfive/starfive.cpp b/src/libcamera/pipeline/starfive/starfive.cpp
+index 159ead04..bb513387 100644
+--- a/src/libcamera/pipeline/starfive/starfive.cpp
++++ b/src/libcamera/pipeline/starfive/starfive.cpp
+@@ -186,7 +186,9 @@ public:
+ 			delete ispSubDev_;
+ 	}
+ 
+-	int init();
++	int init(MediaDevice *media);
++	void addControl(uint32_t cid, const ControlInfo &v4l2Info,
++				ControlInfoMap::Map *ctrls);
+ 	void bufferReady(FrameBuffer *buffer);
+ 	bool haveRaw() const { return haveRaw_; }
+ 	bool rawActive() const { return rawActive_; }
+@@ -299,7 +301,7 @@ private:
+ 	StarFiveCameraData *data_;
+ };
+ 
+-int StarFiveCameraData::init()
++int StarFiveCameraData::init(MediaDevice *media)
+ {
+ 	int ret;
+ 
+@@ -354,7 +356,38 @@ int StarFiveCameraData::init()
+ 					&StarFiveCameraData::bufferReady);
+ 		}
+ 	}
+-	
++
++	// properties_.set(properties::Location, properties::CameraLocationExternal);
++	properties_.set(properties::Model, utils::toAscii(media->model()));
++
++	/*
++	 * Get the current format in order to initialize the sensor array
++	 * properties.
++	 */
++	Size resolution;
++	for (const auto &it : video_->formats()) {
++		const std::vector<SizeRange> &sizeRanges = it.second;
++		for (const SizeRange &sizeRange : sizeRanges) {
++			if (sizeRange.max > resolution)
++				resolution = sizeRange.max;
++		}
++	}
++
++	properties_.set(properties::PixelArraySize, resolution);
++	properties_.set(properties::PixelArrayActiveAreas, { Rectangle(resolution) });
++
++	/* Initialise the supported controls. */
++	ControlInfoMap::Map ctrls;
++
++	for (const auto &ctrl : sensor_->controls()) {
++		uint32_t cid = ctrl.first->id();
++		const ControlInfo &info = ctrl.second;
++
++		addControl(cid, info, &ctrls);
++	}
++
++	controlInfo_ = ControlInfoMap(std::move(ctrls), controls::controls);
++
+ 	return 0;
+ }
+ 
+@@ -736,97 +769,89 @@ int PipelineHandlerStarFive::processControls(StarFiveCameraData *data, const Con
+ 
+ 	for (auto it : controls) {
+ 		unsigned int id = it.first;
+-		unsigned int offset = 0;
+ 		uint32_t cid = 0;
+ 		int32_t value = 0;
+ 
+ 		switch (id) {
+ 		case controls::BRIGHTNESS:
+ 			cid = V4L2_CID_BRIGHTNESS;
+-			offset = 128;
+-			value = lroundf(it.second.get<float>() * 128 + offset);
+-			LOG(STARFIVE, Debug) << "Brightness controls: " << id
+-				<< ", value: " << value;
+ 			break;
+ 		case controls::CONTRAST:
+ 			cid = V4L2_CID_CONTRAST;
+-			offset = 0;
+-			value = lroundf(it.second.get<float>() * 128 + offset);
+-			LOG(STARFIVE, Debug) << "Contrast controls: " << id
+-				<< ", value: " << value;
+ 			break;
+ 		case controls::SATURATION:
+ 			cid = V4L2_CID_SATURATION;
+-			offset = 0;
+-			value = lroundf(it.second.get<float>() * 128 + offset);
+-			LOG(STARFIVE, Debug) << "Saturation controls: " << id
+-				<< ", value: " << value;
+ 			break;
+ 		case controls::SHARPNESS:
+ 			cid = V4L2_CID_SHARPNESS;
+-			offset = 0;
+-			value = lroundf(it.second.get<float>() * 128 + offset);
+-			LOG(STARFIVE, Debug) << "Sharpness controls: " << id
+-				<< ", value: " << value;
+ 			break;
+ 		case controls::COLOUR_GAINS:
+ 			cid = V4L2_CID_RED_BALANCE;
+-			offset = 0;
+-			value = lroundf(it.second.get<Span<const float>>()[0]);
+-			LOG(STARFIVE, Debug) << "ColourGains controls: " << id
+-				<< ", COLOUR_GAINS RED value: " << value;
+-			if (SensorControlsInfoMap.find(cid) != SensorControlsInfoMap.end())
+-				SensorControls.set(cid, value);
+-			else
+-				LOG(STARFIVE, Debug)
+-					<< "SensorControls not supported controls: " << id;
+-			cid = V4L2_CID_BLUE_BALANCE;
+-			value = lroundf(it.second.get<Span<const float>>()[1]);
+-			LOG(STARFIVE, Debug) << "COLOUR_GAINS BLUE value: " << value;
+ 			break;
+ 		case controls::AWB_ENABLE:
+ 			cid = V4L2_CID_AUTO_WHITE_BALANCE;
+-			value = it.second.get<bool>();
+-			LOG(STARFIVE, Debug) << "AwbMode controls: " << id
+-				<< ", value: " << value;
+ 			break;
+ 		case controls::EXPOSURE_VALUE:
+ 			cid = V4L2_CID_EXPOSURE;
+-			offset = 0;
+-			value = lroundf(it.second.get<float>());
+-			LOG(STARFIVE, Debug) << "ExposureValue controls: " << id
+-				<< ", value: " << value;
+ 			break;
+ 		case controls::AE_ENABLE:
+ 			cid = V4L2_CID_EXPOSURE_AUTO;
+-			value = it.second.get<bool>();
+-			LOG(STARFIVE, Debug) << "AeExposureMode controls: " << id
+-				<< ", value: " << value;
+ 			break;
+ 		case controls::ANALOGUE_GAIN:
+ 			cid = V4L2_CID_ANALOGUE_GAIN;
+-			offset = 0;
+-			value = lroundf(it.second.get<float>());
+-			LOG(STARFIVE, Debug) << "AnalogueGain controls: " << id
+-				<< ", value: " << value;
+ 			break;
+ 		case controls::DIGITAL_GAIN:
+-			cid = V4L2_CID_GAIN;
+-			offset = 0;
+-			value = lroundf(it.second.get<float>());
+-			LOG(STARFIVE, Debug) << "AnalogueGain controls: " << id
+-				<< ", value: " << value;
++			cid = V4L2_CID_DIGITAL_GAIN;
+ 			break;
+ 		default:
+-			LOG(STARFIVE, Debug) << "default controls: " << id;
++			continue;
++		}
++
++		if (SensorControlsInfoMap.find(cid) == SensorControlsInfoMap.end()) {
++			LOG(STARFIVE, Debug) << "SensorControls not supported controls: " << id;
++			continue;
++		}
++
++		const ControlInfo &v4l2Info = SensorControlsInfoMap.at(cid);
++		int32_t min = v4l2Info.min().get<int32_t>();
++		int32_t def = v4l2Info.def().get<int32_t>();
++		int32_t max = v4l2Info.max().get<int32_t>();
++
++		switch (id) {
++		case controls::SATURATION:
++		case controls::SHARPNESS:
++		case controls::CONTRAST:
++		case controls::EXPOSURE_VALUE:
++		case controls::ANALOGUE_GAIN:
++		case controls::DIGITAL_GAIN:
++		case controls::BRIGHTNESS: {
++			value = static_cast<int32_t>(lroundf(it.second.get<float>()));
++			break;
++		}
++		case controls::COLOUR_GAINS: {
++			value = static_cast<int32_t>(lroundf(it.second.get<float>()));
++			LOG(STARFIVE, Debug) << "ColourGains controls: " << id
++				<< ", COLOUR_GAINS RED value: " << value;
++			SensorControls.set(cid, value);
++			cid = V4L2_CID_BLUE_BALANCE;
++			value = static_cast<int32_t>(lroundf(it.second.get<float>()));
++			LOG(STARFIVE, Debug) << "COLOUR_GAINS BLUE value: " << value;
++			break;
++		}
++		case controls::AWB_ENABLE:
++		case controls::AE_ENABLE:
++			value = it.second.get<bool>();
+ 			break;
++		default:
++			LOG(STARFIVE, Debug) << "default controls: " << id;
++			continue;
+ 		}
+ 
++		LOG(STARFIVE, Debug) << "controls: " << id << ", value: " << value;
+ 		if (SensorControlsInfoMap.find(cid) != SensorControlsInfoMap.end())
+ 			SensorControls.set(cid, value);
+ 		else
+ 			LOG(STARFIVE, Debug) << "SensorControls not supported controls: " << id;
+-
+ 	}
+ 
+ 	for (const auto &ctrl : SensorControls)
+@@ -967,7 +992,7 @@ int PipelineHandlerStarFive::registerCameras()
+ 					videoEntiryName, sensorEntityName);
+ 
+ 		/* Locate and open the capture video node. */
+-		if (data->init())
++		if (data->init(starFiveMediaDev_))
+ 			continue;
+ 
+ 		/*create ipa module*/
+@@ -1077,6 +1102,92 @@ void StarFiveCameraData::bufferReady(FrameBuffer *buffer)
+ 	pipe->completeRequest(request);
+ }
+ 
++void StarFiveCameraData::addControl(uint32_t cid, const ControlInfo &v4l2Info,
++				ControlInfoMap::Map *ctrls)
++{
++	const ControlId *id;
++	ControlInfo info;
++
++	LOG(STARFIVE, Debug) << __func__ << " cid: " << utils::hex(cid);
++	/* Map the control ID. */
++	switch (cid) {
++	case V4L2_CID_BRIGHTNESS:
++		id = &controls::Brightness;
++		break;
++	case V4L2_CID_CONTRAST:
++		id = &controls::Contrast;
++		break;
++	case V4L2_CID_SATURATION:
++		id = &controls::Saturation;
++		break;
++	case V4L2_CID_SHARPNESS:
++		id = &controls::Sharpness;
++		break;
++	case V4L2_CID_RED_BALANCE:
++	case V4L2_CID_BLUE_BALANCE:
++		id = &controls::ColourGains;
++		break;
++	case V4L2_CID_AUTO_WHITE_BALANCE:
++		id = &controls::AwbMode;
++		break;
++	case V4L2_CID_EXPOSURE:
++		id = &controls::ExposureValue;
++		break;
++	case V4L2_CID_EXPOSURE_AUTO:
++		id = &controls::AeExposureMode;
++		break;
++	case V4L2_CID_ANALOGUE_GAIN:
++		id = &controls::AnalogueGain;
++		break;
++	case V4L2_CID_DIGITAL_GAIN:
++		id = &controls::DigitalGain;
++		break;
++	default:
++		LOG(STARFIVE, Debug) << __func__ << " unsupport cid: " << utils::hex(cid);
++		return;
++	}
++
++	/* Map the control info. */
++	int32_t min = v4l2Info.min().get<int32_t>();
++	int32_t max = v4l2Info.max().get<int32_t>();
++	int32_t def = v4l2Info.def().get<int32_t>();
++
++	switch (cid) {
++	case V4L2_CID_SATURATION:
++	case V4L2_CID_SHARPNESS:
++	case V4L2_CID_CONTRAST:
++	case V4L2_CID_EXPOSURE:
++	case V4L2_CID_ANALOGUE_GAIN:
++	case V4L2_CID_DIGITAL_GAIN:
++	case V4L2_CID_BRIGHTNESS: {
++		info = ControlInfo{
++			{ static_cast<float>(min) },
++			{ static_cast<float>(max) },
++			{ static_cast<float>(def) },
++		};
++		break;
++	}
++
++	case V4L2_CID_EXPOSURE_AUTO:
++		info = ControlInfo{ false, true, true };
++		break;
++
++	case V4L2_CID_EXPOSURE_ABSOLUTE:
++		info = ControlInfo{
++			{ min * 100 },
++			{ max * 100 },
++			{ def * 100 }
++		};
++		break;
++
++	default:
++		info = v4l2Info;
++		break;
++	}
++
++	ctrls->emplace(id, info);
++}
++
+ REGISTER_PIPELINE_HANDLER(PipelineHandlerStarFive)
+ 
+ } /* namespace libcamera */
+-- 
+2.17.1
+

+ 249 - 0
package/libcamera/0008-support-use-yaml-file-to-config-sensor-pipeline.patch

@@ -0,0 +1,249 @@
+From dccc998ecf580e5c41b290a60e6b6f170601dc29 Mon Sep 17 00:00:00 2001
+From: sw.multimedia <sw.multimedia@starfivetech.com>
+Date: Wed, 15 Dec 2021 17:13:58 +0800
+Subject: [PATCH] support use yaml file to config sensor pipeline
+
+---
+ src/libcamera/meson.build                     |   3 +
+ .../pipeline/starfive/sensors_config.yaml     |  35 ++++++
+ src/libcamera/pipeline/starfive/starfive.cpp  | 106 ++++++++++++++++--
+ 3 files changed, 132 insertions(+), 12 deletions(-)
+ create mode 100644 src/libcamera/pipeline/starfive/sensors_config.yaml
+
+diff --git a/src/libcamera/meson.build b/src/libcamera/meson.build
+index 6727a777..39c44d42 100644
+--- a/src/libcamera/meson.build
++++ b/src/libcamera/meson.build
+@@ -113,6 +113,8 @@ if ipa_sign_module
+     libcamera_sources += ipa_pub_key_cpp
+ endif
+ 
++libyamlcpp = cc.find_library('yaml-cpp', required : true)
++
+ libcamera_deps = [
+     libatomic,
+     libcamera_base,
+@@ -121,6 +123,7 @@ libcamera_deps = [
+     libgnutls,
+     liblttng,
+     libudev,
++    libyamlcpp,
+ ]
+ 
+ # We add '/' to the build_rpath as a 'safe' path to act as a boolean flag.
+diff --git a/src/libcamera/pipeline/starfive/sensors_config.yaml b/src/libcamera/pipeline/starfive/sensors_config.yaml
+new file mode 100644
+index 00000000..3446b0ac
+--- /dev/null
++++ b/src/libcamera/pipeline/starfive/sensors_config.yaml
+@@ -0,0 +1,35 @@
++# SPDX-License-Identifier: LGPL-2.1-or-later
++#
++# Copyright (C) 2020, Google Inc.
++#
++%YAML 1.2
++# description:
++# sensors:
++#   - sensorentity: "imx219 0-0010"
++#       ispfwimage: "stf_isp0_fw_dump.bin"  isp fw image name
++#       sensortype: 4
++#
++#       sensortype value:
++#               {
++#               	DVP_YUV = 0,
++#               	MIPICSI0_YUV = 1,
++#               	MIPICSI1_YUV = 2,
++#               	DVP_ISP0 = 3,    // ISP0
++#               	MIPICSI0_ISP0 = 4,
++#               	MIPICSI1_ISP0 = 5,
++#               	DVP_ISP1 = 6,    // ISP1
++#               	MIPICSI0_ISP1 = 7,
++#               	MIPICSI1_ISP1 = 8,
++#               	SENSORTYPE_MAX
++#               }
++#
++---
++sensors:
++  - sensorentity: "imx219 0-0010"
++    ispfwimage: "stf_isp0_fw_dump.bin"
++    sensortype: 4
++
++  - sensorentity: "imx219 2-0010"
++    ispfwimage: "stf_isp0_fw_dump.bin"
++    sensortype: 8
++...
+diff --git a/src/libcamera/pipeline/starfive/starfive.cpp b/src/libcamera/pipeline/starfive/starfive.cpp
+index bb513387..8afe5e89 100644
+--- a/src/libcamera/pipeline/starfive/starfive.cpp
++++ b/src/libcamera/pipeline/starfive/starfive.cpp
+@@ -9,6 +9,8 @@
+ #include <queue>
+ #include <sys/mman.h>
+ #include <math.h>
++#include <yaml-cpp/yaml.h>
++#include <yaml-cpp/node/parse.h>
+ 
+ #include <libcamera/camera.h>
+ #include <libcamera/control_ids.h>
+@@ -40,14 +42,7 @@
+ 
+ #define STF_MAX_CAMERAS   3
+ 
+-namespace libcamera {
+-
+-LOG_DEFINE_CATEGORY(STARFIVE)
+-
+-static constexpr unsigned int BUFFER_COUNT = 4;
+-static constexpr unsigned int MAX_STREAMS = 2;
+-static const Size OUTPUT_MIN_SIZE = { 8, 8 };
+-static const Size OUTPUT_MAX_SIZE = { 8192, 8192 };
++#define PIPELINE_CONFIG_FILENAME "/data/sensors_pipeline.yaml"
+ 
+ namespace {
+ 
+@@ -67,14 +62,16 @@ typedef enum {
+ typedef struct {
+ 	std::string sensorEntityName_;
+ 	std::string sensorFwImageName_;
+-	SensorType sensorType_;
++	int sensorType_;
+ } SensorConfig;
+ 
+-const std::vector<SensorConfig> sensorConfigs = {
++const std::vector<SensorConfig> defaultSensorConfigs = {
+ 	{ "imx219 0-0010", "stf_isp0_fw_dump.bin", MIPICSI0_ISP0 },
+ 	{ "imx219 2-0010", "stf_isp0_fw_dump.bin", MIPICSI1_ISP1 },
+ };
+ 
++std::vector<SensorConfig> sensorConfigs;
++
+ typedef struct {
+ 	std::string source;
+ 	std::string link;
+@@ -140,6 +137,40 @@ const std::vector<PipelineConfigLink> pipelineConfigs[SENSORTYPE_MAX] = {
+ 	mipicsi1raw1Config,
+ };
+ 
++} /* namespace */
++
++namespace YAML {
++	template<>
++	struct convert<SensorConfig> {
++		static Node encode(const SensorConfig& sensor) {
++			Node node;
++			node.push_back(sensor.sensorEntityName_);
++			node.push_back(sensor.sensorFwImageName_);
++			node.push_back(sensor.sensorType_);
++			return node;
++		}
++
++		static bool decode(const Node& node, SensorConfig& sensor) {
++			if(node.size() != 3)
++				return false;
++
++			sensor.sensorEntityName_ = node["sensorentity"].as<std::string>();
++			sensor.sensorFwImageName_ = node["ispfwimage"].as<std::string>();
++			sensor.sensorType_ = node["sensortype"].as<int>();
++			return true;
++		}
++	};
++}
++
++namespace libcamera {
++
++LOG_DEFINE_CATEGORY(STARFIVE)
++
++static constexpr unsigned int BUFFER_COUNT = 4;
++static constexpr unsigned int MAX_STREAMS = 2;
++static const Size OUTPUT_MIN_SIZE = { 8, 8 };
++static const Size OUTPUT_MAX_SIZE = { 8192, 8192 };
++
+ const std::map<uint32_t, PixelFormat> mbusCodesToPixelFormat = {
+ 	{ MEDIA_BUS_FMT_SBGGR10_1X10, formats::SBGGR12 },
+ 	{ MEDIA_BUS_FMT_SGBRG10_1X10, formats::SGBRG12 },
+@@ -147,8 +178,6 @@ const std::map<uint32_t, PixelFormat> mbusCodesToPixelFormat = {
+ 	{ MEDIA_BUS_FMT_SRGGB10_1X10, formats::SRGGB12 },
+ };
+ 
+-} /* namespace */
+-
+ class StarFiveCameraData : public Camera::Private
+ {
+ public:
+@@ -424,6 +453,7 @@ private:
+ 	int enableLinks(std::vector<PipelineConfigLink> config);
+ 
+ 	MediaDevice *starFiveMediaDev_;
++	int parserPipelineConfig(const char *filename);
+ };
+ 
+ StarFiveCameraConfiguration::StarFiveCameraConfiguration(StarFiveCameraData *data)
+@@ -1044,6 +1074,56 @@ int PipelineHandlerStarFive::enableLinks(std::vector<PipelineConfigLink> config)
+ 	return ret;
+ }
+ 
++static void loaddefaultSensorConfig(void)
++{
++	LOG(STARFIVE, Debug) << __func__;
++	sensorConfigs.clear();
++	for (SensorConfig it : defaultSensorConfigs)
++		sensorConfigs.push_back(it);
++}
++
++static void printSensorConfig(void)
++{
++	for (SensorConfig it : sensorConfigs)
++		LOG(STARFIVE, Debug)
++			<< " sensorEntityName: " << it.sensorEntityName_
++			<< ", ispLoadFW: " << it.sensorFwImageName_
++			<< ", sensorType: " << it.sensorType_;
++}
++
++int PipelineHandlerStarFive::parserPipelineConfig(const char *filename)
++{
++	LOG(STARFIVE, Debug) << __func__;
++	try {
++		YAML::Node config = YAML::LoadFile(PIPELINE_CONFIG_FILENAME);
++
++		if (config["sensors"]) {
++			std::vector<SensorConfig> sensors =
++				config["sensors"].as<std::vector<SensorConfig>>();
++			int found = 0;
++
++			for (SensorConfig it : sensors) {
++				found = 0;
++				for (SensorConfig its : sensorConfigs) {
++					if (its.sensorEntityName_ == it.sensorEntityName_) {
++						found = 1;
++						break;
++					}
++				}
++				if (!found)
++					sensorConfigs.push_back(it);
++			}
++		}
++	} catch(std::exception const & e) {
++		LOG(STARFIVE, Debug) << PIPELINE_CONFIG_FILENAME
++			<< " yaml file error, use default config!!!";
++		loaddefaultSensorConfig();
++	}
++
++	printSensorConfig();
++	return 0;
++}
++
+ bool PipelineHandlerStarFive::match(DeviceEnumerator *enumerator)
+ {
+ 	int numCameras = 0;
+@@ -1061,6 +1141,8 @@ bool PipelineHandlerStarFive::match(DeviceEnumerator *enumerator)
+ 	if (starFiveMediaDev_->disableLinks())
+ 		return false;
+ 
++	parserPipelineConfig(PIPELINE_CONFIG_FILENAME);
++
+ 	for (SensorConfig it : sensorConfigs) {
+ 		MediaEntity *sensorEntity =
+ 			starFiveMediaDev_->getEntityByName(it.sensorEntityName_);
+-- 
+2.17.1
+

+ 271 - 0
package/libcamera/0009-csi0-csi1-can-be-configured-to-the-same-ISP.patch

@@ -0,0 +1,271 @@
+From 6d04377edf1307e3f2325db3ecfc9ff86db35233 Mon Sep 17 00:00:00 2001
+From: sw.multimedia <sw.multimedia@starfivetech.com>
+Date: Thu, 16 Dec 2021 17:46:03 +0800
+Subject: [PATCH] csi0/csi1 can be configured to the same ISP
+
+---
+ src/libcamera/pipeline/starfive/starfive.cpp | 146 +++++++++++--------
+ 1 file changed, 89 insertions(+), 57 deletions(-)
+
+diff --git a/src/libcamera/pipeline/starfive/starfive.cpp b/src/libcamera/pipeline/starfive/starfive.cpp
+index 8afe5e89..c433ec0d 100644
+--- a/src/libcamera/pipeline/starfive/starfive.cpp
++++ b/src/libcamera/pipeline/starfive/starfive.cpp
+@@ -183,11 +183,11 @@ class StarFiveCameraData : public Camera::Private
+ public:
+ 	StarFiveCameraData(PipelineHandler *pipe, MediaDevice *media,
+ 			std::string entityName,
+-			std::string sensorEntityName)
++			SensorConfig sensorConfig)
+ 		: Camera::Private(pipe), media_(media)
+ 	{
+ 		LOG(STARFIVE, Debug) << __func__;
+-		sensorEntityName_ = sensorEntityName;
++		sensorConfig_ = sensorConfig;
+ 		videoEntityName_ = entityName;
+ 		if ( videoEntityName_ == "stf_vin0_isp0_video1")
+ 			ispEntityName_ = "stf_isp0";
+@@ -222,10 +222,12 @@ public:
+ 	bool haveRaw() const { return haveRaw_; }
+ 	bool rawActive() const { return rawActive_; }
+ 	void setRawActive(bool val) { rawActive_ = val; }
++	SensorConfig getSensorConfig() {return sensorConfig_; }
+ 	std::vector<SizeRange> sensorSizes() const;
+ 	std::vector<PixelFormat> sensorFormats() const;
+ 	std::vector<PixelFormat> videoFormats() const;
+ 	void paramsFilled(unsigned int id){}
++	int ispLoadFW(const char *filename);
+ 
+ 	MediaDevice *media_;
+ 	V4L2VideoDevice *video_;
+@@ -241,8 +243,8 @@ private:
+ 	bool haveRaw_;
+ 	bool rawActive_;
+ 	std::string videoEntityName_;
+-	std::string sensorEntityName_;
+ 	std::string ispEntityName_;
++	SensorConfig sensorConfig_;
+ 	std::string getRawVideoEntityName()
+ 	{
+ 		LOG(STARFIVE, Debug) << __func__;
+@@ -253,7 +255,6 @@ private:
+ 		else
+ 			return "unknow";
+ 	}
+-	int ispLoadFW(const char *filename);
+ };
+ 
+ std::vector<PixelFormat> StarFiveCameraData::videoFormats() const
+@@ -335,17 +336,12 @@ int StarFiveCameraData::init(MediaDevice *media)
+ 	int ret;
+ 
+ 	LOG(STARFIVE, Debug) << __func__;
+-	if (sensorEntityName_ != "unknow") {
+-		sensor_ =
+-		  new CameraSensor(media_->getEntityByName(sensorEntityName_));
+-		ret = sensor_->init();
+-		if (ret)
+-			return ret;
+-		LOG(STARFIVE, Debug) << "sensor id: " << sensor_->id();
+-	} else {
+-		LOG(STARFIVE, Debug) << " Can't find sensorEntityName!";
+-		return -ENODEV;
+-	}
++	sensor_ =
++	  new CameraSensor(media_->getEntityByName(sensorConfig_.sensorEntityName_));
++	ret = sensor_->init();
++	if (ret)
++		return ret;
++	LOG(STARFIVE, Debug) << "sensor id: " << sensor_->id();
+ 
+ 	if (ispEntityName_ != "unknow") {
+ 		ispSubDev_ =
+@@ -353,13 +349,6 @@ int StarFiveCameraData::init(MediaDevice *media)
+ 		LOG(STARFIVE, Debug) << "ispEntityName: " << ispEntityName_;
+ 		if (ispSubDev_->open())
+ 			return -ENODEV;
+-
+-		for (SensorConfig it : sensorConfigs) {
+-			if (it.sensorEntityName_ == sensorEntityName_) {
+-				ispLoadFW(it.sensorFwImageName_.c_str());
+-				break;
+-			}
+-		}
+ 	}
+ 
+ 	video_ = new V4L2VideoDevice(media_->getEntityByName(videoEntityName_));
+@@ -551,10 +540,20 @@ PipelineHandlerStarFive::generateConfiguration(Camera *camera,
+ 	StarFiveCameraData *data = cameraData(camera);
+ 	StarFiveCameraConfiguration *config =
+ 		new StarFiveCameraConfiguration(data);
++	SensorConfig sensorConfig = data->getSensorConfig();
+ 
+ 	if (roles.empty())
+ 		return config;
+ 
++	int ret = enableLinks(pipelineConfigs[sensorConfig.sensorType_]);
++	if (ret < 0) {
++		LOG(STARFIVE, Error)
++			<< sensorConfig.sensorEntityName_
++			<< " enableLinks failed!";
++		return config;
++	}
++	data->ispLoadFW(sensorConfig.sensorFwImageName_.c_str());
++
+ 	for (const StreamRole role : roles) {
+ 		std::map<PixelFormat, std::vector<SizeRange>> streamFormats;
+ 		unsigned int bufferCount;
+@@ -1004,9 +1003,43 @@ int PipelineHandlerStarFive::registerCameras()
+ 	unsigned int numCameras = 0;
+ 
+ 	LOG(STARFIVE, Debug) << __func__;
+-	for (unsigned int id = 0;
+-			id < STF_MAX_CAMERAS 
+-			&& numCameras < STF_MAX_CAMERAS; ++id) {
++	for (SensorConfig it : sensorConfigs) {
++		std::string cameraName;
++		int id = 0;
++
++		switch (it.sensorType_) {
++		case DVP_YUV:
++		case MIPICSI0_YUV:
++		case MIPICSI1_YUV:
++			id = 0;
++			break;
++		case DVP_ISP0:
++		case MIPICSI0_ISP0:
++		case MIPICSI1_ISP0:
++			id = 1;
++			break;
++		case DVP_ISP1:
++		case MIPICSI0_ISP1:
++		case MIPICSI1_ISP1:
++			id = 2;
++			break;
++		default:
++			continue;
++		}
++
++		MediaEntity *sensorEntity =
++			starFiveMediaDev_->getEntityByName(it.sensorEntityName_);
++		if (sensorEntity != nullptr) {
++			int ret = enableLinks(pipelineConfigs[it.sensorType_]);
++			if (ret < 0) {
++				LOG(STARFIVE, Error)
++					<< it.sensorEntityName_
++					<< " enableLinks failed!";
++				continue;
++			}
++		} else
++			continue;
++
+ 		std::string videoEntiryName;
+ 		videoEntiryName = getVideoEntityNameById(id);
+ 		if (videoEntiryName == "unknow")
+@@ -1014,12 +1047,17 @@ int PipelineHandlerStarFive::registerCameras()
+ 
+ 		std::string sensorEntityName;
+ 		sensorEntityName = findSensorEntityName(videoEntiryName);
+-		if (sensorEntityName == "unknow")
+-			continue;
++		if (sensorEntityName != it.sensorEntityName_)
++		        continue;
++
++		if (id != 0)
++			cameraName = it.sensorEntityName_ + " isp" + std::to_string(id - 1);
++		else
++			cameraName = it.sensorEntityName_ + " wr";
+ 
+ 		std::unique_ptr<StarFiveCameraData> data =
+ 			std::make_unique<StarFiveCameraData>(this, starFiveMediaDev_,
+-					videoEntiryName, sensorEntityName);
++					videoEntiryName, it);
+ 
+ 		/* Locate and open the capture video node. */
+ 		if (data->init(starFiveMediaDev_))
+@@ -1036,18 +1074,17 @@ int PipelineHandlerStarFive::registerCameras()
+ 		data->ipa_->init(IPASettings{ conf, data->sensor_->model() });
+ 
+ 		/* Create and register the camera. */
+-		LOG(STARFIVE, Debug) << "register deviceName: "
+-				     << videoEntiryName;
++		LOG(STARFIVE, Debug) << "register deviceName: " << cameraName;
+ 		if (data->haveRaw()) {
+ 			std::set<Stream *> streams{ &data->outStream_,
+ 						&data->rawStream_ };
+ 			std::shared_ptr<Camera> camera =
+-				Camera::create(std::move(data), videoEntiryName, streams);
++				Camera::create(std::move(data), cameraName, streams);
+ 			registerCamera(std::move(camera));
+ 		} else {
+ 			std::set<Stream *> streams{ &data->outStream_ };
+ 			std::shared_ptr<Camera> camera =
+-				Camera::create(std::move(data), videoEntiryName, streams);
++				Camera::create(std::move(data), cameraName, streams);
+ 			registerCamera(std::move(camera));
+ 		}
+ 		numCameras++;
+@@ -1066,9 +1103,26 @@ int PipelineHandlerStarFive::enableLinks(std::vector<PipelineConfigLink> config)
+ 		if (!link)
+ 			return -ENODEV;
+ 
+-		ret = link->setEnabled(true);
+-		if (ret < 0)
+-			return ret;
++		MediaEntity *remote = link->sink()->entity();
++		for (MediaPad *pad : remote->pads()) {
++			for (MediaLink *e : pad->links()) {
++				if (link == e)
++					continue;
++
++				if ((e->flags() & MEDIA_LNK_FL_ENABLED) &&
++				    !(e->flags() & MEDIA_LNK_FL_IMMUTABLE)) {
++					ret = e->setEnabled(false);
++					if (ret < 0)
++						return ret;
++				}
++			}
++		}
++
++		if (!(link->flags() & MEDIA_LNK_FL_ENABLED)) {
++			ret = link->setEnabled(true);
++			if (ret < 0)
++				return ret;
++		}
+ 	}
+ 
+ 	return ret;
+@@ -1138,30 +1192,8 @@ bool PipelineHandlerStarFive::match(DeviceEnumerator *enumerator)
+ 	if (!starFiveMediaDev_)
+ 		return false;
+ 
+-	if (starFiveMediaDev_->disableLinks())
+-		return false;
+-
+ 	parserPipelineConfig(PIPELINE_CONFIG_FILENAME);
+ 
+-	for (SensorConfig it : sensorConfigs) {
+-		MediaEntity *sensorEntity =
+-			starFiveMediaDev_->getEntityByName(it.sensorEntityName_);
+-		int ret;
+-
+-		if (sensorEntity != nullptr) {
+-			if (it.sensorType_ < DVP_YUV
+-				|| it.sensorType_ >= SENSORTYPE_MAX)
+-				continue;
+-			ret = enableLinks(pipelineConfigs[it.sensorType_]);
+-			if (ret < 0) {
+-				LOG(STARFIVE, Error)
+-					<< it.sensorEntityName_
+-					<< " enableLinks failed!";
+-				continue;
+-			}
+-		}
+-	}
+-
+ 	numCameras = registerCameras();
+ 	if (numCameras)
+ 		LOG(STARFIVE, Debug)
+-- 
+2.17.1
+

+ 12 - 0
package/libcamera/Config.in

@@ -12,6 +12,11 @@ menuconfig BR2_PACKAGE_LIBCAMERA
 	depends on BR2_TOOLCHAIN_GCC_AT_LEAST_7 # C++17
 	depends on !BR2_STATIC_LIBS # gnutls
 	depends on BR2_USE_WCHAR # gnutls
+	depends on BR2_PACKAGE_LIBEVENT # libevent
+	depends on BR2_PACKAGE_PYTHON3
+	select BR2_PACKAGE_PYTHON_JINJA2
+	select BR2_PACKAGE_PYTHON_PLY
+	select BR2_PACKAGE_PYTHON_PYYAML
 	select BR2_PACKAGE_GNUTLS
 	select BR2_PACKAGE_LIBCAMERA_PIPELINE_UVCVIDEO if !BR2_PACKAGE_LIBCAMERA_HAS_PIPELINE
 	help
@@ -68,6 +73,13 @@ config BR2_PACKAGE_LIBCAMERA_PIPELINE_VIMC
 	help
 	  Pipeline for the vimc device.
 
+config BR2_PACKAGE_LIBCAMERA_PIPELINE_STARFIVE
+	bool "starfive pipeline"
+	select BR2_PACKAGE_LIBCAMERA_HAS_PIPELINE
+	select BR2_PACKAGE_YAML_CPP
+	help
+	  Pipeline for the starfive device.
+
 endif # BR2_PACKAGE_LIBCAMERA
 
 comment "libcamera needs a toolchain w/ C++, threads, wchar, dynamic library, gcc >= 7"

+ 2 - 0
package/libcamera/libcamera.hash

@@ -1,4 +1,6 @@
 sha256  a7800b54584dee321874610ef70b1b7690c79d8340c5b134ea35176657681e29  libcamera-e59713c68678f3eb6b6ebe97cabdc88c7042567f.tar.gz
+sha256  5bca8bc504318da05fd238ca48a2519e71721d3417ff6f014b42146d270281a8  libcamera-06e53199c2563105030bda4c72752b853da7edc8.tar.gz
+sha256  9369e7cbc138cd31f782325a4361f4163909d09c7475a1f99e938504c191d5f2  libcamera-d6f4abeead1e86d89dc376e8a303849bdb98d5fd.tar.gz
 
 # license files
 sha256  fd38b2c053c0cce46d9c5ef3545a6e34d157a240ba99c9b8dca5d37a8147da6c  LICENSES/BSD-2-Clause.txt

+ 16 - 3
package/libcamera/libcamera.mk

@@ -5,16 +5,20 @@
 ################################################################################
 
 LIBCAMERA_SITE = https://git.linuxtv.org/libcamera.git
-LIBCAMERA_VERSION = e59713c68678f3eb6b6ebe97cabdc88c7042567f
+# LIBCAMERA_VERSION = e59713c68678f3eb6b6ebe97cabdc88c7042567f
+# LIBCAMERA_VERSION = 06e53199c2563105030bda4c72752b853da7edc8
+LIBCAMERA_VERSION = d6f4abeead1e86d89dc376e8a303849bdb98d5fd
 LIBCAMERA_SITE_METHOD = git
 LIBCAMERA_DEPENDENCIES = \
 	host-openssl \
 	host-pkgconf \
 	host-python3-pyyaml \
+	host-python-jinja2 \
+	host-python-ply \
 	gnutls
 LIBCAMERA_CONF_OPTS = \
-	-Dandroid=false \
-	-Ddocumentation=false \
+	-Dandroid=disabled \
+	-Ddocumentation=disabled \
 	-Dtest=false \
 	-Dwerror=false
 LIBCAMERA_INSTALL_STAGING = YES
@@ -56,6 +60,7 @@ LIBCAMERA_PIPELINES-$(BR2_PACKAGE_LIBCAMERA_PIPELINE_RKISP1) += rkisp1
 LIBCAMERA_PIPELINES-$(BR2_PACKAGE_LIBCAMERA_PIPELINE_SIMPLE) += simple
 LIBCAMERA_PIPELINES-$(BR2_PACKAGE_LIBCAMERA_PIPELINE_UVCVIDEO) += uvcvideo
 LIBCAMERA_PIPELINES-$(BR2_PACKAGE_LIBCAMERA_PIPELINE_VIMC) += vimc
+LIBCAMERA_PIPELINES-$(BR2_PACKAGE_LIBCAMERA_PIPELINE_STARFIVE) += starfive
 
 LIBCAMERA_CONF_OPTS += -Dpipelines=$(subst $(space),$(comma),$(LIBCAMERA_PIPELINES-y))
 
@@ -83,4 +88,12 @@ ifeq ($(BR2_PACKAGE_HAS_UDEV),y)
 LIBCAMERA_DEPENDENCIES += udev
 endif
 
+ifeq ($(BR2_PACKAGE_LIBEVENT),y)
+LIBCAMERA_DEPENDENCIES += libevent
+endif
+
+ifeq ($(BR2_PACKAGE_LIBCAMERA_PIPELINE_STARFIVE),y)
+LIBCAMERA_DEPENDENCIES += yaml-cpp
+endif
+
 $(eval $(meson-package))