Demonstration of ControlList queues

include/libcamera/camera.h

@@ -147,6 +147,7 @@ class Camera final : public Object, public std::enable_shared_from_this<Camera>,
 
 	std::unique_ptr<Request> createRequest(uint64_t cookie = 0);
 	int queueRequest(Request *request);
+	int queueControls(ControlList &&controls);
 
 	int start(const ControlList *controls = nullptr);
 	int stop();

include/libcamera/internal/pipeline_handler.h

@@ -59,6 +59,7 @@ class PipelineHandler : public std::enable_shared_from_this<PipelineHandler>,
 
 	void registerRequest(Request *request);
 	void queueRequest(Request *request);
+	int queueControls(Camera *camera, ControlList controls);
 
 	bool completeBuffer(Request *request, FrameBuffer *buffer);
 	void completeRequest(Request *request);
@@ -85,6 +86,12 @@ class PipelineHandler : public std::enable_shared_from_this<PipelineHandler>,
 	unsigned int useCount() const { return useCount_; }
 
 	virtual int queueRequestDevice(Camera *camera, Request *request) = 0;
+
+	virtual int queueControlsDevice([[maybe_unused]] Camera *camera, [[maybe_unused]] const ControlList &controls)
+	{
+		return -EOPNOTSUPP;
+	}
+
 	virtual void stopDevice(Camera *camera) = 0;
 
 	virtual bool acquireDevice(Camera *camera);

src/libcamera/camera.cpp

@@ -1379,6 +1379,57 @@ int Camera::queueRequest(Request *request)
 	return 0;
 }
 
+/**
+ * \brief Queue controls to be applied as soon as possible
+ * \param[in] controls The list of controls to queue
+ *
+ * This function tries to ensure that the controls in \a controls are applied
+ * to the camera as soon as possible. If there are still pending controls waiting
+ * to be applied (because of previous calls to Camera::queueControls), then
+ * these controls will be applied as soon as possible on a frame after those.
+ *
+ * The exact guarantees are camera dependent, but it is guaranteed that the
+ * controls will be applied no later than with the next \ref Request"request"
+ * that the application \ref Camera::queueRequest() "queues" (after any requests
+ * have been *used up" for sending previously queued controls).
+ *
+ * \context This function is \threadsafe. It may only be called when the camera
+ * is in the Running state as defined in \ref camera_operation.
+ *
+ * \return 0 on success or a negative error code otherwise
+ * \retval -ENODEV The camera has been disconnected from the system
+ * \retval -EACCES The camera is not running
+ */
+int Camera::queueControls(ControlList &&controls)
+{
+	Private *const d = _d();
+
+	/*
+	 * Like requests, controls can't be queued if the camera is not running.
+	 * Controls can be applied immediately when the camera starts using the
+	 * Camera::Start method.
+	 */
+
+	int ret = d->isAccessAllowed(Private::CameraRunning);
+	if (ret < 0)
+		return ret;
+
+	/*
+	 * We want to be able to queue empty control lists, as this gives a way of
+	 * forcing another frame with the same controls as last time, before queueing
+	 * another control list that might change them again.
+	 */
+
+	patchControlList(controls);
+
+	/*
+	 * \todo Or `ConnectionTypeBlocking` to get the return value?
+	 */
+	d->pipe_->invokeMethod(&PipelineHandler::queueControls, ConnectionTypeQueued, this, std::move(controls));
+
+	return 0;
+}
+
 /**
  * \brief Start capture from camera
  * \param[in] controls Controls to be applied before starting the Camera

src/libcamera/control_ids_rpi.yaml

@@ -188,10 +188,18 @@ controls:
       type: int64_t
       direction: out
       description: |
-        This is the sequence number of the request whose control list has
-        just been applied. Controls normally take several frames to apply,
-        so the number here will refer to a request submitted a number of
-        frames earlier.
+        This is the control list id number that is synchonised with this
+        request, meaning that this request is the first one where the images
+        have had the identified list of controls applied.
+
+        Every time a control list is queued using Camera::queueControls, it
+        is assigned a new sequence number, starting at 1 on the first such
+        after the camera is started, and increasing by one on every subsequent
+        call.
+
+        This is the number that is then reported back as the
+        ControlListSequence. When no controls have been sent with
+        Camera::queueControls, the ControlListSequence reports the value zero.
 
   - CnnOutputTensor:
       type: float

src/libcamera/pipeline/rpi/common/pipeline_base.cpp

@@ -793,6 +793,20 @@ int PipelineHandlerBase::queueRequestDevice(Camera *camera, Request *request)
 	return 0;
 }
 
+int PipelineHandlerBase::queueControlsDevice(Camera *camera, const ControlList &controls)
+{
+	CameraData *data = cameraData(camera);
+
+	/*
+	 * For now, just hold these controls in a queue. The front item will get popped
+	 * off and merged into the next request that we pull off our request queue and
+	 * start to process.
+	 */
+	data->controlsQueue_.push(controls);
+
+	return 0;
+}
+
 int PipelineHandlerBase::registerCamera(std::unique_ptr<RPi::CameraData> &cameraData,
 					std::shared_ptr<MediaDevice> frontend,
 					const std::string &frontendName,
@@ -1546,12 +1560,59 @@ void CameraData::handleControlLists(uint32_t delayContext, ControlList &paramCon
 {
 	/*
 	 * The delayContext is the sequence number after it's gone through the various
-	 * pipeline delays, so that's what gets reported as the "ControlListSequence"
-	 * in the metadata, being the sequence number of the request whose ControlList
-	 * has just been applied.
+	 * pipeline delays.
 	 */
 	Request *request = requestQueue_.front();
-	request->_d()->metadata().set(controls::rpi::ControlListSequence, delayContext);
+
+	/*
+	 * Create a merged list of controls from this request, and also from the
+	 * ControlList queue.
+	 */
+	paramControls = request->controls();
+	if (!controlsQueue_.empty()) {
+		paramControls.merge(std::move(controlsQueue_.front()),
+				    ControlList::MergePolicy::OverwriteExisting);
+		controlsQueue_.pop();
+		controlListId_++;
+		LOG(RPI, Debug) << "Popped control list " << controlListId_ << " from queue";
+	}
+
+	/*
+	 * Record which control list corresponds to this ipaCookie. Because setDelayedControls
+	 * now gets called by the IPA from the start of the following frame, we must record
+	 * the previous control list id.
+	 */
+	syncTable_.emplace(SyncTableEntry{ request->sequence(), controlListId_ });
+	LOG(RPI, Debug) << "Add sync table entry: sequence " << request->sequence()
+			<< " control list id " << controlListId_;
+
+	/*
+	 * We know we that we added an entry for every delayContext, so we can
+	 * find the one for this Bayer frame, and this links us to the correct
+	 * control list. Anything ahead of "our" entry in the queue is old, so
+	 * can be dropped.
+	 */
+	while (!syncTable_.empty() &&
+	       syncTable_.front().ipaCookie != delayContext) {
+		LOG(RPI, Debug) << "Pop sync entry: ipa cookie "
+				<< syncTable_.front().ipaCookie << " control id "
+				<< syncTable_.front().controlListId << " for job "
+				<< delayContext;
+		syncTable_.pop();
+	}
+
+	if (syncTable_.empty())
+		LOG(RPI, Warning) << "Unable to find ipa cookie for PFC";
+	else {
+		LOG(RPI, Debug) << "Using sync control id " << syncTable_.front().controlListId;
+		requestControlId_ = syncTable_.front().controlListId;
+	}
+
+	/*
+	 * We report the sequence number of the controls from the ControlList queue
+	 * as the "ControlListSequence" in the metadata.
+	 */
+	request->_d()->metadata().set(controls::rpi::ControlListSequence, requestControlId_);
 
 	/*
 	 * Controls that take effect immediately (typically ISP controls) have to be
@@ -1563,18 +1624,17 @@ void CameraData::handleControlLists(uint32_t delayContext, ControlList &paramCon
 	 * we can pass back the controls that really need to happen now, without
 	 * disturbing the controls that were submitted with the request.
 	 */
-	ASSERT(paramControls.empty());
-	immediateControls_.push({ request->sequence(), {} });
-	for (const auto &ctrl : request->controls()) {
+	ControlList controls = std::move(paramControls);
+	immediateControls_.push({ controlListId_, {} });
+	for (const auto &ctrl : controls) {
 		if (isControlDelayed(ctrl.first))
 			paramControls.set(ctrl.first, ctrl.second);
 		else
 			immediateControls_.back().controls.set(ctrl.first, ctrl.second);
 	}
-
 	/* "Immediate" controls that have become due are now merged back into this request. */
 	while (!immediateControls_.empty() &&
-	       immediateControls_.front().controlListId <= delayContext) {
+	       immediateControls_.front().controlListId <= requestControlId_) {
 		paramControls.merge(immediateControls_.front().controls,
 				    ControlList::MergePolicy::OverwriteExisting);
 		immediateControls_.pop();

src/libcamera/pipeline/rpi/common/pipeline_base.h

@@ -48,7 +48,7 @@ class CameraData : public Camera::Private
 {
 public:
 	CameraData(PipelineHandler *pipe)
-		: Camera::Private(pipe), state_(State::Stopped),
+		: Camera::Private(pipe), state_(State::Stopped), controlListId_(0),
 		  startupFrameCount_(0), invalidFrameCount_(0), buffersAllocated_(false)
 	{
 	}
@@ -130,6 +130,9 @@ class CameraData : public Camera::Private
 	}
 
 	std::queue<Request *> requestQueue_;
+	std::queue<ControlList> controlsQueue_;
+	uint64_t controlListId_;
+	uint64_t requestControlId_;
 
 	/* For handling digital zoom. */
 	IPACameraSensorInfo sensorInfo_;
@@ -180,6 +183,12 @@ class CameraData : public Camera::Private
 
 	ClockRecovery wallClockRecovery_;
 
+	struct SyncTableEntry {
+		uint32_t ipaCookie;
+		uint64_t controlListId;
+	};
+	std::queue<SyncTableEntry> syncTable_;
+
 	struct ImmediateControlsEntry {
 		uint64_t controlListId;
 		ControlList controls;
@@ -235,6 +244,7 @@ class PipelineHandlerBase : public PipelineHandler
 	void releaseDevice(Camera *camera) override;
 
 	int queueRequestDevice(Camera *camera, Request *request) override;
+	int queueControlsDevice(Camera *camera, const ControlList &controls) override;
 
 protected:
 	int registerCamera(std::unique_ptr<RPi::CameraData> &cameraData,

src/libcamera/pipeline/rpi/pisp/pisp.cpp

@@ -2329,7 +2329,7 @@ void PiSPCameraData::tryRunPipeline()
 	params.sensorControls = std::move(job.sensorControls);
 	/* params.requestControls is set by handleControlLists. */
 
-	/* This sorts out synchronisation with ControlLists in earlier requests. */
+	/* This sorts out synchronisation with the ControlList queue. */
 	handleControlLists(job.delayContext, params.requestControls);
 
 	/* Set our state to say the pipeline is active. */

src/libcamera/pipeline/rpi/vc4/vc4.cpp

@@ -997,7 +997,7 @@ void Vc4CameraData::tryRunPipeline()
 	params.buffers.embedded = 0;
 	/* params.requestControls is set by handleControlLists. */
 
-	/* This sorts out synchronisation with ControlLists in earlier requests. */
+	/* This sorts out synchronisation with the ControlList queue. */
 	handleControlLists(bayerFrame.delayContext, params.requestControls);
 
 	/* Set our state to say the pipeline is active. */

src/libcamera/pipeline_handler.cpp

@@ -477,6 +477,26 @@ void PipelineHandler::queueRequest(Request *request)
 	request->_d()->prepare(300ms);
 }
 
+/**
+ * \brief Queue controls to apply as soon as possible
+ * \param[in] camera The camera
+ * \param[in] controls The controls to apply
+ *
+ * This function tries to queue \a controls immediately to the device by
+ * calling queueControlsDevice().
+ *
+ * \context This function is called from the CameraManager thread.
+ */
+int PipelineHandler::queueControls(Camera *camera, ControlList controls)
+{
+	int ret = queueControlsDevice(camera, controls);
+
+	if (ret < 0)
+		LOG(Pipeline, Error) << "Failed to queue controls: " << ret;
+
+	return ret;
+}
+
 /**
  * \brief Queue one requests to the device
  */
@@ -543,6 +563,21 @@ void PipelineHandler::doQueueRequests(Camera *camera)
  * \return 0 on success or a negative error code otherwise
  */
 
+/**
+ * \fn PipelineHandler::queueControlsDevice()
+ * \brief Queue controls to be applied as soon as possible
+ * \param[in] camera The camera
+ * \param[in] controls The controls to apply
+ *
+ * This function queues \a controls to \a camera so that they can be
+ * applied as soon as possible
+ *
+ * \context This function is called from the CameraManager thread.
+ *
+ * \return 0 on success or a negative error code otherwise
+ * \return -EOPNOTSUPP if control queueing is not supported
+ */
+
 /**
  * \brief Complete a buffer for a request
  * \param[in] request The request the buffer belongs to

src/py/libcamera/py_main.cpp

@@ -246,6 +246,20 @@ PYBIND11_MODULE(_libcamera, m)
 			}
 		})
 
+		.def("queue_controls", [](Camera &self, const std::unordered_map<const ControlId *, py::object> &controls) {
+
+			ControlList controlList(self.controls());
+
+			for (const auto &[id, obj] : controls) {
+				auto val = pyToControlValue(obj, id->type());
+				controlList.set(id->id(), val);
+			}
+
+			int ret = self.queueControls(std::move(controlList));
+			if (ret)
+				throw std::system_error(-ret, std::generic_category(),
+							"Failed to queue controls"); }, py::arg("controls") = std::unordered_map<const ControlId *, py::object>())
+
 		.def_property_readonly("streams", [](Camera &self) {
 			py::set set;
 			for (auto &s : self.streams()) {