From f8bb27636f229d00921576aa44161820a729d923 Mon Sep 17 00:00:00 2001 From: David Plowman Date: Thu, 12 Mar 2026 13:31:57 +0000 Subject: [PATCH 1/5] libcamera: Add Camera::queueControls() Add `Camera::queueControls()` whose purpose is to apply controls as soon as possible, without going through `Request::controls()`. A new virtual function `PipelineHandler::queueControlsDevice()` is provided for the pipeline handler to implement fast-tracked application of controls. If the pipeline handler does not support control queueing, an error is signalled and applications should avoid using this mechanism. In future we might consider an alternative fallback scheme where queued controls are dispatched into the pipeline handler with the next request. (Though note that the benefit of by-passing the request queue is then lost.) Note further that we make no attempt to remove controls from requests, thereby maintaining backwards compatability. The controls in a request, and the top ControlList from the queue are simply merged for processing. Again, one could reconsider this at a later date. Signed-off-by: David Plowman --- include/libcamera/camera.h | 1 + include/libcamera/internal/pipeline_handler.h | 7 +++ src/libcamera/camera.cpp | 51 +++++++++++++++++++ src/libcamera/pipeline_handler.cpp | 35 +++++++++++++ 4 files changed, 94 insertions(+) diff --git a/include/libcamera/camera.h b/include/libcamera/camera.h index b24a29740..93a484e44 100644 --- a/include/libcamera/camera.h +++ b/include/libcamera/camera.h @@ -147,6 +147,7 @@ class Camera final : public Object, public std::enable_shared_from_this, std::unique_ptr createRequest(uint64_t cookie = 0); int queueRequest(Request *request); + int queueControls(ControlList &&controls); int start(const ControlList *controls = nullptr); int stop(); diff --git a/include/libcamera/internal/pipeline_handler.h b/include/libcamera/internal/pipeline_handler.h index 6922ce18e..4f97a6f28 100644 --- a/include/libcamera/internal/pipeline_handler.h +++ b/include/libcamera/internal/pipeline_handler.h @@ -59,6 +59,7 @@ class PipelineHandler : public std::enable_shared_from_this, void registerRequest(Request *request); void queueRequest(Request *request); + int queueControls(Camera *camera, ControlList controls); bool completeBuffer(Request *request, FrameBuffer *buffer); void completeRequest(Request *request); @@ -85,6 +86,12 @@ class PipelineHandler : public std::enable_shared_from_this, unsigned int useCount() const { return useCount_; } virtual int queueRequestDevice(Camera *camera, Request *request) = 0; + + virtual int queueControlsDevice([[maybe_unused]] Camera *camera, [[maybe_unused]] const ControlList &controls) + { + return -EOPNOTSUPP; + } + virtual void stopDevice(Camera *camera) = 0; virtual bool acquireDevice(Camera *camera); diff --git a/src/libcamera/camera.cpp b/src/libcamera/camera.cpp index 93b9e603d..8e6042f01 100644 --- a/src/libcamera/camera.cpp +++ b/src/libcamera/camera.cpp @@ -1379,6 +1379,57 @@ int Camera::queueRequest(Request *request) return 0; } +/** + * \brief Queue controls to be applied as soon as possible + * \param[in] controls The list of controls to queue + * + * This function tries to ensure that the controls in \a controls are applied + * to the camera as soon as possible. If there are still pending controls waiting + * to be applied (because of previous calls to Camera::queueControls), then + * these controls will be applied as soon as possible on a frame after those. + * + * The exact guarantees are camera dependent, but it is guaranteed that the + * controls will be applied no later than with the next \ref Request"request" + * that the application \ref Camera::queueRequest() "queues" (after any requests + * have been *used up" for sending previously queued controls). + * + * \context This function is \threadsafe. It may only be called when the camera + * is in the Running state as defined in \ref camera_operation. + * + * \return 0 on success or a negative error code otherwise + * \retval -ENODEV The camera has been disconnected from the system + * \retval -EACCES The camera is not running + */ +int Camera::queueControls(ControlList &&controls) +{ + Private *const d = _d(); + + /* + * Like requests, controls can't be queued if the camera is not running. + * Controls can be applied immediately when the camera starts using the + * Camera::Start method. + */ + + int ret = d->isAccessAllowed(Private::CameraRunning); + if (ret < 0) + return ret; + + /* + * We want to be able to queue empty control lists, as this gives a way of + * forcing another frame with the same controls as last time, before queueing + * another control list that might change them again. + */ + + patchControlList(controls); + + /* + * \todo Or `ConnectionTypeBlocking` to get the return value? + */ + d->pipe_->invokeMethod(&PipelineHandler::queueControls, ConnectionTypeQueued, this, std::move(controls)); + + return 0; +} + /** * \brief Start capture from camera * \param[in] controls Controls to be applied before starting the Camera diff --git a/src/libcamera/pipeline_handler.cpp b/src/libcamera/pipeline_handler.cpp index e7145c1d4..f138498cc 100644 --- a/src/libcamera/pipeline_handler.cpp +++ b/src/libcamera/pipeline_handler.cpp @@ -477,6 +477,26 @@ void PipelineHandler::queueRequest(Request *request) request->_d()->prepare(300ms); } +/** + * \brief Queue controls to apply as soon as possible + * \param[in] camera The camera + * \param[in] controls The controls to apply + * + * This function tries to queue \a controls immediately to the device by + * calling queueControlsDevice(). + * + * \context This function is called from the CameraManager thread. + */ +int PipelineHandler::queueControls(Camera *camera, ControlList controls) +{ + int ret = queueControlsDevice(camera, controls); + + if (ret < 0) + LOG(Pipeline, Error) << "Failed to queue controls: " << ret; + + return ret; +} + /** * \brief Queue one requests to the device */ @@ -543,6 +563,21 @@ void PipelineHandler::doQueueRequests(Camera *camera) * \return 0 on success or a negative error code otherwise */ +/** + * \fn PipelineHandler::queueControlsDevice() + * \brief Queue controls to be applied as soon as possible + * \param[in] camera The camera + * \param[in] controls The controls to apply + * + * This function queues \a controls to \a camera so that they can be + * applied as soon as possible + * + * \context This function is called from the CameraManager thread. + * + * \return 0 on success or a negative error code otherwise + * \return -EOPNOTSUPP if control queueing is not supported + */ + /** * \brief Complete a buffer for a request * \param[in] request The request the buffer belongs to From 552ef40c886701ac9fb7b5866991ff5497d4a091 Mon Sep 17 00:00:00 2001 From: David Plowman Date: Fri, 13 Mar 2026 10:15:56 +0000 Subject: [PATCH 2/5] py: Add bindings for Camera::queueControls Python Camera.queue_controls calls Camera::queueControls. Signed-off-by: David Plowman --- src/py/libcamera/py_main.cpp | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/src/py/libcamera/py_main.cpp b/src/py/libcamera/py_main.cpp index 3b101f996..78cdd3858 100644 --- a/src/py/libcamera/py_main.cpp +++ b/src/py/libcamera/py_main.cpp @@ -246,6 +246,20 @@ PYBIND11_MODULE(_libcamera, m) } }) + .def("queue_controls", [](Camera &self, const std::unordered_map &controls) { + + ControlList controlList(self.controls()); + + for (const auto &[id, obj] : controls) { + auto val = pyToControlValue(obj, id->type()); + controlList.set(id->id(), val); + } + + int ret = self.queueControls(std::move(controlList)); + if (ret) + throw std::system_error(-ret, std::generic_category(), + "Failed to queue controls"); }, py::arg("controls") = std::unordered_map()) + .def_property_readonly("streams", [](Camera &self) { py::set set; for (auto &s : self.streams()) { From f52490bc9d12a2eb2ba2d43a1298a1dfaa74c9f5 Mon Sep 17 00:00:00 2001 From: David Plowman Date: Fri, 13 Mar 2026 14:52:46 +0000 Subject: [PATCH 3/5] pipeline: rpi: Add queueControlsDevice Allows control lists to be queued up independently of requests. Signed-off-by: David Plowman --- .../pipeline/rpi/common/pipeline_base.cpp | 14 ++++++++++++++ src/libcamera/pipeline/rpi/common/pipeline_base.h | 2 ++ 2 files changed, 16 insertions(+) diff --git a/src/libcamera/pipeline/rpi/common/pipeline_base.cpp b/src/libcamera/pipeline/rpi/common/pipeline_base.cpp index 5a5acf6a1..37f0d1e6e 100644 --- a/src/libcamera/pipeline/rpi/common/pipeline_base.cpp +++ b/src/libcamera/pipeline/rpi/common/pipeline_base.cpp @@ -793,6 +793,20 @@ int PipelineHandlerBase::queueRequestDevice(Camera *camera, Request *request) return 0; } +int PipelineHandlerBase::queueControlsDevice(Camera *camera, const ControlList &controls) +{ + CameraData *data = cameraData(camera); + + /* + * For now, just hold these controls in a queue. The front item will get popped + * off and merged into the next request that we pull off our request queue and + * start to process. + */ + data->controlsQueue_.push(controls); + + return 0; +} + int PipelineHandlerBase::registerCamera(std::unique_ptr &cameraData, std::shared_ptr frontend, const std::string &frontendName, diff --git a/src/libcamera/pipeline/rpi/common/pipeline_base.h b/src/libcamera/pipeline/rpi/common/pipeline_base.h index 758155ee0..5130bd1a0 100644 --- a/src/libcamera/pipeline/rpi/common/pipeline_base.h +++ b/src/libcamera/pipeline/rpi/common/pipeline_base.h @@ -130,6 +130,7 @@ class CameraData : public Camera::Private } std::queue requestQueue_; + std::queue controlsQueue_; /* For handling digital zoom. */ IPACameraSensorInfo sensorInfo_; @@ -235,6 +236,7 @@ class PipelineHandlerBase : public PipelineHandler void releaseDevice(Camera *camera) override; int queueRequestDevice(Camera *camera, Request *request) override; + int queueControlsDevice(Camera *camera, const ControlList &controls) override; protected: int registerCamera(std::unique_ptr &cameraData, From e7a3911e02b2f8bb075c971d7b03a03757baf3a3 Mon Sep 17 00:00:00 2001 From: David Plowman Date: Tue, 17 Mar 2026 14:51:01 +0000 Subject: [PATCH 4/5] controls: rpi: Amend the ControlListSequence control The ControlListSequence is amended to identify the sequence number of the ControlList, from the ControlList queue, that has just been applied, and which was submitted using Camera::queueControls. It no longer lists a request sequence number as ControlLists that applications will want to synchronise with are more effectively applied through the ControlList queue. Signed-off-by: David Plowman --- src/libcamera/control_ids_rpi.yaml | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/src/libcamera/control_ids_rpi.yaml b/src/libcamera/control_ids_rpi.yaml index 9d0f30002..c36fdfd6d 100644 --- a/src/libcamera/control_ids_rpi.yaml +++ b/src/libcamera/control_ids_rpi.yaml @@ -188,10 +188,18 @@ controls: type: int64_t direction: out description: | - This is the sequence number of the request whose control list has - just been applied. Controls normally take several frames to apply, - so the number here will refer to a request submitted a number of - frames earlier. + This is the control list id number that is synchonised with this + request, meaning that this request is the first one where the images + have had the identified list of controls applied. + + Every time a control list is queued using Camera::queueControls, it + is assigned a new sequence number, starting at 1 on the first such + after the camera is started, and increasing by one on every subsequent + call. + + This is the number that is then reported back as the + ControlListSequence. When no controls have been sent with + Camera::queueControls, the ControlListSequence reports the value zero. - CnnOutputTensor: type: float From 9ac68ab6456d51781af1520fd154a4ddf6665e4e Mon Sep 17 00:00:00 2001 From: David Plowman Date: Tue, 17 Mar 2026 15:51:42 +0000 Subject: [PATCH 5/5] pipeline: rpi: Amend ControlList reporting to use the ControlList queue Previously, the ControlListSequence reported the request sequence number of the ControlList that has just been applied. However, ControlLists can now be applied more quickly and effectively using the ControlList queue, and this is recommended to applications. Therefore we switch over to reporting the sequence number of the ControlList in the ControlList queue instead. Applications can still put ControlLists into requests as before, so the scheme is entirely backwards compatible, however you can't find out when those controls are applied using this mechanism. Signed-off-by: David Plowman --- .../pipeline/rpi/common/pipeline_base.cpp | 64 ++++++++++++++++--- .../pipeline/rpi/common/pipeline_base.h | 10 ++- src/libcamera/pipeline/rpi/pisp/pisp.cpp | 2 +- src/libcamera/pipeline/rpi/vc4/vc4.cpp | 2 +- 4 files changed, 66 insertions(+), 12 deletions(-) diff --git a/src/libcamera/pipeline/rpi/common/pipeline_base.cpp b/src/libcamera/pipeline/rpi/common/pipeline_base.cpp index 37f0d1e6e..384b5ada9 100644 --- a/src/libcamera/pipeline/rpi/common/pipeline_base.cpp +++ b/src/libcamera/pipeline/rpi/common/pipeline_base.cpp @@ -1560,12 +1560,59 @@ void CameraData::handleControlLists(uint32_t delayContext, ControlList ¶mCon { /* * The delayContext is the sequence number after it's gone through the various - * pipeline delays, so that's what gets reported as the "ControlListSequence" - * in the metadata, being the sequence number of the request whose ControlList - * has just been applied. + * pipeline delays. */ Request *request = requestQueue_.front(); - request->_d()->metadata().set(controls::rpi::ControlListSequence, delayContext); + + /* + * Create a merged list of controls from this request, and also from the + * ControlList queue. + */ + paramControls = request->controls(); + if (!controlsQueue_.empty()) { + paramControls.merge(std::move(controlsQueue_.front()), + ControlList::MergePolicy::OverwriteExisting); + controlsQueue_.pop(); + controlListId_++; + LOG(RPI, Debug) << "Popped control list " << controlListId_ << " from queue"; + } + + /* + * Record which control list corresponds to this ipaCookie. Because setDelayedControls + * now gets called by the IPA from the start of the following frame, we must record + * the previous control list id. + */ + syncTable_.emplace(SyncTableEntry{ request->sequence(), controlListId_ }); + LOG(RPI, Debug) << "Add sync table entry: sequence " << request->sequence() + << " control list id " << controlListId_; + + /* + * We know we that we added an entry for every delayContext, so we can + * find the one for this Bayer frame, and this links us to the correct + * control list. Anything ahead of "our" entry in the queue is old, so + * can be dropped. + */ + while (!syncTable_.empty() && + syncTable_.front().ipaCookie != delayContext) { + LOG(RPI, Debug) << "Pop sync entry: ipa cookie " + << syncTable_.front().ipaCookie << " control id " + << syncTable_.front().controlListId << " for job " + << delayContext; + syncTable_.pop(); + } + + if (syncTable_.empty()) + LOG(RPI, Warning) << "Unable to find ipa cookie for PFC"; + else { + LOG(RPI, Debug) << "Using sync control id " << syncTable_.front().controlListId; + requestControlId_ = syncTable_.front().controlListId; + } + + /* + * We report the sequence number of the controls from the ControlList queue + * as the "ControlListSequence" in the metadata. + */ + request->_d()->metadata().set(controls::rpi::ControlListSequence, requestControlId_); /* * Controls that take effect immediately (typically ISP controls) have to be @@ -1577,18 +1624,17 @@ void CameraData::handleControlLists(uint32_t delayContext, ControlList ¶mCon * we can pass back the controls that really need to happen now, without * disturbing the controls that were submitted with the request. */ - ASSERT(paramControls.empty()); - immediateControls_.push({ request->sequence(), {} }); - for (const auto &ctrl : request->controls()) { + ControlList controls = std::move(paramControls); + immediateControls_.push({ controlListId_, {} }); + for (const auto &ctrl : controls) { if (isControlDelayed(ctrl.first)) paramControls.set(ctrl.first, ctrl.second); else immediateControls_.back().controls.set(ctrl.first, ctrl.second); } - /* "Immediate" controls that have become due are now merged back into this request. */ while (!immediateControls_.empty() && - immediateControls_.front().controlListId <= delayContext) { + immediateControls_.front().controlListId <= requestControlId_) { paramControls.merge(immediateControls_.front().controls, ControlList::MergePolicy::OverwriteExisting); immediateControls_.pop(); diff --git a/src/libcamera/pipeline/rpi/common/pipeline_base.h b/src/libcamera/pipeline/rpi/common/pipeline_base.h index 5130bd1a0..de9529bd5 100644 --- a/src/libcamera/pipeline/rpi/common/pipeline_base.h +++ b/src/libcamera/pipeline/rpi/common/pipeline_base.h @@ -48,7 +48,7 @@ class CameraData : public Camera::Private { public: CameraData(PipelineHandler *pipe) - : Camera::Private(pipe), state_(State::Stopped), + : Camera::Private(pipe), state_(State::Stopped), controlListId_(0), startupFrameCount_(0), invalidFrameCount_(0), buffersAllocated_(false) { } @@ -131,6 +131,8 @@ class CameraData : public Camera::Private std::queue requestQueue_; std::queue controlsQueue_; + uint64_t controlListId_; + uint64_t requestControlId_; /* For handling digital zoom. */ IPACameraSensorInfo sensorInfo_; @@ -181,6 +183,12 @@ class CameraData : public Camera::Private ClockRecovery wallClockRecovery_; + struct SyncTableEntry { + uint32_t ipaCookie; + uint64_t controlListId; + }; + std::queue syncTable_; + struct ImmediateControlsEntry { uint64_t controlListId; ControlList controls; diff --git a/src/libcamera/pipeline/rpi/pisp/pisp.cpp b/src/libcamera/pipeline/rpi/pisp/pisp.cpp index c9d89d58b..902d56cac 100644 --- a/src/libcamera/pipeline/rpi/pisp/pisp.cpp +++ b/src/libcamera/pipeline/rpi/pisp/pisp.cpp @@ -2329,7 +2329,7 @@ void PiSPCameraData::tryRunPipeline() params.sensorControls = std::move(job.sensorControls); /* params.requestControls is set by handleControlLists. */ - /* This sorts out synchronisation with ControlLists in earlier requests. */ + /* This sorts out synchronisation with the ControlList queue. */ handleControlLists(job.delayContext, params.requestControls); /* Set our state to say the pipeline is active. */ diff --git a/src/libcamera/pipeline/rpi/vc4/vc4.cpp b/src/libcamera/pipeline/rpi/vc4/vc4.cpp index ac86a43ad..22e021720 100644 --- a/src/libcamera/pipeline/rpi/vc4/vc4.cpp +++ b/src/libcamera/pipeline/rpi/vc4/vc4.cpp @@ -997,7 +997,7 @@ void Vc4CameraData::tryRunPipeline() params.buffers.embedded = 0; /* params.requestControls is set by handleControlLists. */ - /* This sorts out synchronisation with ControlLists in earlier requests. */ + /* This sorts out synchronisation with the ControlList queue. */ handleControlLists(bayerFrame.delayContext, params.requestControls); /* Set our state to say the pipeline is active. */