From 76c9eb9d8cf2734bcf389445f3dd4c04b862ec47 Mon Sep 17 00:00:00 2001 From: naivedogger Date: Mon, 14 Sep 2026 14:24:32 +0800 Subject: [PATCH 01/19] [c++] Add asynchronous write callbacks --- fluss-rust/bindings/cpp/BUILD.bazel | 7 + fluss-rust/bindings/cpp/build.rs | 4 + fluss-rust/bindings/cpp/include/fluss.hpp | 35 ++ fluss-rust/bindings/cpp/src/lib.rs | 11 + fluss-rust/bindings/cpp/src/table.cpp | 59 +++ .../bindings/cpp/src/write_callback.hpp | 65 +++ fluss-rust/bindings/cpp/src/write_callback.rs | 396 +++++++++++++++++ .../bindings/cpp/test/test_write_callback.cpp | 420 ++++++++++++++++++ .../test/test_write_callback_allocation.cpp | 65 +++ .../docs/user-guide/cpp/api-reference.md | 60 +++ 10 files changed, 1122 insertions(+) create mode 100644 fluss-rust/bindings/cpp/src/write_callback.hpp create mode 100644 fluss-rust/bindings/cpp/src/write_callback.rs create mode 100644 fluss-rust/bindings/cpp/test/test_write_callback.cpp create mode 100644 fluss-rust/bindings/cpp/test/test_write_callback_allocation.cpp diff --git a/fluss-rust/bindings/cpp/BUILD.bazel b/fluss-rust/bindings/cpp/BUILD.bazel index c751266fe63..4f01fc6781d 100644 --- a/fluss-rust/bindings/cpp/BUILD.bazel +++ b/fluss-rust/bindings/cpp/BUILD.bazel @@ -55,6 +55,9 @@ genrule( name = "cargo_build_debug", srcs = glob([ "src/**/*.rs", + "src/**/*.hpp", + "include/**/*.hpp", + "build.rs", "Cargo.toml", ]), outs = [ @@ -121,6 +124,9 @@ genrule( name = "cargo_build_release", srcs = glob([ "src/**/*.rs", + "src/**/*.hpp", + "include/**/*.hpp", + "build.rs", "Cargo.toml", ]), outs = [ @@ -274,6 +280,7 @@ cc_library( textual_hdrs = [ "src/ffi_converter.hpp", "src/type_lowering.hpp", + "src/write_callback.hpp", ":rust_bridge_h_unified", ":lib_rs_h_unified", ":cxx_h_unified", diff --git a/fluss-rust/bindings/cpp/build.rs b/fluss-rust/bindings/cpp/build.rs index ec75e24aebd..7a633c606b2 100644 --- a/fluss-rust/bindings/cpp/build.rs +++ b/fluss-rust/bindings/cpp/build.rs @@ -17,8 +17,12 @@ fn main() { cxx_build::bridge("src/lib.rs") + .include("include") + .include("src") .std("c++17") .compile("fluss-cpp-bridge"); println!("cargo:rerun-if-changed=src/lib.rs"); + println!("cargo:rerun-if-changed=src/write_callback.hpp"); + println!("cargo:rerun-if-changed=include/fluss.hpp"); } diff --git a/fluss-rust/bindings/cpp/include/fluss.hpp b/fluss-rust/bindings/cpp/include/fluss.hpp index d9799e0912a..ac3a4025e96 100644 --- a/fluss-rust/bindings/cpp/include/fluss.hpp +++ b/fluss-rust/bindings/cpp/include/fluss.hpp @@ -21,6 +21,7 @@ #include #include +#include #include #include #include @@ -47,6 +48,7 @@ struct Admin; struct Table; struct AppendWriter; struct WriteResult; +class WriteCallback; struct LogScanner; struct RecordBatchLogReader; struct BatchScanner; @@ -531,6 +533,26 @@ struct Result { bool IsRetriable() const { return ErrorCode::IsRetriable(error_code); } }; +/// Receives the final outcome of an accepted write. Function pointers and lambdas +/// are supported. An empty callback is rejected before submitting the write. +/// +/// The SDK owns the callback until completion and invokes it exactly once on +/// background callback threads, never inline in the submitting call. Callbacks +/// may run concurrently and out of order, including before the call returns. +/// Keep callbacks short and synchronize access to shared state, including writers. +/// Callback overloads do not make writers safe for concurrent access. Captured +/// references must remain valid until the callback finishes; capturing shared +/// ownership is recommended. Keep the connection alive until completion. +/// +/// Callback threads are shared across connections. Do not wait for another +/// callback from a callback: it can exhaust the worker pool. Synchronous SDK +/// calls are supported with exclusive writer access. Limit outstanding callbacks +/// if they are slow: completed callbacks queue in memory, outside writer buffers. +/// +/// Exceptions thrown by callbacks are caught and reported to stderr; they do not +/// change the write outcome. Flush() waits for writes, not for callbacks to finish. +using WriteCallback = std::function; + struct TablePath { std::string database_name; std::string table_name; @@ -1877,6 +1899,7 @@ class WriteResult { friend class UpsertWriter; WriteResult(ffi::WriteResult* inner) noexcept; + Result Notify(std::unique_ptr callback); void Destroy() noexcept; ffi::WriteResult* inner_{nullptr}; }; @@ -1895,8 +1918,14 @@ class AppendWriter { Result Append(const GenericRow& row); Result Append(const GenericRow& row, WriteResult& out); + /// Submit a row and notify callback of its final outcome without waiting for + /// acknowledgment. Returns submission status; on failure no callback runs. + /// Submission can still block on buffer backpressure. See WriteCallback. + Result Append(const GenericRow& row, WriteCallback callback); Result AppendArrowBatch(const std::shared_ptr& batch); Result AppendArrowBatch(const std::shared_ptr& batch, WriteResult& out); + /// Like the callback Append overload, but notifies once for the entire batch. + Result AppendArrowBatch(const std::shared_ptr& batch, WriteCallback callback); Result Flush(); private: @@ -1922,8 +1951,14 @@ class UpsertWriter { Result Upsert(const GenericRow& row); Result Upsert(const GenericRow& row, WriteResult& out); + /// Submit an upsert and notify callback of its final outcome. Returns + /// submission status; on failure no callback runs. Submission may block on + /// buffer backpressure, but does not wait for acknowledgment. See WriteCallback. + Result Upsert(const GenericRow& row, WriteCallback callback); Result Delete(const GenericRow& row); Result Delete(const GenericRow& row, WriteResult& out); + /// Like the callback Upsert overload, but deletes a row by primary key. + Result Delete(const GenericRow& row, WriteCallback callback); Result Flush(); private: diff --git a/fluss-rust/bindings/cpp/src/lib.rs b/fluss-rust/bindings/cpp/src/lib.rs index b9ca1f93b95..ff8df75dce9 100644 --- a/fluss-rust/bindings/cpp/src/lib.rs +++ b/fluss-rust/bindings/cpp/src/lib.rs @@ -16,6 +16,7 @@ // under the License. mod types; +mod write_callback; use std::collections::HashMap; use std::str::FromStr; @@ -42,6 +43,15 @@ static RUNTIME: LazyLock = LazyLock::new(|| { #[cxx::bridge(namespace = "fluss::ffi")] mod ffi { + unsafe extern "C++" { + include!("write_callback.hpp"); + + type WriteCallback; + + #[cxx_name = "Complete"] + fn complete(self: Pin<&mut WriteCallback>, error_code: i32, error_message: &str); + } + struct HashMapValue { key: String, value: String, @@ -670,6 +680,7 @@ mod ffi { // WriteResult unsafe fn delete_write_result(wr: *mut WriteResult); fn wait(self: &mut WriteResult) -> FfiResult; + fn notify(self: &mut WriteResult, callback: UniquePtr) -> FfiResult; // UpsertWriter unsafe fn delete_upsert_writer(writer: *mut UpsertWriter); diff --git a/fluss-rust/bindings/cpp/src/table.cpp b/fluss-rust/bindings/cpp/src/table.cpp index 9601dcfecca..2204c16634b 100644 --- a/fluss-rust/bindings/cpp/src/table.cpp +++ b/fluss-rust/bindings/cpp/src/table.cpp @@ -1605,6 +1605,10 @@ Result WriteResult::Wait() { return utils::from_ffi_result(ffi_result); } +Result WriteResult::Notify(std::unique_ptr callback) { + return utils::from_ffi_result(inner_->notify(std::move(callback))); +} + // ============================================================================ // AppendWriter // ============================================================================ @@ -1658,6 +1662,21 @@ Result AppendWriter::Append(const GenericRow& row, WriteResult& out) { return result; } +Result AppendWriter::Append(const GenericRow& row, WriteCallback callback) { + if (!callback) { + return utils::make_client_error("Write callback must not be empty"); + } + // Allocate before submission so an allocation failure cannot lose an + // already accepted write's completion notification. + auto completion = std::make_unique(std::move(callback)); + WriteResult pending; + auto result = Append(row, pending); + if (result.Ok()) { + return pending.Notify(std::move(completion)); + } + return result; +} + Result AppendWriter::AppendArrowBatch(const std::shared_ptr& batch) { WriteResult wr; return AppendArrowBatch(batch, wr); @@ -1696,6 +1715,20 @@ Result AppendWriter::AppendArrowBatch(const std::shared_ptr& return result; } +Result AppendWriter::AppendArrowBatch(const std::shared_ptr& batch, + WriteCallback callback) { + if (!callback) { + return utils::make_client_error("Write callback must not be empty"); + } + auto completion = std::make_unique(std::move(callback)); + WriteResult pending; + auto result = AppendArrowBatch(batch, pending); + if (result.Ok()) { + return pending.Notify(std::move(completion)); + } + return result; +} + Result AppendWriter::Flush() { if (!Available()) { return utils::make_client_error("AppendWriter not available"); @@ -1758,6 +1791,19 @@ Result UpsertWriter::Upsert(const GenericRow& row, WriteResult& out) { return result; } +Result UpsertWriter::Upsert(const GenericRow& row, WriteCallback callback) { + if (!callback) { + return utils::make_client_error("Write callback must not be empty"); + } + auto completion = std::make_unique(std::move(callback)); + WriteResult pending; + auto result = Upsert(row, pending); + if (result.Ok()) { + return pending.Notify(std::move(completion)); + } + return result; +} + Result UpsertWriter::Delete(const GenericRow& row) { WriteResult wr; return Delete(row, wr); @@ -1779,6 +1825,19 @@ Result UpsertWriter::Delete(const GenericRow& row, WriteResult& out) { return result; } +Result UpsertWriter::Delete(const GenericRow& row, WriteCallback callback) { + if (!callback) { + return utils::make_client_error("Write callback must not be empty"); + } + auto completion = std::make_unique(std::move(callback)); + WriteResult pending; + auto result = Delete(row, pending); + if (result.Ok()) { + return pending.Notify(std::move(completion)); + } + return result; +} + Result UpsertWriter::Flush() { if (!Available()) { return utils::make_client_error("UpsertWriter not available"); diff --git a/fluss-rust/bindings/cpp/src/write_callback.hpp b/fluss-rust/bindings/cpp/src/write_callback.hpp new file mode 100644 index 00000000000..a304f6fdcdc --- /dev/null +++ b/fluss-rust/bindings/cpp/src/write_callback.hpp @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#pragma once + +#include +#include + +#include "fluss.hpp" +#include "rust/cxx.h" + +namespace fluss { +namespace ffi { + +/// Owns a callback transferred to Rust. Access is exclusive, never concurrent. +class WriteCallback { + public: + explicit WriteCallback(fluss::WriteCallback callback) : callback_(std::move(callback)) {} + + /// Invoke once, containing all C++ exceptions on this side of the FFI boundary. + void Complete(int32_t error_code, rust::Str error_message) noexcept { + // Moving std::function alone need not empty the source. Swap with an + // empty function so captures are released even if the callback throws. + fluss::WriteCallback callback; + callback.swap(callback_); + Result result; + result.error_code = error_code; + try { + result.error_message = std::string(error_message); + } catch (...) { + // Error text is best-effort; allocation failure must not skip completion. + std::fprintf(stderr, "Fluss write callback could not copy error text (code %d)\n", + error_code); + } + try { + callback(std::move(result)); + } catch (const std::exception& e) { + std::fprintf(stderr, "Fluss write callback threw an exception: %s\n", e.what()); + } catch (...) { + std::fprintf(stderr, "Fluss write callback threw an unknown exception\n"); + } + } + + private: + fluss::WriteCallback callback_; +}; + +} // namespace ffi +} // namespace fluss diff --git a/fluss-rust/bindings/cpp/src/write_callback.rs b/fluss-rust/bindings/cpp/src/write_callback.rs new file mode 100644 index 00000000000..ca69cea49c9 --- /dev/null +++ b/fluss-rust/bindings/cpp/src/write_callback.rs @@ -0,0 +1,396 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +use std::future::Future; +use std::panic::{AssertUnwindSafe, catch_unwind}; +use std::sync::{Arc, LazyLock, Mutex, mpsc}; +use std::thread::{self, JoinHandle}; + +use crate::{RUNTIME, WriteResult, client_err, err_from_core_error, ffi, ok_result}; + +const CALLBACK_WORKERS: usize = 4; +type Completion = Box; + +// Like RUNTIME, the executor is process-wide and lives until process exit. +// Initialize it on the submitting thread, not an async I/O worker. +static CALLBACK_EXECUTOR: LazyLock> = + LazyLock::new(|| match CallbackExecutor::new(CALLBACK_WORKERS) { + Ok(executor) => Some(executor), + Err(error) => { + // The write has already been accepted. Keep the old dispatch path + // as a resource-exhaustion fallback rather than lose its callback. + eprintln!("Fluss callback worker initialization failed: {error}; using blocking pool"); + None + } + }); + +struct CallbackExecutor { + sender: Option>, + workers: Vec>, +} + +impl CallbackExecutor { + fn new(worker_count: usize) -> std::io::Result { + assert!(worker_count > 0); + let (sender, receiver) = mpsc::channel::(); + let receiver = Arc::new(Mutex::new(receiver)); + let mut executor = Self { + sender: Some(sender), + workers: Vec::with_capacity(worker_count), + }; + for index in 0..worker_count { + let receiver = Arc::clone(&receiver); + executor.workers.push( + thread::Builder::new() + .name(format!("fluss-callback-{index}")) + .spawn(move || { + loop { + let completion = { + // Take one completion, then release the queue lock + // before running user code. + let receiver = receiver.lock().unwrap(); + let Ok(completion) = receiver.recv() else { + break; + }; + completion + }; + // Never hold a queue lock or enter a Tokio runtime + // while running user code. Synchronous SDK calls + // from a callback can safely use RUNTIME.block_on. + if catch_unwind(AssertUnwindSafe(completion)).is_err() { + eprintln!("Fluss callback worker contained a Rust panic"); + } + } + })?, + ); + } + Ok(executor) + } + + fn enqueue(&self, completion: Completion) -> Result<(), Completion> { + // An unbounded completion queue keeps slow user callbacks from blocking + // async I/O workers. Applications must bound outstanding callbacks; + // this caps threads, not queued captures or total process memory. + self.sender + .as_ref() + .unwrap() + .send(completion) + .map_err(|error| error.0) + } +} + +impl Drop for CallbackExecutor { + fn drop(&mut self) { + // Also handles partial worker initialization and lets tests verify + // drain/release. The process-wide static is not dropped at exit. + drop(self.sender.take()); + for worker in self.workers.drain(..) { + let _ = worker.join(); + } + } +} + +// SAFETY: The C++ wrapper is transferred by UniquePtr and accessed exclusively +// by one write task, then one callback worker. It is never shared concurrently. +// The public C++ contract requires captures to support background execution. +unsafe impl Send for ffi::WriteCallback {} + +impl WriteResult { + pub(crate) fn notify( + &mut self, + mut callback: cxx::UniquePtr, + ) -> ffi::FfiResult { + if callback.is_null() { + return client_err("Write callback must not be empty".to_string()); + } + let Some(future) = self.inner.take() else { + return client_err("WriteResult already consumed".to_string()); + }; + dispatch(future, move |result| { + callback + .pin_mut() + .complete(result.error_code, &result.error_message); + }); + ok_result() + } +} + +fn to_ffi_result(result: Result<(), fluss::error::Error>) -> ffi::FfiResult { + match result { + Ok(()) => ok_result(), + Err(e) => err_from_core_error(&e), + } +} + +fn dispatch( + future: impl Future> + Send + 'static, + callback: impl FnOnce(ffi::FfiResult) + Send + 'static, +) { + let executor = CALLBACK_EXECUTOR.as_ref(); + RUNTIME.spawn(async move { + let result = to_ffi_result(future.await); + deliver(executor, Box::new(move || callback(result))); + }); +} + +fn deliver(executor: Option<&CallbackExecutor>, completion: Completion) { + let completion = match executor { + Some(executor) => match executor.enqueue(completion) { + Ok(()) => return, + Err(completion) => completion, + }, + None => completion, + }; + // Preserve completion even if dedicated workers could not be started or + // their channel disconnected. Never execute user code on the I/O worker. + RUNTIME.spawn_blocking(completion); +} + +#[cfg(test)] +mod tests { + use std::collections::HashSet; + use std::sync::atomic::{AtomicUsize, Ordering}; + use std::sync::{Arc, mpsc}; + use std::thread; + use std::time::Duration; + + use super::{CallbackExecutor, deliver, dispatch}; + use crate::{CLIENT_ERROR_CODE, RUNTIME}; + + #[test] + fn test_completed_future_runs_off_runtime_and_releases_capture() { + let (tx, rx) = mpsc::channel(); + let capture = Arc::new(()); + let weak = Arc::downgrade(&capture); + RUNTIME.block_on(async { + dispatch( + fluss::client::WriteResultFuture::join(Vec::new()), + move |r| { + // Empty/previously completed batches must still use the executor. + let answer = RUNTIME.block_on(async { 42 }); + tx.send((r.error_code, answer, capture)).unwrap(); + }, + ); + }); + let (code, answer, capture) = rx.recv_timeout(Duration::from_secs(10)).unwrap(); + assert_eq!((code, answer), (0, 42)); + drop(capture); + assert!(weak.upgrade().is_none()); + assert!(matches!( + rx.try_recv(), + Err(mpsc::TryRecvError::Disconnected) + )); + } + + #[test] + fn test_callback_waits_asynchronously_for_acknowledgment() { + let (ack_tx, ack_rx) = tokio::sync::oneshot::channel(); + let (callback_tx, callback_rx) = mpsc::channel(); + dispatch( + async move { + ack_rx.await.unwrap(); + Ok(()) + }, + move |result| callback_tx.send(result).unwrap(), + ); + assert!(matches!( + callback_rx.try_recv(), + Err(mpsc::TryRecvError::Empty) + )); + ack_tx.send(()).unwrap(); + let result = callback_rx.recv_timeout(Duration::from_secs(10)).unwrap(); + assert_eq!(result.error_code, 0); + assert!(result.error_message.is_empty()); + assert!(matches!( + callback_rx.recv_timeout(Duration::from_secs(10)), + Err(mpsc::RecvTimeoutError::Disconnected) + )); + } + + #[test] + fn test_callback_preserves_server_error() { + let (tx, rx) = mpsc::channel(); + dispatch( + async { + Err(fluss::error::Error::FlussAPIError { + api_error: fluss::rpc::ApiError { + code: 57, + message: "Deletion is disabled".to_string(), + }, + }) + }, + move |result| tx.send(result).unwrap(), + ); + let result = rx.recv_timeout(Duration::from_secs(10)).unwrap(); + assert_eq!(result.error_code, 57); + assert_eq!(result.error_message, "Deletion is disabled"); + } + + #[test] + fn test_callback_preserves_client_error() { + let (tx, rx) = mpsc::channel(); + dispatch( + async { + Err(fluss::error::Error::UnexpectedError { + message: "Writer closed".to_string(), + source: None, + }) + }, + move |result| tx.send(result).unwrap(), + ); + let result = rx.recv_timeout(Duration::from_secs(10)).unwrap(); + assert_eq!(result.error_code, CLIENT_ERROR_CODE); + assert!(result.error_message.contains("Writer closed")); + } + + #[test] + fn test_callback_can_reenter_synchronous_runtime_calls() { + let (tx, rx) = mpsc::channel(); + dispatch(async { Ok(()) }, move |_| { + // block_on would panic if the callback ran on an async worker. + let result = RUNTIME.block_on(async { RUNTIME.spawn(async { 42 }).await.unwrap() }); + tx.send(result).unwrap(); + }); + assert_eq!(rx.recv_timeout(Duration::from_secs(10)).unwrap(), 42); + } + + #[test] + fn test_executor_bounds_workers_and_does_not_block_the_runtime() { + let executor = CallbackExecutor::new(2).unwrap(); + let (started_tx, started_rx) = mpsc::channel(); + let (done_tx, done_rx) = mpsc::channel(); + let mut releases = Vec::new(); + let mut worker_ids = HashSet::new(); + for _ in 0..2 { + let (release_tx, release_rx) = mpsc::channel(); + releases.push(release_tx); + let started_tx = started_tx.clone(); + assert!( + executor + .enqueue(Box::new(move || { + started_tx.send(thread::current().id()).unwrap(); + release_rx.recv_timeout(Duration::from_secs(10)).unwrap(); + })) + .is_ok() + ); + // Ensure this worker is occupied before giving another worker work. + worker_ids.insert(started_rx.recv_timeout(Duration::from_secs(10)).unwrap()); + } + assert_eq!(worker_ids.len(), 2); + for _ in 0..256 { + let done_tx = done_tx.clone(); + assert!( + executor + .enqueue(Box::new(move || { + done_tx.send(thread::current().id()).unwrap(); + })) + .is_ok() + ); + } + assert!(matches!(done_rx.try_recv(), Err(mpsc::TryRecvError::Empty))); + assert_eq!( + RUNTIME.block_on(async { + tokio::time::timeout(Duration::from_secs(5), RUNTIME.spawn(async { 42 })) + .await + .unwrap() + .unwrap() + }), + 42 + ); + for release in releases { + release.send(()).unwrap(); + } + drop(executor); + for _ in 0..256 { + assert!(worker_ids.contains(&done_rx.recv_timeout(Duration::from_secs(10)).unwrap())); + } + } + + #[test] + fn test_executor_drains_and_survives_a_panicking_callback() { + let executor = CallbackExecutor::new(1).unwrap(); + let completed = Arc::new(AtomicUsize::new(0)); + assert!( + executor + .enqueue(Box::new(|| panic!("test callback panic"))) + .is_ok() + ); + for _ in 0..1000 { + let completed = Arc::clone(&completed); + assert!( + executor + .enqueue(Box::new(move || { + completed.fetch_add(1, Ordering::Relaxed); + })) + .is_ok() + ); + } + drop(executor); + assert_eq!(completed.load(Ordering::Relaxed), 1000); + assert_eq!(Arc::strong_count(&completed), 1); + } + + #[test] + fn test_concurrent_producers_complete_each_job_once() { + let executor = Arc::new(CallbackExecutor::new(4).unwrap()); + let (tx, rx) = mpsc::channel(); + let mut producers = Vec::new(); + for producer in 0..4 { + let executor = Arc::clone(&executor); + let tx = tx.clone(); + producers.push(thread::spawn(move || { + for index in 0..1000 { + let tx = tx.clone(); + assert!( + executor + .enqueue(Box::new(move || { + tx.send(producer * 1000 + index).unwrap(); + })) + .is_ok() + ); + } + })); + } + for producer in producers { + producer.join().unwrap(); + } + drop(executor); + drop(tx); + let mut completed: Vec<_> = rx.into_iter().collect(); + completed.sort_unstable(); + assert_eq!(completed, (0..4000).collect::>()); + } + + #[test] + fn test_unavailable_executor_falls_back_off_the_caller_thread() { + let (tx, rx) = mpsc::channel(); + deliver( + None, + Box::new(move || { + let answer = RUNTIME.block_on(async { 42 }); + tx.send((thread::current().id(), answer)).unwrap(); + }), + ); + let (thread_id, answer) = rx.recv_timeout(Duration::from_secs(10)).unwrap(); + assert_ne!(thread_id, thread::current().id()); + assert_eq!(answer, 42); + assert!(matches!( + rx.try_recv(), + Err(mpsc::TryRecvError::Disconnected) + )); + } +} diff --git a/fluss-rust/bindings/cpp/test/test_write_callback.cpp b/fluss-rust/bindings/cpp/test/test_write_callback.cpp new file mode 100644 index 00000000000..8072ef2d552 --- /dev/null +++ b/fluss-rust/bindings/cpp/test/test_write_callback.cpp @@ -0,0 +1,420 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include +#include + +#include +#include +#include +#include + +#include "test_utils.h" +#include "write_callback.hpp" + +namespace { + +class Completion { + public: + void Reset() { + std::lock_guard lock(mutex_); + results_.clear(); + thread_ = std::thread::id{}; + } + + void Record(fluss::Result result) { + std::lock_guard lock(mutex_); + results_.push_back(std::move(result)); + thread_ = std::this_thread::get_id(); + ready_.notify_all(); + } + + bool Await(size_t count = 1) { + std::unique_lock lock(mutex_); + return ready_.wait_for(lock, std::chrono::seconds(10), + [&] { return results_.size() >= count; }); + } + + std::vector Results() { + std::lock_guard lock(mutex_); + return results_; + } + + std::thread::id Thread() { + std::lock_guard lock(mutex_); + return thread_; + } + + private: + std::mutex mutex_; + std::condition_variable ready_; + std::vector results_; + std::thread::id thread_; +}; + +// A plain function pointer has no capture. This state lives for the process. +Completion function_completion; + +void RecordFunctionCallback(fluss::Result result) { function_completion.Record(std::move(result)); } + +struct Lifetime { + std::promise released; + ~Lifetime() { released.set_value(); } +}; + +} // namespace + +class WriteCallbackTest : public ::testing::Test { + protected: + void CreateTable(bool primary_key = false, bool disable_delete = false) { + auto& env = *fluss_test::FlussTestEnvironment::Instance(); + auto builder = fluss::Schema::NewBuilder() + .AddColumn("id", fluss::DataType::Int()) + .AddColumn("value", fluss::DataType::String()); + if (primary_key) { + builder.SetPrimaryKeys({"id"}); + } + auto descriptor_builder = fluss::TableDescriptor::NewBuilder() + .SetSchema(builder.Build()) + .SetBucketCount(3) + .SetBucketKeys({"id"}) + .SetProperty("table.replication.factor", "1"); + if (disable_delete) { + descriptor_builder.SetProperty("table.delete.behavior", "disable"); + } + auto descriptor = descriptor_builder.Build(); + fluss::TablePath path("fluss", + std::string("cpp_callback_") + + ::testing::UnitTest::GetInstance()->current_test_info()->name()); + fluss_test::CreateTable(env.GetAdmin(), path, descriptor); + auto result = env.GetConnection().GetTable(path, table_); + ASSERT_OK(result); + } + + fluss::GenericRow Row(int32_t id = 1) { + fluss::GenericRow row(2); + row.SetInt32(0, id); + row.SetString(1, "callback"); + return row; + } + + fluss::Table table_; +}; + +TEST_F(WriteCallbackTest, AppendAcceptsFunctionPointer) { + function_completion.Reset(); + CreateTable(); + fluss::AppendWriter writer; + ASSERT_OK(table_.NewAppend().CreateWriter(writer)); + + auto submitted = writer.Append(Row(), &RecordFunctionCallback); + ASSERT_OK(submitted); + ASSERT_TRUE(function_completion.Await()); + auto results = function_completion.Results(); + ASSERT_EQ(results.size(), 1u); + EXPECT_OK(results.front()); + EXPECT_NE(function_completion.Thread(), std::this_thread::get_id()); + + // The old acknowledgment and fire-and-forget overloads still work. + fluss::WriteResult pending; + ASSERT_OK(writer.Append(Row(2), pending)); + ASSERT_OK(pending.Wait()); + ASSERT_OK(writer.Append(Row(3))); + ASSERT_OK(writer.Flush()); +} + +TEST_F(WriteCallbackTest, CallbackOwnsCapturesAndDoesNotDelayFlush) { + CreateTable(); + fluss::AppendWriter writer; + ASSERT_OK(table_.NewAppend().CreateWriter(writer)); + auto started = std::make_shared(); + auto finished = std::make_shared(); + auto gate = std::make_shared>(); + auto resume = gate->get_future().share(); + auto lifetime = std::make_shared(); + auto released = lifetime->released.get_future(); + std::weak_ptr weak_lifetime = lifetime; + { + auto row = Row(); + fluss::WriteCallback callback = [started, finished, resume, + owned = std::move(lifetime)](fluss::Result result) { + started->Record(result); + // Bounded even when a preceding assertion fails. + resume.wait_for(std::chrono::seconds(20)); + finished->Record(std::move(result)); + }; + ASSERT_OK(writer.Append(row, std::move(callback))); + } + ASSERT_TRUE(started->Await()); + EXPECT_FALSE(weak_lifetime.expired()); + ASSERT_OK(writer.Flush()); + // Flush must not wait for a user callback that is waiting for us. + EXPECT_TRUE(finished->Results().empty()); + writer = fluss::AppendWriter{}; + EXPECT_FALSE(weak_lifetime.expired()); + gate->set_value(); + ASSERT_TRUE(finished->Await()); + EXPECT_EQ(released.wait_for(std::chrono::seconds(10)), std::future_status::ready); + EXPECT_TRUE(weak_lifetime.expired()); + EXPECT_OK(finished->Results().front()); +} + +TEST_F(WriteCallbackTest, AppendArrowBatchNotifiesOnce) { + CreateTable(); + fluss::AppendWriter writer; + ASSERT_OK(table_.NewAppend().CreateWriter(writer)); + auto completion = std::make_shared(); + { + arrow::Int32Builder ids; + arrow::StringBuilder values; + ASSERT_TRUE(ids.AppendValues({1, 2, 3, 4, 5, 6}).ok()); + ASSERT_TRUE(values.AppendValues({"a", "b", "c", "d", "e", "f"}).ok()); + auto batch = + arrow::RecordBatch::Make(arrow::schema({arrow::field("id", arrow::int32()), + arrow::field("value", arrow::utf8())}), + 6, {ids.Finish().ValueOrDie(), values.Finish().ValueOrDie()}); + ASSERT_OK(writer.AppendArrowBatch( + batch, [completion](fluss::Result result) { completion->Record(std::move(result)); })); + } + ASSERT_TRUE(completion->Await()); + ASSERT_EQ(completion->Results().size(), 1u); + EXPECT_OK(completion->Results().front()); + ASSERT_OK(writer.Flush()); +} + +TEST_F(WriteCallbackTest, UpsertAndDeleteNotifyCompletion) { + CreateTable(true); + fluss::UpsertWriter writer; + ASSERT_OK(table_.NewUpsert().CreateWriter(writer)); + auto completion = std::make_shared(); + fluss::WriteCallback callback = [completion](fluss::Result result) { + completion->Record(std::move(result)); + }; + ASSERT_OK(writer.Upsert(Row(), callback)); + ASSERT_TRUE(completion->Await()); + + fluss::Lookuper lookuper; + ASSERT_OK(table_.NewLookup().CreateLookuper(lookuper)); + fluss::GenericRow key(2); + key.SetInt32(0, 1); + fluss::LookupResult found; + ASSERT_OK(lookuper.Lookup(key, found)); + ASSERT_TRUE(found.Found()); + + ASSERT_OK(writer.Delete(key, callback)); + ASSERT_TRUE(completion->Await(2)); + auto results = completion->Results(); + ASSERT_EQ(results.size(), 2u); + EXPECT_OK(results[0]); + EXPECT_OK(results[1]); + fluss::LookupResult deleted; + ASSERT_OK(lookuper.Lookup(key, deleted)); + EXPECT_FALSE(deleted.Found()); +} + +TEST_F(WriteCallbackTest, ServerRejectionIsReportedThroughCallback) { + CreateTable(true, true); + fluss::UpsertWriter writer; + ASSERT_OK(table_.NewUpsert().CreateWriter(writer)); + fluss::WriteResult pending; + ASSERT_OK(writer.Upsert(Row(), pending)); + ASSERT_OK(pending.Wait()); + + auto completion = std::make_shared(); + auto submitted = writer.Delete( + Row(), [completion](fluss::Result result) { completion->Record(std::move(result)); }); + // The write is accepted locally; only the callback reports server rejection. + ASSERT_OK(submitted); + ASSERT_TRUE(completion->Await()); + auto results = completion->Results(); + ASSERT_EQ(results.size(), 1u); + EXPECT_EQ(results.front().error_code, fluss::ErrorCode::DELETION_DISABLED_EXCEPTION); + EXPECT_NE(results.front().error_message.find("disabled"), std::string::npos); +} + +TEST_F(WriteCallbackTest, MultipleOutstandingWritesEachNotifyOnce) { + CreateTable(); + fluss::AppendWriter writer; + ASSERT_OK(table_.NewAppend().CreateWriter(writer)); + auto completion = std::make_shared(); + for (int32_t id = 0; id < 64; ++id) { + ASSERT_OK(writer.Append(Row(id), [completion](fluss::Result result) { + completion->Record(std::move(result)); + })); + } + ASSERT_TRUE(completion->Await(64)); + ASSERT_OK(writer.Flush()); + auto results = completion->Results(); + ASSERT_EQ(results.size(), 64u); + for (const auto& result : results) { + EXPECT_OK(result); + } +} + +TEST_F(WriteCallbackTest, RejectedSubmissionDoesNotInvokeCallback) { + CreateTable(); + fluss::AppendWriter writer; + ASSERT_OK(table_.NewAppend().CreateWriter(writer)); + auto completion = std::make_shared(); + auto lifetime = std::make_shared(); + auto released = lifetime->released.get_future(); + fluss::GenericRow invalid(1); + invalid.SetInt32(0, 1); // Table requires two columns. + auto result = + writer.Append(invalid, [completion, owned = std::move(lifetime)](fluss::Result completed) { + completion->Record(std::move(completed)); + }); + EXPECT_FALSE(result.Ok()); + EXPECT_EQ(released.wait_for(std::chrono::seconds(10)), std::future_status::ready); + EXPECT_TRUE(completion->Results().empty()); + + result = writer.AppendArrowBatch(nullptr, [completion](fluss::Result completed) { + completion->Record(std::move(completed)); + }); + EXPECT_FALSE(result.Ok()); + EXPECT_TRUE(completion->Results().empty()); + ASSERT_OK(writer.Flush()); +} + +TEST_F(WriteCallbackTest, BatchedCallbacksSurviveExceptionsAndCoexistWithWait) { + CreateTable(); + fluss::AppendWriter writer; + ASSERT_OK(table_.NewAppend().CreateWriter(writer)); + auto completion = std::make_shared(); + constexpr int count = 1024; + for (int i = 0; i < count; ++i) { + // The same bucket key encourages shared internal batches. + ASSERT_OK(writer.Append(Row(1), [completion, i](fluss::Result result) { + completion->Record(std::move(result)); + if (i % 64 == 0) { + throw std::runtime_error("isolated batch callback exception"); + } + })); + } + fluss::WriteResult pending; + ASSERT_OK(writer.Append(Row(1), pending)); + ASSERT_OK(pending.Wait()); + ASSERT_OK(writer.Flush()); + ASSERT_TRUE(completion->Await(count)); + auto results = completion->Results(); + ASSERT_EQ(results.size(), static_cast(count)); + for (const auto& result : results) { + EXPECT_OK(result); + } +} + +TEST_F(WriteCallbackTest, BatchedServerFailureNotifiesEveryAcceptedDelete) { + CreateTable(true, true); + fluss::UpsertWriter writer; + ASSERT_OK(table_.NewUpsert().CreateWriter(writer)); + fluss::WriteResult initial; + ASSERT_OK(writer.Upsert(Row(), initial)); + ASSERT_OK(initial.Wait()); + auto completion = std::make_shared(); + constexpr int count = 257; + for (int i = 0; i < count; ++i) { + ASSERT_OK(writer.Delete( + Row(), [completion](fluss::Result result) { completion->Record(std::move(result)); })); + } + ASSERT_TRUE(completion->Await(count)); + auto results = completion->Results(); + ASSERT_EQ(results.size(), static_cast(count)); + for (const auto& result : results) { + EXPECT_EQ(result.error_code, fluss::ErrorCode::DELETION_DISABLED_EXCEPTION); + } +} + +TEST_F(WriteCallbackTest, EmptyArrowBatchCallbackIsStillAsynchronous) { + CreateTable(); + fluss::AppendWriter writer; + ASSERT_OK(table_.NewAppend().CreateWriter(writer)); + arrow::Int32Builder ids; + arrow::StringBuilder values; + auto batch = arrow::RecordBatch::Make( + arrow::schema({arrow::field("id", arrow::int32()), arrow::field("value", arrow::utf8())}), + 0, {ids.Finish().ValueOrDie(), values.Finish().ValueOrDie()}); + auto completion = std::make_shared(); + ASSERT_OK(writer.AppendArrowBatch( + batch, [completion](fluss::Result result) { completion->Record(std::move(result)); })); + ASSERT_TRUE(completion->Await()); + ASSERT_EQ(completion->Results().size(), 1u); + EXPECT_OK(completion->Results().front()); + EXPECT_NE(completion->Thread(), std::this_thread::get_id()); +} + +TEST_F(WriteCallbackTest, EmptyCallbacksAreRejectedBeforeSubmission) { + CreateTable(); + fluss::AppendWriter writer; + ASSERT_OK(table_.NewAppend().CreateWriter(writer)); + auto result = writer.Append(Row(), nullptr); + EXPECT_FALSE(result.Ok()); + EXPECT_EQ(result.error_message, "Write callback must not be empty"); + result = writer.AppendArrowBatch(nullptr, fluss::WriteCallback{}); + EXPECT_FALSE(result.Ok()); + EXPECT_EQ(result.error_message, "Write callback must not be empty"); + + fluss::UpsertWriter upsert; + result = upsert.Upsert(Row(), nullptr); + EXPECT_FALSE(result.Ok()); + EXPECT_EQ(result.error_message, "Write callback must not be empty"); + result = upsert.Delete(Row(), nullptr); + EXPECT_FALSE(result.Ok()); + EXPECT_EQ(result.error_message, "Write callback must not be empty"); + ASSERT_OK(writer.Flush()); +} + +TEST_F(WriteCallbackTest, UnavailableWritersDoNotInvokeCallbacks) { + auto completion = std::make_shared(); + auto callback = [completion](fluss::Result result) { completion->Record(std::move(result)); }; + fluss::AppendWriter append; + fluss::UpsertWriter upsert; + EXPECT_FALSE(append.Append(Row(), callback).Ok()); + EXPECT_FALSE(append.AppendArrowBatch(nullptr, callback).Ok()); + EXPECT_FALSE(upsert.Upsert(Row(), callback).Ok()); + EXPECT_FALSE(upsert.Delete(Row(), callback).Ok()); + EXPECT_TRUE(completion->Results().empty()); +} + +TEST(WriteCallbackBridgeTest, ForwardsErrorAndReleasesCaptures) { + auto completion = std::make_shared(); + auto lifetime = std::make_shared(); + auto released = lifetime->released.get_future(); + fluss::ffi::WriteCallback callback( + [completion, owned = std::move(lifetime)](fluss::Result result) { + completion->Record(std::move(result)); + }); + callback.Complete(fluss::ErrorCode::DELETION_DISABLED_EXCEPTION, "Deletion is disabled"); + EXPECT_EQ(released.wait_for(std::chrono::seconds(10)), std::future_status::ready); + auto results = completion->Results(); + ASSERT_EQ(results.size(), 1u); + EXPECT_EQ(results.front().error_code, fluss::ErrorCode::DELETION_DISABLED_EXCEPTION); + EXPECT_EQ(results.front().error_message, "Deletion is disabled"); +} + +TEST(WriteCallbackBridgeTest, ContainsCallbackExceptions) { + auto lifetime = std::make_shared(); + auto released = lifetime->released.get_future(); + fluss::ffi::WriteCallback callback([owned = std::move(lifetime)](fluss::Result) { + throw std::runtime_error("callback failure"); + }); + EXPECT_NO_THROW(callback.Complete(0, "")); + EXPECT_EQ(released.wait_for(std::chrono::seconds(10)), std::future_status::ready); + fluss::ffi::WriteCallback unknown([](fluss::Result) { throw 42; }); + EXPECT_NO_THROW(unknown.Complete(0, "")); +} diff --git a/fluss-rust/bindings/cpp/test/test_write_callback_allocation.cpp b/fluss-rust/bindings/cpp/test/test_write_callback_allocation.cpp new file mode 100644 index 00000000000..995d9fc2b93 --- /dev/null +++ b/fluss-rust/bindings/cpp/test/test_write_callback_allocation.cpp @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +#include + +#include +#include + +#include "write_callback.hpp" + +namespace { +// Fault injection is confined to this test thread; production has no allocation hook. +thread_local bool fail_next_allocation = false; +} // namespace + +void* operator new(std::size_t size) { + if (std::exchange(fail_next_allocation, false)) { + throw std::bad_alloc(); + } + if (void* value = std::malloc(size ? size : 1)) { + return value; + } + throw std::bad_alloc(); +} + +void operator delete(void* value) noexcept { std::free(value); } +void operator delete(void* value, std::size_t) noexcept { std::free(value); } + +TEST(WriteCallbackBridgeTest, ErrorTextAllocationFailureStillCompletesAndReleasesCapture) { + const std::string message(4096, 'x'); // Exceeds small-string capacity. + auto lifetime = std::make_shared(42); + std::weak_ptr weak = lifetime; + fluss::Result observed; + int calls = 0; + fluss::ffi::WriteCallback callback([&, owned = std::move(lifetime)](fluss::Result result) { + ++calls; + observed = std::move(result); + }); + const rust::Str text(message); + fail_next_allocation = true; + callback.Complete(fluss::ErrorCode::DELETION_DISABLED_EXCEPTION, text); + const bool allocation_was_attempted = !std::exchange(fail_next_allocation, false); + + EXPECT_TRUE(allocation_was_attempted); + EXPECT_EQ(calls, 1); + EXPECT_EQ(observed.error_code, fluss::ErrorCode::DELETION_DISABLED_EXCEPTION); + EXPECT_TRUE(observed.error_message.empty()); + EXPECT_TRUE(weak.expired()); +} diff --git a/fluss-rust/website/docs/user-guide/cpp/api-reference.md b/fluss-rust/website/docs/user-guide/cpp/api-reference.md index ec60464bb63..d152b51af11 100644 --- a/fluss-rust/website/docs/user-guide/cpp/api-reference.md +++ b/fluss-rust/website/docs/user-guide/cpp/api-reference.md @@ -188,6 +188,10 @@ are retained. Filters apply to Arrow log scans and are rejected by `CreateBucket |-------------------------------------------------------------|----------------------------------------| | `Append(const GenericRow& row) -> Result` | Append a row (fire-and-forget) | | `Append(const GenericRow& row, WriteResult& out) -> Result` | Append a row with write acknowledgment | +| `Append(const GenericRow& row, WriteCallback callback) -> Result` | Append a row with completion notification | +| `AppendArrowBatch(const std::shared_ptr& batch) -> Result` | Append a batch (fire-and-forget) | +| `AppendArrowBatch(const std::shared_ptr& batch, WriteResult& out) -> Result` | Append a batch with write acknowledgment | +| `AppendArrowBatch(const std::shared_ptr& batch, WriteCallback callback) -> Result` | Append a batch with one completion notification | | `Flush() -> Result` | Flush all pending writes | ## `UpsertWriter` @@ -196,8 +200,10 @@ are retained. Filters apply to Arrow log scans and are rejected by `CreateBucket |-------------------------------------------------------------|-----------------------------------------------| | `Upsert(const GenericRow& row) -> Result` | Upsert a row (fire-and-forget) | | `Upsert(const GenericRow& row, WriteResult& out) -> Result` | Upsert a row with write acknowledgment | +| `Upsert(const GenericRow& row, WriteCallback callback) -> Result` | Upsert a row with completion notification | | `Delete(const GenericRow& row) -> Result` | Delete a row by primary key (fire-and-forget) | | `Delete(const GenericRow& row, WriteResult& out) -> Result` | Delete a row with write acknowledgment | +| `Delete(const GenericRow& row, WriteCallback callback) -> Result` | Delete a row with completion notification | | `Flush() -> Result` | Flush all pending operations | ## `WriteResult` @@ -206,6 +212,60 @@ are retained. Filters apply to Arrow log scans and are rejected by `CreateBucket |--------------------|---------------------------------------------| | `Wait() -> Result` | Wait for server acknowledgment of the write | +## `WriteCallback` + +`WriteCallback` is `std::function`. Pass a function pointer or a +lambda to receive the final write outcome without a `WriteResult` handle or a +call to `Wait()`: + +```cpp +void OnWriteComplete(fluss::Result completed) { + if (!completed.Ok()) { + std::cerr << "Write failed: " << completed.error_message << '\n'; + } +} + +auto submitted = writer.Append(row, &OnWriteComplete); +if (!submitted.Ok()) { + // Submission failed; OnWriteComplete will not be called. + std::cerr << "Submission failed: " << submitted.error_message << '\n'; +} +``` + +The immediate return value reports submission status, not acknowledgment. An +empty callback is rejected before submission. Each successfully submitted +operation invokes its callback exactly once with its final success or failure; +`AppendArrowBatch` invokes one callback for the batch, not one per row or bucket. +Submission does not wait for acknowledgment, but may still wait for buffer +space under backpressure. + +The SDK takes ownership of the callback and its captures. Callbacks run on +background callback threads, may execute concurrently and out of submission +order, and may start before the submitting call returns. Keep callbacks short; +synchronize access to shared state and keep captured references valid until the +callback finishes. Callback overloads do not make writers safe for concurrent +access: serialize access if both the caller and a callback use the same writer. +Prefer capturing `std::shared_ptr` by value when sharing +application state. Keep the connection alive until outstanding operations +complete. Exceptions thrown by callbacks are caught and reported to stderr; +they do not change the write outcome. + +The binding asynchronously awaits each write's result, then dispatches its +callback to one of four process-wide callback workers. User callbacks run +outside the runtime's async I/O workers. + +The completion queue is unbounded, so limit outstanding +callbacks when callback processing is slower than writing; writer buffer limits +do not bound memory retained by completed callbacks. +Do not wait for another callback from within a callback, since all callback +workers could become occupied. Synchronous SDK calls remain supported with +exclusive access to the writer. If dedicated workers cannot be initialized, the +SDK falls back to its runtime blocking pool to preserve callback delivery. + +`Flush()` still waits for pending writes, **not** for user callbacks to finish. +Applications that need to drain callbacks must track their completion separately. +The existing fire-and-forget and `WriteResult::Wait()` APIs are unchanged. + ## `Lookuper` | Method | Description | From 587ba946eac773fa00920f5327b98c4226022c4a Mon Sep 17 00:00:00 2001 From: naivedogger Date: Mon, 14 Sep 2026 14:29:58 +0800 Subject: [PATCH 02/19] [rust] Batch write callback completion notifications --- fluss-rust/bindings/cpp/src/write_callback.rs | 45 +- .../fluss/src/client/write/broadcast.rs | 431 +++++++++++++++++- .../crates/fluss/src/client/write/mod.rs | 342 +++++++++++++- .../docs/user-guide/cpp/api-reference.md | 11 +- 4 files changed, 790 insertions(+), 39 deletions(-) diff --git a/fluss-rust/bindings/cpp/src/write_callback.rs b/fluss-rust/bindings/cpp/src/write_callback.rs index ca69cea49c9..cf35c3e75a9 100644 --- a/fluss-rust/bindings/cpp/src/write_callback.rs +++ b/fluss-rust/bindings/cpp/src/write_callback.rs @@ -15,6 +15,7 @@ // specific language governing permissions and limitations // under the License. +#[cfg(test)] use std::future::Future; use std::panic::{AssertUnwindSafe, catch_unwind}; use std::sync::{Arc, LazyLock, Mutex, mpsc}; @@ -23,6 +24,7 @@ use std::thread::{self, JoinHandle}; use crate::{RUNTIME, WriteResult, client_err, err_from_core_error, ffi, ok_result}; const CALLBACK_WORKERS: usize = 4; +const COMPLETION_BATCH_SIZE: usize = 64; type Completion = Box; // Like RUNTIME, the executor is process-wide and lives until process exit. @@ -60,8 +62,9 @@ impl CallbackExecutor { .spawn(move || { loop { let completion = { - // Take one completion, then release the queue lock - // before running user code. + // Each job already contains up to 64 callbacks. + // Do not prefetch 64 jobs here: that would let a + // worker hoard up to 4096 callbacks. let receiver = receiver.lock().unwrap(); let Ok(completion) = receiver.recv() else { break; @@ -105,7 +108,7 @@ impl Drop for CallbackExecutor { } // SAFETY: The C++ wrapper is transferred by UniquePtr and accessed exclusively -// by one write task, then one callback worker. It is never shared concurrently. +// by one batch registration, then one callback worker. It is never shared concurrently. // The public C++ contract requires captures to support background execution. unsafe impl Send for ffi::WriteCallback {} @@ -120,7 +123,7 @@ impl WriteResult { let Some(future) = self.inner.take() else { return client_err("WriteResult already consumed".to_string()); }; - dispatch(future, move |result| { + dispatch_write(future, move |result| { callback .pin_mut() .complete(result.error_code, &result.error_message); @@ -129,6 +132,33 @@ impl WriteResult { } } +fn dispatch_write( + future: fluss::client::WriteResultFuture, + callback: impl FnOnce(ffi::FfiResult) + Send + 'static, +) { + // Force worker initialization before registering with an in-flight batch. + let _ = CALLBACK_EXECUTOR.as_ref(); + let callback = move |result| callback(to_ffi_result(result)); + if let Err((future, callback)) = future.try_on_complete(callback, dispatch_batch) { + // Only futures already polled before registration need this path. + // Normal C++ Append/Upsert/Delete never poll before registering. + RUNTIME.spawn(async move { + let result = future.await; + deliver( + CALLBACK_EXECUTOR.as_ref(), + Box::new(move || callback(result)), + ); + }); + } +} + +fn dispatch_batch(batch: fluss::client::WriteCallbackBatch) { + let executor = CALLBACK_EXECUTOR.as_ref(); + for chunk in batch.into_chunks(COMPLETION_BATCH_SIZE) { + deliver(executor, Box::new(move || chunk.run())); + } +} + fn to_ffi_result(result: Result<(), fluss::error::Error>) -> ffi::FfiResult { match result { Ok(()) => ok_result(), @@ -136,6 +166,7 @@ fn to_ffi_result(result: Result<(), fluss::error::Error>) -> ffi::FfiResult { } } +#[cfg(test)] fn dispatch( future: impl Future> + Send + 'static, callback: impl FnOnce(ffi::FfiResult) + Send + 'static, @@ -168,16 +199,16 @@ mod tests { use std::thread; use std::time::Duration; - use super::{CallbackExecutor, deliver, dispatch}; + use super::{CallbackExecutor, deliver, dispatch, dispatch_write}; use crate::{CLIENT_ERROR_CODE, RUNTIME}; #[test] - fn test_completed_future_runs_off_runtime_and_releases_capture() { + fn test_direct_batch_completion_runs_off_runtime_and_releases_capture() { let (tx, rx) = mpsc::channel(); let capture = Arc::new(()); let weak = Arc::downgrade(&capture); RUNTIME.block_on(async { - dispatch( + dispatch_write( fluss::client::WriteResultFuture::join(Vec::new()), move |r| { // Empty/previously completed batches must still use the executor. diff --git a/fluss-rust/crates/fluss/src/client/write/broadcast.rs b/fluss-rust/crates/fluss/src/client/write/broadcast.rs index 9e00403586f..33a0d5027e0 100644 --- a/fluss-rust/crates/fluss/src/client/write/broadcast.rs +++ b/fluss-rust/crates/fluss/src/client/write/broadcast.rs @@ -15,7 +15,8 @@ // specific language governing permissions and limitations // under the License. -use parking_lot::RwLock; +use parking_lot::{Mutex, RwLock}; +use std::panic::{AssertUnwindSafe, catch_unwind}; use std::sync::Arc; use thiserror::Error; use tokio::sync::Notify; @@ -24,6 +25,60 @@ pub type Result = std::result::Result; pub type BatchWriteResult = Result<(), Error>; +type Callback = Box) + Send + 'static>; +type Dispatcher = fn(CompletionBatch); + +/// An owned set of callbacks sharing one published result. +/// +/// Binding executors split this into bounded jobs and run those jobs off the +/// I/O thread. User callbacks are never invoked by the broadcast itself. +#[doc(hidden)] +pub struct CompletionBatch { + result: Arc>, + callbacks: Vec>, +} + +impl CompletionBatch { + /// Split into jobs of at most `limit` callbacks without waiting to fill one. + pub fn into_chunks(self, limit: usize) -> impl Iterator { + assert!(limit > 0); + let mut callbacks = self.callbacks.into_iter(); + std::iter::from_fn(move || { + let chunk: Vec<_> = callbacks.by_ref().take(limit).collect(); + if chunk.is_empty() { + None + } else { + Some(Self { + result: Arc::clone(&self.result), + callbacks: chunk, + }) + } + }) + } + + /// Execute every callback, isolating panics so later callbacks still run. + pub fn run(self) { + for callback in self.callbacks { + if catch_unwind(AssertUnwindSafe(|| callback(&self.result))).is_err() { + log::error!("Write completion callback panicked"); + } + } + } +} + +struct CallbackGroup { + dispatch: Dispatcher, + callbacks: Vec>, +} + +impl std::fmt::Debug for CallbackGroup { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("CallbackGroup") + .field("len", &self.callbacks.len()) + .finish() + } +} + #[derive(Debug, Error, Clone, PartialEq, Eq)] pub enum Error { #[error("BroadcastOnce dropped")] @@ -42,7 +97,34 @@ pub struct BroadcastOnceReceiver { impl BroadcastOnceReceiver { /// Returns `Some(_)` if data has been produced pub fn peek(&self) -> Option> { - self.shared.data.read().clone() + self.shared.data.read().as_deref().cloned() + } + + /// Register under the result read lock, so publication cannot overtake + /// registration. A late registration is dispatched immediately, not lost. + pub(crate) fn subscribe(&self, callback: Callback, dispatch: Dispatcher) { + let data = self.shared.data.read(); + if let Some(result) = data.as_ref() { + let result = Arc::clone(result); + drop(data); + dispatch(CompletionBatch { + result, + callbacks: vec![callback], + }); + } else { + let mut groups = self.shared.callbacks.lock(); + if let Some(group) = groups + .iter_mut() + .find(|group| std::ptr::fn_addr_eq(group.dispatch, dispatch)) + { + group.callbacks.push(callback); + } else { + groups.push(CallbackGroup { + dispatch, + callbacks: vec![callback], + }); + } + } } /// Waits for [`BroadcastOnce::broadcast`] to be called or returns an error @@ -63,18 +145,38 @@ impl BroadcastOnceReceiver { /// Used by `abort_batches` to fail in-flight handles that can't be /// reached through `WriteBatch::complete`. pub(crate) fn fail(&self, error: Error) { - let mut data = self.shared.data.write(); - if data.is_none() { - *data = Some(Err(error)); - self.shared.notify.notify_waiters(); + let result = Arc::new(Err(error)); + { + let mut data = self.shared.data.write(); + if data.is_some() { + return; + } + *data = Some(Arc::clone(&result)); } + self.shared.notify_completion(result); } } #[derive(Debug)] struct Shared { - data: RwLock>>, + data: RwLock>>>, notify: Notify, + callbacks: Mutex>>, +} + +impl Shared { + fn notify_completion(&self, result: Arc>) { + // Registration takes data then callbacks. Publication has already + // released data, so neither dispatcher nor user code runs under locks. + let groups = std::mem::take(&mut *self.callbacks.lock()); + self.notify.notify_waiters(); + for group in groups { + (group.dispatch)(CompletionBatch { + result: Arc::clone(&result), + callbacks: group.callbacks, + }); + } + } } #[derive(Debug)] @@ -94,6 +196,7 @@ where shared: Arc::new(Shared { data: Default::default(), notify: Default::default(), + callbacks: Default::default(), }), } } @@ -110,11 +213,16 @@ impl BroadcastOnce { /// Broadcast a value to all [`BroadcastOnceReceiver`] handles pub fn broadcast(&self, r: T) { - let mut locked = self.shared.data.write(); - assert!(locked.is_none(), "double publish"); - - *locked = Some(Ok(r)); - self.shared.notify.notify_waiters(); + let result = Arc::new(Ok(r)); + { + let mut locked = self.shared.data.write(); + assert!(locked.is_none(), "double publish"); + *locked = Some(Arc::clone(&result)); + } + // Woken receivers immediately read the result. Publish it and release + // the write lock before waking them, rather than make them contend + // with the notification loop for the same lock. + self.shared.notify_completion(result); } } @@ -123,11 +231,300 @@ where T: Send + Sync, { fn drop(&mut self) { - let mut data = self.shared.data.write(); - if data.is_none() { - log::warn!("BroadcastOnce dropped without producing"); - *data = Some(Err(Error::Dropped)); - self.shared.notify.notify_waiters(); + let result = { + let mut data = self.shared.data.write(); + if data.is_some() { + return; + } + let result = Arc::new(Err(Error::Dropped)); + *data = Some(Arc::clone(&result)); + result + }; + log::warn!("BroadcastOnce dropped without producing"); + self.shared.notify_completion(result); + } +} + +#[cfg(test)] +mod tests { + use super::{BroadcastOnce, CompletionBatch, Error, Shared}; + use std::future::Future; + use std::sync::Arc; + use std::sync::atomic::{AtomicBool, Ordering}; + use std::task::{Context, Poll, Wake, Waker}; + use std::time::Duration; + + #[test] + fn test_callbacks_share_one_dispatch_and_late_registration_is_delivered() { + use std::sync::atomic::AtomicUsize; + static DISPATCHES: AtomicUsize = AtomicUsize::new(0); + fn dispatch(batch: CompletionBatch) { + DISPATCHES.fetch_add(1, Ordering::SeqCst); + batch.run(); + } + let broadcast = BroadcastOnce::default(); + let receiver = broadcast.receiver(); + let (tx, rx) = std::sync::mpsc::channel(); + for index in 0..1000 { + let tx = tx.clone(); + receiver.subscribe( + Box::new(move |result| tx.send((index, result.clone())).unwrap()), + dispatch, + ); + } + assert_eq!(DISPATCHES.load(Ordering::SeqCst), 0); + broadcast.broadcast(42); + assert_eq!(DISPATCHES.load(Ordering::SeqCst), 1); + let late_tx = tx.clone(); + receiver.subscribe( + Box::new(move |result| late_tx.send((1000, result.clone())).unwrap()), + dispatch, + ); + assert_eq!(DISPATCHES.load(Ordering::SeqCst), 2); + drop(tx); + let mut results: Vec<_> = rx.into_iter().collect(); + results.sort_by_key(|(index, _)| *index); + assert_eq!(results, (0..1001).map(|i| (i, Ok(42))).collect::>()); + } + + #[test] + fn test_chunks_bound_work_and_panics_do_not_drop_remaining_callbacks() { + fn dispatch(batch: CompletionBatch) { + let chunks: Vec<_> = batch.into_chunks(64).collect(); + assert_eq!( + chunks + .iter() + .map(|chunk| chunk.callbacks.len()) + .collect::>(), + vec![64, 64, 3] + ); + for chunk in chunks { + chunk.run(); + } + } + let broadcast = BroadcastOnce::default(); + let (tx, rx) = std::sync::mpsc::channel(); + broadcast + .receiver() + .subscribe(Box::new(|_| panic!("isolated callback")), dispatch); + for i in 0..130 { + let tx = tx.clone(); + broadcast + .receiver() + .subscribe(Box::new(move |_| tx.send(i).unwrap()), dispatch); + } + broadcast.broadcast(42); + drop(tx); + assert_eq!( + rx.into_iter().collect::>(), + (0..130).collect::>() + ); + } + + #[test] + fn test_callback_registration_races_all_terminal_paths() { + for mode in 0..3 { + for _ in 0..16 { + let broadcast = BroadcastOnce::::default(); + let receiver = broadcast.receiver(); + let (tx, rx) = std::sync::mpsc::channel(); + let barrier = Arc::new(std::sync::Barrier::new(5)); + let mut threads = Vec::new(); + for producer in 0..4 { + let receiver = receiver.clone(); + let tx = tx.clone(); + let barrier = Arc::clone(&barrier); + threads.push(std::thread::spawn(move || { + barrier.wait(); + for i in 0..32 { + let tx = tx.clone(); + receiver.subscribe( + Box::new(move |r| tx.send((producer * 32 + i, r.clone())).unwrap()), + CompletionBatch::run, + ); + } + })); + } + barrier.wait(); + let expected = match mode { + 0 => { + broadcast.broadcast(42); + Ok(42) + } + 1 => { + receiver.fail(Error::Client { + message: "abort".into(), + }); + Err(Error::Client { + message: "abort".into(), + }) + } + _ => { + drop(broadcast); + Err(Error::Dropped) + } + }; + for thread in threads { + thread.join().unwrap(); + } + drop(tx); + let mut results: Vec<_> = rx.into_iter().collect(); + results.sort_by_key(|(index, _)| *index); + assert_eq!( + results, + (0..128).map(|i| (i, expected.clone())).collect::>() + ); + assert_eq!(receiver.peek(), Some(expected)); + } + } + } + + #[test] + fn test_dispatch_runs_outside_locks_and_failure_cannot_complete_twice() { + let broadcast = BroadcastOnce::::default(); + let receiver = broadcast.receiver(); + let nested = receiver.clone(); + let (tx, rx) = std::sync::mpsc::channel(); + receiver.subscribe( + Box::new(move |result| { + assert_eq!(nested.peek(), Some(result.clone())); + // A synchronous test dispatcher deliberately reenters registration. + nested.subscribe( + Box::new(move |r| tx.send(r.clone()).unwrap()), + CompletionBatch::run, + ); + }), + CompletionBatch::run, + ); + broadcast.broadcast(42); + receiver.fail(Error::Dropped); + drop(broadcast); + assert_eq!(rx.recv_timeout(Duration::from_secs(5)).unwrap(), Ok(42)); + assert!(matches!( + rx.try_recv(), + Err(std::sync::mpsc::TryRecvError::Disconnected) + )); + } + + struct InspectOnWake { + shared: Arc>, + woke: AtomicBool, + result_readable: AtomicBool, + } + + impl InspectOnWake { + fn inspect(&self) { + self.woke.store(true, Ordering::SeqCst); + let readable = self + .shared + .data + .try_read() + .is_some_and(|data| data.is_some()); + self.result_readable.store(readable, Ordering::SeqCst); + } + } + + impl Wake for InspectOnWake { + fn wake(self: Arc) { + self.inspect(); + } + + fn wake_by_ref(self: &Arc) { + self.inspect(); } } + + fn assert_unlocked_on_notification( + action: impl FnOnce(BroadcastOnce), + expected: super::Result, + ) { + let broadcast = BroadcastOnce::::default(); + let receiver = broadcast.receiver(); + let probe = Arc::new(InspectOnWake { + shared: Arc::clone(&receiver.shared), + woke: AtomicBool::new(false), + result_readable: AtomicBool::new(false), + }); + let waker = Waker::from(Arc::clone(&probe)); + let mut context = Context::from_waker(&waker); + let mut future = Box::pin(receiver.receive()); + assert!(future.as_mut().poll(&mut context).is_pending()); + action(broadcast); + assert!(probe.woke.load(Ordering::SeqCst)); + assert!(probe.result_readable.load(Ordering::SeqCst)); + assert_eq!(future.as_mut().poll(&mut context), Poll::Ready(expected)); + } + + #[test] + fn test_broadcast_releases_result_lock_before_waking() { + assert_unlocked_on_notification(|broadcast| broadcast.broadcast(42), Ok(42)); + } + + #[test] + fn test_failure_releases_result_lock_before_waking() { + let error = Error::Client { + message: "writer closed".to_string(), + }; + assert_unlocked_on_notification( + |broadcast| broadcast.receiver().fail(error.clone()), + Err(error.clone()), + ); + } + + #[test] + fn test_drop_releases_result_lock_before_waking() { + assert_unlocked_on_notification(drop, Err(Error::Dropped)); + } + + #[test] + fn test_failure_does_not_overwrite_published_result() { + let broadcast = BroadcastOnce::::default(); + let receiver = broadcast.receiver(); + broadcast.broadcast(42); + receiver.fail(Error::Dropped); + assert_eq!(receiver.peek(), Some(Ok(42))); + drop(broadcast); + assert_eq!(receiver.peek(), Some(Ok(42))); + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn test_receivers_observe_results_before_and_after_notification() { + tokio::time::timeout(Duration::from_secs(10), async { + for round in 0..64 { + let broadcast = BroadcastOnce::::default(); + let late = broadcast.receiver(); + let barrier = Arc::new(tokio::sync::Barrier::new(33)); + let mut readers = Vec::new(); + for _ in 0..32 { + let receiver = broadcast.receiver(); + let barrier = Arc::clone(&barrier); + readers.push(tokio::spawn(async move { + barrier.wait().await; + receiver.receive().await + })); + } + barrier.wait().await; + let expected = match round % 3 { + 0 => { + broadcast.broadcast(round); + Ok(round) + } + 1 => { + broadcast.receiver().fail(Error::Dropped); + Err(Error::Dropped) + } + _ => { + drop(broadcast); + Err(Error::Dropped) + } + }; + for reader in readers { + assert_eq!(reader.await.unwrap(), expected); + } + assert_eq!(late.receive().await, expected); + } + }) + .await + .expect("all receivers must finish without a missed notification"); + } } diff --git a/fluss-rust/crates/fluss/src/client/write/mod.rs b/fluss-rust/crates/fluss/src/client/write/mod.rs index 37bdbffe3fa..398f9699baa 100644 --- a/fluss-rust/crates/fluss/src/client/write/mod.rs +++ b/fluss-rust/crates/fluss/src/client/write/mod.rs @@ -216,6 +216,10 @@ impl ResultHandle { } pub fn result(&self, batch_result: BatchWriteResult) -> Result<(), Error> { + Self::resolve(batch_result) + } + + fn resolve(batch_result: BatchWriteResult) -> Result<(), Error> { batch_result.map_err(|e| match e { client_broadcast::Error::WriteFailed { code, message } => Error::FlussAPIError { api_error: crate::rpc::ApiError { code, message }, @@ -241,9 +245,19 @@ impl ResultHandle { /// This pattern is similar to rdkafka's `DeliveryFuture` and allows for efficient batching /// when users don't need immediate per-record acknowledgment. pub struct WriteResultFuture { - inner: Pin> + Send>>, + state: WriteResultState, +} + +enum WriteResultState { + Single(ResultHandle), + Joined(Vec), + Waiting(Pin> + Send>>), } +/// An opaque group of write completions for language-binding executors. +#[doc(hidden)] +pub type WriteCallbackBatch = broadcast::CompletionBatch; + impl std::fmt::Debug for WriteResultFuture { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { f.debug_struct("WriteResultFuture").finish_non_exhaustive() @@ -254,30 +268,334 @@ impl WriteResultFuture { /// Create a new WriteResultFuture from a ResultHandle. pub fn new(result_handle: ResultHandle) -> Self { Self { - inner: Box::pin(async move { - let result = result_handle.wait().await?; - result_handle.result(result) - }), + state: WriteResultState::Single(result_handle), } } pub fn join(handles: Vec) -> Self { Self { - inner: Box::pin(async move { - for handle in handles { - let result = handle.wait().await?; - handle.result(result)?; + state: WriteResultState::Joined(handles), + } + } + + /// Register completions directly on their owning batches, without spawning + /// per-record waiting tasks. `dispatch` must reliably enqueue every job, + /// must not panic, and must not run callbacks on the calling/I/O thread. + /// + /// A previously polled future is returned with its callback untouched for + /// an executor to await normally. Dispatch may happen before this returns. + #[doc(hidden)] + pub fn try_on_complete( + self, + callback: C, + dispatch: fn(WriteCallbackBatch), + ) -> std::result::Result<(), (Self, C)> + where + C: FnOnce(Result<(), Error>) + Send + 'static, + { + match self.state { + WriteResultState::Single(handle) => { + handle.receiver.subscribe( + Box::new(move |result| callback(resolve_callback_result(result))), + dispatch, + ); + } + WriteResultState::Joined(handles) if handles.is_empty() => { + // Use the same executor even for an empty Arrow RecordBatch. + let completed = broadcast::BroadcastOnce::default(); + completed.receiver().subscribe( + Box::new(move |result| callback(resolve_callback_result(result))), + dispatch, + ); + completed.broadcast(Ok(())); + } + WriteResultState::Joined(handles) => { + // Preserve join's input-order error semantics, even when batches + // finish out of order. Do not call user code under this mutex. + let count = handles.len(); + let joined = Arc::new(parking_lot::Mutex::new(JoinedCallback { + results: (0..count).map(|_| None).collect(), + next: 0, + callback: Some(callback), + })); + for (index, handle) in handles.into_iter().enumerate() { + let joined = Arc::clone(&joined); + handle.receiver.subscribe( + Box::new(move |result| { + let ready = { + let mut joined = joined.lock(); + joined.complete(index, resolve_callback_result(result)) + }; + if let Some((callback, result)) = ready { + callback(result); + } + }), + dispatch, + ); } - Ok(()) - }), + } + WriteResultState::Waiting(_) => return Err((self, callback)), + } + Ok(()) + } +} + +struct JoinedCallback { + results: Vec>>, + next: usize, + callback: Option, +} + +impl JoinedCallback { + fn complete( + &mut self, + index: usize, + result: Result<(), Error>, + ) -> Option<(C, Result<(), Error>)> { + self.callback.as_ref()?; + self.results[index] = Some(result); + while self.next < self.results.len() { + let result = self.results[self.next].take()?; + self.next += 1; + if result.is_err() || self.next == self.results.len() { + return self.callback.take().map(|callback| (callback, result)); + } } + None } } +fn resolve_callback_result( + result: &client_broadcast::Result, +) -> Result<(), Error> { + let result = result.clone().map_err(|e| Error::UnexpectedError { + message: format!("Fail to wait write result {e:?}"), + source: None, + })?; + ResultHandle::resolve(result) +} + impl Future for WriteResultFuture { type Output = Result<(), Error>; fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { - self.inner.as_mut().poll(cx) + loop { + match &mut self.state { + WriteResultState::Waiting(future) => return future.as_mut().poll(cx), + _ => { + let state = + std::mem::replace(&mut self.state, WriteResultState::Joined(Vec::new())); + self.state = WriteResultState::Waiting(Box::pin(async move { + match state { + WriteResultState::Single(handle) => { + let result = handle.wait().await?; + handle.result(result) + } + WriteResultState::Joined(handles) => { + for handle in handles { + let result = handle.wait().await?; + handle.result(result)?; + } + Ok(()) + } + WriteResultState::Waiting(_) => unreachable!(), + } + })); + } + } + } + } +} + +#[cfg(test)] +mod callback_tests { + use super::*; + use broadcast::BroadcastOnce; + use std::sync::mpsc; + use std::time::Duration; + + fn future(batch: &BroadcastOnce) -> WriteResultFuture { + WriteResultFuture::new(ResultHandle::new(batch.receiver())) + } + + fn register(future: WriteResultFuture) -> mpsc::Receiver> { + let (tx, rx) = mpsc::channel(); + assert!( + future + .try_on_complete(move |r| tx.send(r).unwrap(), WriteCallbackBatch::run) + .is_ok() + ); + rx + } + + #[tokio::test] + async fn test_callback_and_wait_share_result_without_consuming_each_other() { + let batch = BroadcastOnce::default(); + let wait = future(&batch); + let rx = register(future(&batch)); + assert!(matches!(rx.try_recv(), Err(mpsc::TryRecvError::Empty))); + batch.broadcast(Ok(())); + assert!(wait.await.is_ok()); + assert!(rx.recv_timeout(Duration::from_secs(5)).unwrap().is_ok()); + assert!(matches!( + rx.try_recv(), + Err(mpsc::TryRecvError::Disconnected) + )); + } + + #[test] + fn test_callbacks_preserve_batch_and_abort_errors() { + let batch = BroadcastOnce::default(); + let rx = register(future(&batch)); + batch.broadcast(Err(client_broadcast::Error::WriteFailed { + code: 57, + message: "Deletion is disabled".into(), + })); + assert!( + matches!(rx.recv().unwrap(), Err(Error::FlussAPIError { api_error }) + if api_error.code == 57 && api_error.message == "Deletion is disabled") + ); + + let batch = BroadcastOnce::default(); + let rx = register(future(&batch)); + batch.receiver().fail(client_broadcast::Error::Client { + message: "abort".into(), + }); + assert!( + rx.recv() + .unwrap() + .unwrap_err() + .to_string() + .contains("abort") + ); + + let batch = BroadcastOnce::default(); + let rx = register(future(&batch)); + drop(batch); + assert!( + rx.recv() + .unwrap() + .unwrap_err() + .to_string() + .contains("Dropped") + ); + } + + #[test] + fn test_join_preserves_input_order_and_short_circuits_error() { + let first = BroadcastOnce::default(); + let second = BroadcastOnce::default(); + let third = BroadcastOnce::default(); + let rx = register(WriteResultFuture::join(vec![ + ResultHandle::new(first.receiver()), + ResultHandle::new(second.receiver()), + ResultHandle::new(third.receiver()), + ])); + second.broadcast(Err(client_broadcast::Error::WriteFailed { + code: 57, + message: "second error".into(), + })); + assert!(matches!(rx.try_recv(), Err(mpsc::TryRecvError::Empty))); + first.broadcast(Ok(())); + assert!( + rx.recv() + .unwrap() + .unwrap_err() + .to_string() + .contains("second error") + ); + // No need to wait for the third batch after the first ordered error. + third.broadcast(Ok(())); + assert!(matches!( + rx.try_recv(), + Err(mpsc::TryRecvError::Disconnected) + )); + } + + #[test] + fn test_join_all_success_empty_and_duplicate_batch_handles() { + assert!( + register(WriteResultFuture::join(Vec::new())) + .recv() + .unwrap() + .is_ok() + ); + let batch = BroadcastOnce::default(); + let rx = register(WriteResultFuture::join(vec![ + ResultHandle::new(batch.receiver()), + ResultHandle::new(batch.receiver()), + ])); + batch.broadcast(Ok(())); + assert!(rx.recv().unwrap().is_ok()); + assert!(matches!( + rx.try_recv(), + Err(mpsc::TryRecvError::Disconnected) + )); + } + + #[test] + fn test_join_first_error_wins_despite_reverse_completion_order() { + let first = BroadcastOnce::default(); + let second = BroadcastOnce::default(); + let rx = register(WriteResultFuture::join(vec![ + ResultHandle::new(first.receiver()), + ResultHandle::new(second.receiver()), + ])); + second.broadcast(Err(client_broadcast::Error::Client { + message: "second".into(), + })); + first.broadcast(Err(client_broadcast::Error::Client { + message: "first".into(), + })); + assert!( + rx.recv() + .unwrap() + .unwrap_err() + .to_string() + .contains("first") + ); + } + + #[test] + fn test_join_concurrent_completions_invoke_callback_once() { + for _ in 0..32 { + let batches: Vec<_> = (0..8).map(|_| BroadcastOnce::default()).collect(); + let rx = register(WriteResultFuture::join( + batches + .iter() + .map(|b| ResultHandle::new(b.receiver())) + .collect(), + )); + std::thread::scope(|scope| { + for batch in &batches { + scope.spawn(move || batch.broadcast(Ok(()))); + } + }); + assert!(rx.recv_timeout(Duration::from_secs(5)).unwrap().is_ok()); + assert!(matches!( + rx.try_recv(), + Err(mpsc::TryRecvError::Disconnected) + )); + } + } + + #[tokio::test] + async fn test_polled_future_returns_untouched_callback_for_fallback() { + let batch = BroadcastOnce::default(); + let mut wait = future(&batch); + assert!( + Pin::new(&mut wait) + .poll(&mut Context::from_waker(std::task::Waker::noop())) + .is_pending() + ); + let (tx, rx) = mpsc::channel(); + let registered = + wait.try_on_complete(move |r| tx.send(r).unwrap(), WriteCallbackBatch::run); + let Err((wait, callback)) = registered else { + panic!("polled future must use fallback") + }; + batch.broadcast(Ok(())); + callback(wait.await); + assert!(rx.recv().unwrap().is_ok()); } } diff --git a/fluss-rust/website/docs/user-guide/cpp/api-reference.md b/fluss-rust/website/docs/user-guide/cpp/api-reference.md index d152b51af11..b86982670fa 100644 --- a/fluss-rust/website/docs/user-guide/cpp/api-reference.md +++ b/fluss-rust/website/docs/user-guide/cpp/api-reference.md @@ -250,9 +250,14 @@ application state. Keep the connection alive until outstanding operations complete. Exceptions thrown by callbacks are caught and reported to stderr; they do not change the write outcome. -The binding asynchronously awaits each write's result, then dispatches its -callback to one of four process-wide callback workers. User callbacks run -outside the runtime's async I/O workers. +Callbacks register directly with their internal write batch, rather than +creating an asynchronous ACK-waiting task for each row. When a batch completes, +its callbacks are dispatched to four process-wide callback workers in jobs of +at most 64 callbacks. Each worker takes one job at a time; every registered +callback still runs individually. Registrations arriving after batch completion +are dispatched separately, without waiting to fill a job. Both Arrow log and KV +write batches use this path. An `AppendArrowBatch` spanning multiple internal +batches aggregates their results into the operation's single callback. The completion queue is unbounded, so limit outstanding callbacks when callback processing is slower than writing; writer buffer limits From 1e9137ead63f29a98980851ddfc2da87ff3d92d1 Mon Sep 17 00:00:00 2001 From: naivedogger Date: Mon, 14 Sep 2026 15:17:37 +0800 Subject: [PATCH 03/19] [docs] Clarify write callback compatibility and lifecycle --- .../docs/user-guide/cpp/api-reference.md | 52 +++++++++++++++++-- 1 file changed, 49 insertions(+), 3 deletions(-) diff --git a/fluss-rust/website/docs/user-guide/cpp/api-reference.md b/fluss-rust/website/docs/user-guide/cpp/api-reference.md index b86982670fa..87dc30a1f06 100644 --- a/fluss-rust/website/docs/user-guide/cpp/api-reference.md +++ b/fluss-rust/website/docs/user-guide/cpp/api-reference.md @@ -233,8 +233,9 @@ if (!submitted.Ok()) { ``` The immediate return value reports submission status, not acknowledgment. An -empty callback is rejected before submission. Each successfully submitted -operation invokes its callback exactly once with its final success or failure; +empty callback is rejected before submission. During normal operation, each +successfully submitted operation invokes its callback exactly once with its +final success or failure, subject to the lifetime and shutdown requirements below; `AppendArrowBatch` invokes one callback for the batch, not one per row or bucket. Submission does not wait for acknowledgment, but may still wait for buffer space under backpressure. @@ -269,7 +270,52 @@ SDK falls back to its runtime blocking pool to preserve callback delivery. `Flush()` still waits for pending writes, **not** for user callbacks to finish. Applications that need to drain callbacks must track their completion separately. -The existing fire-and-forget and `WriteResult::Wait()` APIs are unchanged. + +### Compatibility and operational limits + +- Existing fire-and-forget and `WriteResult::Wait()` overloads retain their + result semantics. Rust callers can still `.await` a `WriteResultFuture`. + Completion follows the configured acknowledgment policy; a callback does not + add a stronger durability guarantee or change retries, request ordering, + wire formats, or storage formats. +- The shared Rust completion path also changes for callers that do not register + callbacks: results are stored behind an `Arc`, callback registration state is + added per batch, and waiters are notified after releasing the result lock. + This changes allocation and scheduling costs, not the reported write outcome. + The boxed waiting future is now allocated on first poll rather than at + construction. Callback workers are initialized only when callbacks are used. +- For an `AppendArrowBatch` spanning multiple internal batches, success requires + all their results to succeed. As with `Wait()`, errors are selected in internal + handle order, not completion order; an error may be reported while later + batches remain pending. This is not an atomic multi-bucket write, and an error + does not imply that no rows were written. A submission error can also follow + partial acceptance of a multi-bucket batch; in that case no callback is + registered, so handle the returned error and do not assume an all-or-nothing retry. +- The four-worker count and 64-callback job size are implementation details, + not ordering or latency guarantees. A slow callback delays other callbacks in + its job, and slow callbacks from one connection can delay another connection. + The blocking-pool fallback is not limited to four callback threads. +- Callback delivery is in memory only. There is no end-to-end callback deadline, + public callback-drain API, or durable recovery of pending notifications. + Connection or writer destruction is not a callback-drain barrier; process exit, + crashes, or fatal resource exhaustion can prevent pending callbacks from running. + The process-wide executor is not automatically drained at exit. +- Before releasing callback state or the connection, stop and join submitting + threads, flush pending writes, and wait separately for tracked callbacks. + Reserve tracking capacity before submission because callbacks may run before + it returns; release that capacity yourself if submission fails. Bound outstanding + operations through callback completion, not just through server acknowledgment. + An application-side wait timeout does not cancel the write or its callback. + Keep referenced state alive if abandoning a wait; use durable application + tracking and a duplicate-safe retry policy when recovery is required. +- C++ exceptions thrown by user callbacks are contained. If copying error text + fails, the callback still receives the error code, but the message may be empty. + Allocation failures while constructing the callback before submission can still + throw a C++ exception rather than return a `Result`. + +The new overloads keep ordinary existing calls source-compatible. Code that +selects an overload by taking a member-function address should specify the +intended function type explicitly. ## `Lookuper` From edc252230db43982374046f8d2aaeb317f3ad7ea Mon Sep 17 00:00:00 2001 From: naivedogger Date: Mon, 14 Sep 2026 20:15:45 +0800 Subject: [PATCH 04/19] [c++] Demonstrate bounded write callbacks in the existing example --- fluss-rust/bindings/cpp/README.md | 4 +- fluss-rust/bindings/cpp/examples/example.cpp | 112 +++++++++++++++++++ 2 files changed, 115 insertions(+), 1 deletion(-) diff --git a/fluss-rust/bindings/cpp/README.md b/fluss-rust/bindings/cpp/README.md index 35c2ecd2771..aea30b66552 100644 --- a/fluss-rust/bindings/cpp/README.md +++ b/fluss-rust/bindings/cpp/README.md @@ -65,7 +65,7 @@ not apply to `CreateBucketBatchScanner()`. ## Examples and Documentation -- [examples/example.cpp](examples/example.cpp) demonstrates log-table writes, continuous scans, +- [examples/example.cpp](examples/example.cpp) demonstrates log-table writes with Wait and bounded callbacks, continuous scans, bounded Arrow record-batch scans, projections, and offset queries. - [examples/admin_example.cpp](examples/admin_example.cpp) demonstrates database, table, partition, and cluster administration. @@ -76,6 +76,8 @@ not apply to `CreateBucketBatchScanner()`. [C++ API reference](../../website/docs/user-guide/cpp/api-reference.md) and [log-table examples](../../website/docs/user-guide/cpp/example/log-tables.md). +The callback section reserves capacity before submission, handles submission and completion errors separately, and drains callbacks after Flush. Its waits have no deadline; production applications need a recovery policy for operations that never complete. A local timeout does not cancel writes or callbacks. + For a bounded log scan, pass the per-bucket offset ranges directly to `TableScan`. The returned reader yields one Arrow batch at a time until every `[starting_offset, stopping_offset)` range is complete: diff --git a/fluss-rust/bindings/cpp/examples/example.cpp b/fluss-rust/bindings/cpp/examples/example.cpp index 14a8380f2a0..f8bda630d44 100644 --- a/fluss-rust/bindings/cpp/examples/example.cpp +++ b/fluss-rust/bindings/cpp/examples/example.cpp @@ -21,8 +21,13 @@ #include #include +#include +#include #include +#include +#include #include +#include #include #include "fluss.hpp" @@ -143,6 +148,113 @@ int main() { std::cout << "Row acknowledged by server" << std::endl; } + // Callback acknowledgment with bounded outstanding operations. + { + struct PendingWrites { + std::mutex mutex; + std::condition_variable changed; + size_t outstanding = 0; + size_t succeeded = 0; + size_t failed = 0; + int32_t first_failed_id = -1; + fluss::Result first_error; + }; + // Use a small limit to demonstrate backpressure with only three rows. + constexpr size_t max_outstanding = 2; + auto pending = std::make_shared(); + size_t accepted = 0; + bool submission_failed = false; + try { + for (const auto& r : rows) { + const int32_t id = 1000 + r.id; + fluss::GenericRow row; + row.SetInt32(0, id); + row.SetString(1, r.name); + row.SetFloat32(2, r.score); + row.SetInt32(3, r.age); + row.SetDate(4, r.date); + row.SetTime(5, r.time); + row.SetTimestampNtz(6, r.ts_ntz); + row.SetTimestampLtz(7, r.ts_ltz); + fluss::WriteCallback callback = [pending, id](fluss::Result result) { + { + std::lock_guard lock(pending->mutex); + if (result.Ok()) { + ++pending->succeeded; + } else { + if (pending->failed == 0) { + pending->first_failed_id = id; + pending->first_error = std::move(result); + } + ++pending->failed; + } + --pending->outstanding; + } + pending->changed.notify_all(); + }; + + // Reserve before Append: the callback may run before Append returns. + { + std::unique_lock lock(pending->mutex); + pending->changed.wait(lock, + [&] { return pending->outstanding < max_outstanding; }); + ++pending->outstanding; + } + fluss::Result submitted; + try { + // Never hold the tracking mutex while calling the SDK. + submitted = writer.Append(row, std::move(callback)); + } catch (...) { + // Callback-wrapper allocation can throw before submission. + std::lock_guard lock(pending->mutex); + --pending->outstanding; + throw; + } + if (!submitted.Ok()) { + // Rejected submissions have no callback; release their slot here. + { + std::lock_guard lock(pending->mutex); + --pending->outstanding; + } + std::cerr << "Submission failed for id=" << id + << ": code=" << submitted.error_code + << " message=" << submitted.error_message << '\n'; + submission_failed = true; + break; + } + ++accepted; + } + } catch (const std::exception& error) { + std::cerr << "Submission stopped: " << error.what() << '\n'; + submission_failed = true; + } catch (...) { + std::cerr << "Submission stopped by an unknown exception\n"; + submission_failed = true; + } + + // Submission has stopped. Flush is not a callback barrier; drain even on error. + auto flushed = writer.Flush(); + if (!flushed.Ok()) { + std::cerr << "Callback write flush failed: " << flushed.error_message << '\n'; + } + std::unique_lock lock(pending->mutex); + // This demonstration waits without a deadline. A timeout would not cancel + // accepted writes or callbacks; production code needs its own recovery policy. + pending->changed.wait(lock, [&] { return pending->outstanding == 0; }); + std::cout << "Callback writes: accepted=" << accepted << " succeeded=" << pending->succeeded + << " failed=" << pending->failed << '\n'; + if (pending->failed != 0) { + std::cerr << "First completion failure: id=" << pending->first_failed_id + << " code=" << pending->first_error.error_code + << " message=" << pending->first_error.error_message << '\n'; + } + // Do not blindly retry failures: an error need not mean nothing was written. + // Shared captures and the connection/writer stay alive through the drain. + if (submission_failed || !flushed.Ok() || pending->failed != 0) { + return 1; + } + } + // Append a row with all fields null (matches Rust log_table.rs all_supported_datatypes) { fluss::GenericRow row; From 91ecc1e6627432710b9e17605f03d70ca66e45a7 Mon Sep 17 00:00:00 2001 From: naivedogger Date: Mon, 14 Sep 2026 23:31:54 +0800 Subject: [PATCH 05/19] [c++] Bound pending write callbacks per writer --- fluss-rust/bindings/cpp/README.md | 2 +- fluss-rust/bindings/cpp/examples/example.cpp | 50 ++-- fluss-rust/bindings/cpp/include/fluss.hpp | 36 ++- fluss-rust/bindings/cpp/src/table.cpp | 55 +++- .../bindings/cpp/src/write_callback.hpp | 84 ++++++ .../bindings/cpp/test/test_write_callback.cpp | 283 +++++++++++++++++- .../test/test_write_callback_allocation.cpp | 5 + .../docs/user-guide/cpp/api-reference.md | 74 ++++- 8 files changed, 529 insertions(+), 60 deletions(-) diff --git a/fluss-rust/bindings/cpp/README.md b/fluss-rust/bindings/cpp/README.md index aea30b66552..206d7173852 100644 --- a/fluss-rust/bindings/cpp/README.md +++ b/fluss-rust/bindings/cpp/README.md @@ -76,7 +76,7 @@ not apply to `CreateBucketBatchScanner()`. [C++ API reference](../../website/docs/user-guide/cpp/api-reference.md) and [log-table examples](../../website/docs/user-guide/cpp/example/log-tables.md). -The callback section reserves capacity before submission, handles submission and completion errors separately, and drains callbacks after Flush. Its waits have no deadline; production applications need a recovery policy for operations that never complete. A local timeout does not cancel writes or callbacks. +The callback section configures `WriteCallbackOptions` so the SDK bounds outstanding callback operations, handles submission and completion errors separately, and waits for result handling after Flush. `max_pending_operations` defaults to 65536 and `enqueue_timeout` to 30 seconds; the example uses a limit of 2 and a 5-second admission timeout. That timeout only covers waiting for callback capacity. Flush and application completion waits still have no deadline; production applications need a recovery policy for operations that never complete. A local timeout does not cancel accepted writes or callbacks. For a bounded log scan, pass the per-bucket offset ranges directly to `TableScan`. The returned reader yields one Arrow batch at a time until every `[starting_offset, stopping_offset)` range diff --git a/fluss-rust/bindings/cpp/examples/example.cpp b/fluss-rust/bindings/cpp/examples/example.cpp index f8bda630d44..5cd74f705d4 100644 --- a/fluss-rust/bindings/cpp/examples/example.cpp +++ b/fluss-rust/bindings/cpp/examples/example.cpp @@ -90,7 +90,12 @@ int main() { // 5) Write rows with scalar and temporal values fluss::AppendWriter writer; - check("new_append_writer", table.NewAppend().CreateWriter(writer)); + fluss::WriteCallbackOptions callback_options; + // A small limit demonstrates SDK backpressure in the callback section below. + // These options do not affect fire-and-forget or WriteResult::Wait(). + callback_options.max_pending_operations = 2; + callback_options.enqueue_timeout = std::chrono::seconds(5); + check("new_append_writer", table.NewAppend().CreateWriter(writer, callback_options)); struct RowData { int id; @@ -153,14 +158,11 @@ int main() { struct PendingWrites { std::mutex mutex; std::condition_variable changed; - size_t outstanding = 0; size_t succeeded = 0; size_t failed = 0; int32_t first_failed_id = -1; fluss::Result first_error; }; - // Use a small limit to demonstrate backpressure with only three rows. - constexpr size_t max_outstanding = 2; auto pending = std::make_shared(); size_t accepted = 0; bool submission_failed = false; @@ -176,7 +178,9 @@ int main() { row.SetTime(5, r.time); row.SetTimestampNtz(6, r.ts_ntz); row.SetTimestampLtz(7, r.ts_ltz); - fluss::WriteCallback callback = [pending, id](fluss::Result result) { + // The SDK waits for callback capacity before submitting. Do not + // hold the tracking mutex here: callbacks need it to finish. + auto submitted = writer.Append(row, [pending, id](fluss::Result result) { { std::lock_guard lock(pending->mutex); if (result.Ok()) { @@ -188,34 +192,11 @@ int main() { } ++pending->failed; } - --pending->outstanding; } - pending->changed.notify_all(); - }; - - // Reserve before Append: the callback may run before Append returns. - { - std::unique_lock lock(pending->mutex); - pending->changed.wait(lock, - [&] { return pending->outstanding < max_outstanding; }); - ++pending->outstanding; - } - fluss::Result submitted; - try { - // Never hold the tracking mutex while calling the SDK. - submitted = writer.Append(row, std::move(callback)); - } catch (...) { - // Callback-wrapper allocation can throw before submission. - std::lock_guard lock(pending->mutex); - --pending->outstanding; - throw; - } + pending->changed.notify_one(); + }); if (!submitted.Ok()) { - // Rejected submissions have no callback; release their slot here. - { - std::lock_guard lock(pending->mutex); - --pending->outstanding; - } + // Rejected submissions have no callback. The SDK returns their capacity. std::cerr << "Submission failed for id=" << id << ": code=" << submitted.error_code << " message=" << submitted.error_message << '\n'; @@ -240,7 +221,9 @@ int main() { std::unique_lock lock(pending->mutex); // This demonstration waits without a deadline. A timeout would not cancel // accepted writes or callbacks; production code needs its own recovery policy. - pending->changed.wait(lock, [&] { return pending->outstanding == 0; }); + // Read accepted only after submission stops, so early callbacks are safe. + pending->changed.wait(lock, + [&] { return pending->succeeded + pending->failed == accepted; }); std::cout << "Callback writes: accepted=" << accepted << " succeeded=" << pending->succeeded << " failed=" << pending->failed << '\n'; if (pending->failed != 0) { @@ -249,7 +232,8 @@ int main() { << " message=" << pending->first_error.error_message << '\n'; } // Do not blindly retry failures: an error need not mean nothing was written. - // Shared captures and the connection/writer stay alive through the drain. + // This wait observes result handling, not SDK callback return. Shared captures + // keep state alive through callback return; the connection stays alive too. if (submission_failed || !flushed.Ok() || pending->failed != 0) { return 1; } diff --git a/fluss-rust/bindings/cpp/include/fluss.hpp b/fluss-rust/bindings/cpp/include/fluss.hpp index ac3a4025e96..7c5e607f9fb 100644 --- a/fluss-rust/bindings/cpp/include/fluss.hpp +++ b/fluss-rust/bindings/cpp/include/fluss.hpp @@ -49,6 +49,7 @@ struct Table; struct AppendWriter; struct WriteResult; class WriteCallback; +class WriteCallbackCapacity; struct LogScanner; struct RecordBatchLogReader; struct BatchScanner; @@ -546,13 +547,25 @@ struct Result { /// /// Callback threads are shared across connections. Do not wait for another /// callback from a callback: it can exhaust the worker pool. Synchronous SDK -/// calls are supported with exclusive writer access. Limit outstanding callbacks -/// if they are slow: completed callbacks queue in memory, outside writer buffers. +/// calls require exclusive writer access. Callback submissions to a full writer +/// fail immediately when called from a callback, instead of blocking the workers. +/// WriteCallbackOptions bounds outstanding callback operations per writer. /// /// Exceptions thrown by callbacks are caught and reported to stderr; they do not /// change the write outcome. Flush() waits for writes, not for callbacks to finish. using WriteCallback = std::function; +/// Admission limits for callback overloads only; Wait and fire-and-forget are unchanged. +struct WriteCallbackOptions { + /// Maximum operations reserved for submission or awaiting callback completion. + /// Must be positive. One AppendArrowBatch call counts as one operation, not its rows. + size_t max_pending_operations = 65536; + + /// Maximum wait for callback capacity; zero rejects immediately when full. + /// Must be nonnegative. Does not bound buffer waits, ACKs, retries, or callback duration. + std::chrono::milliseconds enqueue_timeout{30000}; +}; + struct TablePath { std::string database_name; std::string table_name; @@ -1762,6 +1775,8 @@ class TableAppend { TableAppend& operator=(TableAppend&&) noexcept = default; Result CreateWriter(AppendWriter& out); + /// Create a writer with per-writer callback admission limits. + Result CreateWriter(AppendWriter& out, const WriteCallbackOptions& options); private: friend class Table; @@ -1781,6 +1796,8 @@ class TableUpsert { TableUpsert& PartialUpdateByName(std::vector column_names); Result CreateWriter(UpsertWriter& out); + /// Create a writer with per-writer callback admission limits shared by Upsert and Delete. + Result CreateWriter(UpsertWriter& out, const WriteCallbackOptions& options); private: friend class Table; @@ -1920,21 +1937,24 @@ class AppendWriter { Result Append(const GenericRow& row, WriteResult& out); /// Submit a row and notify callback of its final outcome without waiting for /// acknowledgment. Returns submission status; on failure no callback runs. - /// Submission can still block on buffer backpressure. See WriteCallback. + /// Submission can block on callback capacity and buffer backpressure. See WriteCallbackOptions. Result Append(const GenericRow& row, WriteCallback callback); Result AppendArrowBatch(const std::shared_ptr& batch); Result AppendArrowBatch(const std::shared_ptr& batch, WriteResult& out); /// Like the callback Append overload, but notifies once for the entire batch. - Result AppendArrowBatch(const std::shared_ptr& batch, WriteCallback callback); + Result AppendArrowBatch(const std::shared_ptr& batch, + WriteCallback callback); Result Flush(); private: friend class Table; friend class TableAppend; - AppendWriter(ffi::AppendWriter* writer) noexcept; + AppendWriter(ffi::AppendWriter* writer, + std::shared_ptr callback_capacity) noexcept; void Destroy() noexcept; ffi::AppendWriter* writer_{nullptr}; + std::shared_ptr callback_capacity_; }; class UpsertWriter { @@ -1953,7 +1973,7 @@ class UpsertWriter { Result Upsert(const GenericRow& row, WriteResult& out); /// Submit an upsert and notify callback of its final outcome. Returns /// submission status; on failure no callback runs. Submission may block on - /// buffer backpressure, but does not wait for acknowledgment. See WriteCallback. + /// callback capacity and buffer backpressure, but not acknowledgment. See WriteCallbackOptions. Result Upsert(const GenericRow& row, WriteCallback callback); Result Delete(const GenericRow& row); Result Delete(const GenericRow& row, WriteResult& out); @@ -1964,9 +1984,11 @@ class UpsertWriter { private: friend class Table; friend class TableUpsert; - UpsertWriter(ffi::UpsertWriter* writer) noexcept; + UpsertWriter(ffi::UpsertWriter* writer, + std::shared_ptr callback_capacity) noexcept; void Destroy() noexcept; ffi::UpsertWriter* writer_{nullptr}; + std::shared_ptr callback_capacity_; }; class Lookuper { diff --git a/fluss-rust/bindings/cpp/src/table.cpp b/fluss-rust/bindings/cpp/src/table.cpp index 2204c16634b..4a0878f915a 100644 --- a/fluss-rust/bindings/cpp/src/table.cpp +++ b/fluss-rust/bindings/cpp/src/table.cpp @@ -1281,14 +1281,23 @@ bool Table::HasPrimaryKey() const { TableAppend::TableAppend(ffi::Table* table) noexcept : table_(table) {} Result TableAppend::CreateWriter(AppendWriter& out) { + return CreateWriter(out, WriteCallbackOptions{}); +} + +Result TableAppend::CreateWriter(AppendWriter& out, const WriteCallbackOptions& options) { + auto validation = ffi::WriteCallbackCapacity::Validate(options); + if (!validation.Ok()) { + return validation; + } if (table_ == nullptr) { return utils::make_client_error("Table not available"); } + auto capacity = std::make_shared(options); auto ffi_result = table_->new_append_writer(); auto result = utils::from_ffi_result(ffi_result.result); if (result.Ok()) { - out = AppendWriter(utils::ptr_from_ffi(ffi_result)); + out = AppendWriter(utils::ptr_from_ffi(ffi_result), std::move(capacity)); } return result; } @@ -1339,11 +1348,20 @@ std::vector TableUpsert::ResolveNameProjection() const { } Result TableUpsert::CreateWriter(UpsertWriter& out) { + return CreateWriter(out, WriteCallbackOptions{}); +} + +Result TableUpsert::CreateWriter(UpsertWriter& out, const WriteCallbackOptions& options) { + auto validation = ffi::WriteCallbackCapacity::Validate(options); + if (!validation.Ok()) { + return validation; + } if (table_ == nullptr) { return utils::make_client_error("Table not available"); } try { + auto capacity = std::make_shared(options); auto resolved_indices = !column_names_.empty() ? ResolveNameProjection() : column_indices_; rust::Vec rust_indices; @@ -1353,7 +1371,8 @@ Result TableUpsert::CreateWriter(UpsertWriter& out) { auto ffi_result = table_->create_upsert_writer(std::move(rust_indices)); auto result = utils::from_ffi_result(ffi_result.result); if (result.Ok()) { - out = UpsertWriter(utils::ptr_from_ffi(ffi_result)); + out = UpsertWriter(utils::ptr_from_ffi(ffi_result), + std::move(capacity)); } return result; } catch (const std::exception& e) { @@ -1615,7 +1634,9 @@ Result WriteResult::Notify(std::unique_ptr callback) { AppendWriter::AppendWriter() noexcept = default; -AppendWriter::AppendWriter(ffi::AppendWriter* writer) noexcept : writer_(writer) {} +AppendWriter::AppendWriter(ffi::AppendWriter* writer, + std::shared_ptr callback_capacity) noexcept + : writer_(writer), callback_capacity_(std::move(callback_capacity)) {} AppendWriter::~AppendWriter() noexcept { Destroy(); } @@ -1626,7 +1647,8 @@ void AppendWriter::Destroy() noexcept { } } -AppendWriter::AppendWriter(AppendWriter&& other) noexcept : writer_(other.writer_) { +AppendWriter::AppendWriter(AppendWriter&& other) noexcept + : writer_(other.writer_), callback_capacity_(std::move(other.callback_capacity_)) { other.writer_ = nullptr; } @@ -1634,6 +1656,7 @@ AppendWriter& AppendWriter::operator=(AppendWriter&& other) noexcept { if (this != &other) { Destroy(); writer_ = other.writer_; + callback_capacity_ = std::move(other.callback_capacity_); other.writer_ = nullptr; } return *this; @@ -1669,6 +1692,10 @@ Result AppendWriter::Append(const GenericRow& row, WriteCallback callback) { // Allocate before submission so an allocation failure cannot lose an // already accepted write's completion notification. auto completion = std::make_unique(std::move(callback)); + auto reserved = completion->Reserve(callback_capacity_); + if (!reserved.Ok()) { + return reserved; + } WriteResult pending; auto result = Append(row, pending); if (result.Ok()) { @@ -1721,6 +1748,10 @@ Result AppendWriter::AppendArrowBatch(const std::shared_ptr& return utils::make_client_error("Write callback must not be empty"); } auto completion = std::make_unique(std::move(callback)); + auto reserved = completion->Reserve(callback_capacity_); + if (!reserved.Ok()) { + return reserved; + } WriteResult pending; auto result = AppendArrowBatch(batch, pending); if (result.Ok()) { @@ -1744,7 +1775,9 @@ Result AppendWriter::Flush() { UpsertWriter::UpsertWriter() noexcept = default; -UpsertWriter::UpsertWriter(ffi::UpsertWriter* writer) noexcept : writer_(writer) {} +UpsertWriter::UpsertWriter(ffi::UpsertWriter* writer, + std::shared_ptr callback_capacity) noexcept + : writer_(writer), callback_capacity_(std::move(callback_capacity)) {} UpsertWriter::~UpsertWriter() noexcept { Destroy(); } @@ -1755,7 +1788,8 @@ void UpsertWriter::Destroy() noexcept { } } -UpsertWriter::UpsertWriter(UpsertWriter&& other) noexcept : writer_(other.writer_) { +UpsertWriter::UpsertWriter(UpsertWriter&& other) noexcept + : writer_(other.writer_), callback_capacity_(std::move(other.callback_capacity_)) { other.writer_ = nullptr; } @@ -1763,6 +1797,7 @@ UpsertWriter& UpsertWriter::operator=(UpsertWriter&& other) noexcept { if (this != &other) { Destroy(); writer_ = other.writer_; + callback_capacity_ = std::move(other.callback_capacity_); other.writer_ = nullptr; } return *this; @@ -1796,6 +1831,10 @@ Result UpsertWriter::Upsert(const GenericRow& row, WriteCallback callback) { return utils::make_client_error("Write callback must not be empty"); } auto completion = std::make_unique(std::move(callback)); + auto reserved = completion->Reserve(callback_capacity_); + if (!reserved.Ok()) { + return reserved; + } WriteResult pending; auto result = Upsert(row, pending); if (result.Ok()) { @@ -1830,6 +1869,10 @@ Result UpsertWriter::Delete(const GenericRow& row, WriteCallback callback) { return utils::make_client_error("Write callback must not be empty"); } auto completion = std::make_unique(std::move(callback)); + auto reserved = completion->Reserve(callback_capacity_); + if (!reserved.Ok()) { + return reserved; + } WriteResult pending; auto result = Delete(row, pending); if (result.Ok()) { diff --git a/fluss-rust/bindings/cpp/src/write_callback.hpp b/fluss-rust/bindings/cpp/src/write_callback.hpp index a304f6fdcdc..1ca97dee0ff 100644 --- a/fluss-rust/bindings/cpp/src/write_callback.hpp +++ b/fluss-rust/bindings/cpp/src/write_callback.hpp @@ -19,8 +19,10 @@ #pragma once +#include #include #include +#include #include "fluss.hpp" #include "rust/cxx.h" @@ -28,13 +30,79 @@ namespace fluss { namespace ffi { +/// Per-writer admission control; independent of Rust buffer memory and ACK completion. +class WriteCallbackCapacity { + public: + explicit WriteCallbackCapacity(const WriteCallbackOptions& options) : options_(options) {} + + static Result Validate(const WriteCallbackOptions& options) { + if (options.max_pending_operations == 0) { + return {ErrorCode::CLIENT_ERROR, "max_pending_operations must be positive"}; + } + if (options.enqueue_timeout.count() < 0) { + return {ErrorCode::CLIENT_ERROR, "enqueue_timeout must be nonnegative"}; + } + return {}; + } + + Result Acquire() { + std::unique_lock lock(mutex_); + if (pending_ == options_.max_pending_operations) { + // Applies across writers and also to callbacks on the fallback executor. + if (in_callback_ || options_.enqueue_timeout.count() == 0) { + return {ErrorCode::CLIENT_ERROR, "Write callback capacity is full"}; + } + if (!available_.wait_for(lock, options_.enqueue_timeout, + [&] { return pending_ < options_.max_pending_operations; })) { + return {ErrorCode::CLIENT_ERROR, "Timed out waiting for write callback capacity"}; + } + } + ++pending_; + return {}; + } + + void Release() noexcept { + { + std::lock_guard lock(mutex_); + --pending_; + } + available_.notify_one(); + } + + private: + friend class WriteCallback; + inline static thread_local bool in_callback_ = false; + const WriteCallbackOptions options_; + std::mutex mutex_; + std::condition_variable available_; + size_t pending_ = 0; +}; + /// Owns a callback transferred to Rust. Access is exclusive, never concurrent. class WriteCallback { public: explicit WriteCallback(fluss::WriteCallback callback) : callback_(std::move(callback)) {} + WriteCallback(const WriteCallback&) = delete; + WriteCallback& operator=(const WriteCallback&) = delete; + + /// Reserve before entering Rust. Destruction also returns capacity on submission failure. + Result Reserve(std::shared_ptr capacity) { + if (!capacity) { + return {ErrorCode::CLIENT_ERROR, "Writer not available"}; + } + auto result = capacity->Acquire(); + if (result.Ok()) { + reservation_.capacity = std::move(capacity); + } + return result; + } + /// Invoke once, containing all C++ exceptions on this side of the FFI boundary. void Complete(int32_t error_code, rust::Str error_message) noexcept { + // Release captures before the reservation, even if this wrapper outlives Complete(). + Reservation reservation{std::move(reservation_.capacity)}; + CallbackScope scope; // Moving std::function alone need not empty the source. Swap with an // empty function so captures are released even if the callback throws. fluss::WriteCallback callback; @@ -58,6 +126,22 @@ class WriteCallback { } private: + struct Reservation { + std::shared_ptr capacity; + ~Reservation() { + if (capacity) { + capacity->Release(); + } + } + }; + + struct CallbackScope { + bool previous = std::exchange(WriteCallbackCapacity::in_callback_, true); + ~CallbackScope() { WriteCallbackCapacity::in_callback_ = previous; } + }; + + // Member order keeps captures alive until invocation, but not past capacity release. + Reservation reservation_; fluss::WriteCallback callback_; }; diff --git a/fluss-rust/bindings/cpp/test/test_write_callback.cpp b/fluss-rust/bindings/cpp/test/test_write_callback.cpp index 8072ef2d552..ef571c874a0 100644 --- a/fluss-rust/bindings/cpp/test/test_write_callback.cpp +++ b/fluss-rust/bindings/cpp/test/test_write_callback.cpp @@ -20,6 +20,7 @@ #include #include +#include #include #include #include @@ -270,7 +271,8 @@ TEST_F(WriteCallbackTest, MultipleOutstandingWritesEachNotifyOnce) { TEST_F(WriteCallbackTest, RejectedSubmissionDoesNotInvokeCallback) { CreateTable(); fluss::AppendWriter writer; - ASSERT_OK(table_.NewAppend().CreateWriter(writer)); + ASSERT_OK(table_.NewAppend().CreateWriter( + writer, fluss::WriteCallbackOptions{1, std::chrono::milliseconds(0)})); auto completion = std::make_shared(); auto lifetime = std::make_shared(); auto released = lifetime->released.get_future(); @@ -289,7 +291,131 @@ TEST_F(WriteCallbackTest, RejectedSubmissionDoesNotInvokeCallback) { }); EXPECT_FALSE(result.Ok()); EXPECT_TRUE(completion->Results().empty()); + // Both failed submissions must return the only slot. + ASSERT_OK(writer.Append(Row(), [completion](fluss::Result completed) { + completion->Record(std::move(completed)); + })); + ASSERT_TRUE(completion->Await()); + ASSERT_OK(writer.Flush()); +} + +TEST_F(WriteCallbackTest, ArrowBatchCapacitySurvivesMovesAndDoesNotAffectWaitOrFlush) { + CreateTable(); + fluss::AppendWriter writer; + ASSERT_OK(table_.NewAppend().CreateWriter( + writer, fluss::WriteCallbackOptions{1, std::chrono::milliseconds(250)})); + arrow::Int32Builder ids; + arrow::StringBuilder values; + ASSERT_TRUE(ids.AppendValues({1, 2, 3}).ok()); + ASSERT_TRUE(values.AppendValues({"a", "b", "c"}).ok()); + auto batch = arrow::RecordBatch::Make( + arrow::schema({arrow::field("id", arrow::int32()), arrow::field("value", arrow::utf8())}), + 3, {ids.Finish().ValueOrDie(), values.Finish().ValueOrDie()}); + auto started = std::make_shared(); + auto gate = std::make_shared>(); + auto resume = gate->get_future().share(); + ASSERT_OK(writer.AppendArrowBatch(batch, [started, resume](fluss::Result result) { + started->Record(std::move(result)); + resume.wait_for(std::chrono::seconds(20)); + })); + ASSERT_TRUE(started->Await()); // The three-row batch fits in one slot. + + fluss::AppendWriter moved(std::move(writer)); + writer = std::move(moved); + EXPECT_FALSE(moved.Available()); + auto rejected = std::make_shared(); + auto callback = [rejected](fluss::Result result) { rejected->Record(std::move(result)); }; + auto row_result = writer.Append(Row(4), callback); + auto batch_result = writer.AppendArrowBatch(batch, callback); + EXPECT_NE(row_result.error_message.find("Timed out"), std::string::npos); + EXPECT_NE(batch_result.error_message.find("Timed out"), std::string::npos); + // Independent writers do not share the capacity limit. + fluss::AppendWriter independent; + ASSERT_OK(table_.NewAppend().CreateWriter(independent)); + auto other = std::make_shared(); + ASSERT_OK(independent.Append( + Row(5), [other](fluss::Result result) { other->Record(std::move(result)); })); + ASSERT_TRUE(other->Await()); + + fluss::WriteResult pending; + ASSERT_OK(writer.Append(Row(6), pending)); + ASSERT_OK(pending.Wait()); + ASSERT_OK(writer.Append(Row(7))); + ASSERT_OK(writer.Flush()); // ACK completion does not return callback capacity. + EXPECT_TRUE(rejected->Results().empty()); + EXPECT_NE(writer.Append(Row(8), callback).error_message.find("Timed out"), std::string::npos); + gate->set_value(); + // Admission waits for the previous callback to return, not merely to signal started. + ASSERT_OK(writer.Append(Row(9), callback)); + ASSERT_TRUE(rejected->Await()); ASSERT_OK(writer.Flush()); + EXPECT_EQ(rejected->Results().size(), 1u); +} + +TEST_F(WriteCallbackTest, UpsertAndDeleteShareCapacityAndReturnItOnSubmissionErrors) { + CreateTable(true); + fluss::UpsertWriter writer; + ASSERT_OK(table_.NewUpsert().CreateWriter( + writer, fluss::WriteCallbackOptions{1, std::chrono::milliseconds(250)})); + auto completion = std::make_shared(); + auto callback = [completion](fluss::Result result) { completion->Record(std::move(result)); }; + fluss::GenericRow invalid(2); + invalid.SetString(0, "not an integer primary key"); + invalid.SetString(1, "value"); + EXPECT_FALSE(writer.Upsert(invalid, callback).Ok()); + EXPECT_FALSE(writer.Delete(invalid, callback).Ok()); + EXPECT_TRUE(completion->Results().empty()); + + auto gate = std::make_shared>(); + auto resume = gate->get_future().share(); + ASSERT_OK(writer.Upsert(Row(), [completion, resume](fluss::Result result) { + completion->Record(std::move(result)); + resume.wait_for(std::chrono::seconds(20)); + })); + ASSERT_TRUE(completion->Await()); + fluss::UpsertWriter moved(std::move(writer)); + writer = std::move(moved); + EXPECT_FALSE(moved.Available()); + EXPECT_NE(writer.Upsert(Row(2), callback).error_message.find("Timed out"), std::string::npos); + EXPECT_NE(writer.Delete(Row(), callback).error_message.find("Timed out"), std::string::npos); + ASSERT_OK(writer.Flush()); + gate->set_value(); + ASSERT_OK(writer.Delete(Row(), callback)); + ASSERT_TRUE(completion->Await(2)); + ASSERT_OK(writer.Flush()); + EXPECT_EQ(completion->Results().size(), 2u); +} + +TEST_F(WriteCallbackTest, CallbackDoesNotWaitForCapacityOnAnotherWriter) { + CreateTable(); + auto full_writer = std::make_shared(); + ASSERT_OK(table_.NewAppend().CreateWriter( + *full_writer, fluss::WriteCallbackOptions{1, std::chrono::seconds(5)})); + auto started = std::make_shared(); + auto gate = std::make_shared>(); + auto resume = gate->get_future().share(); + ASSERT_OK(full_writer->Append(Row(), [started, resume](fluss::Result result) { + started->Record(std::move(result)); + resume.wait_for(std::chrono::seconds(20)); + })); + ASSERT_TRUE(started->Await()); + fluss::AppendWriter other; + ASSERT_OK(table_.NewAppend().CreateWriter(other)); + auto attempted = std::make_shared(); + auto unexpected = std::make_shared(); + auto row = std::make_shared(Row(2)); + ASSERT_OK(other.Append(Row(3), [full_writer, row, attempted, unexpected](fluss::Result) { + // No submitting thread accesses full_writer concurrently with this callback. + auto result = full_writer->Append(*row, [unexpected](fluss::Result completed) { + unexpected->Record(std::move(completed)); + }); + attempted->Record(std::move(result)); + })); + ASSERT_TRUE(attempted->Await()); + EXPECT_EQ(attempted->Results().front().error_message, "Write callback capacity is full"); + EXPECT_TRUE(unexpected->Results().empty()); + gate->set_value(); + ASSERT_OK(other.Flush()); } TEST_F(WriteCallbackTest, BatchedCallbacksSurviveExceptionsAndCoexistWithWait) { @@ -418,3 +544,158 @@ TEST(WriteCallbackBridgeTest, ContainsCallbackExceptions) { fluss::ffi::WriteCallback unknown([](fluss::Result) { throw 42; }); EXPECT_NO_THROW(unknown.Complete(0, "")); } + +TEST(WriteCallbackBridgeTest, ValidatesCallbackOptions) { + fluss::WriteCallbackOptions options; + EXPECT_EQ(options.max_pending_operations, 65536u); + EXPECT_EQ(options.enqueue_timeout, std::chrono::seconds(30)); + EXPECT_OK(fluss::ffi::WriteCallbackCapacity::Validate(options)); + options.max_pending_operations = 0; + EXPECT_FALSE(fluss::ffi::WriteCallbackCapacity::Validate(options).Ok()); + fluss::Table table; + fluss::AppendWriter append; + fluss::UpsertWriter upsert; + EXPECT_EQ(table.NewAppend().CreateWriter(append, options).error_message, + "max_pending_operations must be positive"); + EXPECT_EQ(table.NewUpsert().CreateWriter(upsert, options).error_message, + "max_pending_operations must be positive"); + options.max_pending_operations = 1; + options.enqueue_timeout = std::chrono::milliseconds(-1); + EXPECT_FALSE(fluss::ffi::WriteCallbackCapacity::Validate(options).Ok()); + EXPECT_EQ(table.NewAppend().CreateWriter(append, options).error_message, + "enqueue_timeout must be nonnegative"); + EXPECT_EQ(table.NewUpsert().CreateWriter(upsert, options).error_message, + "enqueue_timeout must be nonnegative"); + options.enqueue_timeout = std::chrono::milliseconds(0); + EXPECT_OK(fluss::ffi::WriteCallbackCapacity::Validate(options)); +} + +TEST(WriteCallbackBridgeTest, CapacityLastsThroughCallbackAndCaptureCleanup) { + auto capacity = std::make_shared( + fluss::WriteCallbackOptions{1, std::chrono::milliseconds(0)}); + // This deleter runs after the user callback returns, but before its slot is returned. + auto capture = std::shared_ptr(new int(0), [capacity](int* value) { + EXPECT_FALSE(capacity->Acquire().Ok()); + delete value; + }); + fluss::ffi::WriteCallback callback([capacity, owned = std::move(capture)](fluss::Result) { + EXPECT_FALSE(capacity->Acquire().Ok()); + throw std::runtime_error("callback failure"); + }); + ASSERT_OK(callback.Reserve(capacity)); + EXPECT_FALSE(capacity->Acquire().Ok()); + callback.Complete(0, ""); + // Complete must release the slot even while its Rust-owned wrapper is still alive. + ASSERT_OK(capacity->Acquire()); + capacity->Release(); +} + +TEST(WriteCallbackBridgeTest, UnsubmittedCallbackReturnsCapacityOnException) { + auto capacity = std::make_shared( + fluss::WriteCallbackOptions{1, std::chrono::milliseconds(0)}); + int calls = 0; + try { + fluss::ffi::WriteCallback callback([&](fluss::Result) { ++calls; }); + ASSERT_OK(callback.Reserve(capacity)); + throw std::bad_alloc(); + } catch (const std::bad_alloc&) { + } + EXPECT_EQ(calls, 0); + ASSERT_OK(capacity->Acquire()); + capacity->Release(); +} + +TEST(WriteCallbackBridgeTest, ReservationOutlivesWriterOwnership) { + auto capacity = std::make_shared( + fluss::WriteCallbackOptions{1, std::chrono::milliseconds(0)}); + std::weak_ptr weak = capacity; + fluss::ffi::WriteCallback callback([](fluss::Result) {}); + ASSERT_OK(callback.Reserve(capacity)); + capacity.reset(); + EXPECT_FALSE(weak.expired()); + callback.Complete(0, ""); + EXPECT_TRUE(weak.expired()); +} + +TEST(WriteCallbackBridgeTest, CapacityTimeoutDoesNotDiscardAcceptedCallback) { + auto capacity = std::make_shared( + fluss::WriteCallbackOptions{1, std::chrono::milliseconds(25)}); + int calls = 0; + fluss::ffi::WriteCallback accepted([&](fluss::Result) { ++calls; }); + ASSERT_OK(accepted.Reserve(capacity)); + auto start = std::chrono::steady_clock::now(); + auto result = capacity->Acquire(); + EXPECT_FALSE(result.Ok()); + EXPECT_NE(result.error_message.find("Timed out"), std::string::npos); + EXPECT_GE(std::chrono::steady_clock::now() - start, std::chrono::milliseconds(25)); + EXPECT_EQ(calls, 0); + accepted.Complete(0, ""); + EXPECT_EQ(calls, 1); + ASSERT_OK(capacity->Acquire()); + capacity->Release(); +} + +TEST(WriteCallbackBridgeTest, WaitingSubmitterResumesAfterCompletion) { + auto capacity = std::make_shared( + fluss::WriteCallbackOptions{1, std::chrono::seconds(5)}); + fluss::ffi::WriteCallback accepted([](fluss::Result) {}); + ASSERT_OK(accepted.Reserve(capacity)); + std::promise started; + auto waiter = std::async(std::launch::async, [&] { + started.set_value(); + auto result = capacity->Acquire(); + if (result.Ok()) { + capacity->Release(); + } + return result; + }); + started.get_future().wait(); + EXPECT_EQ(waiter.wait_for(std::chrono::milliseconds(25)), std::future_status::timeout); + accepted.Complete(0, ""); + EXPECT_OK(waiter.get()); +} + +TEST(WriteCallbackBridgeTest, CallbackRejectsFullOtherWriterAndRestoresThreadContext) { + auto capacity = std::make_shared( + fluss::WriteCallbackOptions{1, std::chrono::milliseconds(25)}); + ASSERT_OK(capacity->Acquire()); // Full writer unrelated to the executing callback. + fluss::ffi::WriteCallback callback([capacity](fluss::Result) { + auto result = capacity->Acquire(); + EXPECT_EQ(result.error_message, "Write callback capacity is full"); + throw 42; + }); + callback.Complete(0, ""); + auto start = std::chrono::steady_clock::now(); + // CallbackScope must restore the thread context, including on exceptions. + EXPECT_NE(capacity->Acquire().error_message.find("Timed out"), std::string::npos); + EXPECT_GE(std::chrono::steady_clock::now() - start, std::chrono::milliseconds(25)); + capacity->Release(); +} + +TEST(WriteCallbackBridgeTest, ConcurrentCapacityReservationsStayBounded) { + constexpr size_t limit = 3; + auto capacity = std::make_shared( + fluss::WriteCallbackOptions{limit, std::chrono::seconds(5)}); + std::atomic active{0}; + std::atomic completed{0}; + std::vector threads; + for (int i = 0; i < 8; ++i) { + threads.emplace_back([&] { + for (int j = 0; j < 250; ++j) { + fluss::ffi::WriteCallback callback([&](fluss::Result) { + --active; + ++completed; + }); + ASSERT_OK(callback.Reserve(capacity)); + EXPECT_LE(++active, limit); + std::this_thread::yield(); + callback.Complete(0, ""); + } + }); + } + for (auto& thread : threads) { + thread.join(); + } + EXPECT_EQ(active.load(), 0u); + EXPECT_EQ(completed.load(), 2000u); +} diff --git a/fluss-rust/bindings/cpp/test/test_write_callback_allocation.cpp b/fluss-rust/bindings/cpp/test/test_write_callback_allocation.cpp index 995d9fc2b93..1fb9ef4d377 100644 --- a/fluss-rust/bindings/cpp/test/test_write_callback_allocation.cpp +++ b/fluss-rust/bindings/cpp/test/test_write_callback_allocation.cpp @@ -52,6 +52,9 @@ TEST(WriteCallbackBridgeTest, ErrorTextAllocationFailureStillCompletesAndRelease ++calls; observed = std::move(result); }); + auto capacity = std::make_shared( + fluss::WriteCallbackOptions{1, std::chrono::milliseconds(0)}); + ASSERT_TRUE(callback.Reserve(capacity).Ok()); const rust::Str text(message); fail_next_allocation = true; callback.Complete(fluss::ErrorCode::DELETION_DISABLED_EXCEPTION, text); @@ -62,4 +65,6 @@ TEST(WriteCallbackBridgeTest, ErrorTextAllocationFailureStillCompletesAndRelease EXPECT_EQ(observed.error_code, fluss::ErrorCode::DELETION_DISABLED_EXCEPTION); EXPECT_TRUE(observed.error_message.empty()); EXPECT_TRUE(weak.expired()); + ASSERT_TRUE(capacity->Acquire().Ok()); + capacity->Release(); } diff --git a/fluss-rust/website/docs/user-guide/cpp/api-reference.md b/fluss-rust/website/docs/user-guide/cpp/api-reference.md index 87dc30a1f06..0d4ecc12cdb 100644 --- a/fluss-rust/website/docs/user-guide/cpp/api-reference.md +++ b/fluss-rust/website/docs/user-guide/cpp/api-reference.md @@ -130,6 +130,7 @@ Complete API reference for the Fluss C++ client. | Method | Description | |----------------------------------------------|-------------------------| | `CreateWriter(AppendWriter& out) -> Result` | Create an append writer | +| `CreateWriter(AppendWriter& out, const WriteCallbackOptions& options) -> Result` | Create a writer with callback admission limits | ## `TableUpsert` @@ -138,6 +139,7 @@ Complete API reference for the Fluss C++ client. | `PartialUpdateByIndex(std::vector column_indices) -> TableUpsert&` | Configure partial update by column indices | | `PartialUpdateByName(std::vector column_names) -> TableUpsert&` | Configure partial update by column names | | `CreateWriter(UpsertWriter& out) -> Result` | Create an upsert writer | +| `CreateWriter(UpsertWriter& out, const WriteCallbackOptions& options) -> Result` | Create a writer with callback admission limits | ## `TableLookup` @@ -237,8 +239,47 @@ empty callback is rejected before submission. During normal operation, each successfully submitted operation invokes its callback exactly once with its final success or failure, subject to the lifetime and shutdown requirements below; `AppendArrowBatch` invokes one callback for the batch, not one per row or bucket. -Submission does not wait for acknowledgment, but may still wait for buffer -space under backpressure. +Submission does not wait for acknowledgment, but may still wait for callback +capacity and then for Rust writer buffer space under backpressure. + +### Callback capacity + +```cpp +fluss::WriteCallbackOptions options; +options.max_pending_operations = 65536; +options.enqueue_timeout = std::chrono::seconds(5); +fluss::AppendWriter writer; +auto created = table.NewAppend().CreateWriter(writer, options); +// Check created before using writer. NewUpsert().CreateWriter accepts the same options. +``` + +| Option | Default | Meaning | +|--------|---------|---------| +| `max_pending_operations` | `65536` | Positive, per-writer limit on callback operations reserved for submission or not yet finished | +| `enqueue_timeout` | `30s` | Nonnegative maximum wait for callback capacity; `0ms` rejects immediately when full | + +The existing `CreateWriter(writer)` overload uses these defaults. Each callback +submission reserves one slot **before** submitting to Rust and holds it through +user callback execution and capture cleanup. Submission errors and exceptions +return the slot automatically. `Upsert` and `Delete` share their writer's limit; +`AppendArrowBatch` consumes one slot per call, regardless of row count. Moving +a writer transfers its capacity state; already accepted callbacks retain it +independently of the writer's lifetime. + +When capacity is full, the submitting thread waits up to `enqueue_timeout`. +A capacity rejection returns `CLIENT_ERROR` without submitting any data or +registering a callback; it never discards an accepted notification. Client errors +return false from `IsRetriable()`, including capacity errors. Applications may +reschedule capacity-rejected submissions with backoff, but must not blindly +retry every client error: other submission failures can have different effects, +including partial ArrowBatch acceptance described below. + +This timeout covers **only callback capacity admission**, not the entire +`Append` call, buffer waits, network requests, core retries, or callback duration. +It does not cancel any accepted write. Do not hold a mutex needed by callbacks +while submitting: a full writer can wait for those callbacks to finish. + +### Execution and lifecycle The SDK takes ownership of the callback and its captures. Callbacks run on background callback threads, may execute concurrently and out of submission @@ -260,13 +301,20 @@ are dispatched separately, without waiting to fill a job. Both Arrow log and KV write batches use this path. An `AppendArrowBatch` spanning multiple internal batches aggregates their results into the operation's single callback. -The completion queue is unbounded, so limit outstanding -callbacks when callback processing is slower than writing; writer buffer limits -do not bound memory retained by completed callbacks. +The completion queue remains internally unbounded; per-writer admission limits +outstanding callback operations rather than dropping results from this queue. +This is not a byte or process-wide memory limit: capture sizes, batch sizes, the +number of writers, and application-owned retry queues need separate controls. +An aggregate ArrowBatch callback may report an error while later internal batches +are still pending, so its capacity slot does not bound all underlying batch memory. Do not wait for another callback from within a callback, since all callback -workers could become occupied. Synchronous SDK calls remain supported with -exclusive access to the writer. If dedicated workers cannot be initialized, the -SDK falls back to its runtime blocking pool to preserve callback delivery. +workers could become occupied. A callback submission from within any SDK write +callback fails immediately if its target writer's capacity is full, regardless +of `enqueue_timeout`, to avoid blocking the shared workers on their own capacity. +This applies across writers and on the fallback executor as well. It does not +remove Rust buffer waits or make arbitrary blocking SDK calls deadlock-free. +Exclusive writer access is still required. If dedicated workers cannot be +initialized, the SDK falls back to its runtime blocking pool to preserve delivery. `Flush()` still waits for pending writes, **not** for user callbacks to finish. Applications that need to drain callbacks must track their completion separately. @@ -274,7 +322,9 @@ Applications that need to drain callbacks must track their completion separately ### Compatibility and operational limits - Existing fire-and-forget and `WriteResult::Wait()` overloads retain their - result semantics. Rust callers can still `.await` a `WriteResultFuture`. + result semantics and do not consume callback capacity. Rust callers can still + `.await` a `WriteResultFuture`. Callback overloads now apply bounded admission + by default; existing callback callers may block or receive a capacity error. Completion follows the configured acknowledgment policy; a callback does not add a stronger durability guarantee or change retries, request ordering, wire formats, or storage formats. @@ -302,9 +352,9 @@ Applications that need to drain callbacks must track their completion separately The process-wide executor is not automatically drained at exit. - Before releasing callback state or the connection, stop and join submitting threads, flush pending writes, and wait separately for tracked callbacks. - Reserve tracking capacity before submission because callbacks may run before - it returns; release that capacity yourself if submission fails. Bound outstanding - operations through callback completion, not just through server acknowledgment. + Application tracking must allow callbacks before submission returns and must + exclude rejected submissions, which have no callback. SDK admission does not + wait for work that a callback delegates to application threads or retry queues. An application-side wait timeout does not cancel the write or its callback. Keep referenced state alive if abandoning a wait; use durable application tracking and a duplicate-safe retry policy when recovery is required. From 4b78f440ac67d7cd04c7b58a8e19940a6f8b1212 Mon Sep 17 00:00:00 2001 From: naivedogger Date: Tue, 15 Sep 2026 10:01:49 +0800 Subject: [PATCH 06/19] [c++] Flush waits for pending callbacks to finish --- fluss-rust/bindings/cpp/README.md | 2 +- fluss-rust/bindings/cpp/examples/example.cpp | 107 +++++------------- fluss-rust/bindings/cpp/include/fluss.hpp | 3 +- fluss-rust/bindings/cpp/src/table.cpp | 8 +- .../bindings/cpp/src/write_callback.hpp | 11 ++ .../bindings/cpp/test/test_write_callback.cpp | 25 ++-- .../docs/user-guide/cpp/api-reference.md | 12 +- 7 files changed, 69 insertions(+), 99 deletions(-) diff --git a/fluss-rust/bindings/cpp/README.md b/fluss-rust/bindings/cpp/README.md index 206d7173852..4418e955102 100644 --- a/fluss-rust/bindings/cpp/README.md +++ b/fluss-rust/bindings/cpp/README.md @@ -76,7 +76,7 @@ not apply to `CreateBucketBatchScanner()`. [C++ API reference](../../website/docs/user-guide/cpp/api-reference.md) and [log-table examples](../../website/docs/user-guide/cpp/example/log-tables.md). -The callback section configures `WriteCallbackOptions` so the SDK bounds outstanding callback operations, handles submission and completion errors separately, and waits for result handling after Flush. `max_pending_operations` defaults to 65536 and `enqueue_timeout` to 30 seconds; the example uses a limit of 2 and a 5-second admission timeout. That timeout only covers waiting for callback capacity. Flush and application completion waits still have no deadline; production applications need a recovery policy for operations that never complete. A local timeout does not cancel accepted writes or callbacks. +The callback section configures `WriteCallbackOptions` so the SDK bounds outstanding callback operations and `Flush()` waits for both server acknowledgment and pending callbacks. `max_pending_operations` defaults to 65536 and `enqueue_timeout` to 30 seconds; the example uses a limit of 2 and a 5-second admission timeout. That timeout only covers waiting for callback capacity. Flush timeout defaults to 60 seconds. Production applications still need a recovery policy for operations that never complete. For a bounded log scan, pass the per-bucket offset ranges directly to `TableScan`. The returned reader yields one Arrow batch at a time until every `[starting_offset, stopping_offset)` range diff --git a/fluss-rust/bindings/cpp/examples/example.cpp b/fluss-rust/bindings/cpp/examples/example.cpp index 5cd74f705d4..3df583fd553 100644 --- a/fluss-rust/bindings/cpp/examples/example.cpp +++ b/fluss-rust/bindings/cpp/examples/example.cpp @@ -20,6 +20,7 @@ #include #include +#include #include #include #include @@ -153,88 +154,40 @@ int main() { std::cout << "Row acknowledged by server" << std::endl; } - // Callback acknowledgment with bounded outstanding operations. + // Callback acknowledgment { - struct PendingWrites { - std::mutex mutex; - std::condition_variable changed; - size_t succeeded = 0; - size_t failed = 0; - int32_t first_failed_id = -1; - fluss::Result first_error; - }; - auto pending = std::make_shared(); - size_t accepted = 0; - bool submission_failed = false; - try { - for (const auto& r : rows) { - const int32_t id = 1000 + r.id; - fluss::GenericRow row; - row.SetInt32(0, id); - row.SetString(1, r.name); - row.SetFloat32(2, r.score); - row.SetInt32(3, r.age); - row.SetDate(4, r.date); - row.SetTime(5, r.time); - row.SetTimestampNtz(6, r.ts_ntz); - row.SetTimestampLtz(7, r.ts_ltz); - // The SDK waits for callback capacity before submitting. Do not - // hold the tracking mutex here: callbacks need it to finish. - auto submitted = writer.Append(row, [pending, id](fluss::Result result) { - { - std::lock_guard lock(pending->mutex); - if (result.Ok()) { - ++pending->succeeded; - } else { - if (pending->failed == 0) { - pending->first_failed_id = id; - pending->first_error = std::move(result); - } - ++pending->failed; - } + std::atomic succeeded{0}; + std::atomic failed{0}; + for (const auto& r : rows) { + const int32_t id = 1000 + r.id; + fluss::GenericRow row; + row.SetInt32(0, id); + row.SetString(1, r.name); + row.SetFloat32(2, r.score); + row.SetInt32(3, r.age); + row.SetDate(4, r.date); + row.SetTime(5, r.time); + row.SetTimestampNtz(6, r.ts_ntz); + row.SetTimestampLtz(7, r.ts_ltz); + auto submitted = writer.Append(row, [id, &succeeded, &failed](fluss::Result result) { + if (result.Ok()) { + ++succeeded; + } else { + if (failed.fetch_add(1) == 0) { + std::cerr << "Write failed for id=" << id << ": " << result.error_message + << '\n'; } - pending->changed.notify_one(); - }); - if (!submitted.Ok()) { - // Rejected submissions have no callback. The SDK returns their capacity. - std::cerr << "Submission failed for id=" << id - << ": code=" << submitted.error_code - << " message=" << submitted.error_message << '\n'; - submission_failed = true; - break; } - ++accepted; + }); + if (!submitted.Ok()) { + std::cerr << "Submission failed for id=" << id << ": " << submitted.error_message + << '\n'; + break; } - } catch (const std::exception& error) { - std::cerr << "Submission stopped: " << error.what() << '\n'; - submission_failed = true; - } catch (...) { - std::cerr << "Submission stopped by an unknown exception\n"; - submission_failed = true; } - - // Submission has stopped. Flush is not a callback barrier; drain even on error. - auto flushed = writer.Flush(); - if (!flushed.Ok()) { - std::cerr << "Callback write flush failed: " << flushed.error_message << '\n'; - } - std::unique_lock lock(pending->mutex); - // This demonstration waits without a deadline. A timeout would not cancel - // accepted writes or callbacks; production code needs its own recovery policy. - // Read accepted only after submission stops, so early callbacks are safe. - pending->changed.wait(lock, - [&] { return pending->succeeded + pending->failed == accepted; }); - std::cout << "Callback writes: accepted=" << accepted << " succeeded=" << pending->succeeded - << " failed=" << pending->failed << '\n'; - if (pending->failed != 0) { - std::cerr << "First completion failure: id=" << pending->first_failed_id - << " code=" << pending->first_error.error_code - << " message=" << pending->first_error.error_message << '\n'; - } - // Do not blindly retry failures: an error need not mean nothing was written. - // This wait observes result handling, not SDK callback return. Shared captures - // keep state alive through callback return; the connection stays alive too. - if (submission_failed || !flushed.Ok() || pending->failed != 0) { + check("flush", writer.Flush()); + std::cout << "Callback writes: succeeded=" << succeeded << " failed=" << failed << '\n'; + if (failed != 0) { return 1; } } diff --git a/fluss-rust/bindings/cpp/include/fluss.hpp b/fluss-rust/bindings/cpp/include/fluss.hpp index 7c5e607f9fb..3c28b6fd147 100644 --- a/fluss-rust/bindings/cpp/include/fluss.hpp +++ b/fluss-rust/bindings/cpp/include/fluss.hpp @@ -552,7 +552,8 @@ struct Result { /// WriteCallbackOptions bounds outstanding callback operations per writer. /// /// Exceptions thrown by callbacks are caught and reported to stderr; they do not -/// change the write outcome. Flush() waits for writes, not for callbacks to finish. +/// change the write outcome. Flush() waits for server acknowledgment and then for +/// pending callbacks to finish; it returns immediately when called from a callback. using WriteCallback = std::function; /// Admission limits for callback overloads only; Wait and fire-and-forget are unchanged. diff --git a/fluss-rust/bindings/cpp/src/table.cpp b/fluss-rust/bindings/cpp/src/table.cpp index 4a0878f915a..c769aacbb11 100644 --- a/fluss-rust/bindings/cpp/src/table.cpp +++ b/fluss-rust/bindings/cpp/src/table.cpp @@ -1766,7 +1766,9 @@ Result AppendWriter::Flush() { } auto ffi_result = writer_->flush(); - return utils::from_ffi_result(ffi_result); + auto result = utils::from_ffi_result(ffi_result); + if (!result.Ok()) return result; + return callback_capacity_->AwaitAll(std::chrono::seconds(60)); } // ============================================================================ @@ -1887,7 +1889,9 @@ Result UpsertWriter::Flush() { } auto ffi_result = writer_->upsert_flush(); - return utils::from_ffi_result(ffi_result); + auto result = utils::from_ffi_result(ffi_result); + if (!result.Ok()) return result; + return callback_capacity_->AwaitAll(std::chrono::seconds(60)); } // ============================================================================ diff --git a/fluss-rust/bindings/cpp/src/write_callback.hpp b/fluss-rust/bindings/cpp/src/write_callback.hpp index 1ca97dee0ff..8db84915b30 100644 --- a/fluss-rust/bindings/cpp/src/write_callback.hpp +++ b/fluss-rust/bindings/cpp/src/write_callback.hpp @@ -69,6 +69,17 @@ class WriteCallbackCapacity { available_.notify_one(); } + /// Wait for all reserved operations to finish their callbacks. + /// When called from within a callback this returns immediately to avoid deadlock. + Result AwaitAll(std::chrono::milliseconds timeout) { + std::unique_lock lock(mutex_); + if (in_callback_) return {}; + if (!available_.wait_for(lock, timeout, [&] { return pending_ == 0; })) { + return {ErrorCode::CLIENT_ERROR, "Timed out waiting for pending callbacks"}; + } + return {}; + } + private: friend class WriteCallback; inline static thread_local bool in_callback_ = false; diff --git a/fluss-rust/bindings/cpp/test/test_write_callback.cpp b/fluss-rust/bindings/cpp/test/test_write_callback.cpp index ef571c874a0..97059ffd491 100644 --- a/fluss-rust/bindings/cpp/test/test_write_callback.cpp +++ b/fluss-rust/bindings/cpp/test/test_write_callback.cpp @@ -140,7 +140,7 @@ TEST_F(WriteCallbackTest, AppendAcceptsFunctionPointer) { ASSERT_OK(writer.Flush()); } -TEST_F(WriteCallbackTest, CallbackOwnsCapturesAndDoesNotDelayFlush) { +TEST_F(WriteCallbackTest, FlushWaitsForPendingCallbacks) { CreateTable(); fluss::AppendWriter writer; ASSERT_OK(table_.NewAppend().CreateWriter(writer)); @@ -164,13 +164,12 @@ TEST_F(WriteCallbackTest, CallbackOwnsCapturesAndDoesNotDelayFlush) { } ASSERT_TRUE(started->Await()); EXPECT_FALSE(weak_lifetime.expired()); - ASSERT_OK(writer.Flush()); - // Flush must not wait for a user callback that is waiting for us. - EXPECT_TRUE(finished->Results().empty()); - writer = fluss::AppendWriter{}; - EXPECT_FALSE(weak_lifetime.expired()); + // Flush now waits for pending callbacks, not just for server ACK. + // The callback is blocked on the gate, so Flush must not return yet. gate->set_value(); - ASSERT_TRUE(finished->Await()); + ASSERT_OK(writer.Flush()); + // After Flush returns, the callback has finished and captures are released. + EXPECT_FALSE(finished->Results().empty()); EXPECT_EQ(released.wait_for(std::chrono::seconds(10)), std::future_status::ready); EXPECT_TRUE(weak_lifetime.expired()); EXPECT_OK(finished->Results().front()); @@ -341,12 +340,12 @@ TEST_F(WriteCallbackTest, ArrowBatchCapacitySurvivesMovesAndDoesNotAffectWaitOrF ASSERT_OK(writer.Append(Row(6), pending)); ASSERT_OK(pending.Wait()); ASSERT_OK(writer.Append(Row(7))); - ASSERT_OK(writer.Flush()); // ACK completion does not return callback capacity. - EXPECT_TRUE(rejected->Results().empty()); - EXPECT_NE(writer.Append(Row(8), callback).error_message.find("Timed out"), std::string::npos); + // Release the gate so the first callback finishes and returns capacity. gate->set_value(); - // Admission waits for the previous callback to return, not merely to signal started. - ASSERT_OK(writer.Append(Row(9), callback)); + ASSERT_OK(writer.Flush()); + EXPECT_TRUE(rejected->Results().empty()); + // Capacity is now available after the first callback completed. + ASSERT_OK(writer.Append(Row(8), callback)); ASSERT_TRUE(rejected->Await()); ASSERT_OK(writer.Flush()); EXPECT_EQ(rejected->Results().size(), 1u); @@ -378,8 +377,8 @@ TEST_F(WriteCallbackTest, UpsertAndDeleteShareCapacityAndReturnItOnSubmissionErr EXPECT_FALSE(moved.Available()); EXPECT_NE(writer.Upsert(Row(2), callback).error_message.find("Timed out"), std::string::npos); EXPECT_NE(writer.Delete(Row(), callback).error_message.find("Timed out"), std::string::npos); - ASSERT_OK(writer.Flush()); gate->set_value(); + ASSERT_OK(writer.Flush()); ASSERT_OK(writer.Delete(Row(), callback)); ASSERT_TRUE(completion->Await(2)); ASSERT_OK(writer.Flush()); diff --git a/fluss-rust/website/docs/user-guide/cpp/api-reference.md b/fluss-rust/website/docs/user-guide/cpp/api-reference.md index 0d4ecc12cdb..4b48566c5fc 100644 --- a/fluss-rust/website/docs/user-guide/cpp/api-reference.md +++ b/fluss-rust/website/docs/user-guide/cpp/api-reference.md @@ -316,8 +316,9 @@ remove Rust buffer waits or make arbitrary blocking SDK calls deadlock-free. Exclusive writer access is still required. If dedicated workers cannot be initialized, the SDK falls back to its runtime blocking pool to preserve delivery. -`Flush()` still waits for pending writes, **not** for user callbacks to finish. -Applications that need to drain callbacks must track their completion separately. +`Flush()` waits for server acknowledgment and then for all pending callbacks to +finish. It returns immediately when called from within a callback to avoid +deadlock. ### Compatibility and operational limits @@ -352,9 +353,10 @@ Applications that need to drain callbacks must track their completion separately The process-wide executor is not automatically drained at exit. - Before releasing callback state or the connection, stop and join submitting threads, flush pending writes, and wait separately for tracked callbacks. - Application tracking must allow callbacks before submission returns and must - exclude rejected submissions, which have no callback. SDK admission does not - wait for work that a callback delegates to application threads or retry queues. + Application tracking must allow callbacks before submission returns and + exclude rejected submissions, which have no callback. Flush() waits for + accepted callbacks; it does not wait for work that a callback delegates to + application threads or retry queues. An application-side wait timeout does not cancel the write or its callback. Keep referenced state alive if abandoning a wait; use durable application tracking and a duplicate-safe retry policy when recovery is required. From dfd3d1c39846ea0096652ee23b45e333873923ee Mon Sep 17 00:00:00 2001 From: naivedogger Date: Wed, 16 Sep 2026 12:19:22 +0800 Subject: [PATCH 07/19] [c++] Clarify callback guarantees and recovery guidance --- fluss-rust/bindings/cpp/README.md | 18 +++- fluss-rust/bindings/cpp/examples/example.cpp | 10 +++ fluss-rust/bindings/cpp/include/fluss.hpp | 27 ++++-- .../docs/user-guide/cpp/api-reference.md | 87 +++++++++++++++---- 4 files changed, 120 insertions(+), 22 deletions(-) diff --git a/fluss-rust/bindings/cpp/README.md b/fluss-rust/bindings/cpp/README.md index 4418e955102..f83780276ba 100644 --- a/fluss-rust/bindings/cpp/README.md +++ b/fluss-rust/bindings/cpp/README.md @@ -76,7 +76,23 @@ not apply to `CreateBucketBatchScanner()`. [C++ API reference](../../website/docs/user-guide/cpp/api-reference.md) and [log-table examples](../../website/docs/user-guide/cpp/example/log-tables.md). -The callback section configures `WriteCallbackOptions` so the SDK bounds outstanding callback operations and `Flush()` waits for both server acknowledgment and pending callbacks. `max_pending_operations` defaults to 65536 and `enqueue_timeout` to 30 seconds; the example uses a limit of 2 and a 5-second admission timeout. That timeout only covers waiting for callback capacity. Flush timeout defaults to 60 seconds. Production applications still need a recovery policy for operations that never complete. +The callback section configures `WriteCallbackOptions` to bound outstanding callback +operations. The SDK executes callbacks; applications do not need a waiting thread or +poll loop. `max_pending_operations` defaults to 65536 and `enqueue_timeout` to 30 seconds; +the example uses a limit of 2 and a 5-second admission timeout. That timeout covers only +waiting for callback capacity. + +A failed callback does not prove that the record was not written. Application +resubmission can duplicate it, even with SDK idempotence enabled. The example only +counts and logs outcomes; it does not implement durable recovery. Keep callbacks +short, protect shared state, and handle retries outside the callback with an +application recovery policy. + +After submissions stop, `Flush()` first flushes writes and, on success, waits up to +60 seconds for pending callbacks. This is not a whole-call timeout. If it returns an +error, keep callback state alive; if it succeeds, still check individual write results. +See the [callback guarantees and recovery guidance](../../website/docs/user-guide/cpp/api-reference.md#write-guarantees-and-recovery) +for result semantics, callback implementation, and shutdown requirements. For a bounded log scan, pass the per-bucket offset ranges directly to `TableScan`. The returned reader yields one Arrow batch at a time until every `[starting_offset, stopping_offset)` range diff --git a/fluss-rust/bindings/cpp/examples/example.cpp b/fluss-rust/bindings/cpp/examples/example.cpp index 3df583fd553..f15ae34fcfe 100644 --- a/fluss-rust/bindings/cpp/examples/example.cpp +++ b/fluss-rust/bindings/cpp/examples/example.cpp @@ -156,6 +156,8 @@ int main() { // Callback acknowledgment { + // The SDK runs callbacks; no application waiting thread is required. + // This example counts outcomes only. It does not implement durable recovery. std::atomic succeeded{0}; std::atomic failed{0}; for (const auto& r : rows) { @@ -173,6 +175,10 @@ int main() { if (result.Ok()) { ++succeeded; } else { + // An error does not prove the row was not written. A new Append + // can duplicate it, even with SDK idempotence enabled. + // For recovery, retain id and input in application-owned state + // and schedule duplicate-safe retries outside this callback. if (failed.fetch_add(1) == 0) { std::cerr << "Write failed for id=" << id << ": " << result.error_message << '\n'; @@ -180,11 +186,15 @@ int main() { } }); if (!submitted.Ok()) { + // No callback will run for this submission; handle this path too. std::cerr << "Submission failed for id=" << id << ": " << submitted.error_message << '\n'; break; } } + // Submission has stopped. Wait for callbacks before leaving the counters' + // scope; individual callback failures are checked below. + // check() exits on error; a continuing application must keep callback state alive. check("flush", writer.Flush()); std::cout << "Callback writes: succeeded=" << succeeded << " failed=" << failed << '\n'; if (failed != 0) { diff --git a/fluss-rust/bindings/cpp/include/fluss.hpp b/fluss-rust/bindings/cpp/include/fluss.hpp index 3c28b6fd147..691be4d9a08 100644 --- a/fluss-rust/bindings/cpp/include/fluss.hpp +++ b/fluss-rust/bindings/cpp/include/fluss.hpp @@ -530,30 +530,45 @@ struct Result { bool Ok() const { return error_code == 0; } - /// Returns true if retrying the request may succeed. Client-side errors always return false. + /// Returns true if retrying the request may succeed. Does not guarantee that a failed + /// write had no effect or that application resubmission is duplicate-safe. + /// Client-side errors always return false. bool IsRetriable() const { return ErrorCode::IsRetriable(error_code); } }; /// Receives the final outcome of an accepted write. Function pointers and lambdas /// are supported. An empty callback is rejected before submitting the write. /// -/// The SDK owns the callback until completion and invokes it exactly once on -/// background callback threads, never inline in the submitting call. Callbacks -/// may run concurrently and out of order, including before the call returns. +/// During normal operation, the SDK owns the callback until completion and invokes +/// it exactly once on SDK-managed background threads; no caller polling or waiting +/// thread is needed. Process exit or a crash can prevent delivery. Callbacks never +/// run inline in the submitting call, but may run concurrently and out of order, +/// including before the call returns. /// Keep callbacks short and synchronize access to shared state, including writers. /// Callback overloads do not make writers safe for concurrent access. Captured /// references must remain valid until the callback finishes; capturing shared /// ownership is recommended. Keep the connection alive until completion. /// +/// Success follows the configured acknowledgment policy. Errors are reported after +/// internal retry handling, but do not guarantee that no data was written. +/// Application resubmission is a new operation and can produce duplicates even +/// with SDK idempotence enabled. Retain input identifiers and recovery state as +/// needed; one callback invocation is not an exactly-once delivery guarantee. +/// /// Callback threads are shared across connections. Do not wait for another /// callback from a callback: it can exhaust the worker pool. Synchronous SDK /// calls require exclusive writer access. Callback submissions to a full writer /// fail immediately when called from a callback, instead of blocking the workers. /// WriteCallbackOptions bounds outstanding callback operations per writer. +/// Hand off retries or expensive work without blocking; bound application queues +/// and handle overflow without silently discarding failed operations. /// /// Exceptions thrown by callbacks are caught and reported to stderr; they do not -/// change the write outcome. Flush() waits for server acknowledgment and then for -/// pending callbacks to finish; it returns immediately when called from a callback. +/// change the write outcome or retry the callback. Stop submissions before Flush(). +/// After a successful Rust write flush, Flush() waits up to 60 seconds for pending +/// callbacks to finish. On error, referenced state may still be in use. +/// Inside a callback only the callback wait is skipped; the write flush may block. +/// Flush() does not wait for work handed to application workers or retry queues. using WriteCallback = std::function; /// Admission limits for callback overloads only; Wait and fire-and-forget are unchanged. diff --git a/fluss-rust/website/docs/user-guide/cpp/api-reference.md b/fluss-rust/website/docs/user-guide/cpp/api-reference.md index 4b48566c5fc..9da4d6042e0 100644 --- a/fluss-rust/website/docs/user-guide/cpp/api-reference.md +++ b/fluss-rust/website/docs/user-guide/cpp/api-reference.md @@ -223,6 +223,7 @@ call to `Wait()`: ```cpp void OnWriteComplete(fluss::Result completed) { if (!completed.Ok()) { + // The record may already have been written. Logging alone is not recovery. std::cerr << "Write failed: " << completed.error_message << '\n'; } } @@ -242,6 +243,37 @@ final success or failure, subject to the lifetime and shutdown requirements belo Submission does not wait for acknowledgment, but may still wait for callback capacity and then for Rust writer buffer space under backpressure. +### Write guarantees and recovery + +- A successful submission means the operation was accepted, not that it was + written successfully. Check both the immediate return value and the callback + result. A submission error does not register a callback. +- A successful callback reports completion under the configured acknowledgment + policy. It does not strengthen that policy; for example, `writer_acks = "0"` + does not wait for server acknowledgment. +- The SDK handles retryable write errors internally according to its retry + configuration. The callback reports the resulting completion, not each retry + attempt. A failure can be reported when retries are exhausted or an error + cannot be retried; not every failure goes through the configured retry count. +- **A failed callback does not guarantee that the record was not written.** + Even for a single row, the server may have written it before a response was + lost. Other errors can represent a definite rejection. `Result` has no separate + field that distinguishes these outcomes, and `IsRetriable()` is not proof that + nothing was written or that resubmission is duplicate-safe. +- Calling `Append` again is a new operation. SDK idempotence for internal retries + does not deduplicate application resubmissions of the same logical record. + Decide whether to resubmit using application identifiers, deduplication or + reconciliation, and a bounded retry policy. Do not blindly resubmit every error. +- One callback invocation per accepted operation is not an exactly-once delivery + guarantee. Pending notifications are not persisted and can be lost on process + exit or a crash. Applications requiring recovery across restarts must retain + their source records or durable operation state independently of the callback. + +`AppendArrowBatch` is not atomic across internal batches or buckets. Its single +callback cannot identify which individual rows succeeded. A failed submission +can also follow partial acceptance without registering a callback; see the +operational limits below before retrying a batch. + ### Callback capacity ```cpp @@ -281,6 +313,10 @@ while submitting: a full writer can wait for those callbacks to finish. ### Execution and lifecycle +Implement the callback's application logic; the SDK supplies the execution +threads. There is no need to create a waiting thread, call `Wait()`, or poll for +callback delivery. + The SDK takes ownership of the callback and its captures. Callbacks run on background callback threads, may execute concurrently and out of submission order, and may start before the submitting call returns. Keep callbacks short; @@ -290,7 +326,23 @@ access: serialize access if both the caller and a callback use the same writer. Prefer capturing `std::shared_ptr` by value when sharing application state. Keep the connection alive until outstanding operations complete. Exceptions thrown by callbacks are caught and reported to stderr; -they do not change the write outcome. +they do not change the write outcome or cause the callback to be invoked again. + +A callback should update thread-safe completion state and return promptly. +Capture an operation identifier by value so failures can be associated with +their input; `Result` does not contain the original row. If recovery is needed, +retain the payload or a reference to a durable source until its outcome has been +handled. The example above only logs failures; it is not a recovery implementation. + +For expensive processing or retries, hand off to an application event loop or +worker through a bounded, nonblocking mechanism. Define what happens when that +queue is full: preserve the failed operation and stop or backpressure new +submissions rather than silently dropping it or blocking SDK callback workers. +Do not loop on `Append`, sleep for retry backoff, or call `Flush()` inside a +callback. Schedule retries outside the callback with exclusive writer access. +They do not need to wait for `Flush()`, and remain subject to the duplicate risks +described above. A separate application thread is optional if an existing +submission loop can handle the handoff. Callbacks register directly with their internal write batch, rather than creating an asynchronous ACK-waiting task for each row. When a batch completes, @@ -316,9 +368,19 @@ remove Rust buffer waits or make arbitrary blocking SDK calls deadlock-free. Exclusive writer access is still required. If dedicated workers cannot be initialized, the SDK falls back to its runtime blocking pool to preserve delivery. -`Flush()` waits for server acknowledgment and then for all pending callbacks to -finish. It returns immediately when called from within a callback to avoid -deadlock. +For shutdown, stop and join submitting threads, then call `Flush()` outside a +callback. It first runs the Rust write flush and, if that succeeds, waits up to +60 seconds for this writer's pending callbacks and captures to finish. The +60-second callback wait is not an end-to-end timeout for the entire call. +Check individual callback results as well: successful flushing is not a summary +that every submitted operation succeeded. + +If the write flush fails or the callback wait times out, callbacks may still be +pending; do not release their referenced state. A timeout does not cancel them. +`Flush()` does not wait for work handed to application workers or retry queues; +those need their own shutdown handling. When called inside a callback, only the +callback-wait phase is skipped; the Rust write flush can still block. Do not use +this path as a shutdown barrier. ### Compatibility and operational limits @@ -346,20 +408,15 @@ deadlock. not ordering or latency guarantees. A slow callback delays other callbacks in its job, and slow callbacks from one connection can delay another connection. The blocking-pool fallback is not limited to four callback threads. -- Callback delivery is in memory only. There is no end-to-end callback deadline, - public callback-drain API, or durable recovery of pending notifications. +- Callback delivery is in memory only. There is no end-to-end callback deadline + or durable recovery of pending notifications. Connection or writer destruction is not a callback-drain barrier; process exit, crashes, or fatal resource exhaustion can prevent pending callbacks from running. The process-wide executor is not automatically drained at exit. -- Before releasing callback state or the connection, stop and join submitting - threads, flush pending writes, and wait separately for tracked callbacks. - Application tracking must allow callbacks before submission returns and - exclude rejected submissions, which have no callback. Flush() waits for - accepted callbacks; it does not wait for work that a callback delegates to - application threads or retry queues. - An application-side wait timeout does not cancel the write or its callback. - Keep referenced state alive if abandoning a wait; use durable application - tracking and a duplicate-safe retry policy when recovery is required. +- Application tracking must allow callbacks before submission returns and + exclude rejected submissions, which have no callback. Follow the shutdown + sequence above before releasing callback state or the connection; an + application-side wait timeout does not cancel the write or its callback. - C++ exceptions thrown by user callbacks are contained. If copying error text fails, the callback still receives the error code, but the message may be empty. Allocation failures while constructing the callback before submission can still From 1c987f53e7dc614066474837c524fe290b3de7d0 Mon Sep 17 00:00:00 2001 From: naivedogger Date: Wed, 16 Sep 2026 13:37:50 +0800 Subject: [PATCH 08/19] [c++] Tune callback defaults and document buffer sizing --- fluss-rust/bindings/cpp/README.md | 12 ++- fluss-rust/bindings/cpp/examples/example.cpp | 10 +-- fluss-rust/bindings/cpp/include/fluss.hpp | 7 +- .../bindings/cpp/test/test_write_callback.cpp | 19 ++++- .../docs/user-guide/cpp/api-reference.md | 82 ++++++++++++++++++- 5 files changed, 115 insertions(+), 15 deletions(-) diff --git a/fluss-rust/bindings/cpp/README.md b/fluss-rust/bindings/cpp/README.md index f83780276ba..7737f282cc9 100644 --- a/fluss-rust/bindings/cpp/README.md +++ b/fluss-rust/bindings/cpp/README.md @@ -78,9 +78,15 @@ not apply to `CreateBucketBatchScanner()`. The callback section configures `WriteCallbackOptions` to bound outstanding callback operations. The SDK executes callbacks; applications do not need a waiting thread or -poll loop. `max_pending_operations` defaults to 65536 and `enqueue_timeout` to 30 seconds; -the example uses a limit of 2 and a 5-second admission timeout. That timeout covers only -waiting for callback capacity. +poll loop. The example uses the defaults: `max_pending_operations = 262144` per Writer +and `enqueue_timeout = 30s`. That timeout covers only waiting for callback capacity. + +The callback limit counts operations, not bytes; it does not preallocate 262144 slots. +The separate `Configuration::writer_buffer_memory_size` remains 64 MiB by default, +shared across all tables and writers on a Connection. Neither setting caps process RSS. +For a high-throughput starting configuration, see the +[buffer sizing guidance](../../website/docs/user-guide/cpp/api-reference.md#sizing-callback-capacity-and-write-buffers), +including a 512 MiB per-Connection example and how to budget for multiple writers. A failed callback does not prove that the record was not written. Application resubmission can duplicate it, even with SDK idempotence enabled. The example only diff --git a/fluss-rust/bindings/cpp/examples/example.cpp b/fluss-rust/bindings/cpp/examples/example.cpp index f15ae34fcfe..159dfa02475 100644 --- a/fluss-rust/bindings/cpp/examples/example.cpp +++ b/fluss-rust/bindings/cpp/examples/example.cpp @@ -91,12 +91,10 @@ int main() { // 5) Write rows with scalar and temporal values fluss::AppendWriter writer; - fluss::WriteCallbackOptions callback_options; - // A small limit demonstrates SDK backpressure in the callback section below. - // These options do not affect fire-and-forget or WriteResult::Wait(). - callback_options.max_pending_operations = 2; - callback_options.enqueue_timeout = std::chrono::seconds(5); - check("new_append_writer", table.NewAppend().CreateWriter(writer, callback_options)); + // Defaults: 262144 pending callback operations per Writer, 30s admission wait. + // Pass WriteCallbackOptions to lower the limit for large captures or many writers. + // This count is independent of the Connection's write-buffer byte budget. + check("new_append_writer", table.NewAppend().CreateWriter(writer)); struct RowData { int id; diff --git a/fluss-rust/bindings/cpp/include/fluss.hpp b/fluss-rust/bindings/cpp/include/fluss.hpp index 691be4d9a08..8c5253d6074 100644 --- a/fluss-rust/bindings/cpp/include/fluss.hpp +++ b/fluss-rust/bindings/cpp/include/fluss.hpp @@ -575,7 +575,9 @@ using WriteCallback = std::function; struct WriteCallbackOptions { /// Maximum operations reserved for submission or awaiting callback completion. /// Must be positive. One AppendArrowBatch call counts as one operation, not its rows. - size_t max_pending_operations = 65536; + /// Per-writer count, not preallocated storage or a byte limit. Reduce for many + /// writers or large captures; independent of Configuration::writer_buffer_memory_size. + size_t max_pending_operations = 262144; /// Maximum wait for callback capacity; zero rejects immediately when full. /// Must be nonnegative. Does not bound buffer waits, ACKs, retries, or callback duration. @@ -1604,7 +1606,8 @@ struct Configuration { bool writer_enable_idempotence{true}; // Maximum number of in-flight requests per bucket for idempotent writes size_t writer_max_inflight_requests_per_bucket{5}; - // Total memory available for buffering write batches (default 64MB) + // Shared write-batch memory budget per Connection, across its tables and writers + // (default 64 MiB). Not a process RSS limit or a callback-capture memory budget. size_t writer_buffer_memory_size{64 * 1024 * 1024}; // Maximum time in milliseconds to block waiting for buffer memory uint64_t writer_buffer_wait_timeout_ms{std::numeric_limits::max()}; diff --git a/fluss-rust/bindings/cpp/test/test_write_callback.cpp b/fluss-rust/bindings/cpp/test/test_write_callback.cpp index 97059ffd491..349d2bf9ef6 100644 --- a/fluss-rust/bindings/cpp/test/test_write_callback.cpp +++ b/fluss-rust/bindings/cpp/test/test_write_callback.cpp @@ -546,7 +546,7 @@ TEST(WriteCallbackBridgeTest, ContainsCallbackExceptions) { TEST(WriteCallbackBridgeTest, ValidatesCallbackOptions) { fluss::WriteCallbackOptions options; - EXPECT_EQ(options.max_pending_operations, 65536u); + EXPECT_EQ(options.max_pending_operations, 262144u); EXPECT_EQ(options.enqueue_timeout, std::chrono::seconds(30)); EXPECT_OK(fluss::ffi::WriteCallbackCapacity::Validate(options)); options.max_pending_operations = 0; @@ -569,6 +569,23 @@ TEST(WriteCallbackBridgeTest, ValidatesCallbackOptions) { EXPECT_OK(fluss::ffi::WriteCallbackCapacity::Validate(options)); } +TEST(WriteCallbackBridgeTest, DefaultCapacityRejectsOverflowWithoutDroppingReservations) { + fluss::WriteCallbackOptions options; + options.enqueue_timeout = std::chrono::milliseconds(0); + fluss::ffi::WriteCallbackCapacity capacity(options); + for (size_t i = 0; i < options.max_pending_operations; ++i) { + ASSERT_OK(capacity.Acquire()); + } + EXPECT_FALSE(capacity.Acquire().Ok()); + capacity.Release(); + ASSERT_OK(capacity.Acquire()); + EXPECT_FALSE(capacity.Acquire().Ok()); + for (size_t i = 0; i < options.max_pending_operations; ++i) { + capacity.Release(); + } + EXPECT_OK(capacity.AwaitAll(std::chrono::milliseconds(0))); +} + TEST(WriteCallbackBridgeTest, CapacityLastsThroughCallbackAndCaptureCleanup) { auto capacity = std::make_shared( fluss::WriteCallbackOptions{1, std::chrono::milliseconds(0)}); diff --git a/fluss-rust/website/docs/user-guide/cpp/api-reference.md b/fluss-rust/website/docs/user-guide/cpp/api-reference.md index 9da4d6042e0..b85b7e1204c 100644 --- a/fluss-rust/website/docs/user-guide/cpp/api-reference.md +++ b/fluss-rust/website/docs/user-guide/cpp/api-reference.md @@ -25,6 +25,8 @@ Complete API reference for the Fluss C++ client. | `writer_dynamic_batch_size_enabled` | `bool` | `true` | Enable per-table dynamic batch sizing: target grows 10% above 80% fill, shrinks 5% below 50% | | `writer_dynamic_batch_size_min` | `int32_t` | `262144` (256 KB) | Lower bound for the dynamic batch size estimator (ignored when disabled) | | `writer_batch_timeout_ms` | `int64_t` | `100` | Maximum time in ms to wait for a writer batch to fill up before sending | +| `writer_buffer_memory_size` | `size_t` | `67108864` (64 MiB) | Shared write-batch memory budget per Connection, across all tables and writers; not a process RSS limit | +| `writer_buffer_wait_timeout_ms` | `uint64_t` | `UINT64_MAX` | Maximum wait for Rust write-buffer capacity in ms; separate from callback admission | | `writer_kv_backpressure_max_throttle_ms` | `uint64_t` | `3000` | Maximum per-bucket KV backpressure throttle in milliseconds | | `writer_bucket_no_key_assigner` | `std::string` | `"sticky"` | Bucket assignment strategy for tables without bucket keys: `"sticky"` or `"round_robin"` | | `scanner_remote_log_prefetch_num` | `size_t` | `4` | Number of remote log segments to prefetch | @@ -278,8 +280,9 @@ operational limits below before retrying a batch. ```cpp fluss::WriteCallbackOptions options; -options.max_pending_operations = 65536; -options.enqueue_timeout = std::chrono::seconds(5); +// These are the defaults; CreateWriter(writer) also uses them. +options.max_pending_operations = 262144; +options.enqueue_timeout = std::chrono::seconds(30); fluss::AppendWriter writer; auto created = table.NewAppend().CreateWriter(writer, options); // Check created before using writer. NewUpsert().CreateWriter accepts the same options. @@ -287,7 +290,7 @@ auto created = table.NewAppend().CreateWriter(writer, options); | Option | Default | Meaning | |--------|---------|---------| -| `max_pending_operations` | `65536` | Positive, per-writer limit on callback operations reserved for submission or not yet finished | +| `max_pending_operations` | `262144` | Positive, per-writer limit on callback operations reserved for submission or not yet finished | | `enqueue_timeout` | `30s` | Nonnegative maximum wait for callback capacity; `0ms` rejects immediately when full | The existing `CreateWriter(writer)` overload uses these defaults. Each callback @@ -311,6 +314,79 @@ This timeout covers **only callback capacity admission**, not the entire It does not cancel any accepted write. Do not hold a mutex needed by callbacks while submitting: a full writer can wait for those callbacks to finish. +### Sizing callback capacity and write buffers + +There are two independent budgets: + +| Setting | Scope | What it limits | +|---------|-------|----------------| +| `max_pending_operations` | Each Writer | Callback operations from admission through callback execution and capture cleanup | +| `writer_buffer_memory_size` | Each Connection | Rust write-batch memory accounting shared by all its tables and writers, including writes using `Wait()` or fire-and-forget | + +The callback default of 262144 operations with a 30-second admission wait was +used in an eight-hour load test with 1 KiB rows, short callbacks, and five writers +per Connection. It is a starting point, not a throughput or latency guarantee. +The limit is not preallocated storage. It allows four times as many outstanding +operations as the previous 65536 default, so applications with tighter memory +budgets should explicitly select a smaller value. + +Budget callback capacity across **all writers**, including writers for different +tables. Five writers at the default allow 1310720 operations in total; fifty allow +13107200. If each outstanding operation retains 1 KiB of application data, those +limits permit roughly 1.25 GiB and 12.5 GiB of captures alone, before SDK overhead. +An `AppendArrowBatch` counts as one operation even when its batch contains many +rows, so large batches need a separate application byte budget. + +For callback capacity, estimate each writer's operation rate multiplied by the +time from admission until its callback finishes, then allow headroom for bursts +and tail latency **within the process memory budget**. Measure capture sizes too. +Use smaller limits for many tables, large captures, or tight memory budgets. +If callbacks are slow, shorten or offload their work before increasing capacity; +a larger queue does not fix a sustained completion-rate deficit. + +The Connection buffer accounts for write batches, not callback captures, +application input, or application retry queues. Its accounting is not a precise +bound on actual Arrow builder allocations or process RSS. +Sharing a Connection shares this budget: do not multiply it by the number of +writers, but expect busy or stalled tables to compete for it. Separate Connections +have separate budgets, which must be added when sizing the process or host. + +The following explicit settings were used with the callback defaults in that +high-throughput test. They are **not** new Connection defaults: + +```cpp +fluss::Configuration config; +config.bootstrap_servers = "127.0.0.1:9123"; // Replace with your cluster endpoint. +config.writer_buffer_memory_size = 512ULL * 1024 * 1024; // Per Connection. +config.writer_buffer_wait_timeout_ms = 5000; +config.writer_batch_size = 2 * 1024 * 1024; +config.writer_dynamic_batch_size_min = 1024 * 1024; +config.writer_batch_timeout_ms = 100; +config.writer_request_max_size = 32 * 1024 * 1024; +// Apply config when creating the Connection; use default callback options per Writer. +``` + +Keep the 64 MiB Connection default for a small workload unless measurements show +buffer pressure. For sustained high-throughput writes with several writers, +512 MiB per Connection is a tested starting point if the host has sufficient +headroom. Eight such Connections have a 4 GiB write-buffer budget in total, not a +4 GiB RSS limit. More active buckets and tables can retain more concurrent +batches; tune using buffer pressure, achieved throughput, completion latency, +and RSS together. Increase the budget only when the downstream service can +drain it; larger buffers can otherwise just extend queues and latency. The +eight-hour test also showed RSS growth, so it does not establish long-term memory +stability or a universally safe configuration. + +`enqueue_timeout` and `writer_buffer_wait_timeout_ms` govern different waits +that can occur in the same call. Set them to suit upstream latency and overload +handling, rather than assuming either is a whole-call deadline. Lowering an +admission timeout rejects sooner; it does not cancel accepted writes. Increasing +callback capacity does not increase Rust buffer space, and increasing Rust buffer +space does not prevent slow callbacks from filling their operation limit. +The default `writer_buffer_wait_timeout_ms = UINT64_MAX` permits a much longer +buffer wait than the 30-second callback admission timeout; configure a finite +buffer wait when the application needs to stop waiting and handle overload. + ### Execution and lifecycle Implement the callback's application logic; the SDK supplies the execution From 81174ad7af502c019b8acd84f2aafbab62d97a8d Mon Sep 17 00:00:00 2001 From: naivedogger Date: Sun, 20 Sep 2026 00:13:50 +0800 Subject: [PATCH 09/19] [c++] Use notify_all in write callback capacity Release to avoid stalled waiters --- fluss-rust/bindings/cpp/src/write_callback.hpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/fluss-rust/bindings/cpp/src/write_callback.hpp b/fluss-rust/bindings/cpp/src/write_callback.hpp index 8db84915b30..ea3950659ae 100644 --- a/fluss-rust/bindings/cpp/src/write_callback.hpp +++ b/fluss-rust/bindings/cpp/src/write_callback.hpp @@ -66,7 +66,10 @@ class WriteCallbackCapacity { std::lock_guard lock(mutex_); --pending_; } - available_.notify_one(); + // notify_all: Acquire() waiters and the AwaitAll() waiter share this condvar, + // so waking only one risks waking AwaitAll() (still pending) while an Acquire() + // waiter sleeps until enqueue_timeout despite the freed slot. + available_.notify_all(); } /// Wait for all reserved operations to finish their callbacks. From 582124b9c871b5c406f94021db65fe4e873438fa Mon Sep 17 00:00:00 2001 From: naivedogger Date: Sun, 20 Sep 2026 20:18:33 +0800 Subject: [PATCH 10/19] [c++] Bound write callback submission by enqueue_timeout Make WriteCallbackOptions::enqueue_timeout cover the whole submission, both callback capacity and buffer backpressure, following the Kafka max.block.ms model. The callback path passes a submit budget through the FFI so the Rust buffer-memory wait is bounded by the remaining budget, while the public overloads keep the writer's configured buffer wait timeout. A zero timeout makes submission non-blocking. --- fluss-rust/bindings/cpp/include/fluss.hpp | 36 ++++++++-- fluss-rust/bindings/cpp/src/lib.rs | 58 +++++++++++++--- fluss-rust/bindings/cpp/src/table.cpp | 43 +++++++++--- .../bindings/cpp/src/write_callback.hpp | 13 ++++ .../crates/fluss/src/client/table/append.rs | 35 ++++++++-- .../crates/fluss/src/client/table/upsert.rs | 27 +++++++- .../fluss/src/client/write/accumulator.rs | 67 +++++++++++++++++-- .../crates/fluss/src/client/write/mod.rs | 15 +++++ 8 files changed, 257 insertions(+), 37 deletions(-) diff --git a/fluss-rust/bindings/cpp/include/fluss.hpp b/fluss-rust/bindings/cpp/include/fluss.hpp index 8c5253d6074..b7065530699 100644 --- a/fluss-rust/bindings/cpp/include/fluss.hpp +++ b/fluss-rust/bindings/cpp/include/fluss.hpp @@ -579,8 +579,10 @@ struct WriteCallbackOptions { /// writers or large captures; independent of Configuration::writer_buffer_memory_size. size_t max_pending_operations = 262144; - /// Maximum wait for callback capacity; zero rejects immediately when full. - /// Must be nonnegative. Does not bound buffer waits, ACKs, retries, or callback duration. + /// Maximum wait for the whole callback submission: callback capacity plus + /// buffer backpressure (Kafka max.block.ms style). Zero makes submission + /// non-blocking, rejecting immediately when either is full. Must be nonnegative. + /// Does not bound ACKs, core retries, or callback duration. std::chrono::milliseconds enqueue_timeout{30000}; }; @@ -1955,8 +1957,11 @@ class AppendWriter { Result Append(const GenericRow& row); Result Append(const GenericRow& row, WriteResult& out); /// Submit a row and notify callback of its final outcome without waiting for - /// acknowledgment. Returns submission status; on failure no callback runs. - /// Submission can block on callback capacity and buffer backpressure. See WriteCallbackOptions. + /// acknowledgment. Submission is bounded by WriteCallbackOptions::enqueue_timeout, + /// covering callback capacity and buffer backpressure: within it, Ok means the + /// write was accepted and the callback fires exactly once, an error means + /// submission failed and no callback runs. A zero enqueue_timeout makes + /// submission non-blocking. See WriteCallbackOptions. Result Append(const GenericRow& row, WriteCallback callback); Result AppendArrowBatch(const std::shared_ptr& batch); Result AppendArrowBatch(const std::shared_ptr& batch, WriteResult& out); @@ -1971,6 +1976,14 @@ class AppendWriter { AppendWriter(ffi::AppendWriter* writer, std::shared_ptr callback_capacity) noexcept; + // Submit through FFI bounding the buffer-backpressure wait by submit_budget_ms + // (Kafka max.block.ms style): negative uses the writer's configured buffer wait + // timeout, >= 0 caps the wait at that many ms (0 = fail fast when the buffer is + // full). Only the callback path passes a budget; the public overloads pass -1. + Result AppendWithBudget(const GenericRow& row, WriteResult& out, int64_t submit_budget_ms); + Result AppendArrowBatchWithBudget(const std::shared_ptr& batch, + WriteResult& out, int64_t submit_budget_ms); + void Destroy() noexcept; ffi::AppendWriter* writer_{nullptr}; std::shared_ptr callback_capacity_; @@ -1990,9 +2003,12 @@ class UpsertWriter { Result Upsert(const GenericRow& row); Result Upsert(const GenericRow& row, WriteResult& out); - /// Submit an upsert and notify callback of its final outcome. Returns - /// submission status; on failure no callback runs. Submission may block on - /// callback capacity and buffer backpressure, but not acknowledgment. See WriteCallbackOptions. + /// Submit an upsert and notify callback of its final outcome without waiting + /// for acknowledgment. Submission is bounded by WriteCallbackOptions::enqueue_timeout, + /// covering callback capacity and buffer backpressure: within it, Ok means the + /// write was accepted and the callback fires exactly once, an error means + /// submission failed and no callback runs. A zero enqueue_timeout makes + /// submission non-blocking. See WriteCallbackOptions. Result Upsert(const GenericRow& row, WriteCallback callback); Result Delete(const GenericRow& row); Result Delete(const GenericRow& row, WriteResult& out); @@ -2005,6 +2021,12 @@ class UpsertWriter { friend class TableUpsert; UpsertWriter(ffi::UpsertWriter* writer, std::shared_ptr callback_capacity) noexcept; + + // See AppendWriter::AppendWithBudget for submit_budget_ms semantics. Only the + // callback path passes a budget; the public overloads pass -1. + Result UpsertWithBudget(const GenericRow& row, WriteResult& out, int64_t submit_budget_ms); + Result DeleteWithBudget(const GenericRow& row, WriteResult& out, int64_t submit_budget_ms); + void Destroy() noexcept; ffi::UpsertWriter* writer_{nullptr}; std::shared_ptr callback_capacity_; diff --git a/fluss-rust/bindings/cpp/src/lib.rs b/fluss-rust/bindings/cpp/src/lib.rs index ff8df75dce9..38e0b9c3295 100644 --- a/fluss-rust/bindings/cpp/src/lib.rs +++ b/fluss-rust/bindings/cpp/src/lib.rs @@ -667,13 +667,17 @@ mod ffi { // AppendWriter unsafe fn delete_append_writer(writer: *mut AppendWriter); - fn append(self: &mut AppendWriter, row: &GenericRowInner) -> FfiPtrResult; + // budget_ms bounds the buffer-memory wait: negative uses the writer's + // configured buffer wait timeout, >= 0 caps the wait at that many ms + // (0 = non-blocking), letting callers keep a submit within a fixed budget. + fn append(self: &mut AppendWriter, row: &GenericRowInner, budget_ms: i64) -> FfiPtrResult; // Partition (if partitioned) comes from the first row, so all rows must // share one partition; rows are distributed across buckets by key. fn append_arrow_batch( self: &mut AppendWriter, array_ptr: usize, schema_ptr: usize, + budget_ms: i64, ) -> FfiPtrResult; fn flush(self: &mut AppendWriter) -> FfiResult; @@ -684,8 +688,12 @@ mod ffi { // UpsertWriter unsafe fn delete_upsert_writer(writer: *mut UpsertWriter); - fn upsert(self: &mut UpsertWriter, row: &GenericRowInner) -> FfiPtrResult; - fn delete_row(self: &mut UpsertWriter, row: &GenericRowInner) -> FfiPtrResult; + fn upsert(self: &mut UpsertWriter, row: &GenericRowInner, budget_ms: i64) -> FfiPtrResult; + fn delete_row( + self: &mut UpsertWriter, + row: &GenericRowInner, + budget_ms: i64, + ) -> FfiPtrResult; fn upsert_flush(self: &mut UpsertWriter) -> FfiResult; // Lookuper @@ -2167,15 +2175,29 @@ unsafe fn delete_append_writer(writer: *mut AppendWriter) { } } +/// Convert a C++ submit budget (milliseconds) into an optional buffer-wait deadline. +/// Negative means "no caller budget": fall back to the writer's configured buffer +/// wait timeout. `>= 0` caps the wait (0 makes it non-blocking / fail fast). +fn budget_deadline(budget_ms: i64) -> Option { + if budget_ms < 0 { + None + } else { + Some(std::time::Instant::now() + std::time::Duration::from_millis(budget_ms as u64)) + } +} + impl AppendWriter { - fn append(&mut self, row: &GenericRowInner) -> ffi::FfiPtrResult { + fn append(&mut self, row: &GenericRowInner, budget_ms: i64) -> ffi::FfiPtrResult { let schema = self.table_info.get_schema(); let generic_row = match types::resolve_row_types(&row.row, Some(schema), 0) { Ok(r) => r, Err(e) => return client_err_ptr(e.to_string()), }; - let result_future = match self.inner.append(generic_row.as_ref()) { + let result_future = match self + .inner + .append_with_deadline(generic_row.as_ref(), budget_deadline(budget_ms)) + { Ok(f) => f, Err(e) => return err_ptr_from_core(&e), }; @@ -2186,7 +2208,12 @@ impl AppendWriter { ok_ptr(ptr as usize) } - fn append_arrow_batch(&mut self, array_ptr: usize, schema_ptr: usize) -> ffi::FfiPtrResult { + fn append_arrow_batch( + &mut self, + array_ptr: usize, + schema_ptr: usize, + budget_ms: i64, + ) -> ffi::FfiPtrResult { // Safety: C++ allocates these via `new ArrowArray/ArrowSchema` after a // successful `ExportRecordBatch`, so both pointers are valid heap // allocations that we take ownership of here. @@ -2205,7 +2232,10 @@ impl AppendWriter { let struct_array = arrow::array::StructArray::from(array_data); let batch = arrow::record_batch::RecordBatch::from(struct_array); - let result_future = match self.inner.append_arrow_batch(batch) { + let result_future = match self + .inner + .append_arrow_batch_with_deadline(batch, budget_deadline(budget_ms)) + { Ok(f) => f, Err(e) => return err_ptr_from_core(&e), }; @@ -2258,7 +2288,7 @@ unsafe fn delete_upsert_writer(writer: *mut UpsertWriter) { } impl UpsertWriter { - fn upsert(&mut self, row: &GenericRowInner) -> ffi::FfiPtrResult { + fn upsert(&mut self, row: &GenericRowInner, budget_ms: i64) -> ffi::FfiPtrResult { let schema = self.table_info.get_schema(); // Resolve types and pad to full schema width, so callers may set only // the fields they care about. @@ -2268,7 +2298,10 @@ impl UpsertWriter { Err(e) => return client_err_ptr(e.to_string()), }; - let result_future = match self.inner.upsert(generic_row.as_ref()) { + let result_future = match self + .inner + .upsert_with_deadline(generic_row.as_ref(), budget_deadline(budget_ms)) + { Ok(f) => f, Err(e) => return err_ptr_from_core(&e), }; @@ -2279,7 +2312,7 @@ impl UpsertWriter { ok_ptr(ptr as usize) } - fn delete_row(&mut self, row: &GenericRowInner) -> ffi::FfiPtrResult { + fn delete_row(&mut self, row: &GenericRowInner, budget_ms: i64) -> ffi::FfiPtrResult { let schema = self.table_info.get_schema(); // Resolve types and pad to full schema width, so callers may set only // the fields they care about. @@ -2289,7 +2322,10 @@ impl UpsertWriter { Err(e) => return client_err_ptr(e.to_string()), }; - let result_future = match self.inner.delete(generic_row.as_ref()) { + let result_future = match self + .inner + .delete_with_deadline(generic_row.as_ref(), budget_deadline(budget_ms)) + { Ok(f) => f, Err(e) => return err_ptr_from_core(&e), }; diff --git a/fluss-rust/bindings/cpp/src/table.cpp b/fluss-rust/bindings/cpp/src/table.cpp index c769aacbb11..dc5e48ecfc2 100644 --- a/fluss-rust/bindings/cpp/src/table.cpp +++ b/fluss-rust/bindings/cpp/src/table.cpp @@ -1670,6 +1670,11 @@ Result AppendWriter::Append(const GenericRow& row) { } Result AppendWriter::Append(const GenericRow& row, WriteResult& out) { + return AppendWithBudget(row, out, -1); +} + +Result AppendWriter::AppendWithBudget(const GenericRow& row, WriteResult& out, + int64_t submit_budget_ms) { if (!Available()) { return utils::make_client_error("AppendWriter not available"); } @@ -1677,7 +1682,7 @@ Result AppendWriter::Append(const GenericRow& row, WriteResult& out) { return utils::make_client_error("GenericRow not available"); } - auto ffi_result = writer_->append(*row.inner_); + auto ffi_result = writer_->append(*row.inner_, submit_budget_ms); auto result = utils::from_ffi_result(ffi_result.result); if (result.Ok()) { out = WriteResult(utils::ptr_from_ffi(ffi_result)); @@ -1692,12 +1697,14 @@ Result AppendWriter::Append(const GenericRow& row, WriteCallback callback) { // Allocate before submission so an allocation failure cannot lose an // already accepted write's completion notification. auto completion = std::make_unique(std::move(callback)); + // Bound the whole submit (capacity reservation + buffer wait) by enqueue_timeout. + const auto submit_start = std::chrono::steady_clock::now(); auto reserved = completion->Reserve(callback_capacity_); if (!reserved.Ok()) { return reserved; } WriteResult pending; - auto result = Append(row, pending); + auto result = AppendWithBudget(row, pending, callback_capacity_->RemainingBudgetMs(submit_start)); if (result.Ok()) { return pending.Notify(std::move(completion)); } @@ -1711,6 +1718,11 @@ Result AppendWriter::AppendArrowBatch(const std::shared_ptr& Result AppendWriter::AppendArrowBatch(const std::shared_ptr& batch, WriteResult& out) { + return AppendArrowBatchWithBudget(batch, out, -1); +} + +Result AppendWriter::AppendArrowBatchWithBudget(const std::shared_ptr& batch, + WriteResult& out, int64_t submit_budget_ms) { if (!Available()) { return utils::make_client_error("AppendWriter not available"); } @@ -1733,7 +1745,8 @@ Result AppendWriter::AppendArrowBatch(const std::shared_ptr& // Rust takes ownership of both pointers immediately via Box::from_raw(), // so after this call C++ must NOT free them. auto ffi_result = writer_->append_arrow_batch(reinterpret_cast(array_heap), - reinterpret_cast(schema_heap)); + reinterpret_cast(schema_heap), + submit_budget_ms); auto result = utils::from_ffi_result(ffi_result.result); if (result.Ok()) { out.Destroy(); @@ -1748,12 +1761,14 @@ Result AppendWriter::AppendArrowBatch(const std::shared_ptr& return utils::make_client_error("Write callback must not be empty"); } auto completion = std::make_unique(std::move(callback)); + const auto submit_start = std::chrono::steady_clock::now(); auto reserved = completion->Reserve(callback_capacity_); if (!reserved.Ok()) { return reserved; } WriteResult pending; - auto result = AppendArrowBatch(batch, pending); + auto result = + AppendArrowBatchWithBudget(batch, pending, callback_capacity_->RemainingBudgetMs(submit_start)); if (result.Ok()) { return pending.Notify(std::move(completion)); } @@ -1813,6 +1828,11 @@ Result UpsertWriter::Upsert(const GenericRow& row) { } Result UpsertWriter::Upsert(const GenericRow& row, WriteResult& out) { + return UpsertWithBudget(row, out, -1); +} + +Result UpsertWriter::UpsertWithBudget(const GenericRow& row, WriteResult& out, + int64_t submit_budget_ms) { if (!Available()) { return utils::make_client_error("UpsertWriter not available"); } @@ -1820,7 +1840,7 @@ Result UpsertWriter::Upsert(const GenericRow& row, WriteResult& out) { return utils::make_client_error("GenericRow not available"); } - auto ffi_result = writer_->upsert(*row.inner_); + auto ffi_result = writer_->upsert(*row.inner_, submit_budget_ms); auto result = utils::from_ffi_result(ffi_result.result); if (result.Ok()) { out = WriteResult(utils::ptr_from_ffi(ffi_result)); @@ -1833,12 +1853,13 @@ Result UpsertWriter::Upsert(const GenericRow& row, WriteCallback callback) { return utils::make_client_error("Write callback must not be empty"); } auto completion = std::make_unique(std::move(callback)); + const auto submit_start = std::chrono::steady_clock::now(); auto reserved = completion->Reserve(callback_capacity_); if (!reserved.Ok()) { return reserved; } WriteResult pending; - auto result = Upsert(row, pending); + auto result = UpsertWithBudget(row, pending, callback_capacity_->RemainingBudgetMs(submit_start)); if (result.Ok()) { return pending.Notify(std::move(completion)); } @@ -1851,6 +1872,11 @@ Result UpsertWriter::Delete(const GenericRow& row) { } Result UpsertWriter::Delete(const GenericRow& row, WriteResult& out) { + return DeleteWithBudget(row, out, -1); +} + +Result UpsertWriter::DeleteWithBudget(const GenericRow& row, WriteResult& out, + int64_t submit_budget_ms) { if (!Available()) { return utils::make_client_error("UpsertWriter not available"); } @@ -1858,7 +1884,7 @@ Result UpsertWriter::Delete(const GenericRow& row, WriteResult& out) { return utils::make_client_error("GenericRow not available"); } - auto ffi_result = writer_->delete_row(*row.inner_); + auto ffi_result = writer_->delete_row(*row.inner_, submit_budget_ms); auto result = utils::from_ffi_result(ffi_result.result); if (result.Ok()) { out = WriteResult(utils::ptr_from_ffi(ffi_result)); @@ -1871,12 +1897,13 @@ Result UpsertWriter::Delete(const GenericRow& row, WriteCallback callback) { return utils::make_client_error("Write callback must not be empty"); } auto completion = std::make_unique(std::move(callback)); + const auto submit_start = std::chrono::steady_clock::now(); auto reserved = completion->Reserve(callback_capacity_); if (!reserved.Ok()) { return reserved; } WriteResult pending; - auto result = Delete(row, pending); + auto result = DeleteWithBudget(row, pending, callback_capacity_->RemainingBudgetMs(submit_start)); if (result.Ok()) { return pending.Notify(std::move(completion)); } diff --git a/fluss-rust/bindings/cpp/src/write_callback.hpp b/fluss-rust/bindings/cpp/src/write_callback.hpp index ea3950659ae..2aca7a129e6 100644 --- a/fluss-rust/bindings/cpp/src/write_callback.hpp +++ b/fluss-rust/bindings/cpp/src/write_callback.hpp @@ -20,6 +20,7 @@ #pragma once #include +#include #include #include #include @@ -83,6 +84,18 @@ class WriteCallbackCapacity { return {}; } + /// Milliseconds left in the enqueue_timeout budget since `start`, floored at 0. + /// Used to bound the buffer-backpressure wait so the whole submit stays within + /// enqueue_timeout (Kafka max.block.ms style). A zero budget makes the buffer + /// wait non-blocking (fail fast). + int64_t RemainingBudgetMs(std::chrono::steady_clock::time_point start) const { + auto elapsed = std::chrono::duration_cast( + std::chrono::steady_clock::now() - start); + auto remaining = options_.enqueue_timeout - elapsed; + auto ms = remaining.count(); + return ms > 0 ? ms : 0; + } + private: friend class WriteCallback; inline static thread_local bool in_callback_ = false; diff --git a/fluss-rust/crates/fluss/src/client/table/append.rs b/fluss-rust/crates/fluss/src/client/table/append.rs index 790321100fd..325fb98e98c 100644 --- a/fluss-rust/crates/fluss/src/client/table/append.rs +++ b/fluss-rust/crates/fluss/src/client/table/append.rs @@ -29,6 +29,7 @@ use bytes::Bytes; use parking_lot::Mutex; use std::collections::HashMap; use std::sync::Arc; +use std::time::Instant; pub struct TableAppend { table_path: Arc, @@ -137,6 +138,17 @@ impl AppendWriter { /// A [`WriteResultFuture`] that can be awaited to wait for server acknowledgment, /// or dropped for fire-and-forget behavior (use `flush()` to ensure delivery). pub fn append(&self, row: &R) -> Result { + self.append_with_deadline(row, None) + } + + /// Like [`Self::append`], but bounds the buffer-memory wait by `deadline`. A deadline + /// already in the past makes the submit fail fast when the buffer is full, which + /// lets a caller keep the whole submit within a fixed budget. + pub fn append_with_deadline( + &self, + row: &R, + deadline: Option, + ) -> Result { self.check_field_count(row)?; let physical_table_path = Arc::new(get_physical_path( &self.table_path, @@ -153,7 +165,8 @@ impl AppendWriter { self.table_info.schema_id, row, ) - .with_bucket_key(bucket_key); + .with_bucket_key(bucket_key) + .with_submit_deadline(deadline); let result_handle = self.writer_client.send(&record)?; Ok(WriteResultFuture::new(result_handle)) } @@ -171,6 +184,15 @@ impl AppendWriter { /// A [`WriteResultFuture`] that can be awaited to wait for server acknowledgment, /// or dropped for fire-and-forget behavior (use `flush()` to ensure delivery). pub fn append_arrow_batch(&self, batch: RecordBatch) -> Result { + self.append_arrow_batch_with_deadline(batch, None) + } + + /// Like [`Self::append_arrow_batch`], but bounds the buffer-memory wait by `deadline`. + pub fn append_arrow_batch_with_deadline( + &self, + batch: RecordBatch, + deadline: Option, + ) -> Result { if batch.num_rows() == 0 { // Nothing to write; also avoids a keyless send to a bucket-key table. return Ok(WriteResultFuture::join(Vec::new())); @@ -193,7 +215,7 @@ impl AppendWriter { }; let Some(router) = self.bucket_router.as_ref() else { - return self.send_arrow_batch(batch, physical_table_path, None); + return self.send_arrow_batch(batch, physical_table_path, None, deadline); }; // Group rows by bucket, keeping one key per bucket (it hashes back there). @@ -215,7 +237,7 @@ impl AppendWriter { if groups.len() == 1 { let (_, (_, rep_key)) = groups.into_iter().next().unwrap(); - return self.send_arrow_batch(batch, physical_table_path, Some(rep_key)); + return self.send_arrow_batch(batch, physical_table_path, Some(rep_key), deadline); } let mut handles = Vec::with_capacity(groups.len()); @@ -227,7 +249,8 @@ impl AppendWriter { self.table_info.schema_id, sub_batch, ) - .with_bucket_key(Some(rep_key)); + .with_bucket_key(Some(rep_key)) + .with_submit_deadline(deadline); handles.push(self.writer_client.send(&record)?); } Ok(WriteResultFuture::join(handles)) @@ -238,6 +261,7 @@ impl AppendWriter { batch: RecordBatch, physical_table_path: Arc, bucket_key: Option, + deadline: Option, ) -> Result { let record = WriteRecord::for_append_record_batch( Arc::clone(&self.table_info), @@ -245,7 +269,8 @@ impl AppendWriter { self.table_info.schema_id, batch, ) - .with_bucket_key(bucket_key); + .with_bucket_key(bucket_key) + .with_submit_deadline(deadline); let result_handle = self.writer_client.send(&record)?; Ok(WriteResultFuture::new(result_handle)) } diff --git a/fluss-rust/crates/fluss/src/client/table/upsert.rs b/fluss-rust/crates/fluss/src/client/table/upsert.rs index 28dce4ee799..fb7af2c08d9 100644 --- a/fluss-rust/crates/fluss/src/client/table/upsert.rs +++ b/fluss-rust/crates/fluss/src/client/table/upsert.rs @@ -23,6 +23,7 @@ use crate::row::InternalRow; use crate::row::encode::{KeyEncoder, KeyEncoderFactory, RowEncoder, RowEncoderFactory}; use crate::row::field_getter::FieldGetter; use std::sync::Arc; +use std::time::Instant; use crate::client::table::partition_getter::{PartitionGetter, get_physical_path}; use bitvec::prelude::bitvec; @@ -347,6 +348,16 @@ impl UpsertWriter { /// A [`WriteResultFuture`] that can be awaited to wait for server acknowledgment, /// or dropped for fire-and-forget behavior (use `flush()` to ensure delivery). pub fn upsert(&self, row: &R) -> Result { + self.upsert_with_deadline(row, None) + } + + /// Like [`Self::upsert`], but bounds the buffer-memory wait by `deadline`. A deadline + /// already in the past makes the submit fail fast when the buffer is full. + pub fn upsert_with_deadline( + &self, + row: &R, + deadline: Option, + ) -> Result { self.check_field_count(row)?; let (key, bucket_key) = self.get_keys(row)?; @@ -369,7 +380,8 @@ impl UpsertWriter { self.write_format, self.target_columns.clone(), Some(row_bytes), - ); + ) + .with_submit_deadline(deadline); let result_handle = self.writer_client.send(&write_record)?; Ok(WriteResultFuture::new(result_handle)) @@ -388,6 +400,16 @@ impl UpsertWriter { /// A [`WriteResultFuture`] that can be awaited to wait for server acknowledgment, /// or dropped for fire-and-forget behavior (use `flush()` to ensure delivery). pub fn delete(&self, row: &R) -> Result { + self.delete_with_deadline(row, None) + } + + /// Like [`Self::delete`], but bounds the buffer-memory wait by `deadline`. A deadline + /// already in the past makes the submit fail fast when the buffer is full. + pub fn delete_with_deadline( + &self, + row: &R, + deadline: Option, + ) -> Result { self.check_field_count(row)?; let (key, bucket_key) = self.get_keys(row)?; @@ -405,7 +427,8 @@ impl UpsertWriter { self.write_format, self.target_columns.clone(), None, - ); + ) + .with_submit_deadline(deadline); let result_handle = self.writer_client.send(&write_record)?; Ok(WriteResultFuture::new(result_handle)) diff --git a/fluss-rust/crates/fluss/src/client/write/accumulator.rs b/fluss-rust/crates/fluss/src/client/write/accumulator.rs index dc6b2c9341c..f7a1b704192 100644 --- a/fluss-rust/crates/fluss/src/client/write/accumulator.rs +++ b/fluss-rust/crates/fluss/src/client/write/accumulator.rs @@ -70,7 +70,20 @@ impl MemoryLimiter { /// Try to acquire `size` bytes. Blocks until memory is available, /// the timeout expires, or the limiter is closed. /// Returns a `MemoryPermit` on success. + #[cfg(test)] pub fn acquire(self: &Arc, size: usize) -> Result { + self.acquire_within(size, None) + } + + /// Like [`acquire`], but bounds the wait by `deadline` when provided instead of + /// the limiter's configured `wait_timeout`. A deadline already in the past makes + /// this non-blocking (fail fast if memory is unavailable), which callers use to + /// keep a submit within a caller-supplied budget (e.g. callback enqueue timeout). + pub fn acquire_within( + self: &Arc, + size: usize, + deadline: Option, + ) -> Result { if self.closed.load(Ordering::Acquire) { return Err(Error::WriterClosed { message: "Memory limiter is closed".to_string(), @@ -87,7 +100,7 @@ impl MemoryLimiter { } let mut used = self.state.lock(); - let deadline = Instant::now() + self.wait_timeout; + let deadline = deadline.unwrap_or_else(|| Instant::now() + self.wait_timeout); while *used + size > self.max_memory { self.waiting_count.fetch_add(1, Ordering::Relaxed); let result = self.cond.wait_until(&mut used, deadline); @@ -101,10 +114,9 @@ impl MemoryLimiter { if result.timed_out() && *used + size > self.max_memory { return Err(Error::BufferExhausted { message: format!( - "Failed to allocate {} bytes for write batch within {}ms. \ + "Failed to allocate {} bytes for write batch within the buffer wait budget. \ {} of {} bytes in use, {} threads waiting.", size, - self.wait_timeout.as_millis(), *used, self.max_memory, self.waiting_count.load(Ordering::Relaxed), @@ -412,7 +424,9 @@ impl RecordAccumulator { let batch_size = dynamic_target.unwrap_or(self.config.writer_batch_size as usize); let record_size = record.estimated_record_size(); let alloc_size = batch_size.max(record_size); - let permit = self.memory_limiter.acquire(alloc_size)?; + let permit = self + .memory_limiter + .acquire_within(alloc_size, record.submit_deadline)?; // Re-acquire dq lock after memory is available let mut dq_guard = dq.lock(); @@ -2346,6 +2360,51 @@ mod tests { assert!(elapsed >= Duration::from_millis(80)); // allow some timing slack } + #[test] + fn test_memory_limiter_acquire_within_bounds_wait_by_deadline() { + // Writer default wait is effectively unbounded; a caller deadline must win. + let limiter = Arc::new(MemoryLimiter::new(1024, Duration::from_secs(3600))); + let _permit = limiter.acquire(1024).unwrap(); + + let start = Instant::now(); + let result = limiter.acquire_within(512, Some(Instant::now() + Duration::from_millis(100))); + let elapsed = start.elapsed(); + + // Returns within the caller budget, not the 1h configured wait_timeout. + assert!(matches!(result.unwrap_err(), Error::BufferExhausted { .. })); + assert!(elapsed >= Duration::from_millis(80)); + assert!(elapsed < Duration::from_secs(2)); + } + + #[test] + fn test_memory_limiter_acquire_within_past_deadline_is_nonblocking() { + // A deadline already in the past = try semantics (callback enqueue_timeout == 0). + let limiter = Arc::new(MemoryLimiter::new(1024, Duration::from_secs(3600))); + let _permit = limiter.acquire(1024).unwrap(); + + let start = Instant::now(); + let result = limiter.acquire_within(512, Some(Instant::now() - Duration::from_millis(1))); + let elapsed = start.elapsed(); + + assert!(matches!(result.unwrap_err(), Error::BufferExhausted { .. })); + assert!(elapsed < Duration::from_millis(50)); + } + + #[test] + fn test_memory_limiter_acquire_within_succeeds_when_capacity_available() { + // A bounded deadline must not prevent an allocation that fits right away. + let limiter = Arc::new(MemoryLimiter::new(1024, Duration::from_secs(3600))); + + let start = Instant::now(); + let permit = limiter + .acquire_within(512, Some(Instant::now() + Duration::from_millis(100))) + .unwrap(); + assert!(start.elapsed() < Duration::from_millis(50)); + assert_eq!(*limiter.state.lock(), 512); + drop(permit); + assert_eq!(*limiter.state.lock(), 0); + } + #[test] fn test_memory_limiter_close_fails_immediately() { let limiter = Arc::new(MemoryLimiter::new(1024, Duration::from_secs(60))); diff --git a/fluss-rust/crates/fluss/src/client/write/mod.rs b/fluss-rust/crates/fluss/src/client/write/mod.rs index 398f9699baa..51fd52c224f 100644 --- a/fluss-rust/crates/fluss/src/client/write/mod.rs +++ b/fluss-rust/crates/fluss/src/client/write/mod.rs @@ -32,6 +32,7 @@ use std::future::Future; use std::pin::Pin; use std::sync::Arc; use std::task::{Context, Poll}; +use std::time::Instant; pub(crate) mod broadcast; mod bucket_assigner; @@ -52,6 +53,9 @@ pub struct WriteRecord<'a> { schema_id: i32, write_format: WriteFormat, table_info: Arc, + /// Optional deadline bounding the buffer-memory wait during append. `None` + /// falls back to the writer's configured buffer wait timeout. + submit_deadline: Option, } impl<'a> WriteRecord<'a> { @@ -143,6 +147,7 @@ impl<'a> WriteRecord<'a> { bucket_key: None, schema_id, write_format: WriteFormat::ArrowLog, + submit_deadline: None, } } @@ -159,6 +164,7 @@ impl<'a> WriteRecord<'a> { bucket_key: None, schema_id, write_format: WriteFormat::ArrowLog, + submit_deadline: None, } } @@ -168,6 +174,14 @@ impl<'a> WriteRecord<'a> { self } + /// Sets a submit deadline that bounds how long the buffer-memory wait may block + /// before this record's append fails fast. `None` uses the writer's configured + /// buffer wait timeout. + pub fn with_submit_deadline(mut self, deadline: Option) -> Self { + self.submit_deadline = deadline; + self + } + #[allow(clippy::too_many_arguments)] pub fn for_upsert( table_info: Arc, @@ -186,6 +200,7 @@ impl<'a> WriteRecord<'a> { bucket_key, schema_id, write_format, + submit_deadline: None, } } } From 18af731bf439a21ab489203b19853661345faed2 Mon Sep 17 00:00:00 2001 From: naivedogger Date: Sun, 20 Sep 2026 20:18:47 +0800 Subject: [PATCH 11/19] [c++] Make callback worker count configurable via FLUSS_CALLBACK_WORKERS Read the process-wide callback executor thread count from the advanced FLUSS_CALLBACK_WORKERS environment variable, falling back to the default when it is unset, invalid, or zero. This is a rarely needed escape hatch; the default of four workers is unchanged. --- fluss-rust/bindings/cpp/src/write_callback.rs | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/fluss-rust/bindings/cpp/src/write_callback.rs b/fluss-rust/bindings/cpp/src/write_callback.rs index cf35c3e75a9..154ac7d7158 100644 --- a/fluss-rust/bindings/cpp/src/write_callback.rs +++ b/fluss-rust/bindings/cpp/src/write_callback.rs @@ -23,14 +23,24 @@ use std::thread::{self, JoinHandle}; use crate::{RUNTIME, WriteResult, client_err, err_from_core_error, ffi, ok_result}; -const CALLBACK_WORKERS: usize = 4; +const DEFAULT_CALLBACK_WORKERS: usize = 4; const COMPLETION_BATCH_SIZE: usize = 64; type Completion = Box; +// Process-wide worker count. Override with FLUSS_CALLBACK_WORKERS; invalid or +// zero values fall back to the default. +fn callback_workers() -> usize { + std::env::var("FLUSS_CALLBACK_WORKERS") + .ok() + .and_then(|value| value.trim().parse::().ok()) + .filter(|count| *count > 0) + .unwrap_or(DEFAULT_CALLBACK_WORKERS) +} + // Like RUNTIME, the executor is process-wide and lives until process exit. // Initialize it on the submitting thread, not an async I/O worker. static CALLBACK_EXECUTOR: LazyLock> = - LazyLock::new(|| match CallbackExecutor::new(CALLBACK_WORKERS) { + LazyLock::new(|| match CallbackExecutor::new(callback_workers()) { Ok(executor) => Some(executor), Err(error) => { // The write has already been accepted. Keep the old dispatch path From f332fe37414e9773cd4170c0775f1f96a6a84ec6 Mon Sep 17 00:00:00 2001 From: naivedogger Date: Sun, 20 Sep 2026 20:19:00 +0800 Subject: [PATCH 12/19] [c++] Document callback execution context and recovery guidance Explain that callbacks run on a small shared executor pool and must stay short and non-blocking, and note the advanced FLUSS_CALLBACK_WORKERS knob. Clarify failure handling: record the outcome and either stop or retry outside the callback, deduplicating by identifier, and drive crash recovery from a replayable source that advances only after Flush. Update the enqueue_timeout wording and the example comments to match. --- fluss-rust/bindings/cpp/README.md | 5 +- fluss-rust/bindings/cpp/examples/example.cpp | 17 ++++-- .../docs/user-guide/cpp/api-reference.md | 61 ++++++++++++------- 3 files changed, 56 insertions(+), 27 deletions(-) diff --git a/fluss-rust/bindings/cpp/README.md b/fluss-rust/bindings/cpp/README.md index 7737f282cc9..e4910616703 100644 --- a/fluss-rust/bindings/cpp/README.md +++ b/fluss-rust/bindings/cpp/README.md @@ -79,7 +79,10 @@ not apply to `CreateBucketBatchScanner()`. The callback section configures `WriteCallbackOptions` to bound outstanding callback operations. The SDK executes callbacks; applications do not need a waiting thread or poll loop. The example uses the defaults: `max_pending_operations = 262144` per Writer -and `enqueue_timeout = 30s`. That timeout covers only waiting for callback capacity. +and `enqueue_timeout = 30s`. That timeout bounds the whole callback submission, callback +capacity plus buffer backpressure, so a callback submit returns a definite result within +it and zero makes the submit non-blocking. It does not bound ACKs, retries, or callback +duration. The callback limit counts operations, not bytes; it does not preallocate 262144 slots. The separate `Configuration::writer_buffer_memory_size` remains 64 MiB by default, diff --git a/fluss-rust/bindings/cpp/examples/example.cpp b/fluss-rust/bindings/cpp/examples/example.cpp index 159dfa02475..df7646a32e1 100644 --- a/fluss-rust/bindings/cpp/examples/example.cpp +++ b/fluss-rust/bindings/cpp/examples/example.cpp @@ -91,7 +91,8 @@ int main() { // 5) Write rows with scalar and temporal values fluss::AppendWriter writer; - // Defaults: 262144 pending callback operations per Writer, 30s admission wait. + // Defaults: 262144 pending callback operations per Writer, 30s submission timeout + // (callback capacity plus buffer backpressure; zero makes callback submits non-blocking). // Pass WriteCallbackOptions to lower the limit for large captures or many writers. // This count is independent of the Connection's write-buffer byte budget. check("new_append_writer", table.NewAppend().CreateWriter(writer)); @@ -155,6 +156,8 @@ int main() { // Callback acknowledgment { // The SDK runs callbacks; no application waiting thread is required. + // Callbacks run on a small shared executor pool, so keep them short and + // non-blocking: do not Flush/Wait or retry synchronously inside a callback. // This example counts outcomes only. It does not implement durable recovery. std::atomic succeeded{0}; std::atomic failed{0}; @@ -173,10 +176,12 @@ int main() { if (result.Ok()) { ++succeeded; } else { - // An error does not prove the row was not written. A new Append - // can duplicate it, even with SDK idempotence enabled. - // For recovery, retain id and input in application-owned state - // and schedule duplicate-safe retries outside this callback. + // By now the SDK has exhausted internal retries or hit a + // non-retriable error, so do not retry synchronously here. + // An error does not prove the row was not written, and a new + // Append can duplicate it even with SDK idempotence enabled. + // Record the outcome and either stop or hand id and input to + // your own retry queue; deduplicate by id downstream. if (failed.fetch_add(1) == 0) { std::cerr << "Write failed for id=" << id << ": " << result.error_message << '\n'; @@ -193,6 +198,8 @@ int main() { // Submission has stopped. Wait for callbacks before leaving the counters' // scope; individual callback failures are checked below. // check() exits on error; a continuing application must keep callback state alive. + // A durable pipeline would advance its source position or offset only + // after Flush() succeeds, then replay from there on restart. check("flush", writer.Flush()); std::cout << "Callback writes: succeeded=" << succeeded << " failed=" << failed << '\n'; if (failed != 0) { diff --git a/fluss-rust/website/docs/user-guide/cpp/api-reference.md b/fluss-rust/website/docs/user-guide/cpp/api-reference.md index b85b7e1204c..c0895cdee46 100644 --- a/fluss-rust/website/docs/user-guide/cpp/api-reference.md +++ b/fluss-rust/website/docs/user-guide/cpp/api-reference.md @@ -26,7 +26,7 @@ Complete API reference for the Fluss C++ client. | `writer_dynamic_batch_size_min` | `int32_t` | `262144` (256 KB) | Lower bound for the dynamic batch size estimator (ignored when disabled) | | `writer_batch_timeout_ms` | `int64_t` | `100` | Maximum time in ms to wait for a writer batch to fill up before sending | | `writer_buffer_memory_size` | `size_t` | `67108864` (64 MiB) | Shared write-batch memory budget per Connection, across all tables and writers; not a process RSS limit | -| `writer_buffer_wait_timeout_ms` | `uint64_t` | `UINT64_MAX` | Maximum wait for Rust write-buffer capacity in ms; separate from callback admission | +| `writer_buffer_wait_timeout_ms` | `uint64_t` | `UINT64_MAX` | Maximum wait for Rust write-buffer capacity in ms for Wait and fire-and-forget writes; callback writes instead bound this wait by `enqueue_timeout` | | `writer_kv_backpressure_max_throttle_ms` | `uint64_t` | `3000` | Maximum per-bucket KV backpressure throttle in milliseconds | | `writer_bucket_no_key_assigner` | `std::string` | `"sticky"` | Bucket assignment strategy for tables without bucket keys: `"sticky"` or `"round_robin"` | | `scanner_remote_log_prefetch_num` | `size_t` | `4` | Number of remote log segments to prefetch | @@ -245,6 +245,13 @@ final success or failure, subject to the lifetime and shutdown requirements belo Submission does not wait for acknowledgment, but may still wait for callback capacity and then for Rust writer buffer space under backpressure. +Callbacks run on a small shared executor pool, four threads by default. That +count is process wide and rarely needs changing; the advanced +`FLUSS_CALLBACK_WORKERS` environment variable can override it. Keep each callback +short and non-blocking. Do not call `Flush()` or `WriteResult::Wait()` inside a +callback, and do not retry synchronously there. Record the outcome and run any +retry in application logic outside the callback. + ### Write guarantees and recovery - A successful submission means the operation was accepted, not that it was @@ -264,12 +271,17 @@ capacity and then for Rust writer buffer space under backpressure. nothing was written or that resubmission is duplicate-safe. - Calling `Append` again is a new operation. SDK idempotence for internal retries does not deduplicate application resubmissions of the same logical record. - Decide whether to resubmit using application identifiers, deduplication or - reconciliation, and a bounded retry policy. Do not blindly resubmit every error. + When a callback reports failure, record the outcome and either stop the + pipeline or hand the record to your own retry queue and resubmit from a + separate thread with a bounded policy. Deduplicate resubmissions by application + identifier. A non-retriable error will fail again, so do not blindly resubmit. - One callback invocation per accepted operation is not an exactly-once delivery guarantee. Pending notifications are not persisted and can be lost on process exit or a crash. Applications requiring recovery across restarts must retain their source records or durable operation state independently of the callback. + A simple approach is to drive writes from a replayable source and advance your + source position or offset only after `Flush()` succeeds, then replay from the + last committed position on restart and deduplicate by identifier. `AppendArrowBatch` is not atomic across internal batches or buckets. Its single callback cannot identify which individual rows succeeded. A failed submission @@ -291,7 +303,7 @@ auto created = table.NewAppend().CreateWriter(writer, options); | Option | Default | Meaning | |--------|---------|---------| | `max_pending_operations` | `262144` | Positive, per-writer limit on callback operations reserved for submission or not yet finished | -| `enqueue_timeout` | `30s` | Nonnegative maximum wait for callback capacity; `0ms` rejects immediately when full | +| `enqueue_timeout` | `30s` | Nonnegative maximum wait for the whole callback submission (callback capacity plus buffer backpressure); `0ms` makes submission non-blocking | The existing `CreateWriter(writer)` overload uses these defaults. Each callback submission reserves one slot **before** submitting to Rust and holds it through @@ -301,7 +313,8 @@ return the slot automatically. `Upsert` and `Delete` share their writer's limit; a writer transfers its capacity state; already accepted callbacks retain it independently of the writer's lifetime. -When capacity is full, the submitting thread waits up to `enqueue_timeout`. +When callback capacity or the write buffer is full, the submitting thread waits up +to `enqueue_timeout` for both together. A capacity rejection returns `CLIENT_ERROR` without submitting any data or registering a callback; it never discards an accepted notification. Client errors return false from `IsRetriable()`, including capacity errors. Applications may @@ -309,10 +322,12 @@ reschedule capacity-rejected submissions with backoff, but must not blindly retry every client error: other submission failures can have different effects, including partial ArrowBatch acceptance described below. -This timeout covers **only callback capacity admission**, not the entire -`Append` call, buffer waits, network requests, core retries, or callback duration. -It does not cancel any accepted write. Do not hold a mutex needed by callbacks -while submitting: a full writer can wait for those callbacks to finish. +This timeout bounds the whole callback submission, callback capacity admission plus +buffer backpressure (Kafka max.block.ms style), so a callback submit returns a +definite result within it and `0ms` makes submission non-blocking. It does not cover +network requests, core retries, or callback duration, and does not cancel any +accepted write. Do not hold a mutex needed by callbacks while submitting: a full +writer can wait for those callbacks to finish. ### Sizing callback capacity and write buffers @@ -323,7 +338,7 @@ There are two independent budgets: | `max_pending_operations` | Each Writer | Callback operations from admission through callback execution and capture cleanup | | `writer_buffer_memory_size` | Each Connection | Rust write-batch memory accounting shared by all its tables and writers, including writes using `Wait()` or fire-and-forget | -The callback default of 262144 operations with a 30-second admission wait was +The callback default of 262144 operations with a 30-second submission timeout was used in an eight-hour load test with 1 KiB rows, short callbacks, and five writers per Connection. It is a starting point, not a throughput or latency guarantee. The limit is not preallocated storage. It allows four times as many outstanding @@ -377,15 +392,17 @@ drain it; larger buffers can otherwise just extend queues and latency. The eight-hour test also showed RSS growth, so it does not establish long-term memory stability or a universally safe configuration. -`enqueue_timeout` and `writer_buffer_wait_timeout_ms` govern different waits -that can occur in the same call. Set them to suit upstream latency and overload -handling, rather than assuming either is a whole-call deadline. Lowering an -admission timeout rejects sooner; it does not cancel accepted writes. Increasing -callback capacity does not increase Rust buffer space, and increasing Rust buffer -space does not prevent slow callbacks from filling their operation limit. -The default `writer_buffer_wait_timeout_ms = UINT64_MAX` permits a much longer -buffer wait than the 30-second callback admission timeout; configure a finite -buffer wait when the application needs to stop waiting and handle overload. +For callback writes, `enqueue_timeout` bounds the whole submission, including the +buffer-backpressure wait, so a callback submit returns within it regardless of +`writer_buffer_wait_timeout_ms`. For `Wait()` and fire-and-forget writes, +`writer_buffer_wait_timeout_ms` governs the buffer wait instead. Set them to suit +upstream latency and overload handling, rather than assuming either bounds ACKs, +retries, or callback duration. Lowering `enqueue_timeout` rejects sooner; it does +not cancel accepted writes. Increasing callback capacity does not increase Rust +buffer space, and increasing Rust buffer space does not prevent slow callbacks from +filling their operation limit. The default `writer_buffer_wait_timeout_ms = UINT64_MAX` +permits an unbounded buffer wait for non-callback writes; configure a finite buffer +wait when those paths need to stop waiting and handle overload. ### Execution and lifecycle @@ -462,8 +479,10 @@ this path as a shutdown barrier. - Existing fire-and-forget and `WriteResult::Wait()` overloads retain their result semantics and do not consume callback capacity. Rust callers can still - `.await` a `WriteResultFuture`. Callback overloads now apply bounded admission - by default; existing callback callers may block or receive a capacity error. + `.await` a `WriteResultFuture`. Callback overloads now bound the whole submission + by `enqueue_timeout` (callback capacity plus buffer backpressure) by default; + existing callback callers may block up to that timeout or receive a capacity or + buffer error. Completion follows the configured acknowledgment policy; a callback does not add a stronger durability guarantee or change retries, request ordering, wire formats, or storage formats. From b94213462c83bb831f6c7cc32b50e921841faf07 Mon Sep 17 00:00:00 2001 From: naivedogger Date: Mon, 21 Sep 2026 12:32:31 +0800 Subject: [PATCH 13/19] nit --- fluss-rust/bindings/cpp/examples/example.cpp | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/fluss-rust/bindings/cpp/examples/example.cpp b/fluss-rust/bindings/cpp/examples/example.cpp index df7646a32e1..f68ce21b4e5 100644 --- a/fluss-rust/bindings/cpp/examples/example.cpp +++ b/fluss-rust/bindings/cpp/examples/example.cpp @@ -91,11 +91,17 @@ int main() { // 5) Write rows with scalar and temporal values fluss::AppendWriter writer; - // Defaults: 262144 pending callback operations per Writer, 30s submission timeout - // (callback capacity plus buffer backpressure; zero makes callback submits non-blocking). - // Pass WriteCallbackOptions to lower the limit for large captures or many writers. - // This count is independent of the Connection's write-buffer byte budget. - check("new_append_writer", table.NewAppend().CreateWriter(writer)); + // Callback admission limits, shown with their defaults. They bound the callback + // overloads only; Wait and fire-and-forget writes are unaffected. Passing no + // options uses these same values, so this block is equivalent to CreateWriter(writer). + fluss::WriteCallbackOptions callback_options; + // Pending callback operations per writer. Lower it for large captures or many + // writers; independent of the Connection's write-buffer byte budget. + callback_options.max_pending_operations = 262144; + // Whole-submit budget (callback capacity plus buffer backpressure). Zero makes + // callback submits non-blocking, rejecting immediately when either is full. + callback_options.enqueue_timeout = std::chrono::milliseconds(30000); + check("new_append_writer", table.NewAppend().CreateWriter(writer, callback_options)); struct RowData { int id; From 8c35446c23f2b50f059d28832819a9c642fc7a45 Mon Sep 17 00:00:00 2001 From: naivedogger Date: Mon, 21 Sep 2026 20:18:41 +0800 Subject: [PATCH 14/19] [c++] Bound callback submission by client.writer.buffer.wait-timeout Remove the WriteCallbackOptions enqueue_timeout field and route the whole callback submit through the connection's client.writer.buffer.wait-timeout. The capacity reservation and the buffer-backpressure wait now share one deadline sourced from that setting, so a submit returns within a single timeout instead of two. UINT64_MAX keeps the default unbounded and a zero timeout makes submission non-blocking. --- fluss-rust/bindings/cpp/README.md | 10 +-- fluss-rust/bindings/cpp/examples/example.cpp | 12 ++-- fluss-rust/bindings/cpp/include/fluss.hpp | 20 +++--- fluss-rust/bindings/cpp/src/lib.rs | 7 ++ fluss-rust/bindings/cpp/src/table.cpp | 8 ++- .../bindings/cpp/src/write_callback.hpp | 45 +++++++------ .../bindings/cpp/test/test_write_callback.cpp | 64 ++++++++++--------- .../test/test_write_callback_allocation.cpp | 2 +- .../fluss/src/client/write/accumulator.rs | 2 +- .../docs/user-guide/cpp/api-reference.md | 34 +++++----- 10 files changed, 109 insertions(+), 95 deletions(-) diff --git a/fluss-rust/bindings/cpp/README.md b/fluss-rust/bindings/cpp/README.md index e4910616703..01afa3e0cb0 100644 --- a/fluss-rust/bindings/cpp/README.md +++ b/fluss-rust/bindings/cpp/README.md @@ -78,11 +78,11 @@ not apply to `CreateBucketBatchScanner()`. The callback section configures `WriteCallbackOptions` to bound outstanding callback operations. The SDK executes callbacks; applications do not need a waiting thread or -poll loop. The example uses the defaults: `max_pending_operations = 262144` per Writer -and `enqueue_timeout = 30s`. That timeout bounds the whole callback submission, callback -capacity plus buffer backpressure, so a callback submit returns a definite result within -it and zero makes the submit non-blocking. It does not bound ACKs, retries, or callback -duration. +poll loop. The example uses the default `max_pending_operations = 262144` per Writer. +A callback submit is bounded by `client.writer.buffer.wait-timeout`, covering callback +capacity plus buffer backpressure, so it returns a definite result within that budget +and a zero timeout makes the submit non-blocking. It does not bound ACKs, retries, or +callback duration. The callback limit counts operations, not bytes; it does not preallocate 262144 slots. The separate `Configuration::writer_buffer_memory_size` remains 64 MiB by default, diff --git a/fluss-rust/bindings/cpp/examples/example.cpp b/fluss-rust/bindings/cpp/examples/example.cpp index f68ce21b4e5..08e60d34a98 100644 --- a/fluss-rust/bindings/cpp/examples/example.cpp +++ b/fluss-rust/bindings/cpp/examples/example.cpp @@ -91,16 +91,14 @@ int main() { // 5) Write rows with scalar and temporal values fluss::AppendWriter writer; - // Callback admission limits, shown with their defaults. They bound the callback - // overloads only; Wait and fire-and-forget writes are unaffected. Passing no - // options uses these same values, so this block is equivalent to CreateWriter(writer). + // Callback admission limit, shown with its default. It bounds the callback overloads + // only; Wait and fire-and-forget writes are unaffected. Passing no options uses this + // same value, so this block is equivalent to CreateWriter(writer). fluss::WriteCallbackOptions callback_options; // Pending callback operations per writer. Lower it for large captures or many - // writers; independent of the Connection's write-buffer byte budget. + // writers; independent of the Connection's write-buffer byte budget. Waiting for a + // free slot is bounded by client.writer.buffer.wait-timeout. callback_options.max_pending_operations = 262144; - // Whole-submit budget (callback capacity plus buffer backpressure). Zero makes - // callback submits non-blocking, rejecting immediately when either is full. - callback_options.enqueue_timeout = std::chrono::milliseconds(30000); check("new_append_writer", table.NewAppend().CreateWriter(writer, callback_options)); struct RowData { diff --git a/fluss-rust/bindings/cpp/include/fluss.hpp b/fluss-rust/bindings/cpp/include/fluss.hpp index b7065530699..f04e440ff3d 100644 --- a/fluss-rust/bindings/cpp/include/fluss.hpp +++ b/fluss-rust/bindings/cpp/include/fluss.hpp @@ -577,13 +577,9 @@ struct WriteCallbackOptions { /// Must be positive. One AppendArrowBatch call counts as one operation, not its rows. /// Per-writer count, not preallocated storage or a byte limit. Reduce for many /// writers or large captures; independent of Configuration::writer_buffer_memory_size. + /// Waiting for a free slot is bounded by client.writer.buffer.wait-timeout, the same + /// budget as the buffer-backpressure wait. size_t max_pending_operations = 262144; - - /// Maximum wait for the whole callback submission: callback capacity plus - /// buffer backpressure (Kafka max.block.ms style). Zero makes submission - /// non-blocking, rejecting immediately when either is full. Must be nonnegative. - /// Does not bound ACKs, core retries, or callback duration. - std::chrono::milliseconds enqueue_timeout{30000}; }; struct TablePath { @@ -1957,11 +1953,11 @@ class AppendWriter { Result Append(const GenericRow& row); Result Append(const GenericRow& row, WriteResult& out); /// Submit a row and notify callback of its final outcome without waiting for - /// acknowledgment. Submission is bounded by WriteCallbackOptions::enqueue_timeout, + /// acknowledgment. Submission is bounded by client.writer.buffer.wait-timeout, /// covering callback capacity and buffer backpressure: within it, Ok means the /// write was accepted and the callback fires exactly once, an error means - /// submission failed and no callback runs. A zero enqueue_timeout makes - /// submission non-blocking. See WriteCallbackOptions. + /// submission failed and no callback runs. A zero timeout makes submission + /// non-blocking. See WriteCallbackOptions. Result Append(const GenericRow& row, WriteCallback callback); Result AppendArrowBatch(const std::shared_ptr& batch); Result AppendArrowBatch(const std::shared_ptr& batch, WriteResult& out); @@ -2004,11 +2000,11 @@ class UpsertWriter { Result Upsert(const GenericRow& row); Result Upsert(const GenericRow& row, WriteResult& out); /// Submit an upsert and notify callback of its final outcome without waiting - /// for acknowledgment. Submission is bounded by WriteCallbackOptions::enqueue_timeout, + /// for acknowledgment. Submission is bounded by client.writer.buffer.wait-timeout, /// covering callback capacity and buffer backpressure: within it, Ok means the /// write was accepted and the callback fires exactly once, an error means - /// submission failed and no callback runs. A zero enqueue_timeout makes - /// submission non-blocking. See WriteCallbackOptions. + /// submission failed and no callback runs. A zero timeout makes submission + /// non-blocking. See WriteCallbackOptions. Result Upsert(const GenericRow& row, WriteCallback callback); Result Delete(const GenericRow& row); Result Delete(const GenericRow& row, WriteResult& out); diff --git a/fluss-rust/bindings/cpp/src/lib.rs b/fluss-rust/bindings/cpp/src/lib.rs index 38e0b9c3295..63693920282 100644 --- a/fluss-rust/bindings/cpp/src/lib.rs +++ b/fluss-rust/bindings/cpp/src/lib.rs @@ -531,6 +531,7 @@ mod ffi { unsafe fn get_arrow_schema(self: &Table, out_ptr: usize) -> FfiResult; fn get_table_path(self: &Table) -> FfiTablePath; fn has_primary_key(self: &Table) -> bool; + fn writer_buffer_wait_timeout_ms(self: &Table) -> u64; fn create_upsert_writer(self: &Table, column_indices: Vec) -> FfiPtrResult; fn new_lookuper(self: &Table) -> FfiPtrResult; fn new_prefix_lookuper(self: &Table, lookup_column_names: Vec) -> FfiPtrResult; @@ -2084,6 +2085,12 @@ impl Table { self.has_pk } + /// The connection's configured write-buffer wait timeout (client.writer.buffer.wait-timeout), + /// used by the C++ callback path to bound the whole submit. UINT64_MAX means unbounded. + fn writer_buffer_wait_timeout_ms(&self) -> u64 { + self.connection.config().writer_buffer_wait_timeout_ms + } + fn create_upsert_writer(&self, column_indices: Vec) -> ffi::FfiPtrResult { let _enter = RUNTIME.enter(); diff --git a/fluss-rust/bindings/cpp/src/table.cpp b/fluss-rust/bindings/cpp/src/table.cpp index dc5e48ecfc2..3733a458297 100644 --- a/fluss-rust/bindings/cpp/src/table.cpp +++ b/fluss-rust/bindings/cpp/src/table.cpp @@ -1293,7 +1293,8 @@ Result TableAppend::CreateWriter(AppendWriter& out, const WriteCallbackOptions& return utils::make_client_error("Table not available"); } - auto capacity = std::make_shared(options); + auto capacity = std::make_shared( + options, table_->writer_buffer_wait_timeout_ms()); auto ffi_result = table_->new_append_writer(); auto result = utils::from_ffi_result(ffi_result.result); if (result.Ok()) { @@ -1361,7 +1362,8 @@ Result TableUpsert::CreateWriter(UpsertWriter& out, const WriteCallbackOptions& } try { - auto capacity = std::make_shared(options); + auto capacity = std::make_shared( + options, table_->writer_buffer_wait_timeout_ms()); auto resolved_indices = !column_names_.empty() ? ResolveNameProjection() : column_indices_; rust::Vec rust_indices; @@ -1697,7 +1699,7 @@ Result AppendWriter::Append(const GenericRow& row, WriteCallback callback) { // Allocate before submission so an allocation failure cannot lose an // already accepted write's completion notification. auto completion = std::make_unique(std::move(callback)); - // Bound the whole submit (capacity reservation + buffer wait) by enqueue_timeout. + // Bound the whole submit (capacity reservation + buffer wait) by client.writer.buffer.wait-timeout. const auto submit_start = std::chrono::steady_clock::now(); auto reserved = completion->Reserve(callback_capacity_); if (!reserved.Ok()) { diff --git a/fluss-rust/bindings/cpp/src/write_callback.hpp b/fluss-rust/bindings/cpp/src/write_callback.hpp index 2aca7a129e6..bf939b4c175 100644 --- a/fluss-rust/bindings/cpp/src/write_callback.hpp +++ b/fluss-rust/bindings/cpp/src/write_callback.hpp @@ -23,6 +23,7 @@ #include #include #include +#include #include #include "fluss.hpp" @@ -34,27 +35,31 @@ namespace ffi { /// Per-writer admission control; independent of Rust buffer memory and ACK completion. class WriteCallbackCapacity { public: - explicit WriteCallbackCapacity(const WriteCallbackOptions& options) : options_(options) {} + /// `wait_timeout_ms` is the connection's client.writer.buffer.wait-timeout, used as + /// the shared budget for the whole submit. UINT64_MAX means block until a slot frees. + WriteCallbackCapacity(const WriteCallbackOptions& options, uint64_t wait_timeout_ms) + : max_pending_(options.max_pending_operations), wait_timeout_ms_(wait_timeout_ms) {} static Result Validate(const WriteCallbackOptions& options) { if (options.max_pending_operations == 0) { return {ErrorCode::CLIENT_ERROR, "max_pending_operations must be positive"}; } - if (options.enqueue_timeout.count() < 0) { - return {ErrorCode::CLIENT_ERROR, "enqueue_timeout must be nonnegative"}; - } return {}; } Result Acquire() { std::unique_lock lock(mutex_); - if (pending_ == options_.max_pending_operations) { - // Applies across writers and also to callbacks on the fallback executor. - if (in_callback_ || options_.enqueue_timeout.count() == 0) { + if (pending_ == max_pending_) { + auto has_slot = [&] { return pending_ < max_pending_; }; + // Fail fast from within a callback to avoid stalling the shared workers on + // their own capacity; a zero budget also rejects immediately. + if (in_callback_ || wait_timeout_ms_ == 0) { return {ErrorCode::CLIENT_ERROR, "Write callback capacity is full"}; } - if (!available_.wait_for(lock, options_.enqueue_timeout, - [&] { return pending_ < options_.max_pending_operations; })) { + if (IsUnbounded()) { + available_.wait(lock, has_slot); + } else if (!available_.wait_for(lock, std::chrono::milliseconds(wait_timeout_ms_), + has_slot)) { return {ErrorCode::CLIENT_ERROR, "Timed out waiting for write callback capacity"}; } } @@ -69,7 +74,7 @@ class WriteCallbackCapacity { } // notify_all: Acquire() waiters and the AwaitAll() waiter share this condvar, // so waking only one risks waking AwaitAll() (still pending) while an Acquire() - // waiter sleeps until enqueue_timeout despite the freed slot. + // waiter keeps sleeping despite the freed slot. available_.notify_all(); } @@ -84,22 +89,26 @@ class WriteCallbackCapacity { return {}; } - /// Milliseconds left in the enqueue_timeout budget since `start`, floored at 0. - /// Used to bound the buffer-backpressure wait so the whole submit stays within - /// enqueue_timeout (Kafka max.block.ms style). A zero budget makes the buffer - /// wait non-blocking (fail fast). + /// Milliseconds left in the client.writer.buffer.wait-timeout budget since `start`, so + /// the buffer-backpressure wait plus the capacity reservation stay within one timeout + /// (Kafka max.block.ms style). Floored at 0 (0 = fail fast). Returns -1 when the timeout + /// is unbounded, letting the buffer wait fall back to the writer's configured timeout. int64_t RemainingBudgetMs(std::chrono::steady_clock::time_point start) const { + if (IsUnbounded()) { + return -1; + } auto elapsed = std::chrono::duration_cast( std::chrono::steady_clock::now() - start); - auto remaining = options_.enqueue_timeout - elapsed; - auto ms = remaining.count(); - return ms > 0 ? ms : 0; + int64_t remaining = static_cast(wait_timeout_ms_) - elapsed.count(); + return remaining > 0 ? remaining : 0; } private: friend class WriteCallback; inline static thread_local bool in_callback_ = false; - const WriteCallbackOptions options_; + bool IsUnbounded() const { return wait_timeout_ms_ == std::numeric_limits::max(); } + const size_t max_pending_; + const uint64_t wait_timeout_ms_; std::mutex mutex_; std::condition_variable available_; size_t pending_ = 0; diff --git a/fluss-rust/bindings/cpp/test/test_write_callback.cpp b/fluss-rust/bindings/cpp/test/test_write_callback.cpp index 349d2bf9ef6..702c5e9c79e 100644 --- a/fluss-rust/bindings/cpp/test/test_write_callback.cpp +++ b/fluss-rust/bindings/cpp/test/test_write_callback.cpp @@ -100,14 +100,26 @@ class WriteCallbackTest : public ::testing::Test { descriptor_builder.SetProperty("table.delete.behavior", "disable"); } auto descriptor = descriptor_builder.Build(); - fluss::TablePath path("fluss", - std::string("cpp_callback_") + - ::testing::UnitTest::GetInstance()->current_test_info()->name()); - fluss_test::CreateTable(env.GetAdmin(), path, descriptor); - auto result = env.GetConnection().GetTable(path, table_); + table_path_ = fluss::TablePath( + "fluss", std::string("cpp_callback_") + + ::testing::UnitTest::GetInstance()->current_test_info()->name()); + fluss_test::CreateTable(env.GetAdmin(), table_path_, descriptor); + auto result = env.GetConnection().GetTable(table_path_, table_); ASSERT_OK(result); } + // Reopen table_ through a dedicated connection whose client.writer.buffer.wait-timeout + // bounds callback submission, so a full-capacity wait fails with a definite timeout + // instead of blocking on the shared connection's unbounded default. + void UseWriterBufferWaitTimeout(uint64_t wait_timeout_ms) { + auto& env = *fluss_test::FlussTestEnvironment::Instance(); + fluss::Configuration config; + config.bootstrap_servers = env.GetBootstrapServers(); + config.writer_buffer_wait_timeout_ms = wait_timeout_ms; + ASSERT_OK(fluss::Connection::Create(config, connection_)); + ASSERT_OK(connection_.GetTable(table_path_, table_)); + } + fluss::GenericRow Row(int32_t id = 1) { fluss::GenericRow row(2); row.SetInt32(0, id); @@ -115,6 +127,9 @@ class WriteCallbackTest : public ::testing::Test { return row; } + // Declared before table_ so the table (which references it) is destroyed first. + fluss::Connection connection_; + fluss::TablePath table_path_; fluss::Table table_; }; @@ -270,8 +285,7 @@ TEST_F(WriteCallbackTest, MultipleOutstandingWritesEachNotifyOnce) { TEST_F(WriteCallbackTest, RejectedSubmissionDoesNotInvokeCallback) { CreateTable(); fluss::AppendWriter writer; - ASSERT_OK(table_.NewAppend().CreateWriter( - writer, fluss::WriteCallbackOptions{1, std::chrono::milliseconds(0)})); + ASSERT_OK(table_.NewAppend().CreateWriter(writer, fluss::WriteCallbackOptions{1})); auto completion = std::make_shared(); auto lifetime = std::make_shared(); auto released = lifetime->released.get_future(); @@ -300,9 +314,9 @@ TEST_F(WriteCallbackTest, RejectedSubmissionDoesNotInvokeCallback) { TEST_F(WriteCallbackTest, ArrowBatchCapacitySurvivesMovesAndDoesNotAffectWaitOrFlush) { CreateTable(); + UseWriterBufferWaitTimeout(250); fluss::AppendWriter writer; - ASSERT_OK(table_.NewAppend().CreateWriter( - writer, fluss::WriteCallbackOptions{1, std::chrono::milliseconds(250)})); + ASSERT_OK(table_.NewAppend().CreateWriter(writer, fluss::WriteCallbackOptions{1})); arrow::Int32Builder ids; arrow::StringBuilder values; ASSERT_TRUE(ids.AppendValues({1, 2, 3}).ok()); @@ -353,9 +367,9 @@ TEST_F(WriteCallbackTest, ArrowBatchCapacitySurvivesMovesAndDoesNotAffectWaitOrF TEST_F(WriteCallbackTest, UpsertAndDeleteShareCapacityAndReturnItOnSubmissionErrors) { CreateTable(true); + UseWriterBufferWaitTimeout(250); fluss::UpsertWriter writer; - ASSERT_OK(table_.NewUpsert().CreateWriter( - writer, fluss::WriteCallbackOptions{1, std::chrono::milliseconds(250)})); + ASSERT_OK(table_.NewUpsert().CreateWriter(writer, fluss::WriteCallbackOptions{1})); auto completion = std::make_shared(); auto callback = [completion](fluss::Result result) { completion->Record(std::move(result)); }; fluss::GenericRow invalid(2); @@ -388,8 +402,7 @@ TEST_F(WriteCallbackTest, UpsertAndDeleteShareCapacityAndReturnItOnSubmissionErr TEST_F(WriteCallbackTest, CallbackDoesNotWaitForCapacityOnAnotherWriter) { CreateTable(); auto full_writer = std::make_shared(); - ASSERT_OK(table_.NewAppend().CreateWriter( - *full_writer, fluss::WriteCallbackOptions{1, std::chrono::seconds(5)})); + ASSERT_OK(table_.NewAppend().CreateWriter(*full_writer, fluss::WriteCallbackOptions{1})); auto started = std::make_shared(); auto gate = std::make_shared>(); auto resume = gate->get_future().share(); @@ -547,7 +560,6 @@ TEST(WriteCallbackBridgeTest, ContainsCallbackExceptions) { TEST(WriteCallbackBridgeTest, ValidatesCallbackOptions) { fluss::WriteCallbackOptions options; EXPECT_EQ(options.max_pending_operations, 262144u); - EXPECT_EQ(options.enqueue_timeout, std::chrono::seconds(30)); EXPECT_OK(fluss::ffi::WriteCallbackCapacity::Validate(options)); options.max_pending_operations = 0; EXPECT_FALSE(fluss::ffi::WriteCallbackCapacity::Validate(options).Ok()); @@ -559,20 +571,12 @@ TEST(WriteCallbackBridgeTest, ValidatesCallbackOptions) { EXPECT_EQ(table.NewUpsert().CreateWriter(upsert, options).error_message, "max_pending_operations must be positive"); options.max_pending_operations = 1; - options.enqueue_timeout = std::chrono::milliseconds(-1); - EXPECT_FALSE(fluss::ffi::WriteCallbackCapacity::Validate(options).Ok()); - EXPECT_EQ(table.NewAppend().CreateWriter(append, options).error_message, - "enqueue_timeout must be nonnegative"); - EXPECT_EQ(table.NewUpsert().CreateWriter(upsert, options).error_message, - "enqueue_timeout must be nonnegative"); - options.enqueue_timeout = std::chrono::milliseconds(0); EXPECT_OK(fluss::ffi::WriteCallbackCapacity::Validate(options)); } TEST(WriteCallbackBridgeTest, DefaultCapacityRejectsOverflowWithoutDroppingReservations) { fluss::WriteCallbackOptions options; - options.enqueue_timeout = std::chrono::milliseconds(0); - fluss::ffi::WriteCallbackCapacity capacity(options); + fluss::ffi::WriteCallbackCapacity capacity(options, 0); for (size_t i = 0; i < options.max_pending_operations; ++i) { ASSERT_OK(capacity.Acquire()); } @@ -588,7 +592,7 @@ TEST(WriteCallbackBridgeTest, DefaultCapacityRejectsOverflowWithoutDroppingReser TEST(WriteCallbackBridgeTest, CapacityLastsThroughCallbackAndCaptureCleanup) { auto capacity = std::make_shared( - fluss::WriteCallbackOptions{1, std::chrono::milliseconds(0)}); + fluss::WriteCallbackOptions{1}, 0); // This deleter runs after the user callback returns, but before its slot is returned. auto capture = std::shared_ptr(new int(0), [capacity](int* value) { EXPECT_FALSE(capacity->Acquire().Ok()); @@ -608,7 +612,7 @@ TEST(WriteCallbackBridgeTest, CapacityLastsThroughCallbackAndCaptureCleanup) { TEST(WriteCallbackBridgeTest, UnsubmittedCallbackReturnsCapacityOnException) { auto capacity = std::make_shared( - fluss::WriteCallbackOptions{1, std::chrono::milliseconds(0)}); + fluss::WriteCallbackOptions{1}, 0); int calls = 0; try { fluss::ffi::WriteCallback callback([&](fluss::Result) { ++calls; }); @@ -623,7 +627,7 @@ TEST(WriteCallbackBridgeTest, UnsubmittedCallbackReturnsCapacityOnException) { TEST(WriteCallbackBridgeTest, ReservationOutlivesWriterOwnership) { auto capacity = std::make_shared( - fluss::WriteCallbackOptions{1, std::chrono::milliseconds(0)}); + fluss::WriteCallbackOptions{1}, 0); std::weak_ptr weak = capacity; fluss::ffi::WriteCallback callback([](fluss::Result) {}); ASSERT_OK(callback.Reserve(capacity)); @@ -635,7 +639,7 @@ TEST(WriteCallbackBridgeTest, ReservationOutlivesWriterOwnership) { TEST(WriteCallbackBridgeTest, CapacityTimeoutDoesNotDiscardAcceptedCallback) { auto capacity = std::make_shared( - fluss::WriteCallbackOptions{1, std::chrono::milliseconds(25)}); + fluss::WriteCallbackOptions{1}, 25); int calls = 0; fluss::ffi::WriteCallback accepted([&](fluss::Result) { ++calls; }); ASSERT_OK(accepted.Reserve(capacity)); @@ -653,7 +657,7 @@ TEST(WriteCallbackBridgeTest, CapacityTimeoutDoesNotDiscardAcceptedCallback) { TEST(WriteCallbackBridgeTest, WaitingSubmitterResumesAfterCompletion) { auto capacity = std::make_shared( - fluss::WriteCallbackOptions{1, std::chrono::seconds(5)}); + fluss::WriteCallbackOptions{1}, 5000); fluss::ffi::WriteCallback accepted([](fluss::Result) {}); ASSERT_OK(accepted.Reserve(capacity)); std::promise started; @@ -673,7 +677,7 @@ TEST(WriteCallbackBridgeTest, WaitingSubmitterResumesAfterCompletion) { TEST(WriteCallbackBridgeTest, CallbackRejectsFullOtherWriterAndRestoresThreadContext) { auto capacity = std::make_shared( - fluss::WriteCallbackOptions{1, std::chrono::milliseconds(25)}); + fluss::WriteCallbackOptions{1}, 25); ASSERT_OK(capacity->Acquire()); // Full writer unrelated to the executing callback. fluss::ffi::WriteCallback callback([capacity](fluss::Result) { auto result = capacity->Acquire(); @@ -691,7 +695,7 @@ TEST(WriteCallbackBridgeTest, CallbackRejectsFullOtherWriterAndRestoresThreadCon TEST(WriteCallbackBridgeTest, ConcurrentCapacityReservationsStayBounded) { constexpr size_t limit = 3; auto capacity = std::make_shared( - fluss::WriteCallbackOptions{limit, std::chrono::seconds(5)}); + fluss::WriteCallbackOptions{limit}, 5000); std::atomic active{0}; std::atomic completed{0}; std::vector threads; diff --git a/fluss-rust/bindings/cpp/test/test_write_callback_allocation.cpp b/fluss-rust/bindings/cpp/test/test_write_callback_allocation.cpp index 1fb9ef4d377..a94e1a97d25 100644 --- a/fluss-rust/bindings/cpp/test/test_write_callback_allocation.cpp +++ b/fluss-rust/bindings/cpp/test/test_write_callback_allocation.cpp @@ -53,7 +53,7 @@ TEST(WriteCallbackBridgeTest, ErrorTextAllocationFailureStillCompletesAndRelease observed = std::move(result); }); auto capacity = std::make_shared( - fluss::WriteCallbackOptions{1, std::chrono::milliseconds(0)}); + fluss::WriteCallbackOptions{1}, 0); ASSERT_TRUE(callback.Reserve(capacity).Ok()); const rust::Str text(message); fail_next_allocation = true; diff --git a/fluss-rust/crates/fluss/src/client/write/accumulator.rs b/fluss-rust/crates/fluss/src/client/write/accumulator.rs index f7a1b704192..93dd8895001 100644 --- a/fluss-rust/crates/fluss/src/client/write/accumulator.rs +++ b/fluss-rust/crates/fluss/src/client/write/accumulator.rs @@ -2378,7 +2378,7 @@ mod tests { #[test] fn test_memory_limiter_acquire_within_past_deadline_is_nonblocking() { - // A deadline already in the past = try semantics (callback enqueue_timeout == 0). + // A deadline already in the past = try semantics (zero submit budget). let limiter = Arc::new(MemoryLimiter::new(1024, Duration::from_secs(3600))); let _permit = limiter.acquire(1024).unwrap(); diff --git a/fluss-rust/website/docs/user-guide/cpp/api-reference.md b/fluss-rust/website/docs/user-guide/cpp/api-reference.md index c0895cdee46..fae7886b073 100644 --- a/fluss-rust/website/docs/user-guide/cpp/api-reference.md +++ b/fluss-rust/website/docs/user-guide/cpp/api-reference.md @@ -26,7 +26,7 @@ Complete API reference for the Fluss C++ client. | `writer_dynamic_batch_size_min` | `int32_t` | `262144` (256 KB) | Lower bound for the dynamic batch size estimator (ignored when disabled) | | `writer_batch_timeout_ms` | `int64_t` | `100` | Maximum time in ms to wait for a writer batch to fill up before sending | | `writer_buffer_memory_size` | `size_t` | `67108864` (64 MiB) | Shared write-batch memory budget per Connection, across all tables and writers; not a process RSS limit | -| `writer_buffer_wait_timeout_ms` | `uint64_t` | `UINT64_MAX` | Maximum wait for Rust write-buffer capacity in ms for Wait and fire-and-forget writes; callback writes instead bound this wait by `enqueue_timeout` | +| `writer_buffer_wait_timeout_ms` | `uint64_t` | `UINT64_MAX` | Maximum wait in ms for Rust write-buffer capacity; also bounds the whole callback submission (callback capacity plus buffer backpressure). `UINT64_MAX` waits indefinitely | | `writer_kv_backpressure_max_throttle_ms` | `uint64_t` | `3000` | Maximum per-bucket KV backpressure throttle in milliseconds | | `writer_bucket_no_key_assigner` | `std::string` | `"sticky"` | Bucket assignment strategy for tables without bucket keys: `"sticky"` or `"round_robin"` | | `scanner_remote_log_prefetch_num` | `size_t` | `4` | Number of remote log segments to prefetch | @@ -292,9 +292,8 @@ operational limits below before retrying a batch. ```cpp fluss::WriteCallbackOptions options; -// These are the defaults; CreateWriter(writer) also uses them. +// This is the default; CreateWriter(writer) also uses it. options.max_pending_operations = 262144; -options.enqueue_timeout = std::chrono::seconds(30); fluss::AppendWriter writer; auto created = table.NewAppend().CreateWriter(writer, options); // Check created before using writer. NewUpsert().CreateWriter accepts the same options. @@ -303,7 +302,6 @@ auto created = table.NewAppend().CreateWriter(writer, options); | Option | Default | Meaning | |--------|---------|---------| | `max_pending_operations` | `262144` | Positive, per-writer limit on callback operations reserved for submission or not yet finished | -| `enqueue_timeout` | `30s` | Nonnegative maximum wait for the whole callback submission (callback capacity plus buffer backpressure); `0ms` makes submission non-blocking | The existing `CreateWriter(writer)` overload uses these defaults. Each callback submission reserves one slot **before** submitting to Rust and holds it through @@ -314,7 +312,7 @@ a writer transfers its capacity state; already accepted callbacks retain it independently of the writer's lifetime. When callback capacity or the write buffer is full, the submitting thread waits up -to `enqueue_timeout` for both together. +to `client.writer.buffer.wait-timeout` for both together. A capacity rejection returns `CLIENT_ERROR` without submitting any data or registering a callback; it never discards an accepted notification. Client errors return false from `IsRetriable()`, including capacity errors. Applications may @@ -324,9 +322,9 @@ including partial ArrowBatch acceptance described below. This timeout bounds the whole callback submission, callback capacity admission plus buffer backpressure (Kafka max.block.ms style), so a callback submit returns a -definite result within it and `0ms` makes submission non-blocking. It does not cover -network requests, core retries, or callback duration, and does not cancel any -accepted write. Do not hold a mutex needed by callbacks while submitting: a full +definite result within it and a zero timeout makes submission non-blocking. It does +not cover network requests, core retries, or callback duration, and does not cancel +any accepted write. Do not hold a mutex needed by callbacks while submitting: a full writer can wait for those callbacks to finish. ### Sizing callback capacity and write buffers @@ -392,17 +390,17 @@ drain it; larger buffers can otherwise just extend queues and latency. The eight-hour test also showed RSS growth, so it does not establish long-term memory stability or a universally safe configuration. -For callback writes, `enqueue_timeout` bounds the whole submission, including the -buffer-backpressure wait, so a callback submit returns within it regardless of -`writer_buffer_wait_timeout_ms`. For `Wait()` and fire-and-forget writes, -`writer_buffer_wait_timeout_ms` governs the buffer wait instead. Set them to suit -upstream latency and overload handling, rather than assuming either bounds ACKs, -retries, or callback duration. Lowering `enqueue_timeout` rejects sooner; it does +For callback writes, `writer_buffer_wait_timeout_ms` (client.writer.buffer.wait-timeout) +bounds the whole submission, including both the callback-capacity wait and the +buffer-backpressure wait, so a callback submit returns within it. For `Wait()` and +fire-and-forget writes, the same setting governs the buffer wait. Set it to suit +upstream latency and overload handling, rather than assuming it bounds ACKs, +retries, or callback duration. Lowering it rejects sooner; it does not cancel accepted writes. Increasing callback capacity does not increase Rust buffer space, and increasing Rust buffer space does not prevent slow callbacks from filling their operation limit. The default `writer_buffer_wait_timeout_ms = UINT64_MAX` -permits an unbounded buffer wait for non-callback writes; configure a finite buffer -wait when those paths need to stop waiting and handle overload. +permits an unbounded wait; configure a finite value when callback submission or the +non-callback buffer wait needs to stop waiting and handle overload. ### Execution and lifecycle @@ -455,7 +453,7 @@ are still pending, so its capacity slot does not bound all underlying batch memo Do not wait for another callback from within a callback, since all callback workers could become occupied. A callback submission from within any SDK write callback fails immediately if its target writer's capacity is full, regardless -of `enqueue_timeout`, to avoid blocking the shared workers on their own capacity. +of `client.writer.buffer.wait-timeout`, to avoid blocking the shared workers on their own capacity. This applies across writers and on the fallback executor as well. It does not remove Rust buffer waits or make arbitrary blocking SDK calls deadlock-free. Exclusive writer access is still required. If dedicated workers cannot be @@ -480,7 +478,7 @@ this path as a shutdown barrier. - Existing fire-and-forget and `WriteResult::Wait()` overloads retain their result semantics and do not consume callback capacity. Rust callers can still `.await` a `WriteResultFuture`. Callback overloads now bound the whole submission - by `enqueue_timeout` (callback capacity plus buffer backpressure) by default; + by `client.writer.buffer.wait-timeout` (callback capacity plus buffer backpressure); existing callback callers may block up to that timeout or receive a capacity or buffer error. Completion follows the configured acknowledgment policy; a callback does not From 81eb38a16e5928ec7f5709dde814cb0bd296fa21 Mon Sep 17 00:00:00 2001 From: naivedogger Date: Mon, 21 Sep 2026 20:22:16 +0800 Subject: [PATCH 15/19] [c++] Show writer_buffer_wait_timeout_ms and test bounded callback submit Set writer_buffer_wait_timeout_ms in the example config so its role is visible: it bounds both the write-buffer wait and the whole callback submission. Add an end-to-end test that fills a writer's capacity with a blocking callback and asserts the next submit returns with a timeout error after the configured budget, not after the callback finally releases the slot. --- fluss-rust/bindings/cpp/examples/example.cpp | 7 +++- .../bindings/cpp/test/test_write_callback.cpp | 38 +++++++++++++++++++ 2 files changed, 44 insertions(+), 1 deletion(-) diff --git a/fluss-rust/bindings/cpp/examples/example.cpp b/fluss-rust/bindings/cpp/examples/example.cpp index 08e60d34a98..25b9847049e 100644 --- a/fluss-rust/bindings/cpp/examples/example.cpp +++ b/fluss-rust/bindings/cpp/examples/example.cpp @@ -45,6 +45,11 @@ int main() { // 1) Connect fluss::Configuration config; config.bootstrap_servers = "127.0.0.1:9123"; + // Bounds how long a write blocks when the shared write buffer is full, and also caps + // the whole callback submission (callback capacity plus buffer backpressure). The + // default UINT64_MAX waits indefinitely; a finite value makes overloaded writes and + // callback submits return with an error instead of blocking. Zero fails fast. + config.writer_buffer_wait_timeout_ms = 30000; fluss::Connection conn; check("create", fluss::Connection::Create(config, conn)); @@ -97,7 +102,7 @@ int main() { fluss::WriteCallbackOptions callback_options; // Pending callback operations per writer. Lower it for large captures or many // writers; independent of the Connection's write-buffer byte budget. Waiting for a - // free slot is bounded by client.writer.buffer.wait-timeout. + // free slot is bounded by the connection's writer_buffer_wait_timeout_ms set above. callback_options.max_pending_operations = 262144; check("new_append_writer", table.NewAppend().CreateWriter(writer, callback_options)); diff --git a/fluss-rust/bindings/cpp/test/test_write_callback.cpp b/fluss-rust/bindings/cpp/test/test_write_callback.cpp index 702c5e9c79e..e654b1fdc54 100644 --- a/fluss-rust/bindings/cpp/test/test_write_callback.cpp +++ b/fluss-rust/bindings/cpp/test/test_write_callback.cpp @@ -365,6 +365,44 @@ TEST_F(WriteCallbackTest, ArrowBatchCapacitySurvivesMovesAndDoesNotAffectWaitOrF EXPECT_EQ(rejected->Results().size(), 1u); } +// End-to-end through the public writer API: when the writer's capacity is exhausted, a +// callback submission must return within the connection's client.writer.buffer.wait-timeout +// rather than blocking on the occupied slot until the running callback releases it. +TEST_F(WriteCallbackTest, CapacityFullSubmissionReturnsWithinWaitTimeout) { + CreateTable(); + constexpr uint64_t wait_timeout_ms = 200; + UseWriterBufferWaitTimeout(wait_timeout_ms); + fluss::AppendWriter writer; + ASSERT_OK(table_.NewAppend().CreateWriter(writer, fluss::WriteCallbackOptions{1})); + + // Occupy the single slot with a callback that blocks until released, so the next + // submission has to wait for capacity instead of completing normally. + auto started = std::make_shared(); + auto gate = std::make_shared>(); + auto resume = gate->get_future().share(); + ASSERT_OK(writer.Append(Row(), [started, resume](fluss::Result result) { + started->Record(std::move(result)); + resume.wait_for(std::chrono::seconds(20)); + })); + ASSERT_TRUE(started->Await()); + + auto rejected = std::make_shared(); + auto start = std::chrono::steady_clock::now(); + auto result = writer.Append( + Row(2), [rejected](fluss::Result completed) { rejected->Record(std::move(completed)); }); + auto elapsed = std::chrono::steady_clock::now() - start; + EXPECT_FALSE(result.Ok()); + EXPECT_NE(result.error_message.find("Timed out"), std::string::npos); + // Returned because the shared budget elapsed: not immediately, and well before the + // 20-second callback block that would otherwise gate the slot. + EXPECT_GE(elapsed, std::chrono::milliseconds(wait_timeout_ms)); + EXPECT_LT(elapsed, std::chrono::seconds(5)); + EXPECT_TRUE(rejected->Results().empty()); + + gate->set_value(); + ASSERT_OK(writer.Flush()); +} + TEST_F(WriteCallbackTest, UpsertAndDeleteShareCapacityAndReturnItOnSubmissionErrors) { CreateTable(true); UseWriterBufferWaitTimeout(250); From 019fde8e374847365002fbbe1df5756cea5276e8 Mon Sep 17 00:00:00 2001 From: naivedogger Date: Mon, 21 Sep 2026 21:55:04 +0800 Subject: [PATCH 16/19] [c++] Pass callback capacity limit directly --- fluss-rust/bindings/cpp/src/table.cpp | 4 +-- .../bindings/cpp/src/write_callback.hpp | 4 +-- .../bindings/cpp/test/test_write_callback.cpp | 31 +++++++------------ .../test/test_write_callback_allocation.cpp | 3 +- 4 files changed, 17 insertions(+), 25 deletions(-) diff --git a/fluss-rust/bindings/cpp/src/table.cpp b/fluss-rust/bindings/cpp/src/table.cpp index 3733a458297..5794cace35e 100644 --- a/fluss-rust/bindings/cpp/src/table.cpp +++ b/fluss-rust/bindings/cpp/src/table.cpp @@ -1294,7 +1294,7 @@ Result TableAppend::CreateWriter(AppendWriter& out, const WriteCallbackOptions& } auto capacity = std::make_shared( - options, table_->writer_buffer_wait_timeout_ms()); + options.max_pending_operations, table_->writer_buffer_wait_timeout_ms()); auto ffi_result = table_->new_append_writer(); auto result = utils::from_ffi_result(ffi_result.result); if (result.Ok()) { @@ -1363,7 +1363,7 @@ Result TableUpsert::CreateWriter(UpsertWriter& out, const WriteCallbackOptions& try { auto capacity = std::make_shared( - options, table_->writer_buffer_wait_timeout_ms()); + options.max_pending_operations, table_->writer_buffer_wait_timeout_ms()); auto resolved_indices = !column_names_.empty() ? ResolveNameProjection() : column_indices_; rust::Vec rust_indices; diff --git a/fluss-rust/bindings/cpp/src/write_callback.hpp b/fluss-rust/bindings/cpp/src/write_callback.hpp index bf939b4c175..532a26aa0de 100644 --- a/fluss-rust/bindings/cpp/src/write_callback.hpp +++ b/fluss-rust/bindings/cpp/src/write_callback.hpp @@ -37,8 +37,8 @@ class WriteCallbackCapacity { public: /// `wait_timeout_ms` is the connection's client.writer.buffer.wait-timeout, used as /// the shared budget for the whole submit. UINT64_MAX means block until a slot frees. - WriteCallbackCapacity(const WriteCallbackOptions& options, uint64_t wait_timeout_ms) - : max_pending_(options.max_pending_operations), wait_timeout_ms_(wait_timeout_ms) {} + WriteCallbackCapacity(size_t max_pending_operations, uint64_t wait_timeout_ms) + : max_pending_(max_pending_operations), wait_timeout_ms_(wait_timeout_ms) {} static Result Validate(const WriteCallbackOptions& options) { if (options.max_pending_operations == 0) { diff --git a/fluss-rust/bindings/cpp/test/test_write_callback.cpp b/fluss-rust/bindings/cpp/test/test_write_callback.cpp index e654b1fdc54..f874f4cc8f7 100644 --- a/fluss-rust/bindings/cpp/test/test_write_callback.cpp +++ b/fluss-rust/bindings/cpp/test/test_write_callback.cpp @@ -612,25 +612,24 @@ TEST(WriteCallbackBridgeTest, ValidatesCallbackOptions) { EXPECT_OK(fluss::ffi::WriteCallbackCapacity::Validate(options)); } -TEST(WriteCallbackBridgeTest, DefaultCapacityRejectsOverflowWithoutDroppingReservations) { - fluss::WriteCallbackOptions options; - fluss::ffi::WriteCallbackCapacity capacity(options, 0); - for (size_t i = 0; i < options.max_pending_operations; ++i) { +TEST(WriteCallbackBridgeTest, CapacityRejectsOverflowWithoutDroppingReservations) { + constexpr size_t max_pending_operations = 3; + fluss::ffi::WriteCallbackCapacity capacity(max_pending_operations, 0); + for (size_t i = 0; i < max_pending_operations; ++i) { ASSERT_OK(capacity.Acquire()); } EXPECT_FALSE(capacity.Acquire().Ok()); capacity.Release(); ASSERT_OK(capacity.Acquire()); EXPECT_FALSE(capacity.Acquire().Ok()); - for (size_t i = 0; i < options.max_pending_operations; ++i) { + for (size_t i = 0; i < max_pending_operations; ++i) { capacity.Release(); } EXPECT_OK(capacity.AwaitAll(std::chrono::milliseconds(0))); } TEST(WriteCallbackBridgeTest, CapacityLastsThroughCallbackAndCaptureCleanup) { - auto capacity = std::make_shared( - fluss::WriteCallbackOptions{1}, 0); + auto capacity = std::make_shared(1, 0); // This deleter runs after the user callback returns, but before its slot is returned. auto capture = std::shared_ptr(new int(0), [capacity](int* value) { EXPECT_FALSE(capacity->Acquire().Ok()); @@ -649,8 +648,7 @@ TEST(WriteCallbackBridgeTest, CapacityLastsThroughCallbackAndCaptureCleanup) { } TEST(WriteCallbackBridgeTest, UnsubmittedCallbackReturnsCapacityOnException) { - auto capacity = std::make_shared( - fluss::WriteCallbackOptions{1}, 0); + auto capacity = std::make_shared(1, 0); int calls = 0; try { fluss::ffi::WriteCallback callback([&](fluss::Result) { ++calls; }); @@ -664,8 +662,7 @@ TEST(WriteCallbackBridgeTest, UnsubmittedCallbackReturnsCapacityOnException) { } TEST(WriteCallbackBridgeTest, ReservationOutlivesWriterOwnership) { - auto capacity = std::make_shared( - fluss::WriteCallbackOptions{1}, 0); + auto capacity = std::make_shared(1, 0); std::weak_ptr weak = capacity; fluss::ffi::WriteCallback callback([](fluss::Result) {}); ASSERT_OK(callback.Reserve(capacity)); @@ -676,8 +673,7 @@ TEST(WriteCallbackBridgeTest, ReservationOutlivesWriterOwnership) { } TEST(WriteCallbackBridgeTest, CapacityTimeoutDoesNotDiscardAcceptedCallback) { - auto capacity = std::make_shared( - fluss::WriteCallbackOptions{1}, 25); + auto capacity = std::make_shared(1, 25); int calls = 0; fluss::ffi::WriteCallback accepted([&](fluss::Result) { ++calls; }); ASSERT_OK(accepted.Reserve(capacity)); @@ -694,8 +690,7 @@ TEST(WriteCallbackBridgeTest, CapacityTimeoutDoesNotDiscardAcceptedCallback) { } TEST(WriteCallbackBridgeTest, WaitingSubmitterResumesAfterCompletion) { - auto capacity = std::make_shared( - fluss::WriteCallbackOptions{1}, 5000); + auto capacity = std::make_shared(1, 5000); fluss::ffi::WriteCallback accepted([](fluss::Result) {}); ASSERT_OK(accepted.Reserve(capacity)); std::promise started; @@ -714,8 +709,7 @@ TEST(WriteCallbackBridgeTest, WaitingSubmitterResumesAfterCompletion) { } TEST(WriteCallbackBridgeTest, CallbackRejectsFullOtherWriterAndRestoresThreadContext) { - auto capacity = std::make_shared( - fluss::WriteCallbackOptions{1}, 25); + auto capacity = std::make_shared(1, 25); ASSERT_OK(capacity->Acquire()); // Full writer unrelated to the executing callback. fluss::ffi::WriteCallback callback([capacity](fluss::Result) { auto result = capacity->Acquire(); @@ -732,8 +726,7 @@ TEST(WriteCallbackBridgeTest, CallbackRejectsFullOtherWriterAndRestoresThreadCon TEST(WriteCallbackBridgeTest, ConcurrentCapacityReservationsStayBounded) { constexpr size_t limit = 3; - auto capacity = std::make_shared( - fluss::WriteCallbackOptions{limit}, 5000); + auto capacity = std::make_shared(limit, 5000); std::atomic active{0}; std::atomic completed{0}; std::vector threads; diff --git a/fluss-rust/bindings/cpp/test/test_write_callback_allocation.cpp b/fluss-rust/bindings/cpp/test/test_write_callback_allocation.cpp index a94e1a97d25..7cbbc9c8c0c 100644 --- a/fluss-rust/bindings/cpp/test/test_write_callback_allocation.cpp +++ b/fluss-rust/bindings/cpp/test/test_write_callback_allocation.cpp @@ -52,8 +52,7 @@ TEST(WriteCallbackBridgeTest, ErrorTextAllocationFailureStillCompletesAndRelease ++calls; observed = std::move(result); }); - auto capacity = std::make_shared( - fluss::WriteCallbackOptions{1}, 0); + auto capacity = std::make_shared(1, 0); ASSERT_TRUE(callback.Reserve(capacity).Ok()); const rust::Str text(message); fail_next_allocation = true; From a4a364895cae39d2629b4bb0c1b480fa12b24ad2 Mon Sep 17 00:00:00 2001 From: naivedogger Date: Mon, 21 Sep 2026 22:04:04 +0800 Subject: [PATCH 17/19] [c++] Polish callback timeout documentation --- fluss-rust/bindings/cpp/include/fluss.hpp | 4 +++- fluss-rust/crates/fluss/src/client/write/accumulator.rs | 2 +- fluss-rust/website/docs/user-guide/cpp/api-reference.md | 2 +- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/fluss-rust/bindings/cpp/include/fluss.hpp b/fluss-rust/bindings/cpp/include/fluss.hpp index f04e440ff3d..7120c786bca 100644 --- a/fluss-rust/bindings/cpp/include/fluss.hpp +++ b/fluss-rust/bindings/cpp/include/fluss.hpp @@ -1607,7 +1607,9 @@ struct Configuration { // Shared write-batch memory budget per Connection, across its tables and writers // (default 64 MiB). Not a process RSS limit or a callback-capture memory budget. size_t writer_buffer_memory_size{64 * 1024 * 1024}; - // Maximum time in milliseconds to block waiting for buffer memory + // Maximum time in milliseconds to block waiting for buffer memory. This also bounds + // the whole callback submission, including callback capacity and buffer backpressure. + // UINT64_MAX waits indefinitely; zero fails fast when capacity or memory is unavailable. uint64_t writer_buffer_wait_timeout_ms{std::numeric_limits::max()}; // Maximum KV backpressure throttle in milliseconds uint64_t writer_kv_backpressure_max_throttle_ms{3000}; diff --git a/fluss-rust/crates/fluss/src/client/write/accumulator.rs b/fluss-rust/crates/fluss/src/client/write/accumulator.rs index 93dd8895001..3a961cdcc86 100644 --- a/fluss-rust/crates/fluss/src/client/write/accumulator.rs +++ b/fluss-rust/crates/fluss/src/client/write/accumulator.rs @@ -78,7 +78,7 @@ impl MemoryLimiter { /// Like [`acquire`], but bounds the wait by `deadline` when provided instead of /// the limiter's configured `wait_timeout`. A deadline already in the past makes /// this non-blocking (fail fast if memory is unavailable), which callers use to - /// keep a submit within a caller-supplied budget (e.g. callback enqueue timeout). + /// keep a submit within a caller-supplied budget (e.g. the callback submission budget). pub fn acquire_within( self: &Arc, size: usize, diff --git a/fluss-rust/website/docs/user-guide/cpp/api-reference.md b/fluss-rust/website/docs/user-guide/cpp/api-reference.md index fae7886b073..4159f0144f6 100644 --- a/fluss-rust/website/docs/user-guide/cpp/api-reference.md +++ b/fluss-rust/website/docs/user-guide/cpp/api-reference.md @@ -303,7 +303,7 @@ auto created = table.NewAppend().CreateWriter(writer, options); |--------|---------|---------| | `max_pending_operations` | `262144` | Positive, per-writer limit on callback operations reserved for submission or not yet finished | -The existing `CreateWriter(writer)` overload uses these defaults. Each callback +The existing `CreateWriter(writer)` overload uses this default. Each callback submission reserves one slot **before** submitting to Rust and holds it through user callback execution and capture cleanup. Submission errors and exceptions return the slot automatically. `Upsert` and `Delete` share their writer's limit; From 6d0fd6244a0da02efe5c743e75b20a8cefd16500 Mon Sep 17 00:00:00 2001 From: naivedogger Date: Tue, 22 Sep 2026 17:57:03 +0800 Subject: [PATCH 18/19] [c++] Address write callback review feedback --- fluss-rust/bindings/cpp/README.md | 24 +- fluss-rust/bindings/cpp/examples/example.cpp | 10 +- fluss-rust/bindings/cpp/include/fluss.hpp | 35 +-- fluss-rust/bindings/cpp/src/lib.rs | 19 ++ fluss-rust/bindings/cpp/src/table.cpp | 28 +-- .../bindings/cpp/src/write_callback.hpp | 17 +- fluss-rust/bindings/cpp/src/write_callback.rs | 57 +++-- .../bindings/cpp/test/test_write_callback.cpp | 213 +++--------------- .../crates/fluss/src/client/table/append.rs | 12 +- .../crates/fluss/src/client/table/upsert.rs | 12 +- .../docs/user-guide/cpp/api-reference.md | 109 ++++----- 11 files changed, 189 insertions(+), 347 deletions(-) diff --git a/fluss-rust/bindings/cpp/README.md b/fluss-rust/bindings/cpp/README.md index 01afa3e0cb0..6bfa63b696f 100644 --- a/fluss-rust/bindings/cpp/README.md +++ b/fluss-rust/bindings/cpp/README.md @@ -76,15 +76,14 @@ not apply to `CreateBucketBatchScanner()`. [C++ API reference](../../website/docs/user-guide/cpp/api-reference.md) and [log-table examples](../../website/docs/user-guide/cpp/example/log-tables.md). -The callback section configures `WriteCallbackOptions` to bound outstanding callback -operations. The SDK executes callbacks; applications do not need a waiting thread or -poll loop. The example uses the default `max_pending_operations = 262144` per Writer. -A callback submit is bounded by `client.writer.buffer.wait-timeout`, covering callback -capacity plus buffer backpressure, so it returns a definite result within that budget -and a zero timeout makes the submit non-blocking. It does not bound ACKs, retries, or -callback duration. - -The callback limit counts operations, not bytes; it does not preallocate 262144 slots. +The SDK executes callbacks on a single shared worker, so they fire in the order writes +complete; applications do not need a waiting thread or poll loop. Each Writer bounds its +outstanding callback operations from the write buffer size, so admission tracks memory +backpressure rather than a separate knob. A callback submit is bounded by +`client.writer.buffer.wait-timeout`, covering callback capacity plus buffer backpressure, +so it returns a definite result within that budget and a zero timeout makes the submit +non-blocking. It does not bound ACKs, retries, or callback duration. + The separate `Configuration::writer_buffer_memory_size` remains 64 MiB by default, shared across all tables and writers on a Connection. Neither setting caps process RSS. For a high-throughput starting configuration, see the @@ -97,9 +96,10 @@ counts and logs outcomes; it does not implement durable recovery. Keep callbacks short, protect shared state, and handle retries outside the callback with an application recovery policy. -After submissions stop, `Flush()` first flushes writes and, on success, waits up to -60 seconds for pending callbacks. This is not a whole-call timeout. If it returns an -error, keep callback state alive; if it succeeds, still check individual write results. +After submissions stop, `Flush()` first flushes writes and, on success, blocks until +pending callbacks finish, acting as a barrier. A callback that never returns hangs it. +If it returns an error, the write flush itself failed, so keep callback state alive; if +it succeeds, still check individual write results. See the [callback guarantees and recovery guidance](../../website/docs/user-guide/cpp/api-reference.md#write-guarantees-and-recovery) for result semantics, callback implementation, and shutdown requirements. diff --git a/fluss-rust/bindings/cpp/examples/example.cpp b/fluss-rust/bindings/cpp/examples/example.cpp index 25b9847049e..fad65bd2ab9 100644 --- a/fluss-rust/bindings/cpp/examples/example.cpp +++ b/fluss-rust/bindings/cpp/examples/example.cpp @@ -96,15 +96,7 @@ int main() { // 5) Write rows with scalar and temporal values fluss::AppendWriter writer; - // Callback admission limit, shown with its default. It bounds the callback overloads - // only; Wait and fire-and-forget writes are unaffected. Passing no options uses this - // same value, so this block is equivalent to CreateWriter(writer). - fluss::WriteCallbackOptions callback_options; - // Pending callback operations per writer. Lower it for large captures or many - // writers; independent of the Connection's write-buffer byte budget. Waiting for a - // free slot is bounded by the connection's writer_buffer_wait_timeout_ms set above. - callback_options.max_pending_operations = 262144; - check("new_append_writer", table.NewAppend().CreateWriter(writer, callback_options)); + check("new_append_writer", table.NewAppend().CreateWriter(writer)); struct RowData { int id; diff --git a/fluss-rust/bindings/cpp/include/fluss.hpp b/fluss-rust/bindings/cpp/include/fluss.hpp index 7120c786bca..befb571a3d2 100644 --- a/fluss-rust/bindings/cpp/include/fluss.hpp +++ b/fluss-rust/bindings/cpp/include/fluss.hpp @@ -555,33 +555,24 @@ struct Result { /// with SDK idempotence enabled. Retain input identifiers and recovery state as /// needed; one callback invocation is not an exactly-once delivery guarantee. /// -/// Callback threads are shared across connections. Do not wait for another -/// callback from a callback: it can exhaust the worker pool. Synchronous SDK -/// calls require exclusive writer access. Callback submissions to a full writer -/// fail immediately when called from a callback, instead of blocking the workers. -/// WriteCallbackOptions bounds outstanding callback operations per writer. +/// Callbacks run on a single shared worker, so they fire in the order writes +/// complete. Do not wait for another callback from a callback: it stalls that +/// worker. Synchronous SDK calls require exclusive writer access. Callback +/// submissions to a full writer fail immediately when called from a callback, +/// instead of blocking the worker. Each writer bounds its outstanding callback +/// operations from the write buffer size, so admission tracks memory backpressure. /// Hand off retries or expensive work without blocking; bound application queues /// and handle overflow without silently discarding failed operations. /// /// Exceptions thrown by callbacks are caught and reported to stderr; they do not /// change the write outcome or retry the callback. Stop submissions before Flush(). -/// After a successful Rust write flush, Flush() waits up to 60 seconds for pending -/// callbacks to finish. On error, referenced state may still be in use. +/// After a successful Rust write flush, Flush() blocks until pending callbacks finish, +/// acting as a barrier, so a callback that never returns hangs it. On error, referenced +/// state may still be in use. /// Inside a callback only the callback wait is skipped; the write flush may block. /// Flush() does not wait for work handed to application workers or retry queues. using WriteCallback = std::function; -/// Admission limits for callback overloads only; Wait and fire-and-forget are unchanged. -struct WriteCallbackOptions { - /// Maximum operations reserved for submission or awaiting callback completion. - /// Must be positive. One AppendArrowBatch call counts as one operation, not its rows. - /// Per-writer count, not preallocated storage or a byte limit. Reduce for many - /// writers or large captures; independent of Configuration::writer_buffer_memory_size. - /// Waiting for a free slot is bounded by client.writer.buffer.wait-timeout, the same - /// budget as the buffer-backpressure wait. - size_t max_pending_operations = 262144; -}; - struct TablePath { std::string database_name; std::string table_name; @@ -1794,8 +1785,6 @@ class TableAppend { TableAppend& operator=(TableAppend&&) noexcept = default; Result CreateWriter(AppendWriter& out); - /// Create a writer with per-writer callback admission limits. - Result CreateWriter(AppendWriter& out, const WriteCallbackOptions& options); private: friend class Table; @@ -1815,8 +1804,6 @@ class TableUpsert { TableUpsert& PartialUpdateByName(std::vector column_names); Result CreateWriter(UpsertWriter& out); - /// Create a writer with per-writer callback admission limits shared by Upsert and Delete. - Result CreateWriter(UpsertWriter& out, const WriteCallbackOptions& options); private: friend class Table; @@ -1959,7 +1946,7 @@ class AppendWriter { /// covering callback capacity and buffer backpressure: within it, Ok means the /// write was accepted and the callback fires exactly once, an error means /// submission failed and no callback runs. A zero timeout makes submission - /// non-blocking. See WriteCallbackOptions. + /// non-blocking. Result Append(const GenericRow& row, WriteCallback callback); Result AppendArrowBatch(const std::shared_ptr& batch); Result AppendArrowBatch(const std::shared_ptr& batch, WriteResult& out); @@ -2006,7 +1993,7 @@ class UpsertWriter { /// covering callback capacity and buffer backpressure: within it, Ok means the /// write was accepted and the callback fires exactly once, an error means /// submission failed and no callback runs. A zero timeout makes submission - /// non-blocking. See WriteCallbackOptions. + /// non-blocking. Result Upsert(const GenericRow& row, WriteCallback callback); Result Delete(const GenericRow& row); Result Delete(const GenericRow& row, WriteResult& out); diff --git a/fluss-rust/bindings/cpp/src/lib.rs b/fluss-rust/bindings/cpp/src/lib.rs index 63693920282..c45c5e005a8 100644 --- a/fluss-rust/bindings/cpp/src/lib.rs +++ b/fluss-rust/bindings/cpp/src/lib.rs @@ -532,6 +532,7 @@ mod ffi { fn get_table_path(self: &Table) -> FfiTablePath; fn has_primary_key(self: &Table) -> bool; fn writer_buffer_wait_timeout_ms(self: &Table) -> u64; + fn estimated_callback_capacity(self: &Table) -> usize; fn create_upsert_writer(self: &Table, column_indices: Vec) -> FfiPtrResult; fn new_lookuper(self: &Table) -> FfiPtrResult; fn new_prefix_lookuper(self: &Table, lookup_column_names: Vec) -> FfiPtrResult; @@ -2091,6 +2092,24 @@ impl Table { self.connection.config().writer_buffer_wait_timeout_ms } + /// Per-writer callback admission limit, derived from the write buffer so it stays consistent + /// with memory backpressure instead of being a separate knob. It approximates how many rows + /// fit in the buffer: buffer bytes / estimated row size. The row estimate uses only the + /// schema's fixed-length part (variable-length payloads are not counted), so the limit is + /// generous and mainly guards against a slow-callback backlog; the buffer-memory wait remains + /// the real backpressure. Floored so tiny buffers still allow pipelining. + fn estimated_callback_capacity(&self) -> usize { + const MIN_CALLBACK_CAPACITY: usize = 1024; + let fields = self.table_info.get_row_type().fields(); + let mut row_size = fcore::row::binary_array::calculate_header_in_bytes(fields.len()); + for field in fields { + row_size += fcore::row::binary_array::calculate_fix_length_part_size(field.data_type()); + } + let row_size = row_size.max(1); + let buffer = self.connection.config().writer_buffer_memory_size; + (buffer / row_size).max(MIN_CALLBACK_CAPACITY) + } + fn create_upsert_writer(&self, column_indices: Vec) -> ffi::FfiPtrResult { let _enter = RUNTIME.enter(); diff --git a/fluss-rust/bindings/cpp/src/table.cpp b/fluss-rust/bindings/cpp/src/table.cpp index 5794cace35e..0fb3b2616fb 100644 --- a/fluss-rust/bindings/cpp/src/table.cpp +++ b/fluss-rust/bindings/cpp/src/table.cpp @@ -1281,20 +1281,12 @@ bool Table::HasPrimaryKey() const { TableAppend::TableAppend(ffi::Table* table) noexcept : table_(table) {} Result TableAppend::CreateWriter(AppendWriter& out) { - return CreateWriter(out, WriteCallbackOptions{}); -} - -Result TableAppend::CreateWriter(AppendWriter& out, const WriteCallbackOptions& options) { - auto validation = ffi::WriteCallbackCapacity::Validate(options); - if (!validation.Ok()) { - return validation; - } if (table_ == nullptr) { return utils::make_client_error("Table not available"); } auto capacity = std::make_shared( - options.max_pending_operations, table_->writer_buffer_wait_timeout_ms()); + table_->estimated_callback_capacity(), table_->writer_buffer_wait_timeout_ms()); auto ffi_result = table_->new_append_writer(); auto result = utils::from_ffi_result(ffi_result.result); if (result.Ok()) { @@ -1349,21 +1341,13 @@ std::vector TableUpsert::ResolveNameProjection() const { } Result TableUpsert::CreateWriter(UpsertWriter& out) { - return CreateWriter(out, WriteCallbackOptions{}); -} - -Result TableUpsert::CreateWriter(UpsertWriter& out, const WriteCallbackOptions& options) { - auto validation = ffi::WriteCallbackCapacity::Validate(options); - if (!validation.Ok()) { - return validation; - } if (table_ == nullptr) { return utils::make_client_error("Table not available"); } try { auto capacity = std::make_shared( - options.max_pending_operations, table_->writer_buffer_wait_timeout_ms()); + table_->estimated_callback_capacity(), table_->writer_buffer_wait_timeout_ms()); auto resolved_indices = !column_names_.empty() ? ResolveNameProjection() : column_indices_; rust::Vec rust_indices; @@ -1785,7 +1769,9 @@ Result AppendWriter::Flush() { auto ffi_result = writer_->flush(); auto result = utils::from_ffi_result(ffi_result); if (!result.Ok()) return result; - return callback_capacity_->AwaitAll(std::chrono::seconds(60)); + // Writes are flushed; block until their callbacks drain so Flush is a real barrier. + callback_capacity_->AwaitAll(); + return {}; } // ============================================================================ @@ -1920,7 +1906,9 @@ Result UpsertWriter::Flush() { auto ffi_result = writer_->upsert_flush(); auto result = utils::from_ffi_result(ffi_result); if (!result.Ok()) return result; - return callback_capacity_->AwaitAll(std::chrono::seconds(60)); + // Writes are flushed; block until their callbacks drain so Flush is a real barrier. + callback_capacity_->AwaitAll(); + return {}; } // ============================================================================ diff --git a/fluss-rust/bindings/cpp/src/write_callback.hpp b/fluss-rust/bindings/cpp/src/write_callback.hpp index 532a26aa0de..c5b46aef9c3 100644 --- a/fluss-rust/bindings/cpp/src/write_callback.hpp +++ b/fluss-rust/bindings/cpp/src/write_callback.hpp @@ -40,13 +40,6 @@ class WriteCallbackCapacity { WriteCallbackCapacity(size_t max_pending_operations, uint64_t wait_timeout_ms) : max_pending_(max_pending_operations), wait_timeout_ms_(wait_timeout_ms) {} - static Result Validate(const WriteCallbackOptions& options) { - if (options.max_pending_operations == 0) { - return {ErrorCode::CLIENT_ERROR, "max_pending_operations must be positive"}; - } - return {}; - } - Result Acquire() { std::unique_lock lock(mutex_); if (pending_ == max_pending_) { @@ -89,6 +82,16 @@ class WriteCallbackCapacity { return {}; } + /// Block until every reserved operation has finished its callback. Used as a flush + /// barrier after the write flush already succeeded, so it has no deadline of its own; + /// a callback that never returns would hang here. Returns immediately when called from + /// within a callback to avoid deadlocking the worker on itself. + void AwaitAll() { + std::unique_lock lock(mutex_); + if (in_callback_) return; + available_.wait(lock, [&] { return pending_ == 0; }); + } + /// Milliseconds left in the client.writer.buffer.wait-timeout budget since `start`, so /// the buffer-backpressure wait plus the capacity reservation stay within one timeout /// (Kafka max.block.ms style). Floored at 0 (0 = fail fast). Returns -1 when the timeout diff --git a/fluss-rust/bindings/cpp/src/write_callback.rs b/fluss-rust/bindings/cpp/src/write_callback.rs index 154ac7d7158..09fc1b8f290 100644 --- a/fluss-rust/bindings/cpp/src/write_callback.rs +++ b/fluss-rust/bindings/cpp/src/write_callback.rs @@ -23,24 +23,20 @@ use std::thread::{self, JoinHandle}; use crate::{RUNTIME, WriteResult, client_err, err_from_core_error, ffi, ok_result}; -const DEFAULT_CALLBACK_WORKERS: usize = 4; -const COMPLETION_BATCH_SIZE: usize = 64; type Completion = Box; -// Process-wide worker count. Override with FLUSS_CALLBACK_WORKERS; invalid or -// zero values fall back to the default. -fn callback_workers() -> usize { - std::env::var("FLUSS_CALLBACK_WORKERS") - .ok() - .and_then(|value| value.trim().parse::().ok()) - .filter(|count| *count > 0) - .unwrap_or(DEFAULT_CALLBACK_WORKERS) -} +// A single worker gives callbacks a global FIFO order that matches server +// completion order, so a batch's callbacks never interleave with another's and +// same-bucket writes report in the order they finished. This mirrors the Java +// client, whose completion stages run on the one Sender thread. The future path +// to more parallelism is sharding the executor by writer or bucket key, which +// keeps per-key order while still using CallbackExecutor::new. +const CALLBACK_WORKERS: usize = 1; // Like RUNTIME, the executor is process-wide and lives until process exit. // Initialize it on the submitting thread, not an async I/O worker. static CALLBACK_EXECUTOR: LazyLock> = - LazyLock::new(|| match CallbackExecutor::new(callback_workers()) { + LazyLock::new(|| match CallbackExecutor::new(CALLBACK_WORKERS) { Ok(executor) => Some(executor), Err(error) => { // The write has already been accepted. Keep the old dispatch path @@ -72,9 +68,9 @@ impl CallbackExecutor { .spawn(move || { loop { let completion = { - // Each job already contains up to 64 callbacks. - // Do not prefetch 64 jobs here: that would let a - // worker hoard up to 4096 callbacks. + // Take one job at a time; a job runs a whole + // completed batch. Do not prefetch, so no worker + // hoards queued batches. let receiver = receiver.lock().unwrap(); let Ok(completion) = receiver.recv() else { break; @@ -163,10 +159,9 @@ fn dispatch_write( } fn dispatch_batch(batch: fluss::client::WriteCallbackBatch) { - let executor = CALLBACK_EXECUTOR.as_ref(); - for chunk in batch.into_chunks(COMPLETION_BATCH_SIZE) { - deliver(executor, Box::new(move || chunk.run())); - } + // Deliver the whole batch as one job so its callbacks stay together and the + // single worker runs them in completion order. + deliver(CALLBACK_EXECUTOR.as_ref(), Box::new(move || batch.run())); } fn to_ffi_result(result: Result<(), fluss::error::Error>) -> ffi::FfiResult { @@ -385,6 +380,30 @@ mod tests { assert_eq!(Arc::strong_count(&completed), 1); } + #[test] + fn test_single_worker_runs_callbacks_in_enqueue_order() { + // A single worker must not reorder callbacks. This is the executor-level + // guarantee behind same-bucket completions reporting in the order they + // finished: the results come out exactly as enqueued, not just once each. + let executor = CallbackExecutor::new(1).unwrap(); + let (tx, rx) = mpsc::channel(); + for index in 0..1000 { + let tx = tx.clone(); + assert!( + executor + .enqueue(Box::new(move || { + tx.send(index).unwrap(); + })) + .is_ok() + ); + } + drop(tx); + let observed: Vec<_> = (0..1000) + .map(|_| rx.recv_timeout(Duration::from_secs(10)).unwrap()) + .collect(); + assert_eq!(observed, (0..1000).collect::>()); + } + #[test] fn test_concurrent_producers_complete_each_job_once() { let executor = Arc::new(CallbackExecutor::new(4).unwrap()); diff --git a/fluss-rust/bindings/cpp/test/test_write_callback.cpp b/fluss-rust/bindings/cpp/test/test_write_callback.cpp index f874f4cc8f7..7c02c252397 100644 --- a/fluss-rust/bindings/cpp/test/test_write_callback.cpp +++ b/fluss-rust/bindings/cpp/test/test_write_callback.cpp @@ -108,18 +108,6 @@ class WriteCallbackTest : public ::testing::Test { ASSERT_OK(result); } - // Reopen table_ through a dedicated connection whose client.writer.buffer.wait-timeout - // bounds callback submission, so a full-capacity wait fails with a definite timeout - // instead of blocking on the shared connection's unbounded default. - void UseWriterBufferWaitTimeout(uint64_t wait_timeout_ms) { - auto& env = *fluss_test::FlussTestEnvironment::Instance(); - fluss::Configuration config; - config.bootstrap_servers = env.GetBootstrapServers(); - config.writer_buffer_wait_timeout_ms = wait_timeout_ms; - ASSERT_OK(fluss::Connection::Create(config, connection_)); - ASSERT_OK(connection_.GetTable(table_path_, table_)); - } - fluss::GenericRow Row(int32_t id = 1) { fluss::GenericRow row(2); row.SetInt32(0, id); @@ -127,8 +115,6 @@ class WriteCallbackTest : public ::testing::Test { return row; } - // Declared before table_ so the table (which references it) is destroyed first. - fluss::Connection connection_; fluss::TablePath table_path_; fluss::Table table_; }; @@ -285,7 +271,7 @@ TEST_F(WriteCallbackTest, MultipleOutstandingWritesEachNotifyOnce) { TEST_F(WriteCallbackTest, RejectedSubmissionDoesNotInvokeCallback) { CreateTable(); fluss::AppendWriter writer; - ASSERT_OK(table_.NewAppend().CreateWriter(writer, fluss::WriteCallbackOptions{1})); + ASSERT_OK(table_.NewAppend().CreateWriter(writer)); auto completion = std::make_shared(); auto lifetime = std::make_shared(); auto released = lifetime->released.get_future(); @@ -304,7 +290,7 @@ TEST_F(WriteCallbackTest, RejectedSubmissionDoesNotInvokeCallback) { }); EXPECT_FALSE(result.Ok()); EXPECT_TRUE(completion->Results().empty()); - // Both failed submissions must return the only slot. + // Both failed submissions must not register a callback. ASSERT_OK(writer.Append(Row(), [completion](fluss::Result completed) { completion->Record(std::move(completed)); })); @@ -312,160 +298,38 @@ TEST_F(WriteCallbackTest, RejectedSubmissionDoesNotInvokeCallback) { ASSERT_OK(writer.Flush()); } -TEST_F(WriteCallbackTest, ArrowBatchCapacitySurvivesMovesAndDoesNotAffectWaitOrFlush) { - CreateTable(); - UseWriterBufferWaitTimeout(250); - fluss::AppendWriter writer; - ASSERT_OK(table_.NewAppend().CreateWriter(writer, fluss::WriteCallbackOptions{1})); - arrow::Int32Builder ids; - arrow::StringBuilder values; - ASSERT_TRUE(ids.AppendValues({1, 2, 3}).ok()); - ASSERT_TRUE(values.AppendValues({"a", "b", "c"}).ok()); - auto batch = arrow::RecordBatch::Make( - arrow::schema({arrow::field("id", arrow::int32()), arrow::field("value", arrow::utf8())}), - 3, {ids.Finish().ValueOrDie(), values.Finish().ValueOrDie()}); - auto started = std::make_shared(); - auto gate = std::make_shared>(); - auto resume = gate->get_future().share(); - ASSERT_OK(writer.AppendArrowBatch(batch, [started, resume](fluss::Result result) { - started->Record(std::move(result)); - resume.wait_for(std::chrono::seconds(20)); - })); - ASSERT_TRUE(started->Await()); // The three-row batch fits in one slot. - - fluss::AppendWriter moved(std::move(writer)); - writer = std::move(moved); - EXPECT_FALSE(moved.Available()); - auto rejected = std::make_shared(); - auto callback = [rejected](fluss::Result result) { rejected->Record(std::move(result)); }; - auto row_result = writer.Append(Row(4), callback); - auto batch_result = writer.AppendArrowBatch(batch, callback); - EXPECT_NE(row_result.error_message.find("Timed out"), std::string::npos); - EXPECT_NE(batch_result.error_message.find("Timed out"), std::string::npos); - // Independent writers do not share the capacity limit. - fluss::AppendWriter independent; - ASSERT_OK(table_.NewAppend().CreateWriter(independent)); - auto other = std::make_shared(); - ASSERT_OK(independent.Append( - Row(5), [other](fluss::Result result) { other->Record(std::move(result)); })); - ASSERT_TRUE(other->Await()); - - fluss::WriteResult pending; - ASSERT_OK(writer.Append(Row(6), pending)); - ASSERT_OK(pending.Wait()); - ASSERT_OK(writer.Append(Row(7))); - // Release the gate so the first callback finishes and returns capacity. - gate->set_value(); - ASSERT_OK(writer.Flush()); - EXPECT_TRUE(rejected->Results().empty()); - // Capacity is now available after the first callback completed. - ASSERT_OK(writer.Append(Row(8), callback)); - ASSERT_TRUE(rejected->Await()); - ASSERT_OK(writer.Flush()); - EXPECT_EQ(rejected->Results().size(), 1u); -} - -// End-to-end through the public writer API: when the writer's capacity is exhausted, a -// callback submission must return within the connection's client.writer.buffer.wait-timeout -// rather than blocking on the occupied slot until the running callback releases it. -TEST_F(WriteCallbackTest, CapacityFullSubmissionReturnsWithinWaitTimeout) { +TEST_F(WriteCallbackTest, SameBucketCallbacksFireInSubmissionOrder) { CreateTable(); - constexpr uint64_t wait_timeout_ms = 200; - UseWriterBufferWaitTimeout(wait_timeout_ms); fluss::AppendWriter writer; - ASSERT_OK(table_.NewAppend().CreateWriter(writer, fluss::WriteCallbackOptions{1})); - - // Occupy the single slot with a callback that blocks until released, so the next - // submission has to wait for capacity instead of completing normally. - auto started = std::make_shared(); - auto gate = std::make_shared>(); - auto resume = gate->get_future().share(); - ASSERT_OK(writer.Append(Row(), [started, resume](fluss::Result result) { - started->Record(std::move(result)); - resume.wait_for(std::chrono::seconds(20)); - })); - ASSERT_TRUE(started->Await()); - - auto rejected = std::make_shared(); - auto start = std::chrono::steady_clock::now(); - auto result = writer.Append( - Row(2), [rejected](fluss::Result completed) { rejected->Record(std::move(completed)); }); - auto elapsed = std::chrono::steady_clock::now() - start; - EXPECT_FALSE(result.Ok()); - EXPECT_NE(result.error_message.find("Timed out"), std::string::npos); - // Returned because the shared budget elapsed: not immediately, and well before the - // 20-second callback block that would otherwise gate the slot. - EXPECT_GE(elapsed, std::chrono::milliseconds(wait_timeout_ms)); - EXPECT_LT(elapsed, std::chrono::seconds(5)); - EXPECT_TRUE(rejected->Results().empty()); - - gate->set_value(); - ASSERT_OK(writer.Flush()); -} - -TEST_F(WriteCallbackTest, UpsertAndDeleteShareCapacityAndReturnItOnSubmissionErrors) { - CreateTable(true); - UseWriterBufferWaitTimeout(250); - fluss::UpsertWriter writer; - ASSERT_OK(table_.NewUpsert().CreateWriter(writer, fluss::WriteCallbackOptions{1})); + ASSERT_OK(table_.NewAppend().CreateWriter(writer)); auto completion = std::make_shared(); - auto callback = [completion](fluss::Result result) { completion->Record(std::move(result)); }; - fluss::GenericRow invalid(2); - invalid.SetString(0, "not an integer primary key"); - invalid.SetString(1, "value"); - EXPECT_FALSE(writer.Upsert(invalid, callback).Ok()); - EXPECT_FALSE(writer.Delete(invalid, callback).Ok()); - EXPECT_TRUE(completion->Results().empty()); - - auto gate = std::make_shared>(); - auto resume = gate->get_future().share(); - ASSERT_OK(writer.Upsert(Row(), [completion, resume](fluss::Result result) { - completion->Record(std::move(result)); - resume.wait_for(std::chrono::seconds(20)); - })); - ASSERT_TRUE(completion->Await()); - fluss::UpsertWriter moved(std::move(writer)); - writer = std::move(moved); - EXPECT_FALSE(moved.Available()); - EXPECT_NE(writer.Upsert(Row(2), callback).error_message.find("Timed out"), std::string::npos); - EXPECT_NE(writer.Delete(Row(), callback).error_message.find("Timed out"), std::string::npos); - gate->set_value(); - ASSERT_OK(writer.Flush()); - ASSERT_OK(writer.Delete(Row(), callback)); - ASSERT_TRUE(completion->Await(2)); + auto order_mutex = std::make_shared(); + auto order = std::make_shared>(); + // A shared id keeps every record on one bucket, so completion order must + // match submission order. The single callback worker must not reorder them. + constexpr int32_t kWrites = 128; + for (int32_t index = 0; index < kWrites; ++index) { + ASSERT_OK(writer.Append( + Row(7), [completion, order_mutex, order, index](fluss::Result result) { + { + std::lock_guard lock(*order_mutex); + order->push_back(index); + } + completion->Record(std::move(result)); + })); + } + ASSERT_TRUE(completion->Await(kWrites)); ASSERT_OK(writer.Flush()); - EXPECT_EQ(completion->Results().size(), 2u); -} - -TEST_F(WriteCallbackTest, CallbackDoesNotWaitForCapacityOnAnotherWriter) { - CreateTable(); - auto full_writer = std::make_shared(); - ASSERT_OK(table_.NewAppend().CreateWriter(*full_writer, fluss::WriteCallbackOptions{1})); - auto started = std::make_shared(); - auto gate = std::make_shared>(); - auto resume = gate->get_future().share(); - ASSERT_OK(full_writer->Append(Row(), [started, resume](fluss::Result result) { - started->Record(std::move(result)); - resume.wait_for(std::chrono::seconds(20)); - })); - ASSERT_TRUE(started->Await()); - fluss::AppendWriter other; - ASSERT_OK(table_.NewAppend().CreateWriter(other)); - auto attempted = std::make_shared(); - auto unexpected = std::make_shared(); - auto row = std::make_shared(Row(2)); - ASSERT_OK(other.Append(Row(3), [full_writer, row, attempted, unexpected](fluss::Result) { - // No submitting thread accesses full_writer concurrently with this callback. - auto result = full_writer->Append(*row, [unexpected](fluss::Result completed) { - unexpected->Record(std::move(completed)); - }); - attempted->Record(std::move(result)); - })); - ASSERT_TRUE(attempted->Await()); - EXPECT_EQ(attempted->Results().front().error_message, "Write callback capacity is full"); - EXPECT_TRUE(unexpected->Results().empty()); - gate->set_value(); - ASSERT_OK(other.Flush()); + auto results = completion->Results(); + ASSERT_EQ(results.size(), static_cast(kWrites)); + for (const auto& result : results) { + EXPECT_OK(result); + } + std::lock_guard lock(*order_mutex); + ASSERT_EQ(order->size(), static_cast(kWrites)); + for (int32_t index = 0; index < kWrites; ++index) { + EXPECT_EQ((*order)[index], index); + } } TEST_F(WriteCallbackTest, BatchedCallbacksSurviveExceptionsAndCoexistWithWait) { @@ -595,23 +459,6 @@ TEST(WriteCallbackBridgeTest, ContainsCallbackExceptions) { EXPECT_NO_THROW(unknown.Complete(0, "")); } -TEST(WriteCallbackBridgeTest, ValidatesCallbackOptions) { - fluss::WriteCallbackOptions options; - EXPECT_EQ(options.max_pending_operations, 262144u); - EXPECT_OK(fluss::ffi::WriteCallbackCapacity::Validate(options)); - options.max_pending_operations = 0; - EXPECT_FALSE(fluss::ffi::WriteCallbackCapacity::Validate(options).Ok()); - fluss::Table table; - fluss::AppendWriter append; - fluss::UpsertWriter upsert; - EXPECT_EQ(table.NewAppend().CreateWriter(append, options).error_message, - "max_pending_operations must be positive"); - EXPECT_EQ(table.NewUpsert().CreateWriter(upsert, options).error_message, - "max_pending_operations must be positive"); - options.max_pending_operations = 1; - EXPECT_OK(fluss::ffi::WriteCallbackCapacity::Validate(options)); -} - TEST(WriteCallbackBridgeTest, CapacityRejectsOverflowWithoutDroppingReservations) { constexpr size_t max_pending_operations = 3; fluss::ffi::WriteCallbackCapacity capacity(max_pending_operations, 0); diff --git a/fluss-rust/crates/fluss/src/client/table/append.rs b/fluss-rust/crates/fluss/src/client/table/append.rs index 325fb98e98c..cc6835107ac 100644 --- a/fluss-rust/crates/fluss/src/client/table/append.rs +++ b/fluss-rust/crates/fluss/src/client/table/append.rs @@ -141,9 +141,10 @@ impl AppendWriter { self.append_with_deadline(row, None) } - /// Like [`Self::append`], but bounds the buffer-memory wait by `deadline`. A deadline - /// already in the past makes the submit fail fast when the buffer is full, which - /// lets a caller keep the whole submit within a fixed budget. + /// Internal: bounds the buffer-memory wait by `deadline` so a language binding can keep + /// a whole callback submission within one `client.writer.buffer.wait-timeout` budget. Not a + /// public deadline API; use [`Self::append`], which passes `None` and defers to the config. + #[doc(hidden)] pub fn append_with_deadline( &self, row: &R, @@ -187,7 +188,10 @@ impl AppendWriter { self.append_arrow_batch_with_deadline(batch, None) } - /// Like [`Self::append_arrow_batch`], but bounds the buffer-memory wait by `deadline`. + /// Internal: bounds the buffer-memory wait by `deadline` so a language binding can keep + /// a whole callback submission within one `client.writer.buffer.wait-timeout` budget. Not a + /// public deadline API; use [`Self::append_arrow_batch`], which passes `None`. + #[doc(hidden)] pub fn append_arrow_batch_with_deadline( &self, batch: RecordBatch, diff --git a/fluss-rust/crates/fluss/src/client/table/upsert.rs b/fluss-rust/crates/fluss/src/client/table/upsert.rs index fb7af2c08d9..226fb2c0ab0 100644 --- a/fluss-rust/crates/fluss/src/client/table/upsert.rs +++ b/fluss-rust/crates/fluss/src/client/table/upsert.rs @@ -351,8 +351,10 @@ impl UpsertWriter { self.upsert_with_deadline(row, None) } - /// Like [`Self::upsert`], but bounds the buffer-memory wait by `deadline`. A deadline - /// already in the past makes the submit fail fast when the buffer is full. + /// Internal: bounds the buffer-memory wait by `deadline` so a language binding can keep + /// a whole callback submission within one `client.writer.buffer.wait-timeout` budget. Not a + /// public deadline API; use [`Self::upsert`], which passes `None` and defers to the config. + #[doc(hidden)] pub fn upsert_with_deadline( &self, row: &R, @@ -403,8 +405,10 @@ impl UpsertWriter { self.delete_with_deadline(row, None) } - /// Like [`Self::delete`], but bounds the buffer-memory wait by `deadline`. A deadline - /// already in the past makes the submit fail fast when the buffer is full. + /// Internal: bounds the buffer-memory wait by `deadline` so a language binding can keep + /// a whole callback submission within one `client.writer.buffer.wait-timeout` budget. Not a + /// public deadline API; use [`Self::delete`], which passes `None` and defers to the config. + #[doc(hidden)] pub fn delete_with_deadline( &self, row: &R, diff --git a/fluss-rust/website/docs/user-guide/cpp/api-reference.md b/fluss-rust/website/docs/user-guide/cpp/api-reference.md index 4159f0144f6..0d65b0a6844 100644 --- a/fluss-rust/website/docs/user-guide/cpp/api-reference.md +++ b/fluss-rust/website/docs/user-guide/cpp/api-reference.md @@ -132,7 +132,6 @@ Complete API reference for the Fluss C++ client. | Method | Description | |----------------------------------------------|-------------------------| | `CreateWriter(AppendWriter& out) -> Result` | Create an append writer | -| `CreateWriter(AppendWriter& out, const WriteCallbackOptions& options) -> Result` | Create a writer with callback admission limits | ## `TableUpsert` @@ -141,7 +140,6 @@ Complete API reference for the Fluss C++ client. | `PartialUpdateByIndex(std::vector column_indices) -> TableUpsert&` | Configure partial update by column indices | | `PartialUpdateByName(std::vector column_names) -> TableUpsert&` | Configure partial update by column names | | `CreateWriter(UpsertWriter& out) -> Result` | Create an upsert writer | -| `CreateWriter(UpsertWriter& out, const WriteCallbackOptions& options) -> Result` | Create a writer with callback admission limits | ## `TableLookup` @@ -245,9 +243,8 @@ final success or failure, subject to the lifetime and shutdown requirements belo Submission does not wait for acknowledgment, but may still wait for callback capacity and then for Rust writer buffer space under backpressure. -Callbacks run on a small shared executor pool, four threads by default. That -count is process wide and rarely needs changing; the advanced -`FLUSS_CALLBACK_WORKERS` environment variable can override it. Keep each callback +Callbacks run on a single shared worker, process wide, so they fire in the order +writes complete. Keep each callback short and non-blocking. Do not call `Flush()` or `WriteResult::Wait()` inside a callback, and do not retry synchronously there. Record the outcome and run any retry in application logic outside the callback. @@ -291,23 +288,18 @@ operational limits below before retrying a batch. ### Callback capacity ```cpp -fluss::WriteCallbackOptions options; -// This is the default; CreateWriter(writer) also uses it. -options.max_pending_operations = 262144; fluss::AppendWriter writer; -auto created = table.NewAppend().CreateWriter(writer, options); -// Check created before using writer. NewUpsert().CreateWriter accepts the same options. +auto created = table.NewAppend().CreateWriter(writer); +// Check created before using writer. NewUpsert().CreateWriter behaves the same. ``` -| Option | Default | Meaning | -|--------|---------|---------| -| `max_pending_operations` | `262144` | Positive, per-writer limit on callback operations reserved for submission or not yet finished | - -The existing `CreateWriter(writer)` overload uses this default. Each callback -submission reserves one slot **before** submitting to Rust and holds it through -user callback execution and capture cleanup. Submission errors and exceptions -return the slot automatically. `Upsert` and `Delete` share their writer's limit; -`AppendArrowBatch` consumes one slot per call, regardless of row count. Moving +Each writer bounds its outstanding callback operations automatically, derived from +`writer_buffer_memory_size` divided by an estimate of the table's row size, so callback +admission tracks the same memory backpressure as the write buffer instead of being a +separate knob. Each callback submission reserves one slot **before** submitting to Rust +and holds it through user callback execution and capture cleanup. Submission errors and +exceptions return the slot automatically. `Upsert` and `Delete` share their writer's +limit; `AppendArrowBatch` consumes one slot per call, regardless of row count. Moving a writer transfers its capacity state; already accepted callbacks retain it independently of the writer's lifetime. @@ -327,36 +319,14 @@ not cover network requests, core retries, or callback duration, and does not can any accepted write. Do not hold a mutex needed by callbacks while submitting: a full writer can wait for those callbacks to finish. -### Sizing callback capacity and write buffers +### Sizing write buffers -There are two independent budgets: +Callback admission is derived from the write buffer, so there is one budget to size: | Setting | Scope | What it limits | |---------|-------|----------------| -| `max_pending_operations` | Each Writer | Callback operations from admission through callback execution and capture cleanup | | `writer_buffer_memory_size` | Each Connection | Rust write-batch memory accounting shared by all its tables and writers, including writes using `Wait()` or fire-and-forget | -The callback default of 262144 operations with a 30-second submission timeout was -used in an eight-hour load test with 1 KiB rows, short callbacks, and five writers -per Connection. It is a starting point, not a throughput or latency guarantee. -The limit is not preallocated storage. It allows four times as many outstanding -operations as the previous 65536 default, so applications with tighter memory -budgets should explicitly select a smaller value. - -Budget callback capacity across **all writers**, including writers for different -tables. Five writers at the default allow 1310720 operations in total; fifty allow -13107200. If each outstanding operation retains 1 KiB of application data, those -limits permit roughly 1.25 GiB and 12.5 GiB of captures alone, before SDK overhead. -An `AppendArrowBatch` counts as one operation even when its batch contains many -rows, so large batches need a separate application byte budget. - -For callback capacity, estimate each writer's operation rate multiplied by the -time from admission until its callback finishes, then allow headroom for bursts -and tail latency **within the process memory budget**. Measure capture sizes too. -Use smaller limits for many tables, large captures, or tight memory budgets. -If callbacks are slow, shorten or offload their work before increasing capacity; -a larger queue does not fix a sustained completion-rate deficit. - The Connection buffer accounts for write batches, not callback captures, application input, or application retry queues. Its accounting is not a precise bound on actual Arrow builder allocations or process RSS. @@ -364,8 +334,16 @@ Sharing a Connection shares this budget: do not multiply it by the number of writers, but expect busy or stalled tables to compete for it. Separate Connections have separate budgets, which must be added when sizing the process or host. -The following explicit settings were used with the callback defaults in that -high-throughput test. They are **not** new Connection defaults: +Because each writer's callback admission scales with this buffer, a larger buffer also +allows more outstanding callback operations. If each outstanding operation retains +application data, account for that alongside the buffer when sizing process memory. +An `AppendArrowBatch` counts as one operation even when its batch contains many rows, +so large batches need a separate application byte budget. If callbacks are slow, +shorten or offload their work rather than relying on a larger buffer; a larger queue +does not fix a sustained completion-rate deficit. + +The following explicit settings are a tested high-throughput starting point. They are +**not** new Connection defaults: ```cpp fluss::Configuration config; @@ -376,7 +354,7 @@ config.writer_batch_size = 2 * 1024 * 1024; config.writer_dynamic_batch_size_min = 1024 * 1024; config.writer_batch_timeout_ms = 100; config.writer_request_max_size = 32 * 1024 * 1024; -// Apply config when creating the Connection; use default callback options per Writer. +// Apply config when creating the Connection. ``` Keep the 64 MiB Connection default for a small workload unless measurements show @@ -408,11 +386,12 @@ Implement the callback's application logic; the SDK supplies the execution threads. There is no need to create a waiting thread, call `Wait()`, or poll for callback delivery. -The SDK takes ownership of the callback and its captures. Callbacks run on -background callback threads, may execute concurrently and out of submission -order, and may start before the submitting call returns. Keep callbacks short; -synchronize access to shared state and keep captured references valid until the -callback finishes. Callback overloads do not make writers safe for concurrent +The SDK takes ownership of the callback and its captures. Callbacks run on a single +shared background worker in the order writes complete, and may start before the +submitting call returns. Keep callbacks short; synchronize access to shared state and +keep captured references valid until the callback finishes. A callback that blocks +stalls the worker for every writer, so do not wait for other callbacks from inside one. +Callback overloads do not make writers safe for concurrent access: serialize access if both the caller and a callback use the same writer. Prefer capturing `std::shared_ptr` by value when sharing application state. Keep the connection alive until outstanding operations @@ -450,24 +429,24 @@ This is not a byte or process-wide memory limit: capture sizes, batch sizes, the number of writers, and application-owned retry queues need separate controls. An aggregate ArrowBatch callback may report an error while later internal batches are still pending, so its capacity slot does not bound all underlying batch memory. -Do not wait for another callback from within a callback, since all callback -workers could become occupied. A callback submission from within any SDK write +Do not wait for another callback from within a callback, since the single callback +worker would be occupied. A callback submission from within any SDK write callback fails immediately if its target writer's capacity is full, regardless -of `client.writer.buffer.wait-timeout`, to avoid blocking the shared workers on their own capacity. -This applies across writers and on the fallback executor as well. It does not +of `client.writer.buffer.wait-timeout`, to avoid blocking the worker on its own capacity. +This applies across writers as well. It does not remove Rust buffer waits or make arbitrary blocking SDK calls deadlock-free. -Exclusive writer access is still required. If dedicated workers cannot be +Exclusive writer access is still required. If a dedicated worker cannot be initialized, the SDK falls back to its runtime blocking pool to preserve delivery. For shutdown, stop and join submitting threads, then call `Flush()` outside a -callback. It first runs the Rust write flush and, if that succeeds, waits up to -60 seconds for this writer's pending callbacks and captures to finish. The -60-second callback wait is not an end-to-end timeout for the entire call. +callback. It first runs the Rust write flush and, if that succeeds, blocks until +this writer's pending callbacks and captures finish, acting as a barrier. A callback +that never returns hangs it. Check individual callback results as well: successful flushing is not a summary that every submitted operation succeeded. -If the write flush fails or the callback wait times out, callbacks may still be -pending; do not release their referenced state. A timeout does not cancel them. +If the write flush fails, `Flush()` returns that error and callbacks may still be +pending; do not release their referenced state. `Flush()` does not wait for work handed to application workers or retry queues; those need their own shutdown handling. When called inside a callback, only the callback-wait phase is skipped; the Rust write flush can still block. Do not use @@ -497,10 +476,10 @@ this path as a shutdown barrier. does not imply that no rows were written. A submission error can also follow partial acceptance of a multi-bucket batch; in that case no callback is registered, so handle the returned error and do not assume an all-or-nothing retry. -- The four-worker count and 64-callback job size are implementation details, - not ordering or latency guarantees. A slow callback delays other callbacks in - its job, and slow callbacks from one connection can delay another connection. - The blocking-pool fallback is not limited to four callback threads. +- The single-worker execution and whole-batch job dispatch are implementation details, + not latency guarantees. A slow callback delays the callbacks queued behind it, and + slow callbacks from one connection can delay another connection. + The blocking-pool fallback preserves delivery but is not a second ordered worker. - Callback delivery is in memory only. There is no end-to-end callback deadline or durable recovery of pending notifications. Connection or writer destruction is not a callback-drain barrier; process exit, From cb7258a3e4634b47016566131bc6f7cca5c5c2d5 Mon Sep 17 00:00:00 2001 From: naivedogger Date: Tue, 22 Sep 2026 22:01:16 +0800 Subject: [PATCH 19/19] [c++] Improve write callback ordering and backpressure --- fluss-rust/bindings/cpp/README.md | 35 +-- fluss-rust/bindings/cpp/examples/example.cpp | 80 ++++--- fluss-rust/bindings/cpp/include/fluss.hpp | 59 +++-- fluss-rust/bindings/cpp/src/lib.rs | 23 +- fluss-rust/bindings/cpp/src/table.cpp | 42 +++- .../bindings/cpp/src/write_callback.hpp | 23 +- fluss-rust/bindings/cpp/src/write_callback.rs | 211 ++++++++---------- .../bindings/cpp/test/test_write_callback.cpp | 209 ++++++++++++----- .../test/test_write_callback_allocation.cpp | 10 +- .../crates/fluss/src/client/table/append.rs | 8 +- .../crates/fluss/src/client/table/upsert.rs | 8 +- .../fluss/src/client/write/accumulator.rs | 2 +- .../fluss/src/client/write/broadcast.rs | 168 +++++++++----- .../docs/user-guide/cpp/api-reference.md | 116 ++++++---- 14 files changed, 598 insertions(+), 396 deletions(-) diff --git a/fluss-rust/bindings/cpp/README.md b/fluss-rust/bindings/cpp/README.md index 6bfa63b696f..00dc11a1704 100644 --- a/fluss-rust/bindings/cpp/README.md +++ b/fluss-rust/bindings/cpp/README.md @@ -76,19 +76,25 @@ not apply to `CreateBucketBatchScanner()`. [C++ API reference](../../website/docs/user-guide/cpp/api-reference.md) and [log-table examples](../../website/docs/user-guide/cpp/example/log-tables.md). -The SDK executes callbacks on a single shared worker, so they fire in the order writes -complete; applications do not need a waiting thread or poll loop. Each Writer bounds its -outstanding callback operations from the write buffer size, so admission tracks memory -backpressure rather than a separate knob. A callback submit is bounded by -`client.writer.buffer.wait-timeout`, covering callback capacity plus buffer backpressure, -so it returns a definite result within that budget and a zero timeout makes the submit -non-blocking. It does not bound ACKs, retries, or callback duration. - -The separate `Configuration::writer_buffer_memory_size` remains 64 MiB by default, -shared across all tables and writers on a Connection. Neither setting caps process RSS. -For a high-throughput starting configuration, see the -[buffer sizing guidance](../../website/docs/user-guide/cpp/api-reference.md#sizing-callback-capacity-and-write-buffers), -including a 512 MiB per-Connection example and how to budget for multiple writers. +The SDK executes `WriteCallback` (`void(const WriteCompletion&)`) on one shared +worker, serially in dispatch order and off the I/O threads. `WriteCompletion.result` +is the write outcome; copy it before passing it to another worker. +`CreateWriter(writer)` uses the default `WriteCallbackOptions`; the overload +`CreateWriter(writer, options)` accepts a positive `max_pending_operations` limit +(default 262144) per writer. This operation-count budget is independent of the +Connection's byte-counted write buffer. Slow callbacks can fill it even when the +write buffer has room. The Rust write-buffer permit is released when the batch +completes, before the user callback returns, but the callback object and its +captures remain retained until callback completion. Once the per-writer callback +limit is full, callback-based submissions wait or fail according to +`client.writer.buffer.wait-timeout`; its default is unbounded. + +Callback-capacity and buffer waits share `client.writer.buffer.wait-timeout`. +Zero makes those waits fail fast; this is not a deadline for the entire API call, +ACKs, retries, or callback execution. See the +[buffer sizing guidance](../../website/docs/user-guide/cpp/api-reference.md#sizing-callback-capacity-and-write-buffers) +for independent capacity and byte budgets. Callback worker initialization failure +rejects the submission before any data is accepted; there is no parallel fallback. A failed callback does not prove that the record was not written. Application resubmission can duplicate it, even with SDK idempotence enabled. The example only @@ -98,7 +104,8 @@ application recovery policy. After submissions stop, `Flush()` first flushes writes and, on success, blocks until pending callbacks finish, acting as a barrier. A callback that never returns hangs it. -If it returns an error, the write flush itself failed, so keep callback state alive; if +Calling it from a write callback is rejected before flushing. If a write flush +returns an error, keep callback state alive; if it succeeds, still check individual write results. See the [callback guarantees and recovery guidance](../../website/docs/user-guide/cpp/api-reference.md#write-guarantees-and-recovery) for result semantics, callback implementation, and shutdown requirements. diff --git a/fluss-rust/bindings/cpp/examples/example.cpp b/fluss-rust/bindings/cpp/examples/example.cpp index fad65bd2ab9..08e77a64a67 100644 --- a/fluss-rust/bindings/cpp/examples/example.cpp +++ b/fluss-rust/bindings/cpp/examples/example.cpp @@ -22,8 +22,6 @@ #include #include -#include -#include #include #include #include @@ -45,10 +43,9 @@ int main() { // 1) Connect fluss::Configuration config; config.bootstrap_servers = "127.0.0.1:9123"; - // Bounds how long a write blocks when the shared write buffer is full, and also caps - // the whole callback submission (callback capacity plus buffer backpressure). The - // default UINT64_MAX waits indefinitely; a finite value makes overloaded writes and - // callback submits return with an error instead of blocking. Zero fails fast. + // Callback capacity and buffer waits share this budget. The default UINT64_MAX + // waits indefinitely; a finite value allows handling overload instead. Zero fails + // fast when either resource is unavailable. This is not a whole-call deadline. config.writer_buffer_wait_timeout_ms = 30000; fluss::Connection conn; @@ -96,7 +93,11 @@ int main() { // 5) Write rows with scalar and temporal values fluss::AppendWriter writer; - check("new_append_writer", table.NewAppend().CreateWriter(writer)); + fluss::WriteCallbackOptions callback_options; + // Per writer, independent of config.writer_buffer_memory_size (per Connection). + // CreateWriter(writer) without options uses the default 262144-operation limit. + callback_options.max_pending_operations = 4096; + check("new_append_writer", table.NewAppend().CreateWriter(writer, callback_options)); struct RowData { int id; @@ -157,11 +158,19 @@ int main() { // Callback acknowledgment { // The SDK runs callbacks; no application waiting thread is required. - // Callbacks run on a small shared executor pool, so keep them short and + // Callbacks run on one shared worker, so keep them short and // non-blocking: do not Flush/Wait or retry synchronously inside a callback. // This example counts outcomes only. It does not implement durable recovery. - std::atomic succeeded{0}; - std::atomic failed{0}; + struct CallbackState { + std::atomic succeeded{0}; + std::atomic failed{0}; + std::mutex mutex; + int32_t first_failed_id{0}; + fluss::Result first_failure; + }; + // Shared ownership also keeps state alive if submission throws or flushing fails. + auto state = std::make_shared(); + bool submission_failed = false; for (const auto& r : rows) { const int32_t id = 1000 + r.id; fluss::GenericRow row; @@ -173,37 +182,44 @@ int main() { row.SetTime(5, r.time); row.SetTimestampNtz(6, r.ts_ntz); row.SetTimestampLtz(7, r.ts_ltz); - auto submitted = writer.Append(row, [id, &succeeded, &failed](fluss::Result result) { - if (result.Ok()) { - ++succeeded; - } else { - // By now the SDK has exhausted internal retries or hit a - // non-retriable error, so do not retry synchronously here. - // An error does not prove the row was not written, and a new - // Append can duplicate it even with SDK idempotence enabled. - // Record the outcome and either stop or hand id and input to - // your own retry queue; deduplicate by id downstream. - if (failed.fetch_add(1) == 0) { - std::cerr << "Write failed for id=" << id << ": " << result.error_message - << '\n'; + auto submitted = writer.Append( + row, [id, state](const fluss::WriteCompletion& notification) { + const auto& result = notification.result; + if (result.Ok()) { + ++state->succeeded; + } else { + // Copy the invocation-scoped result. Do not log, perform I/O, + // or retry here: one slow callback delays every writer. + if (state->failed.fetch_add(1) == 0) { + std::lock_guard lock(state->mutex); + state->first_failed_id = id; + state->first_failure = result; + } } - } - }); + }); if (!submitted.Ok()) { // No callback will run for this submission; handle this path too. + submission_failed = true; std::cerr << "Submission failed for id=" << id << ": " << submitted.error_message << '\n'; break; } } - // Submission has stopped. Wait for callbacks before leaving the counters' - // scope; individual callback failures are checked below. - // check() exits on error; a continuing application must keep callback state alive. - // A durable pipeline would advance its source position or offset only - // after Flush() succeeds, then replay from there on restart. + // Stop submissions, then drain accepted callbacks. Flush success alone does + // not mean every write succeeded. Writer/Connection destruction is not a drain. check("flush", writer.Flush()); - std::cout << "Callback writes: succeeded=" << succeeded << " failed=" << failed << '\n'; - if (failed != 0) { + std::cout << "Callback writes: succeeded=" << state->succeeded.load() + << " failed=" << state->failed.load() << '\n'; + if (state->failed.load() != 0) { + std::lock_guard lock(state->mutex); + std::cerr << "First failed id=" << state->first_failed_id + << ": " << state->first_failure.error_message << '\n'; + } + // A failed write may have reached the server. Recover outside the callback + // using retained input or a replayable source and application-level deduplication. + // Advance a source position only after Flush and all relevant writes succeed. + // This example reports failure and exits; it does not implement durable recovery. + if (submission_failed || state->failed.load() != 0) { return 1; } } diff --git a/fluss-rust/bindings/cpp/include/fluss.hpp b/fluss-rust/bindings/cpp/include/fluss.hpp index befb571a3d2..fff1b850773 100644 --- a/fluss-rust/bindings/cpp/include/fluss.hpp +++ b/fluss-rust/bindings/cpp/include/fluss.hpp @@ -536,14 +536,26 @@ struct Result { bool IsRetriable() const { return ErrorCode::IsRetriable(error_code); } }; +/// Write-specific completion metadata. The reference passed to a callback is valid +/// only for that invocation; copy the result when retaining it for later work. +struct WriteCompletion { + Result result; +}; + +/// Per-writer admission control for callback operations, independent of buffer bytes. +struct WriteCallbackOptions { + // Includes accepted writes awaiting completion and callbacks queued or executing. + // Must be greater than zero. This is not a byte limit on callback captures. + size_t max_pending_operations{262144}; +}; + /// Receives the final outcome of an accepted write. Function pointers and lambdas /// are supported. An empty callback is rejected before submitting the write. /// /// During normal operation, the SDK owns the callback until completion and invokes -/// it exactly once on SDK-managed background threads; no caller polling or waiting -/// thread is needed. Process exit or a crash can prevent delivery. Callbacks never -/// run inline in the submitting call, but may run concurrently and out of order, -/// including before the call returns. +/// it exactly once on a shared SDK worker; no caller polling or waiting thread +/// is needed. Process exit or a crash can prevent delivery. Callbacks never run +/// inline in the submitting call, but may start before the call returns. /// Keep callbacks short and synchronize access to shared state, including writers. /// Callback overloads do not make writers safe for concurrent access. Captured /// references must remain valid until the callback finishes; capturing shared @@ -555,12 +567,13 @@ struct Result { /// with SDK idempotence enabled. Retain input identifiers and recovery state as /// needed; one callback invocation is not an exactly-once delivery guarantee. /// -/// Callbacks run on a single shared worker, so they fire in the order writes -/// complete. Do not wait for another callback from a callback: it stalls that +/// Callbacks run serially in dispatch order on a single shared worker. There is +/// no cross-bucket submission-order guarantee; late registrations are queued +/// when registered. Do not wait for another callback from a callback: it stalls that /// worker. Synchronous SDK calls require exclusive writer access. Callback /// submissions to a full writer fail immediately when called from a callback, /// instead of blocking the worker. Each writer bounds its outstanding callback -/// operations from the write buffer size, so admission tracks memory backpressure. +/// operations using WriteCallbackOptions, independently of the write buffer size. /// Hand off retries or expensive work without blocking; bound application queues /// and handle overflow without silently discarding failed operations. /// @@ -569,9 +582,9 @@ struct Result { /// After a successful Rust write flush, Flush() blocks until pending callbacks finish, /// acting as a barrier, so a callback that never returns hangs it. On error, referenced /// state may still be in use. -/// Inside a callback only the callback wait is skipped; the write flush may block. +/// Flush() called inside any write callback returns a client error without flushing. /// Flush() does not wait for work handed to application workers or retry queues. -using WriteCallback = std::function; +using WriteCallback = std::function; struct TablePath { std::string database_name; @@ -1598,8 +1611,8 @@ struct Configuration { // Shared write-batch memory budget per Connection, across its tables and writers // (default 64 MiB). Not a process RSS limit or a callback-capture memory budget. size_t writer_buffer_memory_size{64 * 1024 * 1024}; - // Maximum time in milliseconds to block waiting for buffer memory. This also bounds - // the whole callback submission, including callback capacity and buffer backpressure. + // Shared wait budget in milliseconds for buffer memory and callback capacity. + // Does not bound data conversion, scheduling, ACKs, or callback execution. // UINT64_MAX waits indefinitely; zero fails fast when capacity or memory is unavailable. uint64_t writer_buffer_wait_timeout_ms{std::numeric_limits::max()}; // Maximum KV backpressure throttle in milliseconds @@ -1785,6 +1798,8 @@ class TableAppend { TableAppend& operator=(TableAppend&&) noexcept = default; Result CreateWriter(AppendWriter& out); + /// Create a writer with an independent, positive callback operation limit. + Result CreateWriter(AppendWriter& out, const WriteCallbackOptions& options); private: friend class Table; @@ -1804,6 +1819,8 @@ class TableUpsert { TableUpsert& PartialUpdateByName(std::vector column_names); Result CreateWriter(UpsertWriter& out); + /// Create a writer sharing one callback operation limit across upserts and deletes. + Result CreateWriter(UpsertWriter& out, const WriteCallbackOptions& options); private: friend class Table; @@ -1942,11 +1959,11 @@ class AppendWriter { Result Append(const GenericRow& row); Result Append(const GenericRow& row, WriteResult& out); /// Submit a row and notify callback of its final outcome without waiting for - /// acknowledgment. Submission is bounded by client.writer.buffer.wait-timeout, - /// covering callback capacity and buffer backpressure: within it, Ok means the - /// write was accepted and the callback fires exactly once, an error means - /// submission failed and no callback runs. A zero timeout makes submission - /// non-blocking. + /// acknowledgment. Callback capacity and buffer waits share the budget from + /// client.writer.buffer.wait-timeout. Ok means the write was accepted and the + /// callback fires exactly once during normal operation; an error means + /// submission failed and no callback runs. A zero timeout makes admission + /// fail fast when callback capacity or buffer memory is unavailable. Result Append(const GenericRow& row, WriteCallback callback); Result AppendArrowBatch(const std::shared_ptr& batch); Result AppendArrowBatch(const std::shared_ptr& batch, WriteResult& out); @@ -1989,11 +2006,11 @@ class UpsertWriter { Result Upsert(const GenericRow& row); Result Upsert(const GenericRow& row, WriteResult& out); /// Submit an upsert and notify callback of its final outcome without waiting - /// for acknowledgment. Submission is bounded by client.writer.buffer.wait-timeout, - /// covering callback capacity and buffer backpressure: within it, Ok means the - /// write was accepted and the callback fires exactly once, an error means - /// submission failed and no callback runs. A zero timeout makes submission - /// non-blocking. + /// for acknowledgment. Callback capacity and buffer waits share the budget from + /// client.writer.buffer.wait-timeout. Ok means the write was accepted and the + /// callback fires exactly once during normal operation; an error means + /// submission failed and no callback runs. A zero timeout makes admission + /// fail fast when callback capacity or buffer memory is unavailable. Result Upsert(const GenericRow& row, WriteCallback callback); Result Delete(const GenericRow& row); Result Delete(const GenericRow& row, WriteResult& out); diff --git a/fluss-rust/bindings/cpp/src/lib.rs b/fluss-rust/bindings/cpp/src/lib.rs index c45c5e005a8..04d6945cfc1 100644 --- a/fluss-rust/bindings/cpp/src/lib.rs +++ b/fluss-rust/bindings/cpp/src/lib.rs @@ -17,6 +17,7 @@ mod types; mod write_callback; +use write_callback::ensure_callback_executor; use std::collections::HashMap; use std::str::FromStr; @@ -532,7 +533,7 @@ mod ffi { fn get_table_path(self: &Table) -> FfiTablePath; fn has_primary_key(self: &Table) -> bool; fn writer_buffer_wait_timeout_ms(self: &Table) -> u64; - fn estimated_callback_capacity(self: &Table) -> usize; + fn ensure_callback_executor() -> FfiResult; fn create_upsert_writer(self: &Table, column_indices: Vec) -> FfiPtrResult; fn new_lookuper(self: &Table) -> FfiPtrResult; fn new_prefix_lookuper(self: &Table, lookup_column_names: Vec) -> FfiPtrResult; @@ -2087,29 +2088,11 @@ impl Table { } /// The connection's configured write-buffer wait timeout (client.writer.buffer.wait-timeout), - /// used by the C++ callback path to bound the whole submit. UINT64_MAX means unbounded. + /// shared by callback admission and buffer waits. UINT64_MAX means unbounded. fn writer_buffer_wait_timeout_ms(&self) -> u64 { self.connection.config().writer_buffer_wait_timeout_ms } - /// Per-writer callback admission limit, derived from the write buffer so it stays consistent - /// with memory backpressure instead of being a separate knob. It approximates how many rows - /// fit in the buffer: buffer bytes / estimated row size. The row estimate uses only the - /// schema's fixed-length part (variable-length payloads are not counted), so the limit is - /// generous and mainly guards against a slow-callback backlog; the buffer-memory wait remains - /// the real backpressure. Floored so tiny buffers still allow pipelining. - fn estimated_callback_capacity(&self) -> usize { - const MIN_CALLBACK_CAPACITY: usize = 1024; - let fields = self.table_info.get_row_type().fields(); - let mut row_size = fcore::row::binary_array::calculate_header_in_bytes(fields.len()); - for field in fields { - row_size += fcore::row::binary_array::calculate_fix_length_part_size(field.data_type()); - } - let row_size = row_size.max(1); - let buffer = self.connection.config().writer_buffer_memory_size; - (buffer / row_size).max(MIN_CALLBACK_CAPACITY) - } - fn create_upsert_writer(&self, column_indices: Vec) -> ffi::FfiPtrResult { let _enter = RUNTIME.enter(); diff --git a/fluss-rust/bindings/cpp/src/table.cpp b/fluss-rust/bindings/cpp/src/table.cpp index 0fb3b2616fb..e65953b55ae 100644 --- a/fluss-rust/bindings/cpp/src/table.cpp +++ b/fluss-rust/bindings/cpp/src/table.cpp @@ -1281,12 +1281,19 @@ bool Table::HasPrimaryKey() const { TableAppend::TableAppend(ffi::Table* table) noexcept : table_(table) {} Result TableAppend::CreateWriter(AppendWriter& out) { + return CreateWriter(out, WriteCallbackOptions{}); +} + +Result TableAppend::CreateWriter(AppendWriter& out, const WriteCallbackOptions& options) { + if (options.max_pending_operations == 0) { + return utils::make_client_error("max_pending_operations must be greater than zero"); + } if (table_ == nullptr) { return utils::make_client_error("Table not available"); } auto capacity = std::make_shared( - table_->estimated_callback_capacity(), table_->writer_buffer_wait_timeout_ms()); + options.max_pending_operations, table_->writer_buffer_wait_timeout_ms()); auto ffi_result = table_->new_append_writer(); auto result = utils::from_ffi_result(ffi_result.result); if (result.Ok()) { @@ -1341,13 +1348,20 @@ std::vector TableUpsert::ResolveNameProjection() const { } Result TableUpsert::CreateWriter(UpsertWriter& out) { + return CreateWriter(out, WriteCallbackOptions{}); +} + +Result TableUpsert::CreateWriter(UpsertWriter& out, const WriteCallbackOptions& options) { + if (options.max_pending_operations == 0) { + return utils::make_client_error("max_pending_operations must be greater than zero"); + } if (table_ == nullptr) { return utils::make_client_error("Table not available"); } try { auto capacity = std::make_shared( - table_->estimated_callback_capacity(), table_->writer_buffer_wait_timeout_ms()); + options.max_pending_operations, table_->writer_buffer_wait_timeout_ms()); auto resolved_indices = !column_names_.empty() ? ResolveNameProjection() : column_indices_; rust::Vec rust_indices; @@ -1680,10 +1694,14 @@ Result AppendWriter::Append(const GenericRow& row, WriteCallback callback) { if (!callback) { return utils::make_client_error("Write callback must not be empty"); } + auto executor_status = utils::from_ffi_result(ffi::ensure_callback_executor()); + if (!executor_status.Ok()) { + return executor_status; + } // Allocate before submission so an allocation failure cannot lose an // already accepted write's completion notification. auto completion = std::make_unique(std::move(callback)); - // Bound the whole submit (capacity reservation + buffer wait) by client.writer.buffer.wait-timeout. + // Share the capacity and buffer wait budget from client.writer.buffer.wait-timeout. const auto submit_start = std::chrono::steady_clock::now(); auto reserved = completion->Reserve(callback_capacity_); if (!reserved.Ok()) { @@ -1746,6 +1764,10 @@ Result AppendWriter::AppendArrowBatch(const std::shared_ptr& if (!callback) { return utils::make_client_error("Write callback must not be empty"); } + auto executor_status = utils::from_ffi_result(ffi::ensure_callback_executor()); + if (!executor_status.Ok()) { + return executor_status; + } auto completion = std::make_unique(std::move(callback)); const auto submit_start = std::chrono::steady_clock::now(); auto reserved = completion->Reserve(callback_capacity_); @@ -1762,6 +1784,9 @@ Result AppendWriter::AppendArrowBatch(const std::shared_ptr& } Result AppendWriter::Flush() { + if (ffi::WriteCallbackCapacity::InCallback()) { + return utils::make_client_error("Flush cannot be called from a write callback"); + } if (!Available()) { return utils::make_client_error("AppendWriter not available"); } @@ -1840,6 +1865,10 @@ Result UpsertWriter::Upsert(const GenericRow& row, WriteCallback callback) { if (!callback) { return utils::make_client_error("Write callback must not be empty"); } + auto executor_status = utils::from_ffi_result(ffi::ensure_callback_executor()); + if (!executor_status.Ok()) { + return executor_status; + } auto completion = std::make_unique(std::move(callback)); const auto submit_start = std::chrono::steady_clock::now(); auto reserved = completion->Reserve(callback_capacity_); @@ -1884,6 +1913,10 @@ Result UpsertWriter::Delete(const GenericRow& row, WriteCallback callback) { if (!callback) { return utils::make_client_error("Write callback must not be empty"); } + auto executor_status = utils::from_ffi_result(ffi::ensure_callback_executor()); + if (!executor_status.Ok()) { + return executor_status; + } auto completion = std::make_unique(std::move(callback)); const auto submit_start = std::chrono::steady_clock::now(); auto reserved = completion->Reserve(callback_capacity_); @@ -1899,6 +1932,9 @@ Result UpsertWriter::Delete(const GenericRow& row, WriteCallback callback) { } Result UpsertWriter::Flush() { + if (ffi::WriteCallbackCapacity::InCallback()) { + return utils::make_client_error("Flush cannot be called from a write callback"); + } if (!Available()) { return utils::make_client_error("UpsertWriter not available"); } diff --git a/fluss-rust/bindings/cpp/src/write_callback.hpp b/fluss-rust/bindings/cpp/src/write_callback.hpp index c5b46aef9c3..7eae1b7820e 100644 --- a/fluss-rust/bindings/cpp/src/write_callback.hpp +++ b/fluss-rust/bindings/cpp/src/write_callback.hpp @@ -36,7 +36,7 @@ namespace ffi { class WriteCallbackCapacity { public: /// `wait_timeout_ms` is the connection's client.writer.buffer.wait-timeout, used as - /// the shared budget for the whole submit. UINT64_MAX means block until a slot frees. + /// the shared budget for capacity and buffer waits. UINT64_MAX means block until a slot frees. WriteCallbackCapacity(size_t max_pending_operations, uint64_t wait_timeout_ms) : max_pending_(max_pending_operations), wait_timeout_ms_(wait_timeout_ms) {} @@ -71,24 +71,13 @@ class WriteCallbackCapacity { available_.notify_all(); } - /// Wait for all reserved operations to finish their callbacks. - /// When called from within a callback this returns immediately to avoid deadlock. - Result AwaitAll(std::chrono::milliseconds timeout) { - std::unique_lock lock(mutex_); - if (in_callback_) return {}; - if (!available_.wait_for(lock, timeout, [&] { return pending_ == 0; })) { - return {ErrorCode::CLIENT_ERROR, "Timed out waiting for pending callbacks"}; - } - return {}; - } + /// True only while this thread executes a user callback or destroys its captures. + static bool InCallback() { return in_callback_; } - /// Block until every reserved operation has finished its callback. Used as a flush - /// barrier after the write flush already succeeded, so it has no deadline of its own; - /// a callback that never returns would hang here. Returns immediately when called from - /// within a callback to avoid deadlocking the worker on itself. + /// Wait until every reserved callback and its captures have finished. + /// Flush rejects callback reentry before starting any write flush. void AwaitAll() { std::unique_lock lock(mutex_); - if (in_callback_) return; available_.wait(lock, [&] { return pending_ == 0; }); } @@ -156,7 +145,7 @@ class WriteCallback { error_code); } try { - callback(std::move(result)); + callback(WriteCompletion{std::move(result)}); } catch (const std::exception& e) { std::fprintf(stderr, "Fluss write callback threw an exception: %s\n", e.what()); } catch (...) { diff --git a/fluss-rust/bindings/cpp/src/write_callback.rs b/fluss-rust/bindings/cpp/src/write_callback.rs index 09fc1b8f290..6810a8ef013 100644 --- a/fluss-rust/bindings/cpp/src/write_callback.rs +++ b/fluss-rust/bindings/cpp/src/write_callback.rs @@ -18,82 +18,62 @@ #[cfg(test)] use std::future::Future; use std::panic::{AssertUnwindSafe, catch_unwind}; -use std::sync::{Arc, LazyLock, Mutex, mpsc}; +use std::sync::{LazyLock, mpsc}; use std::thread::{self, JoinHandle}; use crate::{RUNTIME, WriteResult, client_err, err_from_core_error, ffi, ok_result}; type Completion = Box; -// A single worker gives callbacks a global FIFO order that matches server -// completion order, so a batch's callbacks never interleave with another's and -// same-bucket writes report in the order they finished. This mirrors the Java -// client, whose completion stages run on the one Sender thread. The future path -// to more parallelism is sharding the executor by writer or bucket key, which -// keeps per-key order while still using CallbackExecutor::new. -const CALLBACK_WORKERS: usize = 1; +// One process-wide worker preserves dispatch order without running user code +// on I/O threads. Admission is bounded independently for each C++ writer. +// Initialize before accepting any callback write. Failure is sticky and returned +// synchronously; silently switching to a parallel pool would break ordering. +static CALLBACK_EXECUTOR: LazyLock> = + LazyLock::new(CallbackExecutor::new); -// Like RUNTIME, the executor is process-wide and lives until process exit. -// Initialize it on the submitting thread, not an async I/O worker. -static CALLBACK_EXECUTOR: LazyLock> = - LazyLock::new(|| match CallbackExecutor::new(CALLBACK_WORKERS) { - Ok(executor) => Some(executor), - Err(error) => { - // The write has already been accepted. Keep the old dispatch path - // as a resource-exhaustion fallback rather than lose its callback. - eprintln!("Fluss callback worker initialization failed: {error}; using blocking pool"); - None - } - }); +pub(crate) fn ensure_callback_executor() -> ffi::FfiResult { + executor_status(&CALLBACK_EXECUTOR) +} + +fn executor_status(executor: &std::io::Result) -> ffi::FfiResult { + match executor { + Ok(_) => ok_result(), + Err(error) => client_err(format!( + "Cannot initialize write callback executor: {error}" + )), + } +} struct CallbackExecutor { sender: Option>, - workers: Vec>, + worker: Option>, } impl CallbackExecutor { - fn new(worker_count: usize) -> std::io::Result { - assert!(worker_count > 0); + fn new() -> std::io::Result { let (sender, receiver) = mpsc::channel::(); - let receiver = Arc::new(Mutex::new(receiver)); - let mut executor = Self { + let worker = thread::Builder::new() + .name("fluss-callback".to_string()) + .spawn(move || { + while let Ok(completion) = receiver.recv() { + // This is a dedicated OS thread, not a Tokio runtime worker. + // No receiver mutex or user-configurable worker count is needed. + if catch_unwind(AssertUnwindSafe(completion)).is_err() { + eprintln!("Fluss callback worker contained a Rust panic"); + } + } + })?; + Ok(Self { sender: Some(sender), - workers: Vec::with_capacity(worker_count), - }; - for index in 0..worker_count { - let receiver = Arc::clone(&receiver); - executor.workers.push( - thread::Builder::new() - .name(format!("fluss-callback-{index}")) - .spawn(move || { - loop { - let completion = { - // Take one job at a time; a job runs a whole - // completed batch. Do not prefetch, so no worker - // hoards queued batches. - let receiver = receiver.lock().unwrap(); - let Ok(completion) = receiver.recv() else { - break; - }; - completion - }; - // Never hold a queue lock or enter a Tokio runtime - // while running user code. Synchronous SDK calls - // from a callback can safely use RUNTIME.block_on. - if catch_unwind(AssertUnwindSafe(completion)).is_err() { - eprintln!("Fluss callback worker contained a Rust panic"); - } - } - })?, - ); - } - Ok(executor) + worker: Some(worker), + }) } fn enqueue(&self, completion: Completion) -> Result<(), Completion> { // An unbounded completion queue keeps slow user callbacks from blocking - // async I/O workers. Applications must bound outstanding callbacks; - // this caps threads, not queued captures or total process memory. + // async I/O workers. Per-writer admission bounds callback operations, + // not the bytes retained by captures or total process memory. self.sender .as_ref() .unwrap() @@ -104,10 +84,10 @@ impl CallbackExecutor { impl Drop for CallbackExecutor { fn drop(&mut self) { - // Also handles partial worker initialization and lets tests verify - // drain/release. The process-wide static is not dropped at exit. + // Tests own executors and verify drain/release. The process-wide static + // lives until process exit and is not automatically drained there. drop(self.sender.take()); - for worker in self.workers.drain(..) { + if let Some(worker) = self.worker.take() { let _ = worker.join(); } } @@ -143,7 +123,7 @@ fn dispatch_write( callback: impl FnOnce(ffi::FfiResult) + Send + 'static, ) { // Force worker initialization before registering with an in-flight batch. - let _ = CALLBACK_EXECUTOR.as_ref(); + assert_eq!(ensure_callback_executor().error_code, 0); let callback = move |result| callback(to_ffi_result(result)); if let Err((future, callback)) = future.try_on_complete(callback, dispatch_batch) { // Only futures already polled before registration need this path. @@ -151,7 +131,9 @@ fn dispatch_write( RUNTIME.spawn(async move { let result = future.await; deliver( - CALLBACK_EXECUTOR.as_ref(), + CALLBACK_EXECUTOR + .as_ref() + .expect("callback executor initialized before submission"), Box::new(move || callback(result)), ); }); @@ -161,7 +143,12 @@ fn dispatch_write( fn dispatch_batch(batch: fluss::client::WriteCallbackBatch) { // Deliver the whole batch as one job so its callbacks stay together and the // single worker runs them in completion order. - deliver(CALLBACK_EXECUTOR.as_ref(), Box::new(move || batch.run())); + deliver( + CALLBACK_EXECUTOR + .as_ref() + .expect("callback executor initialized before submission"), + Box::new(move || batch.run()), + ); } fn to_ffi_result(result: Result<(), fluss::error::Error>) -> ffi::FfiResult { @@ -176,35 +163,33 @@ fn dispatch( future: impl Future> + Send + 'static, callback: impl FnOnce(ffi::FfiResult) + Send + 'static, ) { - let executor = CALLBACK_EXECUTOR.as_ref(); + let executor = CALLBACK_EXECUTOR + .as_ref() + .expect("callback executor initialized"); RUNTIME.spawn(async move { let result = to_ffi_result(future.await); deliver(executor, Box::new(move || callback(result))); }); } -fn deliver(executor: Option<&CallbackExecutor>, completion: Completion) { - let completion = match executor { - Some(executor) => match executor.enqueue(completion) { - Ok(()) => return, - Err(completion) => completion, - }, - None => completion, - }; - // Preserve completion even if dedicated workers could not be started or - // their channel disconnected. Never execute user code on the I/O worker. - RUNTIME.spawn_blocking(completion); +fn deliver(executor: &CallbackExecutor, completion: Completion) { + if executor.enqueue(completion).is_err() { + // The static executor is never shut down and contains callback panics. + // Disconnection is an internal invariant violation, not an overload policy. + // Do not silently lose accepted notifications or run them out of order. + eprintln!("Fluss callback executor unexpectedly disconnected"); + std::process::abort(); + } } #[cfg(test)] mod tests { - use std::collections::HashSet; use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::{Arc, mpsc}; use std::thread; use std::time::Duration; - use super::{CallbackExecutor, deliver, dispatch, dispatch_write}; + use super::{CallbackExecutor, dispatch, dispatch_write, executor_status}; use crate::{CLIENT_ERROR_CODE, RUNTIME}; #[test] @@ -305,28 +290,21 @@ mod tests { } #[test] - fn test_executor_bounds_workers_and_does_not_block_the_runtime() { - let executor = CallbackExecutor::new(2).unwrap(); + fn test_slow_callback_does_not_block_runtime_or_run_callbacks_concurrently() { + let executor = CallbackExecutor::new().unwrap(); let (started_tx, started_rx) = mpsc::channel(); + let (release_tx, release_rx) = mpsc::channel(); let (done_tx, done_rx) = mpsc::channel(); - let mut releases = Vec::new(); - let mut worker_ids = HashSet::new(); - for _ in 0..2 { - let (release_tx, release_rx) = mpsc::channel(); - releases.push(release_tx); - let started_tx = started_tx.clone(); - assert!( - executor - .enqueue(Box::new(move || { - started_tx.send(thread::current().id()).unwrap(); - release_rx.recv_timeout(Duration::from_secs(10)).unwrap(); - })) - .is_ok() - ); - // Ensure this worker is occupied before giving another worker work. - worker_ids.insert(started_rx.recv_timeout(Duration::from_secs(10)).unwrap()); - } - assert_eq!(worker_ids.len(), 2); + assert!( + executor + .enqueue(Box::new(move || { + started_tx.send(thread::current().id()).unwrap(); + release_rx.recv_timeout(Duration::from_secs(10)).unwrap(); + })) + .is_ok() + ); + let worker_id = started_rx.recv_timeout(Duration::from_secs(10)).unwrap(); + assert_ne!(worker_id, thread::current().id()); for _ in 0..256 { let done_tx = done_tx.clone(); assert!( @@ -347,18 +325,19 @@ mod tests { }), 42 ); - for release in releases { - release.send(()).unwrap(); - } + release_tx.send(()).unwrap(); drop(executor); for _ in 0..256 { - assert!(worker_ids.contains(&done_rx.recv_timeout(Duration::from_secs(10)).unwrap())); + assert_eq!( + done_rx.recv_timeout(Duration::from_secs(10)).unwrap(), + worker_id + ); } } #[test] fn test_executor_drains_and_survives_a_panicking_callback() { - let executor = CallbackExecutor::new(1).unwrap(); + let executor = CallbackExecutor::new().unwrap(); let completed = Arc::new(AtomicUsize::new(0)); assert!( executor @@ -385,7 +364,7 @@ mod tests { // A single worker must not reorder callbacks. This is the executor-level // guarantee behind same-bucket completions reporting in the order they // finished: the results come out exactly as enqueued, not just once each. - let executor = CallbackExecutor::new(1).unwrap(); + let executor = CallbackExecutor::new().unwrap(); let (tx, rx) = mpsc::channel(); for index in 0..1000 { let tx = tx.clone(); @@ -406,7 +385,7 @@ mod tests { #[test] fn test_concurrent_producers_complete_each_job_once() { - let executor = Arc::new(CallbackExecutor::new(4).unwrap()); + let executor = Arc::new(CallbackExecutor::new().unwrap()); let (tx, rx) = mpsc::channel(); let mut producers = Vec::new(); for producer in 0..4 { @@ -436,21 +415,13 @@ mod tests { } #[test] - fn test_unavailable_executor_falls_back_off_the_caller_thread() { - let (tx, rx) = mpsc::channel(); - deliver( - None, - Box::new(move || { - let answer = RUNTIME.block_on(async { 42 }); - tx.send((thread::current().id(), answer)).unwrap(); - }), - ); - let (thread_id, answer) = rx.recv_timeout(Duration::from_secs(10)).unwrap(); - assert_ne!(thread_id, thread::current().id()); - assert_eq!(answer, 42); - assert!(matches!( - rx.try_recv(), - Err(mpsc::TryRecvError::Disconnected) - )); + fn test_executor_initialization_error_is_reported_without_fallback() { + let unavailable = Err(std::io::Error::other("thread creation failed")); + let status = executor_status(&unavailable); + assert_eq!(status.error_code, CLIENT_ERROR_CODE); + assert!(status.error_message.contains("thread creation failed")); + assert!(unavailable.is_err()); + let ready = Ok(CallbackExecutor::new().unwrap()); + assert_eq!(executor_status(&ready).error_code, 0); } } diff --git a/fluss-rust/bindings/cpp/test/test_write_callback.cpp b/fluss-rust/bindings/cpp/test/test_write_callback.cpp index 7c02c252397..308880d4431 100644 --- a/fluss-rust/bindings/cpp/test/test_write_callback.cpp +++ b/fluss-rust/bindings/cpp/test/test_write_callback.cpp @@ -72,7 +72,9 @@ class Completion { // A plain function pointer has no capture. This state lives for the process. Completion function_completion; -void RecordFunctionCallback(fluss::Result result) { function_completion.Record(std::move(result)); } +void RecordFunctionCallback(const fluss::WriteCompletion& completion) { + function_completion.Record(completion.result); +} struct Lifetime { std::promise released; @@ -154,12 +156,13 @@ TEST_F(WriteCallbackTest, FlushWaitsForPendingCallbacks) { std::weak_ptr weak_lifetime = lifetime; { auto row = Row(); - fluss::WriteCallback callback = [started, finished, resume, - owned = std::move(lifetime)](fluss::Result result) { + fluss::WriteCallback callback = [started, finished, resume, owned = std::move(lifetime)]( + const fluss::WriteCompletion& notification) { + const auto& result = notification.result; started->Record(result); // Bounded even when a preceding assertion fails. resume.wait_for(std::chrono::seconds(20)); - finished->Record(std::move(result)); + finished->Record(result); }; ASSERT_OK(writer.Append(row, std::move(callback))); } @@ -167,8 +170,15 @@ TEST_F(WriteCallbackTest, FlushWaitsForPendingCallbacks) { EXPECT_FALSE(weak_lifetime.expired()); // Flush now waits for pending callbacks, not just for server ACK. // The callback is blocked on the gate, so Flush must not return yet. + std::promise flush_started; + auto flush = std::async(std::launch::async, [&] { + flush_started.set_value(); + return writer.Flush(); + }); + flush_started.get_future().wait(); + EXPECT_EQ(flush.wait_for(std::chrono::milliseconds(25)), std::future_status::timeout); gate->set_value(); - ASSERT_OK(writer.Flush()); + ASSERT_OK(flush.get()); // After Flush returns, the callback has finished and captures are released. EXPECT_FALSE(finished->Results().empty()); EXPECT_EQ(released.wait_for(std::chrono::seconds(10)), std::future_status::ready); @@ -190,8 +200,11 @@ TEST_F(WriteCallbackTest, AppendArrowBatchNotifiesOnce) { arrow::RecordBatch::Make(arrow::schema({arrow::field("id", arrow::int32()), arrow::field("value", arrow::utf8())}), 6, {ids.Finish().ValueOrDie(), values.Finish().ValueOrDie()}); - ASSERT_OK(writer.AppendArrowBatch( - batch, [completion](fluss::Result result) { completion->Record(std::move(result)); })); + ASSERT_OK(writer.AppendArrowBatch(batch, + [completion](const fluss::WriteCompletion& notification) { + const auto& result = notification.result; + completion->Record(result); + })); } ASSERT_TRUE(completion->Await()); ASSERT_EQ(completion->Results().size(), 1u); @@ -204,8 +217,9 @@ TEST_F(WriteCallbackTest, UpsertAndDeleteNotifyCompletion) { fluss::UpsertWriter writer; ASSERT_OK(table_.NewUpsert().CreateWriter(writer)); auto completion = std::make_shared(); - fluss::WriteCallback callback = [completion](fluss::Result result) { - completion->Record(std::move(result)); + fluss::WriteCallback callback = [completion](const fluss::WriteCompletion& notification) { + const auto& result = notification.result; + completion->Record(result); }; ASSERT_OK(writer.Upsert(Row(), callback)); ASSERT_TRUE(completion->Await()); @@ -238,8 +252,10 @@ TEST_F(WriteCallbackTest, ServerRejectionIsReportedThroughCallback) { ASSERT_OK(pending.Wait()); auto completion = std::make_shared(); - auto submitted = writer.Delete( - Row(), [completion](fluss::Result result) { completion->Record(std::move(result)); }); + auto submitted = writer.Delete(Row(), [completion](const fluss::WriteCompletion& notification) { + const auto& result = notification.result; + completion->Record(result); + }); // The write is accepted locally; only the callback reports server rejection. ASSERT_OK(submitted); ASSERT_TRUE(completion->Await()); @@ -255,8 +271,9 @@ TEST_F(WriteCallbackTest, MultipleOutstandingWritesEachNotifyOnce) { ASSERT_OK(table_.NewAppend().CreateWriter(writer)); auto completion = std::make_shared(); for (int32_t id = 0; id < 64; ++id) { - ASSERT_OK(writer.Append(Row(id), [completion](fluss::Result result) { - completion->Record(std::move(result)); + ASSERT_OK(writer.Append(Row(id), [completion](const fluss::WriteCompletion& notification) { + const auto& result = notification.result; + completion->Record(result); })); } ASSERT_TRUE(completion->Await(64)); @@ -277,22 +294,26 @@ TEST_F(WriteCallbackTest, RejectedSubmissionDoesNotInvokeCallback) { auto released = lifetime->released.get_future(); fluss::GenericRow invalid(1); invalid.SetInt32(0, 1); // Table requires two columns. - auto result = - writer.Append(invalid, [completion, owned = std::move(lifetime)](fluss::Result completed) { - completion->Record(std::move(completed)); - }); + auto result = writer.Append(invalid, [completion, owned = std::move(lifetime)]( + const fluss::WriteCompletion& notification) { + const auto& completed = notification.result; + completion->Record(completed); + }); EXPECT_FALSE(result.Ok()); EXPECT_EQ(released.wait_for(std::chrono::seconds(10)), std::future_status::ready); EXPECT_TRUE(completion->Results().empty()); - result = writer.AppendArrowBatch(nullptr, [completion](fluss::Result completed) { - completion->Record(std::move(completed)); - }); + result = + writer.AppendArrowBatch(nullptr, [completion](const fluss::WriteCompletion& notification) { + const auto& completed = notification.result; + completion->Record(completed); + }); EXPECT_FALSE(result.Ok()); EXPECT_TRUE(completion->Results().empty()); // Both failed submissions must not register a callback. - ASSERT_OK(writer.Append(Row(), [completion](fluss::Result completed) { - completion->Record(std::move(completed)); + ASSERT_OK(writer.Append(Row(), [completion](const fluss::WriteCompletion& notification) { + const auto& completed = notification.result; + completion->Record(completed); })); ASSERT_TRUE(completion->Await()); ASSERT_OK(writer.Flush()); @@ -309,14 +330,15 @@ TEST_F(WriteCallbackTest, SameBucketCallbacksFireInSubmissionOrder) { // match submission order. The single callback worker must not reorder them. constexpr int32_t kWrites = 128; for (int32_t index = 0; index < kWrites; ++index) { - ASSERT_OK(writer.Append( - Row(7), [completion, order_mutex, order, index](fluss::Result result) { - { - std::lock_guard lock(*order_mutex); - order->push_back(index); - } - completion->Record(std::move(result)); - })); + ASSERT_OK(writer.Append(Row(7), [completion, order_mutex, order, + index](const fluss::WriteCompletion& notification) { + const auto& result = notification.result; + { + std::lock_guard lock(*order_mutex); + order->push_back(index); + } + completion->Record(result); + })); } ASSERT_TRUE(completion->Await(kWrites)); ASSERT_OK(writer.Flush()); @@ -340,12 +362,14 @@ TEST_F(WriteCallbackTest, BatchedCallbacksSurviveExceptionsAndCoexistWithWait) { constexpr int count = 1024; for (int i = 0; i < count; ++i) { // The same bucket key encourages shared internal batches. - ASSERT_OK(writer.Append(Row(1), [completion, i](fluss::Result result) { - completion->Record(std::move(result)); - if (i % 64 == 0) { - throw std::runtime_error("isolated batch callback exception"); - } - })); + ASSERT_OK( + writer.Append(Row(1), [completion, i](const fluss::WriteCompletion& notification) { + const auto& result = notification.result; + completion->Record(result); + if (i % 64 == 0) { + throw std::runtime_error("isolated batch callback exception"); + } + })); } fluss::WriteResult pending; ASSERT_OK(writer.Append(Row(1), pending)); @@ -369,8 +393,10 @@ TEST_F(WriteCallbackTest, BatchedServerFailureNotifiesEveryAcceptedDelete) { auto completion = std::make_shared(); constexpr int count = 257; for (int i = 0; i < count; ++i) { - ASSERT_OK(writer.Delete( - Row(), [completion](fluss::Result result) { completion->Record(std::move(result)); })); + ASSERT_OK(writer.Delete(Row(), [completion](const fluss::WriteCompletion& notification) { + const auto& result = notification.result; + completion->Record(result); + })); } ASSERT_TRUE(completion->Await(count)); auto results = completion->Results(); @@ -390,8 +416,11 @@ TEST_F(WriteCallbackTest, EmptyArrowBatchCallbackIsStillAsynchronous) { arrow::schema({arrow::field("id", arrow::int32()), arrow::field("value", arrow::utf8())}), 0, {ids.Finish().ValueOrDie(), values.Finish().ValueOrDie()}); auto completion = std::make_shared(); - ASSERT_OK(writer.AppendArrowBatch( - batch, [completion](fluss::Result result) { completion->Record(std::move(result)); })); + ASSERT_OK( + writer.AppendArrowBatch(batch, [completion](const fluss::WriteCompletion& notification) { + const auto& result = notification.result; + completion->Record(result); + })); ASSERT_TRUE(completion->Await()); ASSERT_EQ(completion->Results().size(), 1u); EXPECT_OK(completion->Results().front()); @@ -421,7 +450,10 @@ TEST_F(WriteCallbackTest, EmptyCallbacksAreRejectedBeforeSubmission) { TEST_F(WriteCallbackTest, UnavailableWritersDoNotInvokeCallbacks) { auto completion = std::make_shared(); - auto callback = [completion](fluss::Result result) { completion->Record(std::move(result)); }; + auto callback = [completion](const fluss::WriteCompletion& notification) { + const auto& result = notification.result; + completion->Record(result); + }; fluss::AppendWriter append; fluss::UpsertWriter upsert; EXPECT_FALSE(append.Append(Row(), callback).Ok()); @@ -436,8 +468,9 @@ TEST(WriteCallbackBridgeTest, ForwardsErrorAndReleasesCaptures) { auto lifetime = std::make_shared(); auto released = lifetime->released.get_future(); fluss::ffi::WriteCallback callback( - [completion, owned = std::move(lifetime)](fluss::Result result) { - completion->Record(std::move(result)); + [completion, owned = std::move(lifetime)](const fluss::WriteCompletion& notification) { + const auto& result = notification.result; + completion->Record(result); }); callback.Complete(fluss::ErrorCode::DELETION_DISABLED_EXCEPTION, "Deletion is disabled"); EXPECT_EQ(released.wait_for(std::chrono::seconds(10)), std::future_status::ready); @@ -450,12 +483,13 @@ TEST(WriteCallbackBridgeTest, ForwardsErrorAndReleasesCaptures) { TEST(WriteCallbackBridgeTest, ContainsCallbackExceptions) { auto lifetime = std::make_shared(); auto released = lifetime->released.get_future(); - fluss::ffi::WriteCallback callback([owned = std::move(lifetime)](fluss::Result) { - throw std::runtime_error("callback failure"); - }); + fluss::ffi::WriteCallback callback( + [owned = std::move(lifetime)](const fluss::WriteCompletion&) { + throw std::runtime_error("callback failure"); + }); EXPECT_NO_THROW(callback.Complete(0, "")); EXPECT_EQ(released.wait_for(std::chrono::seconds(10)), std::future_status::ready); - fluss::ffi::WriteCallback unknown([](fluss::Result) { throw 42; }); + fluss::ffi::WriteCallback unknown([](const fluss::WriteCompletion&) { throw 42; }); EXPECT_NO_THROW(unknown.Complete(0, "")); } @@ -472,7 +506,7 @@ TEST(WriteCallbackBridgeTest, CapacityRejectsOverflowWithoutDroppingReservations for (size_t i = 0; i < max_pending_operations; ++i) { capacity.Release(); } - EXPECT_OK(capacity.AwaitAll(std::chrono::milliseconds(0))); + capacity.AwaitAll(); } TEST(WriteCallbackBridgeTest, CapacityLastsThroughCallbackAndCaptureCleanup) { @@ -482,10 +516,11 @@ TEST(WriteCallbackBridgeTest, CapacityLastsThroughCallbackAndCaptureCleanup) { EXPECT_FALSE(capacity->Acquire().Ok()); delete value; }); - fluss::ffi::WriteCallback callback([capacity, owned = std::move(capture)](fluss::Result) { - EXPECT_FALSE(capacity->Acquire().Ok()); - throw std::runtime_error("callback failure"); - }); + fluss::ffi::WriteCallback callback( + [capacity, owned = std::move(capture)](const fluss::WriteCompletion&) { + EXPECT_FALSE(capacity->Acquire().Ok()); + throw std::runtime_error("callback failure"); + }); ASSERT_OK(callback.Reserve(capacity)); EXPECT_FALSE(capacity->Acquire().Ok()); callback.Complete(0, ""); @@ -498,7 +533,7 @@ TEST(WriteCallbackBridgeTest, UnsubmittedCallbackReturnsCapacityOnException) { auto capacity = std::make_shared(1, 0); int calls = 0; try { - fluss::ffi::WriteCallback callback([&](fluss::Result) { ++calls; }); + fluss::ffi::WriteCallback callback([&](const fluss::WriteCompletion&) { ++calls; }); ASSERT_OK(callback.Reserve(capacity)); throw std::bad_alloc(); } catch (const std::bad_alloc&) { @@ -511,7 +546,7 @@ TEST(WriteCallbackBridgeTest, UnsubmittedCallbackReturnsCapacityOnException) { TEST(WriteCallbackBridgeTest, ReservationOutlivesWriterOwnership) { auto capacity = std::make_shared(1, 0); std::weak_ptr weak = capacity; - fluss::ffi::WriteCallback callback([](fluss::Result) {}); + fluss::ffi::WriteCallback callback([](const fluss::WriteCompletion&) {}); ASSERT_OK(callback.Reserve(capacity)); capacity.reset(); EXPECT_FALSE(weak.expired()); @@ -522,7 +557,7 @@ TEST(WriteCallbackBridgeTest, ReservationOutlivesWriterOwnership) { TEST(WriteCallbackBridgeTest, CapacityTimeoutDoesNotDiscardAcceptedCallback) { auto capacity = std::make_shared(1, 25); int calls = 0; - fluss::ffi::WriteCallback accepted([&](fluss::Result) { ++calls; }); + fluss::ffi::WriteCallback accepted([&](const fluss::WriteCompletion&) { ++calls; }); ASSERT_OK(accepted.Reserve(capacity)); auto start = std::chrono::steady_clock::now(); auto result = capacity->Acquire(); @@ -538,7 +573,7 @@ TEST(WriteCallbackBridgeTest, CapacityTimeoutDoesNotDiscardAcceptedCallback) { TEST(WriteCallbackBridgeTest, WaitingSubmitterResumesAfterCompletion) { auto capacity = std::make_shared(1, 5000); - fluss::ffi::WriteCallback accepted([](fluss::Result) {}); + fluss::ffi::WriteCallback accepted([](const fluss::WriteCompletion&) {}); ASSERT_OK(accepted.Reserve(capacity)); std::promise started; auto waiter = std::async(std::launch::async, [&] { @@ -558,7 +593,7 @@ TEST(WriteCallbackBridgeTest, WaitingSubmitterResumesAfterCompletion) { TEST(WriteCallbackBridgeTest, CallbackRejectsFullOtherWriterAndRestoresThreadContext) { auto capacity = std::make_shared(1, 25); ASSERT_OK(capacity->Acquire()); // Full writer unrelated to the executing callback. - fluss::ffi::WriteCallback callback([capacity](fluss::Result) { + fluss::ffi::WriteCallback callback([capacity](const fluss::WriteCompletion&) { auto result = capacity->Acquire(); EXPECT_EQ(result.error_message, "Write callback capacity is full"); throw 42; @@ -580,7 +615,7 @@ TEST(WriteCallbackBridgeTest, ConcurrentCapacityReservationsStayBounded) { for (int i = 0; i < 8; ++i) { threads.emplace_back([&] { for (int j = 0; j < 250; ++j) { - fluss::ffi::WriteCallback callback([&](fluss::Result) { + fluss::ffi::WriteCallback callback([&](const fluss::WriteCompletion&) { --active; ++completed; }); @@ -597,3 +632,61 @@ TEST(WriteCallbackBridgeTest, ConcurrentCapacityReservationsStayBounded) { EXPECT_EQ(active.load(), 0u); EXPECT_EQ(completed.load(), 2000u); } + +TEST(WriteCallbackBridgeTest, FlushRejectsCallbackReentryBeforeTouchingEitherWriter) { + fluss::AppendWriter append; + fluss::UpsertWriter upsert; + EXPECT_FALSE(fluss::ffi::WriteCallbackCapacity::InCallback()); + fluss::ffi::WriteCallback callback([&](const fluss::WriteCompletion&) { + EXPECT_TRUE(fluss::ffi::WriteCallbackCapacity::InCallback()); + EXPECT_EQ(append.Flush().error_message, "Flush cannot be called from a write callback"); + EXPECT_EQ(upsert.Flush().error_message, "Flush cannot be called from a write callback"); + throw std::runtime_error("restore callback context after exception"); + }); + callback.Complete(0, ""); + EXPECT_FALSE(fluss::ffi::WriteCallbackCapacity::InCallback()); + EXPECT_EQ(append.Flush().error_message, "AppendWriter not available"); + EXPECT_EQ(upsert.Flush().error_message, "UpsertWriter not available"); +} + +TEST_F(WriteCallbackTest, RejectsZeroCapacityWithoutCreatingWriter) { + CreateTable(true); + fluss::WriteCallbackOptions options; + EXPECT_EQ(options.max_pending_operations, 262144u); + options.max_pending_operations = 0; + fluss::AppendWriter append; + fluss::UpsertWriter upsert; + auto appended = table_.NewAppend().CreateWriter(append, options); + auto upserted = table_.NewUpsert().CreateWriter(upsert, options); + EXPECT_EQ(appended.error_message, "max_pending_operations must be greater than zero"); + EXPECT_EQ(upserted.error_message, "max_pending_operations must be greater than zero"); + EXPECT_FALSE(append.Available()); + EXPECT_FALSE(upsert.Available()); +} + +TEST_F(WriteCallbackTest, ConfiguredCapacityBlocksUntilCallbackFinishes) { + CreateTable(); + fluss::WriteCallbackOptions options; + options.max_pending_operations = 1; + fluss::AppendWriter writer; + ASSERT_OK(table_.NewAppend().CreateWriter(writer, options)); + auto gate = std::make_shared>(); + auto resume = gate->get_future().share(); + auto started = std::make_shared>(); + auto ready = started->get_future(); + ASSERT_OK(writer.Append(Row(), [started, resume](const fluss::WriteCompletion&) { + started->set_value(); + resume.wait_for(std::chrono::seconds(10)); + })); + ASSERT_EQ(ready.wait_for(std::chrono::seconds(10)), std::future_status::ready); + std::promise submitting; + auto submitted = std::async(std::launch::async, [&] { + submitting.set_value(); + return writer.Append(Row(2), [](const fluss::WriteCompletion&) {}); + }); + submitting.get_future().wait(); + EXPECT_EQ(submitted.wait_for(std::chrono::milliseconds(25)), std::future_status::timeout); + gate->set_value(); + ASSERT_OK(submitted.get()); + ASSERT_OK(writer.Flush()); +} diff --git a/fluss-rust/bindings/cpp/test/test_write_callback_allocation.cpp b/fluss-rust/bindings/cpp/test/test_write_callback_allocation.cpp index 7cbbc9c8c0c..19cbe5c90f3 100644 --- a/fluss-rust/bindings/cpp/test/test_write_callback_allocation.cpp +++ b/fluss-rust/bindings/cpp/test/test_write_callback_allocation.cpp @@ -48,10 +48,12 @@ TEST(WriteCallbackBridgeTest, ErrorTextAllocationFailureStillCompletesAndRelease std::weak_ptr weak = lifetime; fluss::Result observed; int calls = 0; - fluss::ffi::WriteCallback callback([&, owned = std::move(lifetime)](fluss::Result result) { - ++calls; - observed = std::move(result); - }); + fluss::ffi::WriteCallback callback( + [&, owned = std::move(lifetime)](const fluss::WriteCompletion& notification) { + const auto& result = notification.result; + ++calls; + observed = result; + }); auto capacity = std::make_shared(1, 0); ASSERT_TRUE(callback.Reserve(capacity).Ok()); const rust::Str text(message); diff --git a/fluss-rust/crates/fluss/src/client/table/append.rs b/fluss-rust/crates/fluss/src/client/table/append.rs index cc6835107ac..62eb8e32182 100644 --- a/fluss-rust/crates/fluss/src/client/table/append.rs +++ b/fluss-rust/crates/fluss/src/client/table/append.rs @@ -141,8 +141,8 @@ impl AppendWriter { self.append_with_deadline(row, None) } - /// Internal: bounds the buffer-memory wait by `deadline` so a language binding can keep - /// a whole callback submission within one `client.writer.buffer.wait-timeout` budget. Not a + /// Internal: bounds the buffer-memory wait by `deadline` so a language binding can share + /// one `client.writer.buffer.wait-timeout` budget with callback admission waits. Not a /// public deadline API; use [`Self::append`], which passes `None` and defers to the config. #[doc(hidden)] pub fn append_with_deadline( @@ -188,8 +188,8 @@ impl AppendWriter { self.append_arrow_batch_with_deadline(batch, None) } - /// Internal: bounds the buffer-memory wait by `deadline` so a language binding can keep - /// a whole callback submission within one `client.writer.buffer.wait-timeout` budget. Not a + /// Internal: bounds the buffer-memory wait by `deadline` so a language binding can share + /// one `client.writer.buffer.wait-timeout` budget with callback admission waits. Not a /// public deadline API; use [`Self::append_arrow_batch`], which passes `None`. #[doc(hidden)] pub fn append_arrow_batch_with_deadline( diff --git a/fluss-rust/crates/fluss/src/client/table/upsert.rs b/fluss-rust/crates/fluss/src/client/table/upsert.rs index 226fb2c0ab0..efb8bf48151 100644 --- a/fluss-rust/crates/fluss/src/client/table/upsert.rs +++ b/fluss-rust/crates/fluss/src/client/table/upsert.rs @@ -351,8 +351,8 @@ impl UpsertWriter { self.upsert_with_deadline(row, None) } - /// Internal: bounds the buffer-memory wait by `deadline` so a language binding can keep - /// a whole callback submission within one `client.writer.buffer.wait-timeout` budget. Not a + /// Internal: bounds the buffer-memory wait by `deadline` so a language binding can share + /// one `client.writer.buffer.wait-timeout` budget with callback admission waits. Not a /// public deadline API; use [`Self::upsert`], which passes `None` and defers to the config. #[doc(hidden)] pub fn upsert_with_deadline( @@ -405,8 +405,8 @@ impl UpsertWriter { self.delete_with_deadline(row, None) } - /// Internal: bounds the buffer-memory wait by `deadline` so a language binding can keep - /// a whole callback submission within one `client.writer.buffer.wait-timeout` budget. Not a + /// Internal: bounds the buffer-memory wait by `deadline` so a language binding can share + /// one `client.writer.buffer.wait-timeout` budget with callback admission waits. Not a /// public deadline API; use [`Self::delete`], which passes `None` and defers to the config. #[doc(hidden)] pub fn delete_with_deadline( diff --git a/fluss-rust/crates/fluss/src/client/write/accumulator.rs b/fluss-rust/crates/fluss/src/client/write/accumulator.rs index 3a961cdcc86..72c22fb5f61 100644 --- a/fluss-rust/crates/fluss/src/client/write/accumulator.rs +++ b/fluss-rust/crates/fluss/src/client/write/accumulator.rs @@ -78,7 +78,7 @@ impl MemoryLimiter { /// Like [`acquire`], but bounds the wait by `deadline` when provided instead of /// the limiter's configured `wait_timeout`. A deadline already in the past makes /// this non-blocking (fail fast if memory is unavailable), which callers use to - /// keep a submit within a caller-supplied budget (e.g. the callback submission budget). + /// share the buffer wait budget with callback admission, not bound the whole API call. pub fn acquire_within( self: &Arc, size: usize, diff --git a/fluss-rust/crates/fluss/src/client/write/broadcast.rs b/fluss-rust/crates/fluss/src/client/write/broadcast.rs index 33a0d5027e0..765d76ae068 100644 --- a/fluss-rust/crates/fluss/src/client/write/broadcast.rs +++ b/fluss-rust/crates/fluss/src/client/write/broadcast.rs @@ -30,8 +30,8 @@ type Dispatcher = fn(CompletionBatch); /// An owned set of callbacks sharing one published result. /// -/// Binding executors split this into bounded jobs and run those jobs off the -/// I/O thread. User callbacks are never invoked by the broadcast itself. +/// Binding executors enqueue this batch and run it off the I/O thread. +/// User callbacks are never invoked by the broadcast itself. #[doc(hidden)] pub struct CompletionBatch { result: Arc>, @@ -39,23 +39,6 @@ pub struct CompletionBatch { } impl CompletionBatch { - /// Split into jobs of at most `limit` callbacks without waiting to fill one. - pub fn into_chunks(self, limit: usize) -> impl Iterator { - assert!(limit > 0); - let mut callbacks = self.callbacks.into_iter(); - std::iter::from_fn(move || { - let chunk: Vec<_> = callbacks.by_ref().take(limit).collect(); - if chunk.is_empty() { - None - } else { - Some(Self { - result: Arc::clone(&self.result), - callbacks: chunk, - }) - } - }) - } - /// Execute every callback, isolating panics so later callbacks still run. pub fn run(self) { for callback in self.callbacks { @@ -100,31 +83,30 @@ impl BroadcastOnceReceiver { self.shared.data.read().as_deref().cloned() } - /// Register under the result read lock, so publication cannot overtake - /// registration. A late registration is dispatched immediately, not lost. + /// Register under the result read lock. Late registrations join the same + /// dispatch queue, so they cannot overtake callbacks awaiting dispatch. pub(crate) fn subscribe(&self, callback: Callback, dispatch: Dispatcher) { let data = self.shared.data.read(); - if let Some(result) = data.as_ref() { - let result = Arc::clone(result); - drop(data); - dispatch(CompletionBatch { - result, - callbacks: vec![callback], - }); - } else { - let mut groups = self.shared.callbacks.lock(); - if let Some(group) = groups - .iter_mut() - .find(|group| std::ptr::fn_addr_eq(group.dispatch, dispatch)) + let result = data.as_ref().map(Arc::clone); + { + let mut state = self.shared.callbacks.lock(); + if let Some(group) = state + .groups + .last_mut() + .filter(|group| std::ptr::fn_addr_eq(group.dispatch, dispatch)) { group.callbacks.push(callback); } else { - groups.push(CallbackGroup { + state.groups.push(CallbackGroup { dispatch, callbacks: vec![callback], }); } } + drop(data); + if let Some(result) = result { + self.shared.dispatch_callbacks(result, false); + } } /// Waits for [`BroadcastOnce::broadcast`] to be called or returns an error @@ -161,20 +143,60 @@ impl BroadcastOnceReceiver { struct Shared { data: RwLock>>>, notify: Notify, - callbacks: Mutex>>, + callbacks: Mutex>, +} + +#[derive(Debug)] +struct CallbackState { + groups: Vec>, + dispatching: bool, + publication_notified: bool, +} + +impl Default for CallbackState { + fn default() -> Self { + Self { + groups: Vec::new(), + dispatching: false, + publication_notified: false, + } + } } impl Shared { fn notify_completion(&self, result: Arc>) { - // Registration takes data then callbacks. Publication has already - // released data, so neither dispatcher nor user code runs under locks. - let groups = std::mem::take(&mut *self.callbacks.lock()); self.notify.notify_waiters(); - for group in groups { - (group.dispatch)(CompletionBatch { - result: Arc::clone(&result), - callbacks: group.callbacks, - }); + self.dispatch_callbacks(result, true); + } + + fn dispatch_callbacks(&self, result: Arc>, publishing: bool) { + let mut state = self.callbacks.lock(); + state.publication_notified |= publishing; + // The publishing thread must own the first drain. A late subscriber + // must not steal it while publication is between storing the result + // and notifying: the publisher could otherwise return and complete + // the next batch before this batch has actually been dispatched. + if !state.publication_notified || state.dispatching { + return; + } + state.dispatching = true; + loop { + let groups = std::mem::take(&mut state.groups); + if groups.is_empty() { + state.dispatching = false; + return; + } + drop(state); + // Exactly one drainer invokes dispatchers, outside every lock. + // Registrations during dispatch (including reentrant ones) queue + // behind this batch rather than dispatching ahead of it. + for group in groups { + (group.dispatch)(CompletionBatch { + result: Arc::clone(&result), + callbacks: group.callbacks, + }); + } + state = self.callbacks.lock(); } } } @@ -288,19 +310,9 @@ mod tests { } #[test] - fn test_chunks_bound_work_and_panics_do_not_drop_remaining_callbacks() { + fn test_panics_do_not_drop_remaining_callbacks() { fn dispatch(batch: CompletionBatch) { - let chunks: Vec<_> = batch.into_chunks(64).collect(); - assert_eq!( - chunks - .iter() - .map(|chunk| chunk.callbacks.len()) - .collect::>(), - vec![64, 64, 3] - ); - for chunk in chunks { - chunk.run(); - } + batch.run(); } let broadcast = BroadcastOnce::default(); let (tx, rx) = std::sync::mpsc::channel(); @@ -406,6 +418,52 @@ mod tests { )); } + #[test] + fn test_late_registration_cannot_overtake_published_callbacks() { + let broadcast = BroadcastOnce::::default(); + let receiver = broadcast.receiver(); + let (tx, rx) = std::sync::mpsc::channel(); + let first = tx.clone(); + receiver.subscribe( + Box::new(move |_| first.send(1).unwrap()), + CompletionBatch::run, + ); + // Pause publication exactly between making data visible and notifying. + let result = Arc::new(Ok(42)); + *receiver.shared.data.write() = Some(Arc::clone(&result)); + receiver.subscribe(Box::new(move |_| tx.send(2).unwrap()), CompletionBatch::run); + // Merely observing the published data must not take ownership of the + // publisher's first dispatch, including its earlier registered callback. + assert!(matches!( + rx.try_recv(), + Err(std::sync::mpsc::TryRecvError::Empty) + )); + receiver.shared.notify_completion(result); + assert_eq!(rx.into_iter().collect::>(), vec![1, 2]); + } + + #[test] + fn test_reentrant_registration_runs_after_existing_callbacks() { + let broadcast = BroadcastOnce::::default(); + let receiver = broadcast.receiver(); + let nested = receiver.clone(); + let (tx, rx) = std::sync::mpsc::channel(); + let first = tx.clone(); + receiver.subscribe( + Box::new(move |_| { + first.send(1).unwrap(); + nested.subscribe( + Box::new(move |_| first.send(3).unwrap()), + CompletionBatch::run, + ); + }), + CompletionBatch::run, + ); + receiver.subscribe(Box::new(move |_| tx.send(2).unwrap()), CompletionBatch::run); + broadcast.broadcast(42); + assert_eq!(rx.into_iter().collect::>(), vec![1, 2, 3]); + } + struct InspectOnWake { shared: Arc>, woke: AtomicBool, diff --git a/fluss-rust/website/docs/user-guide/cpp/api-reference.md b/fluss-rust/website/docs/user-guide/cpp/api-reference.md index 0d65b0a6844..fa1c03caa33 100644 --- a/fluss-rust/website/docs/user-guide/cpp/api-reference.md +++ b/fluss-rust/website/docs/user-guide/cpp/api-reference.md @@ -26,7 +26,7 @@ Complete API reference for the Fluss C++ client. | `writer_dynamic_batch_size_min` | `int32_t` | `262144` (256 KB) | Lower bound for the dynamic batch size estimator (ignored when disabled) | | `writer_batch_timeout_ms` | `int64_t` | `100` | Maximum time in ms to wait for a writer batch to fill up before sending | | `writer_buffer_memory_size` | `size_t` | `67108864` (64 MiB) | Shared write-batch memory budget per Connection, across all tables and writers; not a process RSS limit | -| `writer_buffer_wait_timeout_ms` | `uint64_t` | `UINT64_MAX` | Maximum wait in ms for Rust write-buffer capacity; also bounds the whole callback submission (callback capacity plus buffer backpressure). `UINT64_MAX` waits indefinitely | +| `writer_buffer_wait_timeout_ms` | `uint64_t` | `UINT64_MAX` | Maximum wait in ms for Rust write-buffer capacity; also provides a shared budget for callback-capacity and buffer waits. `UINT64_MAX` waits indefinitely | | `writer_kv_backpressure_max_throttle_ms` | `uint64_t` | `3000` | Maximum per-bucket KV backpressure throttle in milliseconds | | `writer_bucket_no_key_assigner` | `std::string` | `"sticky"` | Bucket assignment strategy for tables without bucket keys: `"sticky"` or `"round_robin"` | | `scanner_remote_log_prefetch_num` | `size_t` | `4` | Number of remote log segments to prefetch | @@ -132,6 +132,7 @@ Complete API reference for the Fluss C++ client. | Method | Description | |----------------------------------------------|-------------------------| | `CreateWriter(AppendWriter& out) -> Result` | Create an append writer | +| `CreateWriter(AppendWriter& out, const WriteCallbackOptions& options) -> Result` | Create with a per-writer callback operation limit | ## `TableUpsert` @@ -140,6 +141,7 @@ Complete API reference for the Fluss C++ client. | `PartialUpdateByIndex(std::vector column_indices) -> TableUpsert&` | Configure partial update by column indices | | `PartialUpdateByName(std::vector column_names) -> TableUpsert&` | Configure partial update by column names | | `CreateWriter(UpsertWriter& out) -> Result` | Create an upsert writer | +| `CreateWriter(UpsertWriter& out, const WriteCallbackOptions& options) -> Result` | Create with a callback limit shared by upserts and deletes | ## `TableLookup` @@ -216,12 +218,13 @@ are retained. Filters apply to Arrow log scans and are rejected by `CreateBucket ## `WriteCallback` -`WriteCallback` is `std::function`. Pass a function pointer or a +`WriteCallback` is `std::function`. Pass a function pointer or a lambda to receive the final write outcome without a `WriteResult` handle or a call to `Wait()`: ```cpp -void OnWriteComplete(fluss::Result completed) { +void OnWriteComplete(const fluss::WriteCompletion& completion) { + const auto& completed = completion.result; if (!completed.Ok()) { // The record may already have been written. Logging alone is not recovery. std::cerr << "Write failed: " << completed.error_message << '\n'; @@ -243,9 +246,13 @@ final success or failure, subject to the lifetime and shutdown requirements belo Submission does not wait for acknowledgment, but may still wait for callback capacity and then for Rust writer buffer space under backpressure. -Callbacks run on a single shared worker, process wide, so they fire in the order -writes complete. Keep each callback -short and non-blocking. Do not call `Flush()` or `WriteResult::Wait()` inside a +`WriteCompletion` currently contains a `Result result` field. Its reference is valid +only during the callback; copy the result if application workers need it later. + +Callbacks run serially on one process-wide worker in dispatch order, off the I/O +threads. There is no cross-bucket submission-order guarantee. Late registrations +are queued when registered, behind callbacks already awaiting dispatch for that +batch. Keep each callback short and non-blocking. Do not call `Flush()` or `WriteResult::Wait()` inside a callback, and do not retry synchronously there. Record the outcome and run any retry in application logic outside the callback. @@ -277,7 +284,8 @@ retry in application logic outside the callback. exit or a crash. Applications requiring recovery across restarts must retain their source records or durable operation state independently of the callback. A simple approach is to drive writes from a replayable source and advance your - source position or offset only after `Flush()` succeeds, then replay from the + source position or offset only after `Flush()` succeeds and the relevant write + outcomes are confirmed successful, then replay from the last committed position on restart and deduplicate by identifier. `AppendArrowBatch` is not atomic across internal batches or buckets. Its single @@ -289,14 +297,17 @@ operational limits below before retrying a batch. ```cpp fluss::AppendWriter writer; -auto created = table.NewAppend().CreateWriter(writer); -// Check created before using writer. NewUpsert().CreateWriter behaves the same. +fluss::WriteCallbackOptions options; +options.max_pending_operations = 262144; // Default, per writer; must be positive. +auto created = table.NewAppend().CreateWriter(writer, options); +// CreateWriter(writer) uses the defaults. Check created before using writer. +// NewUpsert().CreateWriter accepts the same options. ``` -Each writer bounds its outstanding callback operations automatically, derived from -`writer_buffer_memory_size` divided by an estimate of the table's row size, so callback -admission tracks the same memory backpressure as the write buffer instead of being a -separate knob. Each callback submission reserves one slot **before** submitting to Rust +Each writer independently limits outstanding callback operations with +`max_pending_operations`. This bounds operation count, not bytes, and is not derived +from the schema or `writer_buffer_memory_size`. A callback submission reserves one +slot **before** submitting to Rust and holds it through user callback execution and capture cleanup. Submission errors and exceptions return the slot automatically. `Upsert` and `Delete` share their writer's limit; `AppendArrowBatch` consumes one slot per call, regardless of row count. Moving @@ -312,19 +323,20 @@ reschedule capacity-rejected submissions with backoff, but must not blindly retry every client error: other submission failures can have different effects, including partial ArrowBatch acceptance described below. -This timeout bounds the whole callback submission, callback capacity admission plus -buffer backpressure (Kafka max.block.ms style), so a callback submit returns a -definite result within it and a zero timeout makes submission non-blocking. It does -not cover network requests, core retries, or callback duration, and does not cancel -any accepted write. Do not hold a mutex needed by callbacks while submitting: a full +Callback capacity admission and buffer backpressure share this wait budget. Zero +makes these waits fail fast if resources are unavailable; it does not make the +entire API call non-blocking. The budget does not bound conversion, scheduling, +network requests, core retries, or callback duration, and does not cancel accepted +writes. Do not hold a mutex needed by callbacks while submitting: a full writer can wait for those callbacks to finish. -### Sizing write buffers +### Sizing callback capacity and write buffers -Callback admission is derived from the write buffer, so there is one budget to size: +Callback operations and buffered bytes have separate budgets: | Setting | Scope | What it limits | |---------|-------|----------------| +| `max_pending_operations` | Each writer | Operations awaiting write completion, callback execution, or capture cleanup | | `writer_buffer_memory_size` | Each Connection | Rust write-batch memory accounting shared by all its tables and writers, including writes using `Wait()` or fire-and-forget | The Connection buffer accounts for write batches, not callback captures, @@ -334,13 +346,28 @@ Sharing a Connection shares this budget: do not multiply it by the number of writers, but expect busy or stalled tables to compete for it. Separate Connections have separate budgets, which must be added when sizing the process or host. -Because each writer's callback admission scales with this buffer, a larger buffer also -allows more outstanding callback operations. If each outstanding operation retains -application data, account for that alongside the buffer when sizing process memory. -An `AppendArrowBatch` counts as one operation even when its batch contains many rows, -so large batches need a separate application byte budget. If callbacks are slow, -shorten or offload their work rather than relying on a larger buffer; a larger queue -does not fix a sustained completion-rate deficit. +Callback capacity can bind even while buffer space remains available. The byte +budget is released on batch completion, whereas a callback slot remains held +through callback execution and capture cleanup. With N writers, the sum of their +limits bounds outstanding operations, not a single process-wide limit. +The callback queue does not keep the Rust write-buffer permit after the batch +completes, so a slow callback does not by itself keep acknowledged write bytes +in the Rust buffer. It can still retain callback objects and application captures. +Once a writer reaches `max_pending_operations`, callback-based submissions to that +writer wait or fail according to `client.writer.buffer.wait-timeout`; with the +default unbounded timeout, they can wait indefinitely. + +For an initial estimate, multiply peak accepted operations/second by measured +submit-to-callback-finish latency, including ACK latency and queueing, then allow +headroom for bursts. Check that this count times retained bytes per operation fits +the callback memory budget, including wrapper overhead and captured objects. This +is a sizing estimate, not a worst-case memory guarantee. The default 262144 is a +starting limit, not a value derived from record size. Under sustained overload no +finite queue compensates for callbacks being slower than incoming completions. + +An `AppendArrowBatch` counts as one operation even when it contains many rows, so +large captures need a separate application byte budget. Increase either budget +only after measuring its pressure; increasing one does not increase the other. The following explicit settings are a tested high-throughput starting point. They are **not** new Connection defaults: @@ -369,8 +396,7 @@ eight-hour test also showed RSS growth, so it does not establish long-term memor stability or a universally safe configuration. For callback writes, `writer_buffer_wait_timeout_ms` (client.writer.buffer.wait-timeout) -bounds the whole submission, including both the callback-capacity wait and the -buffer-backpressure wait, so a callback submit returns within it. For `Wait()` and +provides one shared budget for callback-capacity and buffer-backpressure waits. For `Wait()` and fire-and-forget writes, the same setting governs the buffer wait. Set it to suit upstream latency and overload handling, rather than assuming it bounds ACKs, retries, or callback duration. Lowering it rejects sooner; it does @@ -387,7 +413,7 @@ threads. There is no need to create a waiting thread, call `Wait()`, or poll for callback delivery. The SDK takes ownership of the callback and its captures. Callbacks run on a single -shared background worker in the order writes complete, and may start before the +shared background worker in dispatch order, and may start before the submitting call returns. Keep callbacks short; synchronize access to shared state and keep captured references valid until the callback finishes. A callback that blocks stalls the worker for every writer, so do not wait for other callbacks from inside one. @@ -416,10 +442,10 @@ submission loop can handle the handoff. Callbacks register directly with their internal write batch, rather than creating an asynchronous ACK-waiting task for each row. When a batch completes, -its callbacks are dispatched to four process-wide callback workers in jobs of -at most 64 callbacks. Each worker takes one job at a time; every registered -callback still runs individually. Registrations arriving after batch completion -are dispatched separately, without waiting to fill a job. Both Arrow log and KV +its registered callbacks are dispatched together to the single process-wide +worker. Every callback still runs individually. Late registrations join the batch's +dispatch queue without overtaking its callbacks already awaiting dispatch. +Both Arrow log and KV write batches use this path. An `AppendArrowBatch` spanning multiple internal batches aggregates their results into the operation's single callback. @@ -435,8 +461,12 @@ callback fails immediately if its target writer's capacity is full, regardless of `client.writer.buffer.wait-timeout`, to avoid blocking the worker on its own capacity. This applies across writers as well. It does not remove Rust buffer waits or make arbitrary blocking SDK calls deadlock-free. -Exclusive writer access is still required. If a dedicated worker cannot be -initialized, the SDK falls back to its runtime blocking pool to preserve delivery. +Exclusive writer access is still required. The SDK initializes its callback worker +before accepting a callback write. Initialization failure returns a synchronous +client error without submitting data or registering a callback. It does not fall +back to a parallel pool; initialization failure remains in effect for the process. +An unexpected disconnection of the process-lifetime worker is a fatal internal +error, not a policy for discarding or reordering notifications. For shutdown, stop and join submitting threads, then call `Flush()` outside a callback. It first runs the Rust write flush and, if that succeeds, blocks until @@ -448,16 +478,15 @@ that every submitted operation succeeded. If the write flush fails, `Flush()` returns that error and callbacks may still be pending; do not release their referenced state. `Flush()` does not wait for work handed to application workers or retry queues; -those need their own shutdown handling. When called inside a callback, only the -callback-wait phase is skipped; the Rust write flush can still block. Do not use -this path as a shutdown barrier. +those need their own shutdown handling. Calling `Flush()` inside any SDK write +callback returns a client error before flushing, including calls on another writer. ### Compatibility and operational limits - Existing fire-and-forget and `WriteResult::Wait()` overloads retain their result semantics and do not consume callback capacity. Rust callers can still - `.await` a `WriteResultFuture`. Callback overloads now bound the whole submission - by `client.writer.buffer.wait-timeout` (callback capacity plus buffer backpressure); + `.await` a `WriteResultFuture`. Callback overloads share + `client.writer.buffer.wait-timeout` across capacity and buffer waits; existing callback callers may block up to that timeout or receive a capacity or buffer error. Completion follows the configured acknowledgment policy; a callback does not @@ -478,8 +507,9 @@ this path as a shutdown barrier. registered, so handle the returned error and do not assume an all-or-nothing retry. - The single-worker execution and whole-batch job dispatch are implementation details, not latency guarantees. A slow callback delays the callbacks queued behind it, and - slow callbacks from one connection can delay another connection. - The blocking-pool fallback preserves delivery but is not a second ordered worker. + slow callbacks from one connection can delay another connection. The worker is + initialized before callback writes are accepted; initialization failure rejects the + submission rather than switching to an unordered fallback. - Callback delivery is in memory only. There is no end-to-end callback deadline or durable recovery of pending notifications. Connection or writer destruction is not a callback-drain barrier; process exit,